drivers/dma/sf-pdma/sf-pdma.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * SiFive FU540 Platform DMA driver
   4  * Copyright (C) 2019 SiFive
   5  *
   6  * Based partially on:
   7  * - drivers/dma/fsl-edma.c
   8  * - drivers/dma/dw-edma/
   9  * - drivers/dma/pxa-dma.c
  10  *
  11  * See the following sources for further documentation:
  12  * - Chapter 12 "Platform DMA Engine (PDMA)" of
  13  *   SiFive FU540-C000 v1.0
  14  *   https://static.dev.sifive.com/FU540-C000-v1.0.pdf
  15  */
  16 #include <linux/module.h>
  17 #include <linux/device.h>
  18 #include <linux/kernel.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/mod_devicetable.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/of.h>
  23
  24 #include "sf-pdma.h"
  25
  26 #ifndef readq
  27 static inline unsigned long long readq(void __iomem *addr)
  28 {
  29         return readl(addr) | (((unsigned long long)readl(addr + 4)) << 32LL);
  30 }
  31 #endif
  32
  33 #ifndef writeq
  34 static inline void writeq(unsigned long long v, void __iomem *addr)
  35 {
  36         writel(lower_32_bits(v), addr);
  37         writel(upper_32_bits(v), addr + 4);
  38 }
  39 #endif
  40
  41 static inline struct sf_pdma_chan *to_sf_pdma_chan(struct dma_chan *dchan)
  42 {
  43         return container_of(dchan, struct sf_pdma_chan, vchan.chan);
  44 }
  45
  46 static inline struct sf_pdma_desc *to_sf_pdma_desc(struct virt_dma_desc *vd)
  47 {
  48         return container_of(vd, struct sf_pdma_desc, vdesc);
  49 }
  50
  51 static struct sf_pdma_desc *sf_pdma_alloc_desc(struct sf_pdma_chan *chan)
  52 {
  53         struct sf_pdma_desc *desc;
  54         unsigned long flags;
  55
  56         spin_lock_irqsave(&chan->lock, flags);
  57
  58         if (chan->desc && !chan->desc->in_use) {
  59                 spin_unlock_irqrestore(&chan->lock, flags);
  60                 return chan->desc;
  61         }
  62
  63         spin_unlock_irqrestore(&chan->lock, flags);
  64
  65         desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
  66         if (!desc)
  67                 return NULL;
  68
  69         desc->chan = chan;
  70
  71         return desc;
  72 }
  73
  74 static void sf_pdma_fill_desc(struct sf_pdma_desc *desc,
  75                               u64 dst, u64 src, u64 size)
  76 {
  77         desc->xfer_type = PDMA_FULL_SPEED;
  78         desc->xfer_size = size;
  79         desc->dst_addr = dst;
  80         desc->src_addr = src;
  81 }
  82
  83 static void sf_pdma_disclaim_chan(struct sf_pdma_chan *chan)
  84 {
  85         struct pdma_regs *regs = &chan->regs;
  86
  87         writel(PDMA_CLEAR_CTRL, regs->ctrl);
  88 }
  89
  90 static struct dma_async_tx_descriptor *
  91 sf_pdma_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dest, dma_addr_t src,
  92                         size_t len, unsigned long flags)
  93 {
  94         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
  95         struct sf_pdma_desc *desc;
  96
  97         if (chan && (!len || !dest || !src)) {
  98                 dev_err(chan->pdma->dma_dev.dev,
  99                         "Please check dma len, dest, src!\n");
 100                 return NULL;
 101         }
 102
 103         desc = sf_pdma_alloc_desc(chan);
 104         if (!desc)
 105                 return NULL;
 106
 107         desc->in_use = true;
 108         desc->dirn = DMA_MEM_TO_MEM;
 109         desc->async_tx = vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 110
 111         spin_lock_irqsave(&chan->vchan.lock, flags);
 112         chan->desc = desc;
 113         sf_pdma_fill_desc(desc, dest, src, len);
 114         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 115
 116         return desc->async_tx;
 117 }
 118
 119 static int sf_pdma_slave_config(struct dma_chan *dchan,
 120                                 struct dma_slave_config *cfg)
 121 {
 122         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 123
 124         memcpy(&chan->cfg, cfg, sizeof(*cfg));
 125
 126         return 0;
 127 }
 128
 129 static int sf_pdma_alloc_chan_resources(struct dma_chan *dchan)
 130 {
 131         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 132         struct pdma_regs *regs = &chan->regs;
 133
 134         dma_cookie_init(dchan);
 135         writel(PDMA_CLAIM_MASK, regs->ctrl);
 136
 137         return 0;
 138 }
 139
 140 static void sf_pdma_disable_request(struct sf_pdma_chan *chan)
 141 {
 142         struct pdma_regs *regs = &chan->regs;
 143
 144         writel(readl(regs->ctrl) & ~PDMA_RUN_MASK, regs->ctrl);
 145 }
 146
 147 static void sf_pdma_free_chan_resources(struct dma_chan *dchan)
 148 {
 149         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 150         unsigned long flags;
 151         LIST_HEAD(head);
 152
 153         spin_lock_irqsave(&chan->vchan.lock, flags);
 154         sf_pdma_disable_request(chan);
 155         kfree(chan->desc);
 156         chan->desc = NULL;
 157         vchan_get_all_descriptors(&chan->vchan, &head);
 158         sf_pdma_disclaim_chan(chan);
 159         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 160         vchan_dma_desc_free_list(&chan->vchan, &head);
 161 }
 162
 163 static size_t sf_pdma_desc_residue(struct sf_pdma_chan *chan,
 164                                    dma_cookie_t cookie)
 165 {
 166         struct virt_dma_desc *vd = NULL;
 167         struct pdma_regs *regs = &chan->regs;
 168         unsigned long flags;
 169         u64 residue = 0;
 170         struct sf_pdma_desc *desc;
 171         struct dma_async_tx_descriptor *tx;
 172
 173         spin_lock_irqsave(&chan->vchan.lock, flags);
 174
 175         tx = &chan->desc->vdesc.tx;
 176         if (cookie == tx->chan->completed_cookie)
 177                 goto out;
 178
 179         if (cookie == tx->cookie) {
 180                 residue = readq(regs->residue);
 181         } else {
 182                 vd = vchan_find_desc(&chan->vchan, cookie);
 183                 if (!vd)
 184                         goto out;
 185
 186                 desc = to_sf_pdma_desc(vd);
 187                 residue = desc->xfer_size;
 188         }
 189
 190 out:
 191         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 192         return residue;
 193 }
 194
 195 static enum dma_status
 196 sf_pdma_tx_status(struct dma_chan *dchan,
 197                   dma_cookie_t cookie,
 198                   struct dma_tx_state *txstate)
 199 {
 200         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 201         enum dma_status status;
 202
 203         status = dma_cookie_status(dchan, cookie, txstate);
 204
 205         if (txstate && status != DMA_ERROR)
 206                 dma_set_residue(txstate, sf_pdma_desc_residue(chan, cookie));
 207
 208         return status;
 209 }
 210
 211 static int sf_pdma_terminate_all(struct dma_chan *dchan)
 212 {
 213         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 214         unsigned long flags;
 215         LIST_HEAD(head);
 216
 217         spin_lock_irqsave(&chan->vchan.lock, flags);
 218         sf_pdma_disable_request(chan);
 219         kfree(chan->desc);
 220         chan->desc = NULL;
 221         chan->xfer_err = false;
 222         vchan_get_all_descriptors(&chan->vchan, &head);
 223         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 224         vchan_dma_desc_free_list(&chan->vchan, &head);
 225
 226         return 0;
 227 }
 228
 229 static void sf_pdma_enable_request(struct sf_pdma_chan *chan)
 230 {
 231         struct pdma_regs *regs = &chan->regs;
 232         u32 v;
 233
 234         v = PDMA_CLAIM_MASK |
 235                 PDMA_ENABLE_DONE_INT_MASK |
 236                 PDMA_ENABLE_ERR_INT_MASK |
 237                 PDMA_RUN_MASK;
 238
 239         writel(v, regs->ctrl);
 240 }
 241
 242 static void sf_pdma_xfer_desc(struct sf_pdma_chan *chan)
 243 {
 244         struct sf_pdma_desc *desc = chan->desc;
 245         struct pdma_regs *regs = &chan->regs;
 246
 247         if (!desc) {
 248                 dev_err(chan->pdma->dma_dev.dev, "NULL desc.\n");
 249                 return;
 250         }
 251
 252         writel(desc->xfer_type, regs->xfer_type);
 253         writeq(desc->xfer_size, regs->xfer_size);
 254         writeq(desc->dst_addr, regs->dst_addr);
 255         writeq(desc->src_addr, regs->src_addr);
 256
 257         chan->desc = desc;
 258         chan->status = DMA_IN_PROGRESS;
 259         sf_pdma_enable_request(chan);
 260 }
 261
 262 static void sf_pdma_issue_pending(struct dma_chan *dchan)
 263 {
 264         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 265         unsigned long flags;
 266
 267         spin_lock_irqsave(&chan->vchan.lock, flags);
 268
 269         if (vchan_issue_pending(&chan->vchan) && chan->desc)
 270                 sf_pdma_xfer_desc(chan);
 271
 272         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 273 }
 274
 275 static void sf_pdma_free_desc(struct virt_dma_desc *vdesc)
 276 {
 277         struct sf_pdma_desc *desc;
 278
 279         desc = to_sf_pdma_desc(vdesc);
 280         desc->in_use = false;
 281 }
 282
 283 static void sf_pdma_donebh_tasklet(unsigned long arg)
 284 {
 285         struct sf_pdma_chan *chan = (struct sf_pdma_chan *)arg;
 286         struct sf_pdma_desc *desc = chan->desc;
 287         unsigned long flags;
 288
 289         spin_lock_irqsave(&chan->lock, flags);
 290         if (chan->xfer_err) {
 291                 chan->retries = MAX_RETRY;
 292                 chan->status = DMA_COMPLETE;
 293                 chan->xfer_err = false;
 294         }
 295         spin_unlock_irqrestore(&chan->lock, flags);
 296
 297         dmaengine_desc_get_callback_invoke(desc->async_tx, NULL);
 298 }
 299
 300 static void sf_pdma_errbh_tasklet(unsigned long arg)
 301 {
 302         struct sf_pdma_chan *chan = (struct sf_pdma_chan *)arg;
 303         struct sf_pdma_desc *desc = chan->desc;
 304         unsigned long flags;
 305
 306         spin_lock_irqsave(&chan->lock, flags);
 307         if (chan->retries <= 0) {
 308                 /* fail to recover */
 309                 spin_unlock_irqrestore(&chan->lock, flags);
 310                 dmaengine_desc_get_callback_invoke(desc->async_tx, NULL);
 311         } else {
 312                 /* retry */
 313                 chan->retries--;
 314                 chan->xfer_err = true;
 315                 chan->status = DMA_ERROR;
 316
 317                 sf_pdma_enable_request(chan);
 318                 spin_unlock_irqrestore(&chan->lock, flags);
 319         }
 320 }
 321
 322 static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id)
 323 {
 324         struct sf_pdma_chan *chan = dev_id;
 325         struct pdma_regs *regs = &chan->regs;
 326         unsigned long flags;
 327         u64 residue;
 328
 329         spin_lock_irqsave(&chan->vchan.lock, flags);
 330         writel((readl(regs->ctrl)) & ~PDMA_DONE_STATUS_MASK, regs->ctrl);
 331         residue = readq(regs->residue);
 332
 333         if (!residue) {
 334                 list_del(&chan->desc->vdesc.node);
 335                 vchan_cookie_complete(&chan->desc->vdesc);
 336         } else {
 337                 /* submit next trascatioin if possible */
 338                 struct sf_pdma_desc *desc = chan->desc;
 339
 340                 desc->src_addr += desc->xfer_size - residue;
 341                 desc->dst_addr += desc->xfer_size - residue;
 342                 desc->xfer_size = residue;
 343
 344                 sf_pdma_xfer_desc(chan);
 345         }
 346
 347         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 348
 349         tasklet_hi_schedule(&chan->done_tasklet);
 350
 351         return IRQ_HANDLED;
 352 }
 353
 354 static irqreturn_t sf_pdma_err_isr(int irq, void *dev_id)
 355 {
 356         struct sf_pdma_chan *chan = dev_id;
 357         struct pdma_regs *regs = &chan->regs;
 358         unsigned long flags;
 359
 360         spin_lock_irqsave(&chan->lock, flags);
 361         writel((readl(regs->ctrl)) & ~PDMA_ERR_STATUS_MASK, regs->ctrl);
 362         spin_unlock_irqrestore(&chan->lock, flags);
 363
 364         tasklet_schedule(&chan->err_tasklet);
 365
 366         return IRQ_HANDLED;
 367 }
 368
 369 /**
 370  * sf_pdma_irq_init() - Init PDMA IRQ Handlers
 371  * @pdev: pointer of platform_device
 372  * @pdma: pointer of PDMA engine. Caller should check NULL
 373  *
 374  * Initialize DONE and ERROR interrupt handler for 4 channels. Caller should
 375  * make sure the pointer passed in are non-NULL. This function should be called
 376  * only one time during the device probe.
 377  *
 378  * Context: Any context.
 379  *
 380  * Return:
 381  * * 0          - OK to init all IRQ handlers
 382  * * -EINVAL    - Fail to request IRQ
 383  */
 384 static int sf_pdma_irq_init(struct platform_device *pdev, struct sf_pdma *pdma)
 385 {
 386         int irq, r, i;
 387         struct sf_pdma_chan *chan;
 388
 389         for (i = 0; i < pdma->n_chans; i++) {
 390                 chan = &pdma->chans[i];
 391
 392                 irq = platform_get_irq(pdev, i * 2);
 393                 if (irq < 0) {
 394                         dev_err(&pdev->dev, "ch(%d) Can't get done irq.\n", i);
 395                         return -EINVAL;
 396                 }
 397
 398                 r = devm_request_irq(&pdev->dev, irq, sf_pdma_done_isr, 0,
 399                                      dev_name(&pdev->dev), (void *)chan);
 400                 if (r) {
 401                         dev_err(&pdev->dev, "Fail to attach done ISR: %d\n", r);
 402                         return -EINVAL;
 403                 }
 404
 405                 chan->txirq = irq;
 406
 407                 irq = platform_get_irq(pdev, (i * 2) + 1);
 408                 if (irq < 0) {
 409                         dev_err(&pdev->dev, "ch(%d) Can't get err irq.\n", i);
 410                         return -EINVAL;
 411                 }
 412
 413                 r = devm_request_irq(&pdev->dev, irq, sf_pdma_err_isr, 0,
 414                                      dev_name(&pdev->dev), (void *)chan);
 415                 if (r) {
 416                         dev_err(&pdev->dev, "Fail to attach err ISR: %d\n", r);
 417                         return -EINVAL;
 418                 }
 419
 420                 chan->errirq = irq;
 421         }
 422
 423         return 0;
 424 }
 425
 426 /**
 427  * sf_pdma_setup_chans() - Init settings of each channel
 428  * @pdma: pointer of PDMA engine. Caller should check NULL
 429  *
 430  * Initialize all data structure and register base. Caller should make sure
 431  * the pointer passed in are non-NULL. This function should be called only
 432  * one time during the device probe.
 433  *
 434  * Context: Any context.
 435  *
 436  * Return: none
 437  */
 438 static void sf_pdma_setup_chans(struct sf_pdma *pdma)
 439 {
 440         int i;
 441         struct sf_pdma_chan *chan;
 442
 443         INIT_LIST_HEAD(&pdma->dma_dev.channels);
 444
 445         for (i = 0; i < pdma->n_chans; i++) {
 446                 chan = &pdma->chans[i];
 447
 448                 chan->regs.ctrl =
 449                         SF_PDMA_REG_BASE(i) + PDMA_CTRL;
 450                 chan->regs.xfer_type =
 451                         SF_PDMA_REG_BASE(i) + PDMA_XFER_TYPE;
 452                 chan->regs.xfer_size =
 453                         SF_PDMA_REG_BASE(i) + PDMA_XFER_SIZE;
 454                 chan->regs.dst_addr =
 455                         SF_PDMA_REG_BASE(i) + PDMA_DST_ADDR;
 456                 chan->regs.src_addr =
 457                         SF_PDMA_REG_BASE(i) + PDMA_SRC_ADDR;
 458                 chan->regs.act_type =
 459                         SF_PDMA_REG_BASE(i) + PDMA_ACT_TYPE;
 460                 chan->regs.residue =
 461                         SF_PDMA_REG_BASE(i) + PDMA_REMAINING_BYTE;
 462                 chan->regs.cur_dst_addr =
 463                         SF_PDMA_REG_BASE(i) + PDMA_CUR_DST_ADDR;
 464                 chan->regs.cur_src_addr =
 465                         SF_PDMA_REG_BASE(i) + PDMA_CUR_SRC_ADDR;
 466
 467                 chan->pdma = pdma;
 468                 chan->pm_state = RUNNING;
 469                 chan->slave_id = i;
 470                 chan->xfer_err = false;
 471                 spin_lock_init(&chan->lock);
 472
 473                 chan->vchan.desc_free = sf_pdma_free_desc;
 474                 vchan_init(&chan->vchan, &pdma->dma_dev);
 475
 476                 writel(PDMA_CLEAR_CTRL, chan->regs.ctrl);
 477
 478                 tasklet_init(&chan->done_tasklet,
 479                              sf_pdma_donebh_tasklet, (unsigned long)chan);
 480                 tasklet_init(&chan->err_tasklet,
 481                              sf_pdma_errbh_tasklet, (unsigned long)chan);
 482         }
 483 }
 484
 485 static int sf_pdma_probe(struct platform_device *pdev)
 486 {
 487         struct sf_pdma *pdma;
 488         struct sf_pdma_chan *chan;
 489         struct resource *res;
 490         int len, chans;
 491         int ret;
 492         const enum dma_slave_buswidth widths =
 493                 DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES |
 494                 DMA_SLAVE_BUSWIDTH_4_BYTES | DMA_SLAVE_BUSWIDTH_8_BYTES |
 495                 DMA_SLAVE_BUSWIDTH_16_BYTES | DMA_SLAVE_BUSWIDTH_32_BYTES |
 496                 DMA_SLAVE_BUSWIDTH_64_BYTES;
 497
 498         chans = PDMA_NR_CH;
 499         len = sizeof(*pdma) + sizeof(*chan) * chans;
 500         pdma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
 501         if (!pdma)
 502                 return -ENOMEM;
 503
 504         pdma->n_chans = chans;
 505
 506         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 507         pdma->membase = devm_ioremap_resource(&pdev->dev, res);
 508         if (IS_ERR(pdma->membase))
 509                 return PTR_ERR(pdma->membase);
 510
 511         ret = sf_pdma_irq_init(pdev, pdma);
 512         if (ret)
 513                 return ret;
 514
 515         sf_pdma_setup_chans(pdma);
 516
 517         pdma->dma_dev.dev = &pdev->dev;
 518
 519         /* Setup capability */
 520         dma_cap_set(DMA_MEMCPY, pdma->dma_dev.cap_mask);
 521         pdma->dma_dev.copy_align = 2;
 522         pdma->dma_dev.src_addr_widths = widths;
 523         pdma->dma_dev.dst_addr_widths = widths;
 524         pdma->dma_dev.directions = BIT(DMA_MEM_TO_MEM);
 525         pdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 526         pdma->dma_dev.descriptor_reuse = true;
 527
 528         /* Setup DMA APIs */
 529         pdma->dma_dev.device_alloc_chan_resources =
 530                 sf_pdma_alloc_chan_resources;
 531         pdma->dma_dev.device_free_chan_resources =
 532                 sf_pdma_free_chan_resources;
 533         pdma->dma_dev.device_tx_status = sf_pdma_tx_status;
 534         pdma->dma_dev.device_prep_dma_memcpy = sf_pdma_prep_dma_memcpy;
 535         pdma->dma_dev.device_config = sf_pdma_slave_config;
 536         pdma->dma_dev.device_terminate_all = sf_pdma_terminate_all;
 537         pdma->dma_dev.device_issue_pending = sf_pdma_issue_pending;
 538
 539         platform_set_drvdata(pdev, pdma);
 540
 541         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 542         if (ret)
 543                 dev_warn(&pdev->dev,
 544                          "Failed to set DMA mask. Fall back to default.\n");
 545
 546         ret = dma_async_device_register(&pdma->dma_dev);
 547         if (ret) {
 548                 dev_err(&pdev->dev,
 549                         "Can't register SiFive Platform DMA. (%d)\n", ret);
 550                 return ret;
 551         }
 552
 553         return 0;
 554 }
 555
 556 static int sf_pdma_remove(struct platform_device *pdev)
 557 {
 558         struct sf_pdma *pdma = platform_get_drvdata(pdev);
 559         struct sf_pdma_chan *ch;
 560         int i;
 561
 562         for (i = 0; i < PDMA_NR_CH; i++) {
 563                 ch = &pdma->chans[i];
 564
 565                 devm_free_irq(&pdev->dev, ch->txirq, ch);
 566                 devm_free_irq(&pdev->dev, ch->errirq, ch);
 567                 list_del(&ch->vchan.chan.device_node);
 568                 tasklet_kill(&ch->vchan.task);
 569                 tasklet_kill(&ch->done_tasklet);
 570                 tasklet_kill(&ch->err_tasklet);
 571         }
 572
 573         dma_async_device_unregister(&pdma->dma_dev);
 574
 575         return 0;
 576 }
 577
 578 static const struct of_device_id sf_pdma_dt_ids[] = {
 579         { .compatible = "sifive,fu540-c000-pdma" },
 580         {},
 581 };
 582 MODULE_DEVICE_TABLE(of, sf_pdma_dt_ids);
 583
 584 static struct platform_driver sf_pdma_driver = {
 585         .probe          = sf_pdma_probe,
 586         .remove         = sf_pdma_remove,
 587         .driver         = {
 588                 .name   = "sf-pdma",
 589                 .of_match_table = of_match_ptr(sf_pdma_dt_ids),
 590         },
 591 };
 592
 593 static int __init sf_pdma_init(void)
 594 {
 595         return platform_driver_register(&sf_pdma_driver);
 596 }
 597
 598 static void __exit sf_pdma_exit(void)
 599 {
 600         platform_driver_unregister(&sf_pdma_driver);
 601 }
 602
 603 /* do early init */
 604 subsys_initcall(sf_pdma_init);
 605 module_exit(sf_pdma_exit);
 606
 607 MODULE_LICENSE("GPL v2");
 608 MODULE_DESCRIPTION("SiFive Platform DMA driver");
 609 MODULE_AUTHOR("Green Wan <green.wan@sifive.com>");