drivers/spi/spi-stm32-qspi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) STMicroelectronics 2018 - All Rights Reserved
   4  * Author: Ludovic Barre <ludovic.barre@st.com> for STMicroelectronics.
   5  */
   6 #include <linux/bitfield.h>
   7 #include <linux/clk.h>
   8 #include <linux/dmaengine.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/errno.h>
  11 #include <linux/gpio/consumer.h>
  12 #include <linux/io.h>
  13 #include <linux/iopoll.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/module.h>
  16 #include <linux/mutex.h>
  17 #include <linux/of.h>
  18 #include <linux/pinctrl/consumer.h>
  19 #include <linux/pm_runtime.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/reset.h>
  22 #include <linux/sizes.h>
  23 #include <linux/spi/spi-mem.h>
  24
  25 #define QSPI_CR                 0x00
  26 #define CR_EN                   BIT(0)
  27 #define CR_ABORT                BIT(1)
  28 #define CR_DMAEN                BIT(2)
  29 #define CR_TCEN                 BIT(3)
  30 #define CR_SSHIFT               BIT(4)
  31 #define CR_DFM                  BIT(6)
  32 #define CR_FSEL                 BIT(7)
  33 #define CR_FTHRES_SHIFT         8
  34 #define CR_TEIE                 BIT(16)
  35 #define CR_TCIE                 BIT(17)
  36 #define CR_FTIE                 BIT(18)
  37 #define CR_SMIE                 BIT(19)
  38 #define CR_TOIE                 BIT(20)
  39 #define CR_APMS                 BIT(22)
  40 #define CR_PRESC_MASK           GENMASK(31, 24)
  41
  42 #define QSPI_DCR                0x04
  43 #define DCR_FSIZE_MASK          GENMASK(20, 16)
  44
  45 #define QSPI_SR                 0x08
  46 #define SR_TEF                  BIT(0)
  47 #define SR_TCF                  BIT(1)
  48 #define SR_FTF                  BIT(2)
  49 #define SR_SMF                  BIT(3)
  50 #define SR_TOF                  BIT(4)
  51 #define SR_BUSY                 BIT(5)
  52 #define SR_FLEVEL_MASK          GENMASK(13, 8)
  53
  54 #define QSPI_FCR                0x0c
  55 #define FCR_CTEF                BIT(0)
  56 #define FCR_CTCF                BIT(1)
  57 #define FCR_CSMF                BIT(3)
  58
  59 #define QSPI_DLR                0x10
  60
  61 #define QSPI_CCR                0x14
  62 #define CCR_INST_MASK           GENMASK(7, 0)
  63 #define CCR_IMODE_MASK          GENMASK(9, 8)
  64 #define CCR_ADMODE_MASK         GENMASK(11, 10)
  65 #define CCR_ADSIZE_MASK         GENMASK(13, 12)
  66 #define CCR_DCYC_MASK           GENMASK(22, 18)
  67 #define CCR_DMODE_MASK          GENMASK(25, 24)
  68 #define CCR_FMODE_MASK          GENMASK(27, 26)
  69 #define CCR_FMODE_INDW          (0U << 26)
  70 #define CCR_FMODE_INDR          (1U << 26)
  71 #define CCR_FMODE_APM           (2U << 26)
  72 #define CCR_FMODE_MM            (3U << 26)
  73 #define CCR_BUSWIDTH_0          0x0
  74 #define CCR_BUSWIDTH_1          0x1
  75 #define CCR_BUSWIDTH_2          0x2
  76 #define CCR_BUSWIDTH_4          0x3
  77
  78 #define QSPI_AR                 0x18
  79 #define QSPI_ABR                0x1c
  80 #define QSPI_DR                 0x20
  81 #define QSPI_PSMKR              0x24
  82 #define QSPI_PSMAR              0x28
  83 #define QSPI_PIR                0x2c
  84 #define QSPI_LPTR               0x30
  85
  86 #define STM32_QSPI_MAX_MMAP_SZ  SZ_256M
  87 #define STM32_QSPI_MAX_NORCHIP  2
  88
  89 #define STM32_FIFO_TIMEOUT_US 30000
  90 #define STM32_BUSY_TIMEOUT_US 100000
  91 #define STM32_ABT_TIMEOUT_US 100000
  92 #define STM32_COMP_TIMEOUT_MS 1000
  93 #define STM32_AUTOSUSPEND_DELAY -1
  94
  95 struct stm32_qspi_flash {
  96         u32 cs;
  97         u32 presc;
  98 };
  99
 100 struct stm32_qspi {
 101         struct device *dev;
 102         struct spi_controller *ctrl;
 103         phys_addr_t phys_base;
 104         void __iomem *io_base;
 105         void __iomem *mm_base;
 106         resource_size_t mm_size;
 107         struct clk *clk;
 108         u32 clk_rate;
 109         struct stm32_qspi_flash flash[STM32_QSPI_MAX_NORCHIP];
 110         struct completion data_completion;
 111         struct completion match_completion;
 112         u32 fmode;
 113
 114         struct dma_chan *dma_chtx;
 115         struct dma_chan *dma_chrx;
 116         struct completion dma_completion;
 117
 118         u32 cr_reg;
 119         u32 dcr_reg;
 120         unsigned long status_timeout;
 121
 122         /*
 123          * to protect device configuration, could be different between
 124          * 2 flash access (bk1, bk2)
 125          */
 126         struct mutex lock;
 127 };
 128
 129 static irqreturn_t stm32_qspi_irq(int irq, void *dev_id)
 130 {
 131         struct stm32_qspi *qspi = (struct stm32_qspi *)dev_id;
 132         u32 cr, sr;
 133
 134         cr = readl_relaxed(qspi->io_base + QSPI_CR);
 135         sr = readl_relaxed(qspi->io_base + QSPI_SR);
 136
 137         if (cr & CR_SMIE && sr & SR_SMF) {
 138                 /* disable irq */
 139                 cr &= ~CR_SMIE;
 140                 writel_relaxed(cr, qspi->io_base + QSPI_CR);
 141                 complete(&qspi->match_completion);
 142
 143                 return IRQ_HANDLED;
 144         }
 145
 146         if (sr & (SR_TEF | SR_TCF)) {
 147                 /* disable irq */
 148                 cr &= ~CR_TCIE & ~CR_TEIE;
 149                 writel_relaxed(cr, qspi->io_base + QSPI_CR);
 150                 complete(&qspi->data_completion);
 151         }
 152
 153         return IRQ_HANDLED;
 154 }
 155
 156 static void stm32_qspi_read_fifo(u8 *val, void __iomem *addr)
 157 {
 158         *val = readb_relaxed(addr);
 159 }
 160
 161 static void stm32_qspi_write_fifo(u8 *val, void __iomem *addr)
 162 {
 163         writeb_relaxed(*val, addr);
 164 }
 165
 166 static int stm32_qspi_tx_poll(struct stm32_qspi *qspi,
 167                               const struct spi_mem_op *op)
 168 {
 169         void (*tx_fifo)(u8 *val, void __iomem *addr);
 170         u32 len = op->data.nbytes, sr;
 171         u8 *buf;
 172         int ret;
 173
 174         if (op->data.dir == SPI_MEM_DATA_IN) {
 175                 tx_fifo = stm32_qspi_read_fifo;
 176                 buf = op->data.buf.in;
 177
 178         } else {
 179                 tx_fifo = stm32_qspi_write_fifo;
 180                 buf = (u8 *)op->data.buf.out;
 181         }
 182
 183         while (len--) {
 184                 ret = readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_SR,
 185                                                         sr, (sr & SR_FTF), 1,
 186                                                         STM32_FIFO_TIMEOUT_US);
 187                 if (ret) {
 188                         dev_err(qspi->dev, "fifo timeout (len:%d stat:%#x)\n",
 189                                 len, sr);
 190                         return ret;
 191                 }
 192                 tx_fifo(buf++, qspi->io_base + QSPI_DR);
 193         }
 194
 195         return 0;
 196 }
 197
 198 static int stm32_qspi_tx_mm(struct stm32_qspi *qspi,
 199                             const struct spi_mem_op *op)
 200 {
 201         memcpy_fromio(op->data.buf.in, qspi->mm_base + op->addr.val,
 202                       op->data.nbytes);
 203         return 0;
 204 }
 205
 206 static void stm32_qspi_dma_callback(void *arg)
 207 {
 208         struct completion *dma_completion = arg;
 209
 210         complete(dma_completion);
 211 }
 212
 213 static int stm32_qspi_tx_dma(struct stm32_qspi *qspi,
 214                              const struct spi_mem_op *op)
 215 {
 216         struct dma_async_tx_descriptor *desc;
 217         enum dma_transfer_direction dma_dir;
 218         struct dma_chan *dma_ch;
 219         struct sg_table sgt;
 220         dma_cookie_t cookie;
 221         u32 cr, t_out;
 222         int err;
 223
 224         if (op->data.dir == SPI_MEM_DATA_IN) {
 225                 dma_dir = DMA_DEV_TO_MEM;
 226                 dma_ch = qspi->dma_chrx;
 227         } else {
 228                 dma_dir = DMA_MEM_TO_DEV;
 229                 dma_ch = qspi->dma_chtx;
 230         }
 231
 232         /*
 233          * spi_map_buf return -EINVAL if the buffer is not DMA-able
 234          * (DMA-able: in vmalloc | kmap | virt_addr_valid)
 235          */
 236         err = spi_controller_dma_map_mem_op_data(qspi->ctrl, op, &sgt);
 237         if (err)
 238                 return err;
 239
 240         desc = dmaengine_prep_slave_sg(dma_ch, sgt.sgl, sgt.nents,
 241                                        dma_dir, DMA_PREP_INTERRUPT);
 242         if (!desc) {
 243                 err = -ENOMEM;
 244                 goto out_unmap;
 245         }
 246
 247         cr = readl_relaxed(qspi->io_base + QSPI_CR);
 248
 249         reinit_completion(&qspi->dma_completion);
 250         desc->callback = stm32_qspi_dma_callback;
 251         desc->callback_param = &qspi->dma_completion;
 252         cookie = dmaengine_submit(desc);
 253         err = dma_submit_error(cookie);
 254         if (err)
 255                 goto out;
 256
 257         dma_async_issue_pending(dma_ch);
 258
 259         writel_relaxed(cr | CR_DMAEN, qspi->io_base + QSPI_CR);
 260
 261         t_out = sgt.nents * STM32_COMP_TIMEOUT_MS;
 262         if (!wait_for_completion_timeout(&qspi->dma_completion,
 263                                          msecs_to_jiffies(t_out)))
 264                 err = -ETIMEDOUT;
 265
 266         if (err)
 267                 dmaengine_terminate_all(dma_ch);
 268
 269 out:
 270         writel_relaxed(cr & ~CR_DMAEN, qspi->io_base + QSPI_CR);
 271 out_unmap:
 272         spi_controller_dma_unmap_mem_op_data(qspi->ctrl, op, &sgt);
 273
 274         return err;
 275 }
 276
 277 static int stm32_qspi_tx(struct stm32_qspi *qspi, const struct spi_mem_op *op)
 278 {
 279         if (!op->data.nbytes)
 280                 return 0;
 281
 282         if (qspi->fmode == CCR_FMODE_MM)
 283                 return stm32_qspi_tx_mm(qspi, op);
 284         else if (((op->data.dir == SPI_MEM_DATA_IN && qspi->dma_chrx) ||
 285                  (op->data.dir == SPI_MEM_DATA_OUT && qspi->dma_chtx)) &&
 286                   op->data.nbytes > 4)
 287                 if (!stm32_qspi_tx_dma(qspi, op))
 288                         return 0;
 289
 290         return stm32_qspi_tx_poll(qspi, op);
 291 }
 292
 293 static int stm32_qspi_wait_nobusy(struct stm32_qspi *qspi)
 294 {
 295         u32 sr;
 296
 297         return readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_SR, sr,
 298                                                  !(sr & SR_BUSY), 1,
 299                                                  STM32_BUSY_TIMEOUT_US);
 300 }
 301
 302 static int stm32_qspi_wait_cmd(struct stm32_qspi *qspi)
 303 {
 304         u32 cr, sr;
 305         int err = 0;
 306
 307         if ((readl_relaxed(qspi->io_base + QSPI_SR) & SR_TCF) ||
 308             qspi->fmode == CCR_FMODE_APM)
 309                 goto out;
 310
 311         reinit_completion(&qspi->data_completion);
 312         cr = readl_relaxed(qspi->io_base + QSPI_CR);
 313         writel_relaxed(cr | CR_TCIE | CR_TEIE, qspi->io_base + QSPI_CR);
 314
 315         if (!wait_for_completion_timeout(&qspi->data_completion,
 316                                 msecs_to_jiffies(STM32_COMP_TIMEOUT_MS))) {
 317                 err = -ETIMEDOUT;
 318         } else {
 319                 sr = readl_relaxed(qspi->io_base + QSPI_SR);
 320                 if (sr & SR_TEF)
 321                         err = -EIO;
 322         }
 323
 324 out:
 325         /* clear flags */
 326         writel_relaxed(FCR_CTCF | FCR_CTEF, qspi->io_base + QSPI_FCR);
 327         if (!err)
 328                 err = stm32_qspi_wait_nobusy(qspi);
 329
 330         return err;
 331 }
 332
 333 static int stm32_qspi_wait_poll_status(struct stm32_qspi *qspi)
 334 {
 335         u32 cr;
 336
 337         reinit_completion(&qspi->match_completion);
 338         cr = readl_relaxed(qspi->io_base + QSPI_CR);
 339         writel_relaxed(cr | CR_SMIE, qspi->io_base + QSPI_CR);
 340
 341         if (!wait_for_completion_timeout(&qspi->match_completion,
 342                                 msecs_to_jiffies(qspi->status_timeout)))
 343                 return -ETIMEDOUT;
 344
 345         writel_relaxed(FCR_CSMF, qspi->io_base + QSPI_FCR);
 346
 347         return 0;
 348 }
 349
 350 static int stm32_qspi_get_mode(u8 buswidth)
 351 {
 352         if (buswidth == 4)
 353                 return CCR_BUSWIDTH_4;
 354
 355         return buswidth;
 356 }
 357
 358 static int stm32_qspi_send(struct spi_device *spi, const struct spi_mem_op *op)
 359 {
 360         struct stm32_qspi *qspi = spi_controller_get_devdata(spi->controller);
 361         struct stm32_qspi_flash *flash = &qspi->flash[spi_get_chipselect(spi, 0)];
 362         u32 ccr, cr;
 363         int timeout, err = 0, err_poll_status = 0;
 364
 365         dev_dbg(qspi->dev, "cmd:%#x mode:%d.%d.%d.%d addr:%#llx len:%#x\n",
 366                 op->cmd.opcode, op->cmd.buswidth, op->addr.buswidth,
 367                 op->dummy.buswidth, op->data.buswidth,
 368                 op->addr.val, op->data.nbytes);
 369
 370         cr = readl_relaxed(qspi->io_base + QSPI_CR);
 371         cr &= ~CR_PRESC_MASK & ~CR_FSEL;
 372         cr |= FIELD_PREP(CR_PRESC_MASK, flash->presc);
 373         cr |= FIELD_PREP(CR_FSEL, flash->cs);
 374         writel_relaxed(cr, qspi->io_base + QSPI_CR);
 375
 376         if (op->data.nbytes)
 377                 writel_relaxed(op->data.nbytes - 1,
 378                                qspi->io_base + QSPI_DLR);
 379
 380         ccr = qspi->fmode;
 381         ccr |= FIELD_PREP(CCR_INST_MASK, op->cmd.opcode);
 382         ccr |= FIELD_PREP(CCR_IMODE_MASK,
 383                           stm32_qspi_get_mode(op->cmd.buswidth));
 384
 385         if (op->addr.nbytes) {
 386                 ccr |= FIELD_PREP(CCR_ADMODE_MASK,
 387                                   stm32_qspi_get_mode(op->addr.buswidth));
 388                 ccr |= FIELD_PREP(CCR_ADSIZE_MASK, op->addr.nbytes - 1);
 389         }
 390
 391         if (op->dummy.nbytes)
 392                 ccr |= FIELD_PREP(CCR_DCYC_MASK,
 393                                   op->dummy.nbytes * 8 / op->dummy.buswidth);
 394
 395         if (op->data.nbytes) {
 396                 ccr |= FIELD_PREP(CCR_DMODE_MASK,
 397                                   stm32_qspi_get_mode(op->data.buswidth));
 398         }
 399
 400         writel_relaxed(ccr, qspi->io_base + QSPI_CCR);
 401
 402         if (op->addr.nbytes && qspi->fmode != CCR_FMODE_MM)
 403                 writel_relaxed(op->addr.val, qspi->io_base + QSPI_AR);
 404
 405         if (qspi->fmode == CCR_FMODE_APM)
 406                 err_poll_status = stm32_qspi_wait_poll_status(qspi);
 407
 408         err = stm32_qspi_tx(qspi, op);
 409
 410         /*
 411          * Abort in:
 412          * -error case
 413          * -read memory map: prefetching must be stopped if we read the last
 414          *  byte of device (device size - fifo size). like device size is not
 415          *  knows, the prefetching is always stop.
 416          */
 417         if (err || err_poll_status || qspi->fmode == CCR_FMODE_MM)
 418                 goto abort;
 419
 420         /* wait end of tx in indirect mode */
 421         err = stm32_qspi_wait_cmd(qspi);
 422         if (err)
 423                 goto abort;
 424
 425         return 0;
 426
 427 abort:
 428         cr = readl_relaxed(qspi->io_base + QSPI_CR) | CR_ABORT;
 429         writel_relaxed(cr, qspi->io_base + QSPI_CR);
 430
 431         /* wait clear of abort bit by hw */
 432         timeout = readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_CR,
 433                                                     cr, !(cr & CR_ABORT), 1,
 434                                                     STM32_ABT_TIMEOUT_US);
 435
 436         writel_relaxed(FCR_CTCF | FCR_CSMF, qspi->io_base + QSPI_FCR);
 437
 438         if (err || err_poll_status || timeout)
 439                 dev_err(qspi->dev, "%s err:%d err_poll_status:%d abort timeout:%d\n",
 440                         __func__, err, err_poll_status, timeout);
 441
 442         return err;
 443 }
 444
 445 static int stm32_qspi_poll_status(struct spi_mem *mem, const struct spi_mem_op *op,
 446                                   u16 mask, u16 match,
 447                                   unsigned long initial_delay_us,
 448                                   unsigned long polling_rate_us,
 449                                   unsigned long timeout_ms)
 450 {
 451         struct stm32_qspi *qspi = spi_controller_get_devdata(mem->spi->controller);
 452         int ret;
 453
 454         if (!spi_mem_supports_op(mem, op))
 455                 return -EOPNOTSUPP;
 456
 457         ret = pm_runtime_resume_and_get(qspi->dev);
 458         if (ret < 0)
 459                 return ret;
 460
 461         mutex_lock(&qspi->lock);
 462
 463         writel_relaxed(mask, qspi->io_base + QSPI_PSMKR);
 464         writel_relaxed(match, qspi->io_base + QSPI_PSMAR);
 465         qspi->fmode = CCR_FMODE_APM;
 466         qspi->status_timeout = timeout_ms;
 467
 468         ret = stm32_qspi_send(mem->spi, op);
 469         mutex_unlock(&qspi->lock);
 470
 471         pm_runtime_mark_last_busy(qspi->dev);
 472         pm_runtime_put_autosuspend(qspi->dev);
 473
 474         return ret;
 475 }
 476
 477 static int stm32_qspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
 478 {
 479         struct stm32_qspi *qspi = spi_controller_get_devdata(mem->spi->controller);
 480         int ret;
 481
 482         ret = pm_runtime_resume_and_get(qspi->dev);
 483         if (ret < 0)
 484                 return ret;
 485
 486         mutex_lock(&qspi->lock);
 487         if (op->data.dir == SPI_MEM_DATA_IN && op->data.nbytes)
 488                 qspi->fmode = CCR_FMODE_INDR;
 489         else
 490                 qspi->fmode = CCR_FMODE_INDW;
 491
 492         ret = stm32_qspi_send(mem->spi, op);
 493         mutex_unlock(&qspi->lock);
 494
 495         pm_runtime_mark_last_busy(qspi->dev);
 496         pm_runtime_put_autosuspend(qspi->dev);
 497
 498         return ret;
 499 }
 500
 501 static int stm32_qspi_dirmap_create(struct spi_mem_dirmap_desc *desc)
 502 {
 503         struct stm32_qspi *qspi = spi_controller_get_devdata(desc->mem->spi->controller);
 504
 505         if (desc->info.op_tmpl.data.dir == SPI_MEM_DATA_OUT)
 506                 return -EOPNOTSUPP;
 507
 508         /* should never happen, as mm_base == null is an error probe exit condition */
 509         if (!qspi->mm_base && desc->info.op_tmpl.data.dir == SPI_MEM_DATA_IN)
 510                 return -EOPNOTSUPP;
 511
 512         if (!qspi->mm_size)
 513                 return -EOPNOTSUPP;
 514
 515         return 0;
 516 }
 517
 518 static ssize_t stm32_qspi_dirmap_read(struct spi_mem_dirmap_desc *desc,
 519                                       u64 offs, size_t len, void *buf)
 520 {
 521         struct stm32_qspi *qspi = spi_controller_get_devdata(desc->mem->spi->controller);
 522         struct spi_mem_op op;
 523         u32 addr_max;
 524         int ret;
 525
 526         ret = pm_runtime_resume_and_get(qspi->dev);
 527         if (ret < 0)
 528                 return ret;
 529
 530         mutex_lock(&qspi->lock);
 531         /* make a local copy of desc op_tmpl and complete dirmap rdesc
 532          * spi_mem_op template with offs, len and *buf in  order to get
 533          * all needed transfer information into struct spi_mem_op
 534          */
 535         memcpy(&op, &desc->info.op_tmpl, sizeof(struct spi_mem_op));
 536         dev_dbg(qspi->dev, "%s len = 0x%zx offs = 0x%llx buf = 0x%p\n", __func__, len, offs, buf);
 537
 538         op.data.nbytes = len;
 539         op.addr.val = desc->info.offset + offs;
 540         op.data.buf.in = buf;
 541
 542         addr_max = op.addr.val + op.data.nbytes + 1;
 543         if (addr_max < qspi->mm_size && op.addr.buswidth)
 544                 qspi->fmode = CCR_FMODE_MM;
 545         else
 546                 qspi->fmode = CCR_FMODE_INDR;
 547
 548         ret = stm32_qspi_send(desc->mem->spi, &op);
 549         mutex_unlock(&qspi->lock);
 550
 551         pm_runtime_mark_last_busy(qspi->dev);
 552         pm_runtime_put_autosuspend(qspi->dev);
 553
 554         return ret ?: len;
 555 }
 556
 557 static int stm32_qspi_transfer_one_message(struct spi_controller *ctrl,
 558                                            struct spi_message *msg)
 559 {
 560         struct stm32_qspi *qspi = spi_controller_get_devdata(ctrl);
 561         struct spi_transfer *transfer;
 562         struct spi_device *spi = msg->spi;
 563         struct spi_mem_op op;
 564         int ret = 0;
 565
 566         if (!spi_get_csgpiod(spi, 0))
 567                 return -EOPNOTSUPP;
 568
 569         ret = pm_runtime_resume_and_get(qspi->dev);
 570         if (ret < 0)
 571                 return ret;
 572
 573         mutex_lock(&qspi->lock);
 574
 575         gpiod_set_value_cansleep(spi_get_csgpiod(spi, 0), true);
 576
 577         list_for_each_entry(transfer, &msg->transfers, transfer_list) {
 578                 u8 dummy_bytes = 0;
 579
 580                 memset(&op, 0, sizeof(op));
 581
 582                 dev_dbg(qspi->dev, "tx_buf:%p tx_nbits:%d rx_buf:%p rx_nbits:%d len:%d dummy_data:%d\n",
 583                         transfer->tx_buf, transfer->tx_nbits,
 584                         transfer->rx_buf, transfer->rx_nbits,
 585                         transfer->len, transfer->dummy_data);
 586
 587                 /*
 588                  * QSPI hardware supports dummy bytes transfer.
 589                  * If current transfer is dummy byte, merge it with the next
 590                  * transfer in order to take into account QSPI block constraint
 591                  */
 592                 if (transfer->dummy_data) {
 593                         op.dummy.buswidth = transfer->tx_nbits;
 594                         op.dummy.nbytes = transfer->len;
 595                         dummy_bytes = transfer->len;
 596
 597                         /* if happens, means that message is not correctly built */
 598                         if (list_is_last(&transfer->transfer_list, &msg->transfers)) {
 599                                 ret = -EINVAL;
 600                                 goto end_of_transfer;
 601                         }
 602
 603                         transfer = list_next_entry(transfer, transfer_list);
 604                 }
 605
 606                 op.data.nbytes = transfer->len;
 607
 608                 if (transfer->rx_buf) {
 609                         qspi->fmode = CCR_FMODE_INDR;
 610                         op.data.buswidth = transfer->rx_nbits;
 611                         op.data.dir = SPI_MEM_DATA_IN;
 612                         op.data.buf.in = transfer->rx_buf;
 613                 } else {
 614                         qspi->fmode = CCR_FMODE_INDW;
 615                         op.data.buswidth = transfer->tx_nbits;
 616                         op.data.dir = SPI_MEM_DATA_OUT;
 617                         op.data.buf.out = transfer->tx_buf;
 618                 }
 619
 620                 ret = stm32_qspi_send(spi, &op);
 621                 if (ret)
 622                         goto end_of_transfer;
 623
 624                 msg->actual_length += transfer->len + dummy_bytes;
 625         }
 626
 627 end_of_transfer:
 628         gpiod_set_value_cansleep(spi_get_csgpiod(spi, 0), false);
 629
 630         mutex_unlock(&qspi->lock);
 631
 632         msg->status = ret;
 633         spi_finalize_current_message(ctrl);
 634
 635         pm_runtime_mark_last_busy(qspi->dev);
 636         pm_runtime_put_autosuspend(qspi->dev);
 637
 638         return ret;
 639 }
 640
 641 static int stm32_qspi_setup(struct spi_device *spi)
 642 {
 643         struct spi_controller *ctrl = spi->controller;
 644         struct stm32_qspi *qspi = spi_controller_get_devdata(ctrl);
 645         struct stm32_qspi_flash *flash;
 646         u32 presc, mode;
 647         int ret;
 648
 649         if (ctrl->busy)
 650                 return -EBUSY;
 651
 652         if (!spi->max_speed_hz)
 653                 return -EINVAL;
 654
 655         mode = spi->mode & (SPI_TX_OCTAL | SPI_RX_OCTAL);
 656         if ((mode == SPI_TX_OCTAL || mode == SPI_RX_OCTAL) ||
 657             ((mode == (SPI_TX_OCTAL | SPI_RX_OCTAL)) &&
 658             gpiod_count(qspi->dev, "cs") == -ENOENT)) {
 659                 dev_err(qspi->dev, "spi-rx-bus-width\\/spi-tx-bus-width\\/cs-gpios\n");
 660                 dev_err(qspi->dev, "configuration not supported\n");
 661
 662                 return -EINVAL;
 663         }
 664
 665         ret = pm_runtime_resume_and_get(qspi->dev);
 666         if (ret < 0)
 667                 return ret;
 668
 669         presc = DIV_ROUND_UP(qspi->clk_rate, spi->max_speed_hz) - 1;
 670
 671         flash = &qspi->flash[spi_get_chipselect(spi, 0)];
 672         flash->cs = spi_get_chipselect(spi, 0);
 673         flash->presc = presc;
 674
 675         mutex_lock(&qspi->lock);
 676         qspi->cr_reg = CR_APMS | 3 << CR_FTHRES_SHIFT | CR_SSHIFT | CR_EN;
 677
 678         /*
 679          * Dual flash mode is only enable in case SPI_TX_OCTAL and SPI_TX_OCTAL
 680          * are both set in spi->mode and "cs-gpios" properties is found in DT
 681          */
 682         if (mode == (SPI_TX_OCTAL | SPI_RX_OCTAL)) {
 683                 qspi->cr_reg |= CR_DFM;
 684                 dev_dbg(qspi->dev, "Dual flash mode enable");
 685         }
 686
 687         writel_relaxed(qspi->cr_reg, qspi->io_base + QSPI_CR);
 688
 689         /* set dcr fsize to max address */
 690         qspi->dcr_reg = DCR_FSIZE_MASK;
 691         writel_relaxed(qspi->dcr_reg, qspi->io_base + QSPI_DCR);
 692         mutex_unlock(&qspi->lock);
 693
 694         pm_runtime_mark_last_busy(qspi->dev);
 695         pm_runtime_put_autosuspend(qspi->dev);
 696
 697         return 0;
 698 }
 699
 700 static int stm32_qspi_dma_setup(struct stm32_qspi *qspi)
 701 {
 702         struct dma_slave_config dma_cfg;
 703         struct device *dev = qspi->dev;
 704         int ret = 0;
 705
 706         memset(&dma_cfg, 0, sizeof(dma_cfg));
 707
 708         dma_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 709         dma_cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 710         dma_cfg.src_addr = qspi->phys_base + QSPI_DR;
 711         dma_cfg.dst_addr = qspi->phys_base + QSPI_DR;
 712         dma_cfg.src_maxburst = 4;
 713         dma_cfg.dst_maxburst = 4;
 714
 715         qspi->dma_chrx = dma_request_chan(dev, "rx");
 716         if (IS_ERR(qspi->dma_chrx)) {
 717                 ret = PTR_ERR(qspi->dma_chrx);
 718                 qspi->dma_chrx = NULL;
 719                 if (ret == -EPROBE_DEFER)
 720                         goto out;
 721         } else {
 722                 if (dmaengine_slave_config(qspi->dma_chrx, &dma_cfg)) {
 723                         dev_err(dev, "dma rx config failed\n");
 724                         dma_release_channel(qspi->dma_chrx);
 725                         qspi->dma_chrx = NULL;
 726                 }
 727         }
 728
 729         qspi->dma_chtx = dma_request_chan(dev, "tx");
 730         if (IS_ERR(qspi->dma_chtx)) {
 731                 ret = PTR_ERR(qspi->dma_chtx);
 732                 qspi->dma_chtx = NULL;
 733         } else {
 734                 if (dmaengine_slave_config(qspi->dma_chtx, &dma_cfg)) {
 735                         dev_err(dev, "dma tx config failed\n");
 736                         dma_release_channel(qspi->dma_chtx);
 737                         qspi->dma_chtx = NULL;
 738                 }
 739         }
 740
 741 out:
 742         init_completion(&qspi->dma_completion);
 743
 744         if (ret != -EPROBE_DEFER)
 745                 ret = 0;
 746
 747         return ret;
 748 }
 749
 750 static void stm32_qspi_dma_free(struct stm32_qspi *qspi)
 751 {
 752         if (qspi->dma_chtx)
 753                 dma_release_channel(qspi->dma_chtx);
 754         if (qspi->dma_chrx)
 755                 dma_release_channel(qspi->dma_chrx);
 756 }
 757
 758 /*
 759  * no special host constraint, so use default spi_mem_default_supports_op
 760  * to check supported mode.
 761  */
 762 static const struct spi_controller_mem_ops stm32_qspi_mem_ops = {
 763         .exec_op        = stm32_qspi_exec_op,
 764         .dirmap_create  = stm32_qspi_dirmap_create,
 765         .dirmap_read    = stm32_qspi_dirmap_read,
 766         .poll_status    = stm32_qspi_poll_status,
 767 };
 768
 769 static int stm32_qspi_probe(struct platform_device *pdev)
 770 {
 771         struct device *dev = &pdev->dev;
 772         struct spi_controller *ctrl;
 773         struct reset_control *rstc;
 774         struct stm32_qspi *qspi;
 775         struct resource *res;
 776         int ret, irq;
 777
 778         ctrl = devm_spi_alloc_host(dev, sizeof(*qspi));
 779         if (!ctrl)
 780                 return -ENOMEM;
 781
 782         qspi = spi_controller_get_devdata(ctrl);
 783         qspi->ctrl = ctrl;
 784
 785         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "qspi");
 786         qspi->io_base = devm_ioremap_resource(dev, res);
 787         if (IS_ERR(qspi->io_base))
 788                 return PTR_ERR(qspi->io_base);
 789
 790         qspi->phys_base = res->start;
 791
 792         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "qspi_mm");
 793         qspi->mm_base = devm_ioremap_resource(dev, res);
 794         if (IS_ERR(qspi->mm_base))
 795                 return PTR_ERR(qspi->mm_base);
 796
 797         qspi->mm_size = resource_size(res);
 798         if (qspi->mm_size > STM32_QSPI_MAX_MMAP_SZ)
 799                 return -EINVAL;
 800
 801         irq = platform_get_irq(pdev, 0);
 802         if (irq < 0)
 803                 return irq;
 804
 805         ret = devm_request_irq(dev, irq, stm32_qspi_irq, 0,
 806                                dev_name(dev), qspi);
 807         if (ret) {
 808                 dev_err(dev, "failed to request irq\n");
 809                 return ret;
 810         }
 811
 812         init_completion(&qspi->data_completion);
 813         init_completion(&qspi->match_completion);
 814
 815         qspi->clk = devm_clk_get(dev, NULL);
 816         if (IS_ERR(qspi->clk))
 817                 return PTR_ERR(qspi->clk);
 818
 819         qspi->clk_rate = clk_get_rate(qspi->clk);
 820         if (!qspi->clk_rate)
 821                 return -EINVAL;
 822
 823         ret = clk_prepare_enable(qspi->clk);
 824         if (ret) {
 825                 dev_err(dev, "can not enable the clock\n");
 826                 return ret;
 827         }
 828
 829         rstc = devm_reset_control_get_exclusive(dev, NULL);
 830         if (IS_ERR(rstc)) {
 831                 ret = PTR_ERR(rstc);
 832                 if (ret == -EPROBE_DEFER)
 833                         goto err_clk_disable;
 834         } else {
 835                 reset_control_assert(rstc);
 836                 udelay(2);
 837                 reset_control_deassert(rstc);
 838         }
 839
 840         qspi->dev = dev;
 841         platform_set_drvdata(pdev, qspi);
 842         ret = stm32_qspi_dma_setup(qspi);
 843         if (ret)
 844                 goto err_dma_free;
 845
 846         mutex_init(&qspi->lock);
 847
 848         ctrl->mode_bits = SPI_RX_DUAL | SPI_RX_QUAD | SPI_TX_OCTAL
 849                 | SPI_TX_DUAL | SPI_TX_QUAD | SPI_RX_OCTAL;
 850         ctrl->setup = stm32_qspi_setup;
 851         ctrl->bus_num = -1;
 852         ctrl->mem_ops = &stm32_qspi_mem_ops;
 853         ctrl->use_gpio_descriptors = true;
 854         ctrl->transfer_one_message = stm32_qspi_transfer_one_message;
 855         ctrl->num_chipselect = STM32_QSPI_MAX_NORCHIP;
 856         ctrl->dev.of_node = dev->of_node;
 857
 858         pm_runtime_set_autosuspend_delay(dev, STM32_AUTOSUSPEND_DELAY);
 859         pm_runtime_use_autosuspend(dev);
 860         pm_runtime_set_active(dev);
 861         pm_runtime_enable(dev);
 862         pm_runtime_get_noresume(dev);
 863
 864         ret = spi_register_controller(ctrl);
 865         if (ret)
 866                 goto err_pm_runtime_free;
 867
 868         pm_runtime_mark_last_busy(dev);
 869         pm_runtime_put_autosuspend(dev);
 870
 871         return 0;
 872
 873 err_pm_runtime_free:
 874         pm_runtime_get_sync(qspi->dev);
 875         /* disable qspi */
 876         writel_relaxed(0, qspi->io_base + QSPI_CR);
 877         mutex_destroy(&qspi->lock);
 878         pm_runtime_put_noidle(qspi->dev);
 879         pm_runtime_disable(qspi->dev);
 880         pm_runtime_set_suspended(qspi->dev);
 881         pm_runtime_dont_use_autosuspend(qspi->dev);
 882 err_dma_free:
 883         stm32_qspi_dma_free(qspi);
 884 err_clk_disable:
 885         clk_disable_unprepare(qspi->clk);
 886
 887         return ret;
 888 }
 889
 890 static void stm32_qspi_remove(struct platform_device *pdev)
 891 {
 892         struct stm32_qspi *qspi = platform_get_drvdata(pdev);
 893
 894         pm_runtime_get_sync(qspi->dev);
 895         spi_unregister_controller(qspi->ctrl);
 896         /* disable qspi */
 897         writel_relaxed(0, qspi->io_base + QSPI_CR);
 898         stm32_qspi_dma_free(qspi);
 899         mutex_destroy(&qspi->lock);
 900         pm_runtime_put_noidle(qspi->dev);
 901         pm_runtime_disable(qspi->dev);
 902         pm_runtime_set_suspended(qspi->dev);
 903         pm_runtime_dont_use_autosuspend(qspi->dev);
 904         clk_disable_unprepare(qspi->clk);
 905 }
 906
 907 static int __maybe_unused stm32_qspi_runtime_suspend(struct device *dev)
 908 {
 909         struct stm32_qspi *qspi = dev_get_drvdata(dev);
 910
 911         clk_disable_unprepare(qspi->clk);
 912
 913         return 0;
 914 }
 915
 916 static int __maybe_unused stm32_qspi_runtime_resume(struct device *dev)
 917 {
 918         struct stm32_qspi *qspi = dev_get_drvdata(dev);
 919
 920         return clk_prepare_enable(qspi->clk);
 921 }
 922
 923 static int __maybe_unused stm32_qspi_suspend(struct device *dev)
 924 {
 925         pinctrl_pm_select_sleep_state(dev);
 926
 927         return pm_runtime_force_suspend(dev);
 928 }
 929
 930 static int __maybe_unused stm32_qspi_resume(struct device *dev)
 931 {
 932         struct stm32_qspi *qspi = dev_get_drvdata(dev);
 933         int ret;
 934
 935         ret = pm_runtime_force_resume(dev);
 936         if (ret < 0)
 937                 return ret;
 938
 939         pinctrl_pm_select_default_state(dev);
 940
 941         ret = pm_runtime_resume_and_get(dev);
 942         if (ret < 0)
 943                 return ret;
 944
 945         writel_relaxed(qspi->cr_reg, qspi->io_base + QSPI_CR);
 946         writel_relaxed(qspi->dcr_reg, qspi->io_base + QSPI_DCR);
 947
 948         pm_runtime_mark_last_busy(dev);
 949         pm_runtime_put_autosuspend(dev);
 950
 951         return 0;
 952 }
 953
 954 static const struct dev_pm_ops stm32_qspi_pm_ops = {
 955         SET_RUNTIME_PM_OPS(stm32_qspi_runtime_suspend,
 956                            stm32_qspi_runtime_resume, NULL)
 957         SET_SYSTEM_SLEEP_PM_OPS(stm32_qspi_suspend, stm32_qspi_resume)
 958 };
 959
 960 static const struct of_device_id stm32_qspi_match[] = {
 961         {.compatible = "st,stm32f469-qspi"},
 962         {}
 963 };
 964 MODULE_DEVICE_TABLE(of, stm32_qspi_match);
 965
 966 static struct platform_driver stm32_qspi_driver = {
 967         .probe  = stm32_qspi_probe,
 968         .remove_new = stm32_qspi_remove,
 969         .driver = {
 970                 .name = "stm32-qspi",
 971                 .of_match_table = stm32_qspi_match,
 972                 .pm = &stm32_qspi_pm_ops,
 973         },
 974 };
 975 module_platform_driver(stm32_qspi_driver);
 976
 977 MODULE_AUTHOR("Ludovic Barre <ludovic.barre@st.com>");
 978 MODULE_DESCRIPTION("STMicroelectronics STM32 quad spi driver");
 979 MODULE_LICENSE("GPL v2");