drivers/spi/spi-qcom-qspi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (c) 2017-2018, The Linux foundation. All rights reserved.
   3
   4 #include <linux/clk.h>
   5 #include <linux/interconnect.h>
   6 #include <linux/interrupt.h>
   7 #include <linux/io.h>
   8 #include <linux/module.h>
   9 #include <linux/of.h>
  10 #include <linux/of_platform.h>
  11 #include <linux/pm_runtime.h>
  12 #include <linux/pm_opp.h>
  13 #include <linux/spi/spi.h>
  14 #include <linux/spi/spi-mem.h>
  15
  16
  17 #define QSPI_NUM_CS             2
  18 #define QSPI_BYTES_PER_WORD     4
  19
  20 #define MSTR_CONFIG             0x0000
  21 #define FULL_CYCLE_MODE         BIT(3)
  22 #define FB_CLK_EN               BIT(4)
  23 #define PIN_HOLDN               BIT(6)
  24 #define PIN_WPN                 BIT(7)
  25 #define DMA_ENABLE              BIT(8)
  26 #define BIG_ENDIAN_MODE         BIT(9)
  27 #define SPI_MODE_MSK            0xc00
  28 #define SPI_MODE_SHFT           10
  29 #define CHIP_SELECT_NUM         BIT(12)
  30 #define SBL_EN                  BIT(13)
  31 #define LPA_BASE_MSK            0x3c000
  32 #define LPA_BASE_SHFT           14
  33 #define TX_DATA_DELAY_MSK       0xc0000
  34 #define TX_DATA_DELAY_SHFT      18
  35 #define TX_CLK_DELAY_MSK        0x300000
  36 #define TX_CLK_DELAY_SHFT       20
  37 #define TX_CS_N_DELAY_MSK       0xc00000
  38 #define TX_CS_N_DELAY_SHFT      22
  39 #define TX_DATA_OE_DELAY_MSK    0x3000000
  40 #define TX_DATA_OE_DELAY_SHFT   24
  41
  42 #define AHB_MASTER_CFG                          0x0004
  43 #define HMEM_TYPE_START_MID_TRANS_MSK           0x7
  44 #define HMEM_TYPE_START_MID_TRANS_SHFT          0
  45 #define HMEM_TYPE_LAST_TRANS_MSK                0x38
  46 #define HMEM_TYPE_LAST_TRANS_SHFT               3
  47 #define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_MSK  0xc0
  48 #define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_SHFT 6
  49 #define HMEMTYPE_READ_TRANS_MSK                 0x700
  50 #define HMEMTYPE_READ_TRANS_SHFT                8
  51 #define HSHARED                                 BIT(11)
  52 #define HINNERSHARED                            BIT(12)
  53
  54 #define MSTR_INT_EN             0x000C
  55 #define MSTR_INT_STATUS         0x0010
  56 #define RESP_FIFO_UNDERRUN      BIT(0)
  57 #define RESP_FIFO_NOT_EMPTY     BIT(1)
  58 #define RESP_FIFO_RDY           BIT(2)
  59 #define HRESP_FROM_NOC_ERR      BIT(3)
  60 #define WR_FIFO_EMPTY           BIT(9)
  61 #define WR_FIFO_FULL            BIT(10)
  62 #define WR_FIFO_OVERRUN         BIT(11)
  63 #define TRANSACTION_DONE        BIT(16)
  64 #define QSPI_ERR_IRQS           (RESP_FIFO_UNDERRUN | HRESP_FROM_NOC_ERR | \
  65                                  WR_FIFO_OVERRUN)
  66 #define QSPI_ALL_IRQS           (QSPI_ERR_IRQS | RESP_FIFO_RDY | \
  67                                  WR_FIFO_EMPTY | WR_FIFO_FULL | \
  68                                  TRANSACTION_DONE)
  69
  70 #define PIO_XFER_CTRL           0x0014
  71 #define REQUEST_COUNT_MSK       0xffff
  72
  73 #define PIO_XFER_CFG            0x0018
  74 #define TRANSFER_DIRECTION      BIT(0)
  75 #define MULTI_IO_MODE_MSK       0xe
  76 #define MULTI_IO_MODE_SHFT      1
  77 #define TRANSFER_FRAGMENT       BIT(8)
  78 #define SDR_1BIT                1
  79 #define SDR_2BIT                2
  80 #define SDR_4BIT                3
  81 #define DDR_1BIT                5
  82 #define DDR_2BIT                6
  83 #define DDR_4BIT                7
  84 #define DMA_DESC_SINGLE_SPI     1
  85 #define DMA_DESC_DUAL_SPI       2
  86 #define DMA_DESC_QUAD_SPI       3
  87
  88 #define PIO_XFER_STATUS         0x001c
  89 #define WR_FIFO_BYTES_MSK       0xffff0000
  90 #define WR_FIFO_BYTES_SHFT      16
  91
  92 #define PIO_DATAOUT_1B          0x0020
  93 #define PIO_DATAOUT_4B          0x0024
  94
  95 #define RD_FIFO_CFG             0x0028
  96 #define CONTINUOUS_MODE         BIT(0)
  97
  98 #define RD_FIFO_STATUS  0x002c
  99 #define FIFO_EMPTY      BIT(11)
 100 #define WR_CNTS_MSK     0x7f0
 101 #define WR_CNTS_SHFT    4
 102 #define RDY_64BYTE      BIT(3)
 103 #define RDY_32BYTE      BIT(2)
 104 #define RDY_16BYTE      BIT(1)
 105 #define FIFO_RDY        BIT(0)
 106
 107 #define RD_FIFO_RESET           0x0030
 108 #define RESET_FIFO              BIT(0)
 109
 110 #define CUR_MEM_ADDR            0x0048
 111 #define HW_VERSION              0x004c
 112 #define RD_FIFO                 0x0050
 113 #define SAMPLING_CLK_CFG        0x0090
 114 #define SAMPLING_CLK_STATUS     0x0094
 115
 116
 117 enum qspi_dir {
 118         QSPI_READ,
 119         QSPI_WRITE,
 120 };
 121
 122 struct qspi_xfer {
 123         union {
 124                 const void *tx_buf;
 125                 void *rx_buf;
 126         };
 127         unsigned int rem_bytes;
 128         unsigned int buswidth;
 129         enum qspi_dir dir;
 130         bool is_last;
 131 };
 132
 133 enum qspi_clocks {
 134         QSPI_CLK_CORE,
 135         QSPI_CLK_IFACE,
 136         QSPI_NUM_CLKS
 137 };
 138
 139 struct qcom_qspi {
 140         void __iomem *base;
 141         struct device *dev;
 142         struct clk_bulk_data *clks;
 143         struct qspi_xfer xfer;
 144         struct icc_path *icc_path_cpu_to_qspi;
 145         struct opp_table *opp_table;
 146         bool has_opp_table;
 147         unsigned long last_speed;
 148         /* Lock to protect data accessed by IRQs */
 149         spinlock_t lock;
 150 };
 151
 152 static u32 qspi_buswidth_to_iomode(struct qcom_qspi *ctrl,
 153                                    unsigned int buswidth)
 154 {
 155         switch (buswidth) {
 156         case 1:
 157                 return SDR_1BIT << MULTI_IO_MODE_SHFT;
 158         case 2:
 159                 return SDR_2BIT << MULTI_IO_MODE_SHFT;
 160         case 4:
 161                 return SDR_4BIT << MULTI_IO_MODE_SHFT;
 162         default:
 163                 dev_warn_once(ctrl->dev,
 164                                 "Unexpected bus width: %u\n", buswidth);
 165                 return SDR_1BIT << MULTI_IO_MODE_SHFT;
 166         }
 167 }
 168
 169 static void qcom_qspi_pio_xfer_cfg(struct qcom_qspi *ctrl)
 170 {
 171         u32 pio_xfer_cfg;
 172         const struct qspi_xfer *xfer;
 173
 174         xfer = &ctrl->xfer;
 175         pio_xfer_cfg = readl(ctrl->base + PIO_XFER_CFG);
 176         pio_xfer_cfg &= ~TRANSFER_DIRECTION;
 177         pio_xfer_cfg |= xfer->dir;
 178         if (xfer->is_last)
 179                 pio_xfer_cfg &= ~TRANSFER_FRAGMENT;
 180         else
 181                 pio_xfer_cfg |= TRANSFER_FRAGMENT;
 182         pio_xfer_cfg &= ~MULTI_IO_MODE_MSK;
 183         pio_xfer_cfg |= qspi_buswidth_to_iomode(ctrl, xfer->buswidth);
 184
 185         writel(pio_xfer_cfg, ctrl->base + PIO_XFER_CFG);
 186 }
 187
 188 static void qcom_qspi_pio_xfer_ctrl(struct qcom_qspi *ctrl)
 189 {
 190         u32 pio_xfer_ctrl;
 191
 192         pio_xfer_ctrl = readl(ctrl->base + PIO_XFER_CTRL);
 193         pio_xfer_ctrl &= ~REQUEST_COUNT_MSK;
 194         pio_xfer_ctrl |= ctrl->xfer.rem_bytes;
 195         writel(pio_xfer_ctrl, ctrl->base + PIO_XFER_CTRL);
 196 }
 197
 198 static void qcom_qspi_pio_xfer(struct qcom_qspi *ctrl)
 199 {
 200         u32 ints;
 201
 202         qcom_qspi_pio_xfer_cfg(ctrl);
 203
 204         /* Ack any previous interrupts that might be hanging around */
 205         writel(QSPI_ALL_IRQS, ctrl->base + MSTR_INT_STATUS);
 206
 207         /* Setup new interrupts */
 208         if (ctrl->xfer.dir == QSPI_WRITE)
 209                 ints = QSPI_ERR_IRQS | WR_FIFO_EMPTY;
 210         else
 211                 ints = QSPI_ERR_IRQS | RESP_FIFO_RDY;
 212         writel(ints, ctrl->base + MSTR_INT_EN);
 213
 214         /* Kick off the transfer */
 215         qcom_qspi_pio_xfer_ctrl(ctrl);
 216 }
 217
 218 static void qcom_qspi_handle_err(struct spi_master *master,
 219                                  struct spi_message *msg)
 220 {
 221         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 222         unsigned long flags;
 223
 224         spin_lock_irqsave(&ctrl->lock, flags);
 225         writel(0, ctrl->base + MSTR_INT_EN);
 226         ctrl->xfer.rem_bytes = 0;
 227         spin_unlock_irqrestore(&ctrl->lock, flags);
 228 }
 229
 230 static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz)
 231 {
 232         int ret;
 233         unsigned int avg_bw_cpu;
 234
 235         if (speed_hz == ctrl->last_speed)
 236                 return 0;
 237
 238         /* In regular operation (SBL_EN=1) core must be 4x transfer clock */
 239         ret = dev_pm_opp_set_rate(ctrl->dev, speed_hz * 4);
 240         if (ret) {
 241                 dev_err(ctrl->dev, "Failed to set core clk %d\n", ret);
 242                 return ret;
 243         }
 244
 245         /*
 246          * Set BW quota for CPU as driver supports FIFO mode only.
 247          * We don't have explicit peak requirement so keep it equal to avg_bw.
 248          */
 249         avg_bw_cpu = Bps_to_icc(speed_hz);
 250         ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, avg_bw_cpu, avg_bw_cpu);
 251         if (ret) {
 252                 dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
 253                         __func__, ret);
 254                 return ret;
 255         }
 256
 257         ctrl->last_speed = speed_hz;
 258
 259         return 0;
 260 }
 261
 262 static int qcom_qspi_transfer_one(struct spi_master *master,
 263                                   struct spi_device *slv,
 264                                   struct spi_transfer *xfer)
 265 {
 266         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 267         int ret;
 268         unsigned long speed_hz;
 269         unsigned long flags;
 270
 271         speed_hz = slv->max_speed_hz;
 272         if (xfer->speed_hz)
 273                 speed_hz = xfer->speed_hz;
 274
 275         ret = qcom_qspi_set_speed(ctrl, speed_hz);
 276         if (ret)
 277                 return ret;
 278
 279         spin_lock_irqsave(&ctrl->lock, flags);
 280
 281         /* We are half duplex, so either rx or tx will be set */
 282         if (xfer->rx_buf) {
 283                 ctrl->xfer.dir = QSPI_READ;
 284                 ctrl->xfer.buswidth = xfer->rx_nbits;
 285                 ctrl->xfer.rx_buf = xfer->rx_buf;
 286         } else {
 287                 ctrl->xfer.dir = QSPI_WRITE;
 288                 ctrl->xfer.buswidth = xfer->tx_nbits;
 289                 ctrl->xfer.tx_buf = xfer->tx_buf;
 290         }
 291         ctrl->xfer.is_last = list_is_last(&xfer->transfer_list,
 292                                           &master->cur_msg->transfers);
 293         ctrl->xfer.rem_bytes = xfer->len;
 294         qcom_qspi_pio_xfer(ctrl);
 295
 296         spin_unlock_irqrestore(&ctrl->lock, flags);
 297
 298         /* We'll call spi_finalize_current_transfer() when done */
 299         return 1;
 300 }
 301
 302 static int qcom_qspi_prepare_message(struct spi_master *master,
 303                                      struct spi_message *message)
 304 {
 305         u32 mstr_cfg;
 306         struct qcom_qspi *ctrl;
 307         int tx_data_oe_delay = 1;
 308         int tx_data_delay = 1;
 309         unsigned long flags;
 310
 311         ctrl = spi_master_get_devdata(master);
 312         spin_lock_irqsave(&ctrl->lock, flags);
 313
 314         mstr_cfg = readl(ctrl->base + MSTR_CONFIG);
 315         mstr_cfg &= ~CHIP_SELECT_NUM;
 316         if (message->spi->chip_select)
 317                 mstr_cfg |= CHIP_SELECT_NUM;
 318
 319         mstr_cfg |= FB_CLK_EN | PIN_WPN | PIN_HOLDN | SBL_EN | FULL_CYCLE_MODE;
 320         mstr_cfg &= ~(SPI_MODE_MSK | TX_DATA_OE_DELAY_MSK | TX_DATA_DELAY_MSK);
 321         mstr_cfg |= message->spi->mode << SPI_MODE_SHFT;
 322         mstr_cfg |= tx_data_oe_delay << TX_DATA_OE_DELAY_SHFT;
 323         mstr_cfg |= tx_data_delay << TX_DATA_DELAY_SHFT;
 324         mstr_cfg &= ~DMA_ENABLE;
 325
 326         writel(mstr_cfg, ctrl->base + MSTR_CONFIG);
 327         spin_unlock_irqrestore(&ctrl->lock, flags);
 328
 329         return 0;
 330 }
 331
 332 static irqreturn_t pio_read(struct qcom_qspi *ctrl)
 333 {
 334         u32 rd_fifo_status;
 335         u32 rd_fifo;
 336         unsigned int wr_cnts;
 337         unsigned int bytes_to_read;
 338         unsigned int words_to_read;
 339         u32 *word_buf;
 340         u8 *byte_buf;
 341         int i;
 342
 343         rd_fifo_status = readl(ctrl->base + RD_FIFO_STATUS);
 344
 345         if (!(rd_fifo_status & FIFO_RDY)) {
 346                 dev_dbg(ctrl->dev, "Spurious IRQ %#x\n", rd_fifo_status);
 347                 return IRQ_NONE;
 348         }
 349
 350         wr_cnts = (rd_fifo_status & WR_CNTS_MSK) >> WR_CNTS_SHFT;
 351         wr_cnts = min(wr_cnts, ctrl->xfer.rem_bytes);
 352
 353         words_to_read = wr_cnts / QSPI_BYTES_PER_WORD;
 354         bytes_to_read = wr_cnts % QSPI_BYTES_PER_WORD;
 355
 356         if (words_to_read) {
 357                 word_buf = ctrl->xfer.rx_buf;
 358                 ctrl->xfer.rem_bytes -= words_to_read * QSPI_BYTES_PER_WORD;
 359                 ioread32_rep(ctrl->base + RD_FIFO, word_buf, words_to_read);
 360                 ctrl->xfer.rx_buf = word_buf + words_to_read;
 361         }
 362
 363         if (bytes_to_read) {
 364                 byte_buf = ctrl->xfer.rx_buf;
 365                 rd_fifo = readl(ctrl->base + RD_FIFO);
 366                 ctrl->xfer.rem_bytes -= bytes_to_read;
 367                 for (i = 0; i < bytes_to_read; i++)
 368                         *byte_buf++ = rd_fifo >> (i * BITS_PER_BYTE);
 369                 ctrl->xfer.rx_buf = byte_buf;
 370         }
 371
 372         return IRQ_HANDLED;
 373 }
 374
 375 static irqreturn_t pio_write(struct qcom_qspi *ctrl)
 376 {
 377         const void *xfer_buf = ctrl->xfer.tx_buf;
 378         const int *word_buf;
 379         const char *byte_buf;
 380         unsigned int wr_fifo_bytes;
 381         unsigned int wr_fifo_words;
 382         unsigned int wr_size;
 383         unsigned int rem_words;
 384
 385         wr_fifo_bytes = readl(ctrl->base + PIO_XFER_STATUS);
 386         wr_fifo_bytes >>= WR_FIFO_BYTES_SHFT;
 387
 388         if (ctrl->xfer.rem_bytes < QSPI_BYTES_PER_WORD) {
 389                 /* Process the last 1-3 bytes */
 390                 wr_size = min(wr_fifo_bytes, ctrl->xfer.rem_bytes);
 391                 ctrl->xfer.rem_bytes -= wr_size;
 392
 393                 byte_buf = xfer_buf;
 394                 while (wr_size--)
 395                         writel(*byte_buf++,
 396                                ctrl->base + PIO_DATAOUT_1B);
 397                 ctrl->xfer.tx_buf = byte_buf;
 398         } else {
 399                 /*
 400                  * Process all the whole words; to keep things simple we'll
 401                  * just wait for the next interrupt to handle the last 1-3
 402                  * bytes if we don't have an even number of words.
 403                  */
 404                 rem_words = ctrl->xfer.rem_bytes / QSPI_BYTES_PER_WORD;
 405                 wr_fifo_words = wr_fifo_bytes / QSPI_BYTES_PER_WORD;
 406
 407                 wr_size = min(rem_words, wr_fifo_words);
 408                 ctrl->xfer.rem_bytes -= wr_size * QSPI_BYTES_PER_WORD;
 409
 410                 word_buf = xfer_buf;
 411                 iowrite32_rep(ctrl->base + PIO_DATAOUT_4B, word_buf, wr_size);
 412                 ctrl->xfer.tx_buf = word_buf + wr_size;
 413
 414         }
 415
 416         return IRQ_HANDLED;
 417 }
 418
 419 static irqreturn_t qcom_qspi_irq(int irq, void *dev_id)
 420 {
 421         u32 int_status;
 422         struct qcom_qspi *ctrl = dev_id;
 423         irqreturn_t ret = IRQ_NONE;
 424         unsigned long flags;
 425
 426         spin_lock_irqsave(&ctrl->lock, flags);
 427
 428         int_status = readl(ctrl->base + MSTR_INT_STATUS);
 429         writel(int_status, ctrl->base + MSTR_INT_STATUS);
 430
 431         if (ctrl->xfer.dir == QSPI_WRITE) {
 432                 if (int_status & WR_FIFO_EMPTY)
 433                         ret = pio_write(ctrl);
 434         } else {
 435                 if (int_status & RESP_FIFO_RDY)
 436                         ret = pio_read(ctrl);
 437         }
 438
 439         if (int_status & QSPI_ERR_IRQS) {
 440                 if (int_status & RESP_FIFO_UNDERRUN)
 441                         dev_err(ctrl->dev, "IRQ error: FIFO underrun\n");
 442                 if (int_status & WR_FIFO_OVERRUN)
 443                         dev_err(ctrl->dev, "IRQ error: FIFO overrun\n");
 444                 if (int_status & HRESP_FROM_NOC_ERR)
 445                         dev_err(ctrl->dev, "IRQ error: NOC response error\n");
 446                 ret = IRQ_HANDLED;
 447         }
 448
 449         if (!ctrl->xfer.rem_bytes) {
 450                 writel(0, ctrl->base + MSTR_INT_EN);
 451                 spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev));
 452         }
 453
 454         spin_unlock_irqrestore(&ctrl->lock, flags);
 455         return ret;
 456 }
 457
 458 static int qcom_qspi_probe(struct platform_device *pdev)
 459 {
 460         int ret;
 461         struct device *dev;
 462         struct spi_master *master;
 463         struct qcom_qspi *ctrl;
 464
 465         dev = &pdev->dev;
 466
 467         master = spi_alloc_master(dev, sizeof(*ctrl));
 468         if (!master)
 469                 return -ENOMEM;
 470
 471         platform_set_drvdata(pdev, master);
 472
 473         ctrl = spi_master_get_devdata(master);
 474
 475         spin_lock_init(&ctrl->lock);
 476         ctrl->dev = dev;
 477         ctrl->base = devm_platform_ioremap_resource(pdev, 0);
 478         if (IS_ERR(ctrl->base)) {
 479                 ret = PTR_ERR(ctrl->base);
 480                 goto exit_probe_master_put;
 481         }
 482
 483         ctrl->clks = devm_kcalloc(dev, QSPI_NUM_CLKS,
 484                                   sizeof(*ctrl->clks), GFP_KERNEL);
 485         if (!ctrl->clks) {
 486                 ret = -ENOMEM;
 487                 goto exit_probe_master_put;
 488         }
 489
 490         ctrl->clks[QSPI_CLK_CORE].id = "core";
 491         ctrl->clks[QSPI_CLK_IFACE].id = "iface";
 492         ret = devm_clk_bulk_get(dev, QSPI_NUM_CLKS, ctrl->clks);
 493         if (ret)
 494                 goto exit_probe_master_put;
 495
 496         ctrl->icc_path_cpu_to_qspi = devm_of_icc_get(dev, "qspi-config");
 497         if (IS_ERR(ctrl->icc_path_cpu_to_qspi)) {
 498                 ret = PTR_ERR(ctrl->icc_path_cpu_to_qspi);
 499                 if (ret != -EPROBE_DEFER)
 500                         dev_err(dev, "Failed to get cpu path: %d\n", ret);
 501                 goto exit_probe_master_put;
 502         }
 503         /* Set BW vote for register access */
 504         ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, Bps_to_icc(1000),
 505                                 Bps_to_icc(1000));
 506         if (ret) {
 507                 dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
 508                                 __func__, ret);
 509                 goto exit_probe_master_put;
 510         }
 511
 512         ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
 513         if (ret) {
 514                 dev_err(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
 515                                 __func__, ret);
 516                 goto exit_probe_master_put;
 517         }
 518
 519         ret = platform_get_irq(pdev, 0);
 520         if (ret < 0)
 521                 goto exit_probe_master_put;
 522         ret = devm_request_irq(dev, ret, qcom_qspi_irq,
 523                         IRQF_TRIGGER_HIGH, dev_name(dev), ctrl);
 524         if (ret) {
 525                 dev_err(dev, "Failed to request irq %d\n", ret);
 526                 goto exit_probe_master_put;
 527         }
 528
 529         master->max_speed_hz = 300000000;
 530         master->num_chipselect = QSPI_NUM_CS;
 531         master->bus_num = -1;
 532         master->dev.of_node = pdev->dev.of_node;
 533         master->mode_bits = SPI_MODE_0 |
 534                             SPI_TX_DUAL | SPI_RX_DUAL |
 535                             SPI_TX_QUAD | SPI_RX_QUAD;
 536         master->flags = SPI_MASTER_HALF_DUPLEX;
 537         master->prepare_message = qcom_qspi_prepare_message;
 538         master->transfer_one = qcom_qspi_transfer_one;
 539         master->handle_err = qcom_qspi_handle_err;
 540         master->auto_runtime_pm = true;
 541
 542         ctrl->opp_table = dev_pm_opp_set_clkname(&pdev->dev, "core");
 543         if (IS_ERR(ctrl->opp_table)) {
 544                 ret = PTR_ERR(ctrl->opp_table);
 545                 goto exit_probe_master_put;
 546         }
 547         /* OPP table is optional */
 548         ret = dev_pm_opp_of_add_table(&pdev->dev);
 549         if (!ret) {
 550                 ctrl->has_opp_table = true;
 551         } else if (ret != -ENODEV) {
 552                 dev_err(&pdev->dev, "invalid OPP table in device tree\n");
 553                 goto exit_probe_master_put;
 554         }
 555
 556         pm_runtime_use_autosuspend(dev);
 557         pm_runtime_set_autosuspend_delay(dev, 250);
 558         pm_runtime_enable(dev);
 559
 560         ret = spi_register_master(master);
 561         if (!ret)
 562                 return 0;
 563
 564         pm_runtime_disable(dev);
 565         if (ctrl->has_opp_table)
 566                 dev_pm_opp_of_remove_table(&pdev->dev);
 567         dev_pm_opp_put_clkname(ctrl->opp_table);
 568
 569 exit_probe_master_put:
 570         spi_master_put(master);
 571
 572         return ret;
 573 }
 574
 575 static int qcom_qspi_remove(struct platform_device *pdev)
 576 {
 577         struct spi_master *master = platform_get_drvdata(pdev);
 578         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 579
 580         /* Unregister _before_ disabling pm_runtime() so we stop transfers */
 581         spi_unregister_master(master);
 582
 583         pm_runtime_disable(&pdev->dev);
 584         if (ctrl->has_opp_table)
 585                 dev_pm_opp_of_remove_table(&pdev->dev);
 586         dev_pm_opp_put_clkname(ctrl->opp_table);
 587
 588         return 0;
 589 }
 590
 591 static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev)
 592 {
 593         struct spi_master *master = dev_get_drvdata(dev);
 594         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 595         int ret;
 596
 597         /* Drop the performance state vote */
 598         dev_pm_opp_set_rate(dev, 0);
 599         clk_bulk_disable_unprepare(QSPI_NUM_CLKS, ctrl->clks);
 600
 601         ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
 602         if (ret) {
 603                 dev_err_ratelimited(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
 604                         __func__, ret);
 605                 return ret;
 606         }
 607
 608         return 0;
 609 }
 610
 611 static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev)
 612 {
 613         struct spi_master *master = dev_get_drvdata(dev);
 614         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 615         int ret;
 616
 617         ret = icc_enable(ctrl->icc_path_cpu_to_qspi);
 618         if (ret) {
 619                 dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n",
 620                         __func__, ret);
 621                 return ret;
 622         }
 623
 624         ret = clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks);
 625         if (ret)
 626                 return ret;
 627
 628         return dev_pm_opp_set_rate(dev, ctrl->last_speed * 4);
 629 }
 630
 631 static int __maybe_unused qcom_qspi_suspend(struct device *dev)
 632 {
 633         struct spi_master *master = dev_get_drvdata(dev);
 634         int ret;
 635
 636         ret = spi_master_suspend(master);
 637         if (ret)
 638                 return ret;
 639
 640         ret = pm_runtime_force_suspend(dev);
 641         if (ret)
 642                 spi_master_resume(master);
 643
 644         return ret;
 645 }
 646
 647 static int __maybe_unused qcom_qspi_resume(struct device *dev)
 648 {
 649         struct spi_master *master = dev_get_drvdata(dev);
 650         int ret;
 651
 652         ret = pm_runtime_force_resume(dev);
 653         if (ret)
 654                 return ret;
 655
 656         ret = spi_master_resume(master);
 657         if (ret)
 658                 pm_runtime_force_suspend(dev);
 659
 660         return ret;
 661 }
 662
 663 static const struct dev_pm_ops qcom_qspi_dev_pm_ops = {
 664         SET_RUNTIME_PM_OPS(qcom_qspi_runtime_suspend,
 665                            qcom_qspi_runtime_resume, NULL)
 666         SET_SYSTEM_SLEEP_PM_OPS(qcom_qspi_suspend, qcom_qspi_resume)
 667 };
 668
 669 static const struct of_device_id qcom_qspi_dt_match[] = {
 670         { .compatible = "qcom,qspi-v1", },
 671         { }
 672 };
 673 MODULE_DEVICE_TABLE(of, qcom_qspi_dt_match);
 674
 675 static struct platform_driver qcom_qspi_driver = {
 676         .driver = {
 677                 .name           = "qcom_qspi",
 678                 .pm             = &qcom_qspi_dev_pm_ops,
 679                 .of_match_table = qcom_qspi_dt_match,
 680         },
 681         .probe = qcom_qspi_probe,
 682         .remove = qcom_qspi_remove,
 683 };
 684 module_platform_driver(qcom_qspi_driver);
 685
 686 MODULE_DESCRIPTION("SPI driver for QSPI cores");
 687 MODULE_LICENSE("GPL v2");