drivers/spi/spi-dw-core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Designware SPI core controller driver (refer pxa2xx_spi.c)
   4  *
   5  * Copyright (c) 2009, Intel Corporation.
   6  */
   7
   8 #include <linux/bitfield.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/module.h>
  12 #include <linux/preempt.h>
  13 #include <linux/highmem.h>
  14 #include <linux/delay.h>
  15 #include <linux/slab.h>
  16 #include <linux/spi/spi.h>
  17 #include <linux/spi/spi-mem.h>
  18 #include <linux/string.h>
  19 #include <linux/of.h>
  20
  21 #include "spi-dw.h"
  22
  23 #ifdef CONFIG_DEBUG_FS
  24 #include <linux/debugfs.h>
  25 #endif
  26
  27 /* Slave spi_device related */
  28 struct dw_spi_chip_data {
  29         u32 cr0;
  30         u32 rx_sample_dly;      /* RX sample delay */
  31 };
  32
  33 #ifdef CONFIG_DEBUG_FS
  34
  35 #define DW_SPI_DBGFS_REG(_name, _off)   \
  36 {                                       \
  37         .name = _name,                  \
  38         .offset = _off,                 \
  39 }
  40
  41 static const struct debugfs_reg32 dw_spi_dbgfs_regs[] = {
  42         DW_SPI_DBGFS_REG("CTRLR0", DW_SPI_CTRLR0),
  43         DW_SPI_DBGFS_REG("CTRLR1", DW_SPI_CTRLR1),
  44         DW_SPI_DBGFS_REG("SSIENR", DW_SPI_SSIENR),
  45         DW_SPI_DBGFS_REG("SER", DW_SPI_SER),
  46         DW_SPI_DBGFS_REG("BAUDR", DW_SPI_BAUDR),
  47         DW_SPI_DBGFS_REG("TXFTLR", DW_SPI_TXFTLR),
  48         DW_SPI_DBGFS_REG("RXFTLR", DW_SPI_RXFTLR),
  49         DW_SPI_DBGFS_REG("TXFLR", DW_SPI_TXFLR),
  50         DW_SPI_DBGFS_REG("RXFLR", DW_SPI_RXFLR),
  51         DW_SPI_DBGFS_REG("SR", DW_SPI_SR),
  52         DW_SPI_DBGFS_REG("IMR", DW_SPI_IMR),
  53         DW_SPI_DBGFS_REG("ISR", DW_SPI_ISR),
  54         DW_SPI_DBGFS_REG("DMACR", DW_SPI_DMACR),
  55         DW_SPI_DBGFS_REG("DMATDLR", DW_SPI_DMATDLR),
  56         DW_SPI_DBGFS_REG("DMARDLR", DW_SPI_DMARDLR),
  57         DW_SPI_DBGFS_REG("RX_SAMPLE_DLY", DW_SPI_RX_SAMPLE_DLY),
  58 };
  59
  60 static int dw_spi_debugfs_init(struct dw_spi *dws)
  61 {
  62         char name[32];
  63
  64         snprintf(name, 32, "dw_spi%d", dws->master->bus_num);
  65         dws->debugfs = debugfs_create_dir(name, NULL);
  66         if (!dws->debugfs)
  67                 return -ENOMEM;
  68
  69         dws->regset.regs = dw_spi_dbgfs_regs;
  70         dws->regset.nregs = ARRAY_SIZE(dw_spi_dbgfs_regs);
  71         dws->regset.base = dws->regs;
  72         debugfs_create_regset32("registers", 0400, dws->debugfs, &dws->regset);
  73
  74         return 0;
  75 }
  76
  77 static void dw_spi_debugfs_remove(struct dw_spi *dws)
  78 {
  79         debugfs_remove_recursive(dws->debugfs);
  80 }
  81
  82 #else
  83 static inline int dw_spi_debugfs_init(struct dw_spi *dws)
  84 {
  85         return 0;
  86 }
  87
  88 static inline void dw_spi_debugfs_remove(struct dw_spi *dws)
  89 {
  90 }
  91 #endif /* CONFIG_DEBUG_FS */
  92
  93 void dw_spi_set_cs(struct spi_device *spi, bool enable)
  94 {
  95         struct dw_spi *dws = spi_controller_get_devdata(spi->controller);
  96         bool cs_high = !!(spi->mode & SPI_CS_HIGH);
  97
  98         /*
  99          * DW SPI controller demands any native CS being set in order to
 100          * proceed with data transfer. So in order to activate the SPI
 101          * communications we must set a corresponding bit in the Slave
 102          * Enable register no matter whether the SPI core is configured to
 103          * support active-high or active-low CS level.
 104          */
 105         if (cs_high == enable)
 106                 dw_writel(dws, DW_SPI_SER, BIT(spi->chip_select));
 107         else
 108                 dw_writel(dws, DW_SPI_SER, 0);
 109 }
 110 EXPORT_SYMBOL_NS_GPL(dw_spi_set_cs, SPI_DW_CORE);
 111
 112 /* Return the max entries we can fill into tx fifo */
 113 static inline u32 dw_spi_tx_max(struct dw_spi *dws)
 114 {
 115         u32 tx_room, rxtx_gap;
 116
 117         tx_room = dws->fifo_len - dw_readl(dws, DW_SPI_TXFLR);
 118
 119         /*
 120          * Another concern is about the tx/rx mismatch, we
 121          * though to use (dws->fifo_len - rxflr - txflr) as
 122          * one maximum value for tx, but it doesn't cover the
 123          * data which is out of tx/rx fifo and inside the
 124          * shift registers. So a control from sw point of
 125          * view is taken.
 126          */
 127         rxtx_gap = dws->fifo_len - (dws->rx_len - dws->tx_len);
 128
 129         return min3((u32)dws->tx_len, tx_room, rxtx_gap);
 130 }
 131
 132 /* Return the max entries we should read out of rx fifo */
 133 static inline u32 dw_spi_rx_max(struct dw_spi *dws)
 134 {
 135         return min_t(u32, dws->rx_len, dw_readl(dws, DW_SPI_RXFLR));
 136 }
 137
 138 static void dw_writer(struct dw_spi *dws)
 139 {
 140         u32 max = dw_spi_tx_max(dws);
 141         u32 txw = 0;
 142
 143         while (max--) {
 144                 if (dws->tx) {
 145                         if (dws->n_bytes == 1)
 146                                 txw = *(u8 *)(dws->tx);
 147                         else if (dws->n_bytes == 2)
 148                                 txw = *(u16 *)(dws->tx);
 149                         else
 150                                 txw = *(u32 *)(dws->tx);
 151
 152                         dws->tx += dws->n_bytes;
 153                 }
 154                 dw_write_io_reg(dws, DW_SPI_DR, txw);
 155                 --dws->tx_len;
 156         }
 157 }
 158
 159 static void dw_reader(struct dw_spi *dws)
 160 {
 161         u32 max = dw_spi_rx_max(dws);
 162         u32 rxw;
 163
 164         while (max--) {
 165                 rxw = dw_read_io_reg(dws, DW_SPI_DR);
 166                 if (dws->rx) {
 167                         if (dws->n_bytes == 1)
 168                                 *(u8 *)(dws->rx) = rxw;
 169                         else if (dws->n_bytes == 2)
 170                                 *(u16 *)(dws->rx) = rxw;
 171                         else
 172                                 *(u32 *)(dws->rx) = rxw;
 173
 174                         dws->rx += dws->n_bytes;
 175                 }
 176                 --dws->rx_len;
 177         }
 178 }
 179
 180 int dw_spi_check_status(struct dw_spi *dws, bool raw)
 181 {
 182         u32 irq_status;
 183         int ret = 0;
 184
 185         if (raw)
 186                 irq_status = dw_readl(dws, DW_SPI_RISR);
 187         else
 188                 irq_status = dw_readl(dws, DW_SPI_ISR);
 189
 190         if (irq_status & DW_SPI_INT_RXOI) {
 191                 dev_err(&dws->master->dev, "RX FIFO overflow detected\n");
 192                 ret = -EIO;
 193         }
 194
 195         if (irq_status & DW_SPI_INT_RXUI) {
 196                 dev_err(&dws->master->dev, "RX FIFO underflow detected\n");
 197                 ret = -EIO;
 198         }
 199
 200         if (irq_status & DW_SPI_INT_TXOI) {
 201                 dev_err(&dws->master->dev, "TX FIFO overflow detected\n");
 202                 ret = -EIO;
 203         }
 204
 205         /* Generically handle the erroneous situation */
 206         if (ret) {
 207                 dw_spi_reset_chip(dws);
 208                 if (dws->master->cur_msg)
 209                         dws->master->cur_msg->status = ret;
 210         }
 211
 212         return ret;
 213 }
 214 EXPORT_SYMBOL_NS_GPL(dw_spi_check_status, SPI_DW_CORE);
 215
 216 static irqreturn_t dw_spi_transfer_handler(struct dw_spi *dws)
 217 {
 218         u16 irq_status = dw_readl(dws, DW_SPI_ISR);
 219
 220         if (dw_spi_check_status(dws, false)) {
 221                 spi_finalize_current_transfer(dws->master);
 222                 return IRQ_HANDLED;
 223         }
 224
 225         /*
 226          * Read data from the Rx FIFO every time we've got a chance executing
 227          * this method. If there is nothing left to receive, terminate the
 228          * procedure. Otherwise adjust the Rx FIFO Threshold level if it's a
 229          * final stage of the transfer. By doing so we'll get the next IRQ
 230          * right when the leftover incoming data is received.
 231          */
 232         dw_reader(dws);
 233         if (!dws->rx_len) {
 234                 dw_spi_mask_intr(dws, 0xff);
 235                 spi_finalize_current_transfer(dws->master);
 236         } else if (dws->rx_len <= dw_readl(dws, DW_SPI_RXFTLR)) {
 237                 dw_writel(dws, DW_SPI_RXFTLR, dws->rx_len - 1);
 238         }
 239
 240         /*
 241          * Send data out if Tx FIFO Empty IRQ is received. The IRQ will be
 242          * disabled after the data transmission is finished so not to
 243          * have the TXE IRQ flood at the final stage of the transfer.
 244          */
 245         if (irq_status & DW_SPI_INT_TXEI) {
 246                 dw_writer(dws);
 247                 if (!dws->tx_len)
 248                         dw_spi_mask_intr(dws, DW_SPI_INT_TXEI);
 249         }
 250
 251         return IRQ_HANDLED;
 252 }
 253
 254 static irqreturn_t dw_spi_irq(int irq, void *dev_id)
 255 {
 256         struct spi_controller *master = dev_id;
 257         struct dw_spi *dws = spi_controller_get_devdata(master);
 258         u16 irq_status = dw_readl(dws, DW_SPI_ISR) & DW_SPI_INT_MASK;
 259
 260         if (!irq_status)
 261                 return IRQ_NONE;
 262
 263         if (!master->cur_msg) {
 264                 dw_spi_mask_intr(dws, 0xff);
 265                 return IRQ_HANDLED;
 266         }
 267
 268         return dws->transfer_handler(dws);
 269 }
 270
 271 static u32 dw_spi_prepare_cr0(struct dw_spi *dws, struct spi_device *spi)
 272 {
 273         u32 cr0 = 0;
 274
 275         if (dw_spi_ip_is(dws, PSSI)) {
 276                 /* CTRLR0[ 5: 4] Frame Format */
 277                 cr0 |= FIELD_PREP(DW_PSSI_CTRLR0_FRF_MASK, DW_SPI_CTRLR0_FRF_MOTO_SPI);
 278
 279                 /*
 280                  * SPI mode (SCPOL|SCPH)
 281                  * CTRLR0[ 6] Serial Clock Phase
 282                  * CTRLR0[ 7] Serial Clock Polarity
 283                  */
 284                 if (spi->mode & SPI_CPOL)
 285                         cr0 |= DW_PSSI_CTRLR0_SCPOL;
 286                 if (spi->mode & SPI_CPHA)
 287                         cr0 |= DW_PSSI_CTRLR0_SCPHA;
 288
 289                 /* CTRLR0[11] Shift Register Loop */
 290                 if (spi->mode & SPI_LOOP)
 291                         cr0 |= DW_PSSI_CTRLR0_SRL;
 292         } else {
 293                 /* CTRLR0[ 7: 6] Frame Format */
 294                 cr0 |= FIELD_PREP(DW_HSSI_CTRLR0_FRF_MASK, DW_SPI_CTRLR0_FRF_MOTO_SPI);
 295
 296                 /*
 297                  * SPI mode (SCPOL|SCPH)
 298                  * CTRLR0[ 8] Serial Clock Phase
 299                  * CTRLR0[ 9] Serial Clock Polarity
 300                  */
 301                 if (spi->mode & SPI_CPOL)
 302                         cr0 |= DW_HSSI_CTRLR0_SCPOL;
 303                 if (spi->mode & SPI_CPHA)
 304                         cr0 |= DW_HSSI_CTRLR0_SCPHA;
 305
 306                 /* CTRLR0[13] Shift Register Loop */
 307                 if (spi->mode & SPI_LOOP)
 308                         cr0 |= DW_HSSI_CTRLR0_SRL;
 309
 310                 if (dws->caps & DW_SPI_CAP_KEEMBAY_MST)
 311                         cr0 |= DW_HSSI_CTRLR0_KEEMBAY_MST;
 312         }
 313
 314         return cr0;
 315 }
 316
 317 void dw_spi_update_config(struct dw_spi *dws, struct spi_device *spi,
 318                           struct dw_spi_cfg *cfg)
 319 {
 320         struct dw_spi_chip_data *chip = spi_get_ctldata(spi);
 321         u32 cr0 = chip->cr0;
 322         u32 speed_hz;
 323         u16 clk_div;
 324
 325         /* CTRLR0[ 4/3: 0] or CTRLR0[ 20: 16] Data Frame Size */
 326         cr0 |= (cfg->dfs - 1) << dws->dfs_offset;
 327
 328         if (dw_spi_ip_is(dws, PSSI))
 329                 /* CTRLR0[ 9:8] Transfer Mode */
 330                 cr0 |= FIELD_PREP(DW_PSSI_CTRLR0_TMOD_MASK, cfg->tmode);
 331         else
 332                 /* CTRLR0[11:10] Transfer Mode */
 333                 cr0 |= FIELD_PREP(DW_HSSI_CTRLR0_TMOD_MASK, cfg->tmode);
 334
 335         dw_writel(dws, DW_SPI_CTRLR0, cr0);
 336
 337         if (cfg->tmode == DW_SPI_CTRLR0_TMOD_EPROMREAD ||
 338             cfg->tmode == DW_SPI_CTRLR0_TMOD_RO)
 339                 dw_writel(dws, DW_SPI_CTRLR1, cfg->ndf ? cfg->ndf - 1 : 0);
 340
 341         /* Note DW APB SSI clock divider doesn't support odd numbers */
 342         clk_div = (DIV_ROUND_UP(dws->max_freq, cfg->freq) + 1) & 0xfffe;
 343         speed_hz = dws->max_freq / clk_div;
 344
 345         if (dws->current_freq != speed_hz) {
 346                 dw_spi_set_clk(dws, clk_div);
 347                 dws->current_freq = speed_hz;
 348         }
 349
 350         /* Update RX sample delay if required */
 351         if (dws->cur_rx_sample_dly != chip->rx_sample_dly) {
 352                 dw_writel(dws, DW_SPI_RX_SAMPLE_DLY, chip->rx_sample_dly);
 353                 dws->cur_rx_sample_dly = chip->rx_sample_dly;
 354         }
 355 }
 356 EXPORT_SYMBOL_NS_GPL(dw_spi_update_config, SPI_DW_CORE);
 357
 358 static void dw_spi_irq_setup(struct dw_spi *dws)
 359 {
 360         u16 level;
 361         u8 imask;
 362
 363         /*
 364          * Originally Tx and Rx data lengths match. Rx FIFO Threshold level
 365          * will be adjusted at the final stage of the IRQ-based SPI transfer
 366          * execution so not to lose the leftover of the incoming data.
 367          */
 368         level = min_t(u16, dws->fifo_len / 2, dws->tx_len);
 369         dw_writel(dws, DW_SPI_TXFTLR, level);
 370         dw_writel(dws, DW_SPI_RXFTLR, level - 1);
 371
 372         dws->transfer_handler = dw_spi_transfer_handler;
 373
 374         imask = DW_SPI_INT_TXEI | DW_SPI_INT_TXOI |
 375                 DW_SPI_INT_RXUI | DW_SPI_INT_RXOI | DW_SPI_INT_RXFI;
 376         dw_spi_umask_intr(dws, imask);
 377 }
 378
 379 /*
 380  * The iterative procedure of the poll-based transfer is simple: write as much
 381  * as possible to the Tx FIFO, wait until the pending to receive data is ready
 382  * to be read, read it from the Rx FIFO and check whether the performed
 383  * procedure has been successful.
 384  *
 385  * Note this method the same way as the IRQ-based transfer won't work well for
 386  * the SPI devices connected to the controller with native CS due to the
 387  * automatic CS assertion/de-assertion.
 388  */
 389 static int dw_spi_poll_transfer(struct dw_spi *dws,
 390                                 struct spi_transfer *transfer)
 391 {
 392         struct spi_delay delay;
 393         u16 nbits;
 394         int ret;
 395
 396         delay.unit = SPI_DELAY_UNIT_SCK;
 397         nbits = dws->n_bytes * BITS_PER_BYTE;
 398
 399         do {
 400                 dw_writer(dws);
 401
 402                 delay.value = nbits * (dws->rx_len - dws->tx_len);
 403                 spi_delay_exec(&delay, transfer);
 404
 405                 dw_reader(dws);
 406
 407                 ret = dw_spi_check_status(dws, true);
 408                 if (ret)
 409                         return ret;
 410         } while (dws->rx_len);
 411
 412         return 0;
 413 }
 414
 415 static int dw_spi_transfer_one(struct spi_controller *master,
 416                                struct spi_device *spi,
 417                                struct spi_transfer *transfer)
 418 {
 419         struct dw_spi *dws = spi_controller_get_devdata(master);
 420         struct dw_spi_cfg cfg = {
 421                 .tmode = DW_SPI_CTRLR0_TMOD_TR,
 422                 .dfs = transfer->bits_per_word,
 423                 .freq = transfer->speed_hz,
 424         };
 425         int ret;
 426
 427         dws->dma_mapped = 0;
 428         dws->n_bytes = DIV_ROUND_UP(transfer->bits_per_word, BITS_PER_BYTE);
 429         dws->tx = (void *)transfer->tx_buf;
 430         dws->tx_len = transfer->len / dws->n_bytes;
 431         dws->rx = transfer->rx_buf;
 432         dws->rx_len = dws->tx_len;
 433
 434         /* Ensure the data above is visible for all CPUs */
 435         smp_mb();
 436
 437         dw_spi_enable_chip(dws, 0);
 438
 439         dw_spi_update_config(dws, spi, &cfg);
 440
 441         transfer->effective_speed_hz = dws->current_freq;
 442
 443         /* Check if current transfer is a DMA transaction */
 444         if (master->can_dma && master->can_dma(master, spi, transfer))
 445                 dws->dma_mapped = master->cur_msg_mapped;
 446
 447         /* For poll mode just disable all interrupts */
 448         dw_spi_mask_intr(dws, 0xff);
 449
 450         if (dws->dma_mapped) {
 451                 ret = dws->dma_ops->dma_setup(dws, transfer);
 452                 if (ret)
 453                         return ret;
 454         }
 455
 456         dw_spi_enable_chip(dws, 1);
 457
 458         if (dws->dma_mapped)
 459                 return dws->dma_ops->dma_transfer(dws, transfer);
 460         else if (dws->irq == IRQ_NOTCONNECTED)
 461                 return dw_spi_poll_transfer(dws, transfer);
 462
 463         dw_spi_irq_setup(dws);
 464
 465         return 1;
 466 }
 467
 468 static void dw_spi_handle_err(struct spi_controller *master,
 469                               struct spi_message *msg)
 470 {
 471         struct dw_spi *dws = spi_controller_get_devdata(master);
 472
 473         if (dws->dma_mapped)
 474                 dws->dma_ops->dma_stop(dws);
 475
 476         dw_spi_reset_chip(dws);
 477 }
 478
 479 static int dw_spi_adjust_mem_op_size(struct spi_mem *mem, struct spi_mem_op *op)
 480 {
 481         if (op->data.dir == SPI_MEM_DATA_IN)
 482                 op->data.nbytes = clamp_val(op->data.nbytes, 0, DW_SPI_NDF_MASK + 1);
 483
 484         return 0;
 485 }
 486
 487 static bool dw_spi_supports_mem_op(struct spi_mem *mem,
 488                                    const struct spi_mem_op *op)
 489 {
 490         if (op->data.buswidth > 1 || op->addr.buswidth > 1 ||
 491             op->dummy.buswidth > 1 || op->cmd.buswidth > 1)
 492                 return false;
 493
 494         return spi_mem_default_supports_op(mem, op);
 495 }
 496
 497 static int dw_spi_init_mem_buf(struct dw_spi *dws, const struct spi_mem_op *op)
 498 {
 499         unsigned int i, j, len;
 500         u8 *out;
 501
 502         /*
 503          * Calculate the total length of the EEPROM command transfer and
 504          * either use the pre-allocated buffer or create a temporary one.
 505          */
 506         len = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes;
 507         if (op->data.dir == SPI_MEM_DATA_OUT)
 508                 len += op->data.nbytes;
 509
 510         if (len <= DW_SPI_BUF_SIZE) {
 511                 out = dws->buf;
 512         } else {
 513                 out = kzalloc(len, GFP_KERNEL);
 514                 if (!out)
 515                         return -ENOMEM;
 516         }
 517
 518         /*
 519          * Collect the operation code, address and dummy bytes into the single
 520          * buffer. If it's a transfer with data to be sent, also copy it into the
 521          * single buffer in order to speed the data transmission up.
 522          */
 523         for (i = 0; i < op->cmd.nbytes; ++i)
 524                 out[i] = DW_SPI_GET_BYTE(op->cmd.opcode, op->cmd.nbytes - i - 1);
 525         for (j = 0; j < op->addr.nbytes; ++i, ++j)
 526                 out[i] = DW_SPI_GET_BYTE(op->addr.val, op->addr.nbytes - j - 1);
 527         for (j = 0; j < op->dummy.nbytes; ++i, ++j)
 528                 out[i] = 0x0;
 529
 530         if (op->data.dir == SPI_MEM_DATA_OUT)
 531                 memcpy(&out[i], op->data.buf.out, op->data.nbytes);
 532
 533         dws->n_bytes = 1;
 534         dws->tx = out;
 535         dws->tx_len = len;
 536         if (op->data.dir == SPI_MEM_DATA_IN) {
 537                 dws->rx = op->data.buf.in;
 538                 dws->rx_len = op->data.nbytes;
 539         } else {
 540                 dws->rx = NULL;
 541                 dws->rx_len = 0;
 542         }
 543
 544         return 0;
 545 }
 546
 547 static void dw_spi_free_mem_buf(struct dw_spi *dws)
 548 {
 549         if (dws->tx != dws->buf)
 550                 kfree(dws->tx);
 551 }
 552
 553 static int dw_spi_write_then_read(struct dw_spi *dws, struct spi_device *spi)
 554 {
 555         u32 room, entries, sts;
 556         unsigned int len;
 557         u8 *buf;
 558
 559         /*
 560          * At initial stage we just pre-fill the Tx FIFO in with no rush,
 561          * since native CS hasn't been enabled yet and the automatic data
 562          * transmission won't start til we do that.
 563          */
 564         len = min(dws->fifo_len, dws->tx_len);
 565         buf = dws->tx;
 566         while (len--)
 567                 dw_write_io_reg(dws, DW_SPI_DR, *buf++);
 568
 569         /*
 570          * After setting any bit in the SER register the transmission will
 571          * start automatically. We have to keep up with that procedure
 572          * otherwise the CS de-assertion will happen whereupon the memory
 573          * operation will be pre-terminated.
 574          */
 575         len = dws->tx_len - ((void *)buf - dws->tx);
 576         dw_spi_set_cs(spi, false);
 577         while (len) {
 578                 entries = readl_relaxed(dws->regs + DW_SPI_TXFLR);
 579                 if (!entries) {
 580                         dev_err(&dws->master->dev, "CS de-assertion on Tx\n");
 581                         return -EIO;
 582                 }
 583                 room = min(dws->fifo_len - entries, len);
 584                 for (; room; --room, --len)
 585                         dw_write_io_reg(dws, DW_SPI_DR, *buf++);
 586         }
 587
 588         /*
 589          * Data fetching will start automatically if the EEPROM-read mode is
 590          * activated. We have to keep up with the incoming data pace to
 591          * prevent the Rx FIFO overflow causing the inbound data loss.
 592          */
 593         len = dws->rx_len;
 594         buf = dws->rx;
 595         while (len) {
 596                 entries = readl_relaxed(dws->regs + DW_SPI_RXFLR);
 597                 if (!entries) {
 598                         sts = readl_relaxed(dws->regs + DW_SPI_RISR);
 599                         if (sts & DW_SPI_INT_RXOI) {
 600                                 dev_err(&dws->master->dev, "FIFO overflow on Rx\n");
 601                                 return -EIO;
 602                         }
 603                         continue;
 604                 }
 605                 entries = min(entries, len);
 606                 for (; entries; --entries, --len)
 607                         *buf++ = dw_read_io_reg(dws, DW_SPI_DR);
 608         }
 609
 610         return 0;
 611 }
 612
 613 static inline bool dw_spi_ctlr_busy(struct dw_spi *dws)
 614 {
 615         return dw_readl(dws, DW_SPI_SR) & DW_SPI_SR_BUSY;
 616 }
 617
 618 static int dw_spi_wait_mem_op_done(struct dw_spi *dws)
 619 {
 620         int retry = DW_SPI_WAIT_RETRIES;
 621         struct spi_delay delay;
 622         unsigned long ns, us;
 623         u32 nents;
 624
 625         nents = dw_readl(dws, DW_SPI_TXFLR);
 626         ns = NSEC_PER_SEC / dws->current_freq * nents;
 627         ns *= dws->n_bytes * BITS_PER_BYTE;
 628         if (ns <= NSEC_PER_USEC) {
 629                 delay.unit = SPI_DELAY_UNIT_NSECS;
 630                 delay.value = ns;
 631         } else {
 632                 us = DIV_ROUND_UP(ns, NSEC_PER_USEC);
 633                 delay.unit = SPI_DELAY_UNIT_USECS;
 634                 delay.value = clamp_val(us, 0, USHRT_MAX);
 635         }
 636
 637         while (dw_spi_ctlr_busy(dws) && retry--)
 638                 spi_delay_exec(&delay, NULL);
 639
 640         if (retry < 0) {
 641                 dev_err(&dws->master->dev, "Mem op hanged up\n");
 642                 return -EIO;
 643         }
 644
 645         return 0;
 646 }
 647
 648 static void dw_spi_stop_mem_op(struct dw_spi *dws, struct spi_device *spi)
 649 {
 650         dw_spi_enable_chip(dws, 0);
 651         dw_spi_set_cs(spi, true);
 652         dw_spi_enable_chip(dws, 1);
 653 }
 654
 655 /*
 656  * The SPI memory operation implementation below is the best choice for the
 657  * devices, which are selected by the native chip-select lane. It's
 658  * specifically developed to workaround the problem with automatic chip-select
 659  * lane toggle when there is no data in the Tx FIFO buffer. Luckily the current
 660  * SPI-mem core calls exec_op() callback only if the GPIO-based CS is
 661  * unavailable.
 662  */
 663 static int dw_spi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
 664 {
 665         struct dw_spi *dws = spi_controller_get_devdata(mem->spi->controller);
 666         struct dw_spi_cfg cfg;
 667         unsigned long flags;
 668         int ret;
 669
 670         /*
 671          * Collect the outbound data into a single buffer to speed the
 672          * transmission up at least on the initial stage.
 673          */
 674         ret = dw_spi_init_mem_buf(dws, op);
 675         if (ret)
 676                 return ret;
 677
 678         /*
 679          * DW SPI EEPROM-read mode is required only for the SPI memory Data-IN
 680          * operation. Transmit-only mode is suitable for the rest of them.
 681          */
 682         cfg.dfs = 8;
 683         cfg.freq = clamp(mem->spi->max_speed_hz, 0U, dws->max_mem_freq);
 684         if (op->data.dir == SPI_MEM_DATA_IN) {
 685                 cfg.tmode = DW_SPI_CTRLR0_TMOD_EPROMREAD;
 686                 cfg.ndf = op->data.nbytes;
 687         } else {
 688                 cfg.tmode = DW_SPI_CTRLR0_TMOD_TO;
 689         }
 690
 691         dw_spi_enable_chip(dws, 0);
 692
 693         dw_spi_update_config(dws, mem->spi, &cfg);
 694
 695         dw_spi_mask_intr(dws, 0xff);
 696
 697         dw_spi_enable_chip(dws, 1);
 698
 699         /*
 700          * DW APB SSI controller has very nasty peculiarities. First originally
 701          * (without any vendor-specific modifications) it doesn't provide a
 702          * direct way to set and clear the native chip-select signal. Instead
 703          * the controller asserts the CS lane if Tx FIFO isn't empty and a
 704          * transmission is going on, and automatically de-asserts it back to
 705          * the high level if the Tx FIFO doesn't have anything to be pushed
 706          * out. Due to that a multi-tasking or heavy IRQs activity might be
 707          * fatal, since the transfer procedure preemption may cause the Tx FIFO
 708          * getting empty and sudden CS de-assertion, which in the middle of the
 709          * transfer will most likely cause the data loss. Secondly the
 710          * EEPROM-read or Read-only DW SPI transfer modes imply the incoming
 711          * data being automatically pulled in into the Rx FIFO. So if the
 712          * driver software is late in fetching the data from the FIFO before
 713          * it's overflown, new incoming data will be lost. In order to make
 714          * sure the executed memory operations are CS-atomic and to prevent the
 715          * Rx FIFO overflow we have to disable the local interrupts so to block
 716          * any preemption during the subsequent IO operations.
 717          *
 718          * Note. At some circumstances disabling IRQs may not help to prevent
 719          * the problems described above. The CS de-assertion and Rx FIFO
 720          * overflow may still happen due to the relatively slow system bus or
 721          * CPU not working fast enough, so the write-then-read algo implemented
 722          * here just won't keep up with the SPI bus data transfer. Such
 723          * situation is highly platform specific and is supposed to be fixed by
 724          * manually restricting the SPI bus frequency using the
 725          * dws->max_mem_freq parameter.
 726          */
 727         local_irq_save(flags);
 728         preempt_disable();
 729
 730         ret = dw_spi_write_then_read(dws, mem->spi);
 731
 732         local_irq_restore(flags);
 733         preempt_enable();
 734
 735         /*
 736          * Wait for the operation being finished and check the controller
 737          * status only if there hasn't been any run-time error detected. In the
 738          * former case it's just pointless. In the later one to prevent an
 739          * additional error message printing since any hw error flag being set
 740          * would be due to an error detected on the data transfer.
 741          */
 742         if (!ret) {
 743                 ret = dw_spi_wait_mem_op_done(dws);
 744                 if (!ret)
 745                         ret = dw_spi_check_status(dws, true);
 746         }
 747
 748         dw_spi_stop_mem_op(dws, mem->spi);
 749
 750         dw_spi_free_mem_buf(dws);
 751
 752         return ret;
 753 }
 754
 755 /*
 756  * Initialize the default memory operations if a glue layer hasn't specified
 757  * custom ones. Direct mapping operations will be preserved anyway since DW SPI
 758  * controller doesn't have an embedded dirmap interface. Note the memory
 759  * operations implemented in this driver is the best choice only for the DW APB
 760  * SSI controller with standard native CS functionality. If a hardware vendor
 761  * has fixed the automatic CS assertion/de-assertion peculiarity, then it will
 762  * be safer to use the normal SPI-messages-based transfers implementation.
 763  */
 764 static void dw_spi_init_mem_ops(struct dw_spi *dws)
 765 {
 766         if (!dws->mem_ops.exec_op && !(dws->caps & DW_SPI_CAP_CS_OVERRIDE) &&
 767             !dws->set_cs) {
 768                 dws->mem_ops.adjust_op_size = dw_spi_adjust_mem_op_size;
 769                 dws->mem_ops.supports_op = dw_spi_supports_mem_op;
 770                 dws->mem_ops.exec_op = dw_spi_exec_mem_op;
 771                 if (!dws->max_mem_freq)
 772                         dws->max_mem_freq = dws->max_freq;
 773         }
 774 }
 775
 776 /* This may be called twice for each spi dev */
 777 static int dw_spi_setup(struct spi_device *spi)
 778 {
 779         struct dw_spi *dws = spi_controller_get_devdata(spi->controller);
 780         struct dw_spi_chip_data *chip;
 781
 782         /* Only alloc on first setup */
 783         chip = spi_get_ctldata(spi);
 784         if (!chip) {
 785                 struct dw_spi *dws = spi_controller_get_devdata(spi->controller);
 786                 u32 rx_sample_dly_ns;
 787
 788                 chip = kzalloc(sizeof(*chip), GFP_KERNEL);
 789                 if (!chip)
 790                         return -ENOMEM;
 791                 spi_set_ctldata(spi, chip);
 792                 /* Get specific / default rx-sample-delay */
 793                 if (device_property_read_u32(&spi->dev,
 794                                              "rx-sample-delay-ns",
 795                                              &rx_sample_dly_ns) != 0)
 796                         /* Use default controller value */
 797                         rx_sample_dly_ns = dws->def_rx_sample_dly_ns;
 798                 chip->rx_sample_dly = DIV_ROUND_CLOSEST(rx_sample_dly_ns,
 799                                                         NSEC_PER_SEC /
 800                                                         dws->max_freq);
 801         }
 802
 803         /*
 804          * Update CR0 data each time the setup callback is invoked since
 805          * the device parameters could have been changed, for instance, by
 806          * the MMC SPI driver or something else.
 807          */
 808         chip->cr0 = dw_spi_prepare_cr0(dws, spi);
 809
 810         return 0;
 811 }
 812
 813 static void dw_spi_cleanup(struct spi_device *spi)
 814 {
 815         struct dw_spi_chip_data *chip = spi_get_ctldata(spi);
 816
 817         kfree(chip);
 818         spi_set_ctldata(spi, NULL);
 819 }
 820
 821 /* Restart the controller, disable all interrupts, clean rx fifo */
 822 static void dw_spi_hw_init(struct device *dev, struct dw_spi *dws)
 823 {
 824         dw_spi_reset_chip(dws);
 825
 826         /*
 827          * Retrieve the Synopsys component version if it hasn't been specified
 828          * by the platform. CoreKit version ID is encoded as a 3-chars ASCII
 829          * code enclosed with '*' (typical for the most of Synopsys IP-cores).
 830          */
 831         if (!dws->ver) {
 832                 dws->ver = dw_readl(dws, DW_SPI_VERSION);
 833
 834                 dev_dbg(dev, "Synopsys DWC%sSSI v%c.%c%c\n",
 835                         dw_spi_ip_is(dws, PSSI) ? " APB " : " ",
 836                         DW_SPI_GET_BYTE(dws->ver, 3), DW_SPI_GET_BYTE(dws->ver, 2),
 837                         DW_SPI_GET_BYTE(dws->ver, 1));
 838         }
 839
 840         /*
 841          * Try to detect the FIFO depth if not set by interface driver,
 842          * the depth could be from 2 to 256 from HW spec
 843          */
 844         if (!dws->fifo_len) {
 845                 u32 fifo;
 846
 847                 for (fifo = 1; fifo < 256; fifo++) {
 848                         dw_writel(dws, DW_SPI_TXFTLR, fifo);
 849                         if (fifo != dw_readl(dws, DW_SPI_TXFTLR))
 850                                 break;
 851                 }
 852                 dw_writel(dws, DW_SPI_TXFTLR, 0);
 853
 854                 dws->fifo_len = (fifo == 1) ? 0 : fifo;
 855                 dev_dbg(dev, "Detected FIFO size: %u bytes\n", dws->fifo_len);
 856         }
 857
 858         /*
 859          * Detect CTRLR0.DFS field size and offset by testing the lowest bits
 860          * writability. Note DWC SSI controller also has the extended DFS, but
 861          * with zero offset.
 862          */
 863         if (dw_spi_ip_is(dws, PSSI)) {
 864                 u32 cr0, tmp = dw_readl(dws, DW_SPI_CTRLR0);
 865
 866                 dw_spi_enable_chip(dws, 0);
 867                 dw_writel(dws, DW_SPI_CTRLR0, 0xffffffff);
 868                 cr0 = dw_readl(dws, DW_SPI_CTRLR0);
 869                 dw_writel(dws, DW_SPI_CTRLR0, tmp);
 870                 dw_spi_enable_chip(dws, 1);
 871
 872                 if (!(cr0 & DW_PSSI_CTRLR0_DFS_MASK)) {
 873                         dws->caps |= DW_SPI_CAP_DFS32;
 874                         dws->dfs_offset = __bf_shf(DW_PSSI_CTRLR0_DFS32_MASK);
 875                         dev_dbg(dev, "Detected 32-bits max data frame size\n");
 876                 }
 877         } else {
 878                 dws->caps |= DW_SPI_CAP_DFS32;
 879         }
 880
 881         /* enable HW fixup for explicit CS deselect for Amazon's alpine chip */
 882         if (dws->caps & DW_SPI_CAP_CS_OVERRIDE)
 883                 dw_writel(dws, DW_SPI_CS_OVERRIDE, 0xF);
 884 }
 885
 886 int dw_spi_add_host(struct device *dev, struct dw_spi *dws)
 887 {
 888         struct spi_controller *master;
 889         int ret;
 890
 891         if (!dws)
 892                 return -EINVAL;
 893
 894         master = spi_alloc_master(dev, 0);
 895         if (!master)
 896                 return -ENOMEM;
 897
 898         device_set_node(&master->dev, dev_fwnode(dev));
 899
 900         dws->master = master;
 901         dws->dma_addr = (dma_addr_t)(dws->paddr + DW_SPI_DR);
 902
 903         spi_controller_set_devdata(master, dws);
 904
 905         /* Basic HW init */
 906         dw_spi_hw_init(dev, dws);
 907
 908         ret = request_irq(dws->irq, dw_spi_irq, IRQF_SHARED, dev_name(dev),
 909                           master);
 910         if (ret < 0 && ret != -ENOTCONN) {
 911                 dev_err(dev, "can not get IRQ\n");
 912                 goto err_free_master;
 913         }
 914
 915         dw_spi_init_mem_ops(dws);
 916
 917         master->use_gpio_descriptors = true;
 918         master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LOOP;
 919         if (dws->caps & DW_SPI_CAP_DFS32)
 920                 master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
 921         else
 922                 master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 16);
 923         master->bus_num = dws->bus_num;
 924         master->num_chipselect = dws->num_cs;
 925         master->setup = dw_spi_setup;
 926         master->cleanup = dw_spi_cleanup;
 927         if (dws->set_cs)
 928                 master->set_cs = dws->set_cs;
 929         else
 930                 master->set_cs = dw_spi_set_cs;
 931         master->transfer_one = dw_spi_transfer_one;
 932         master->handle_err = dw_spi_handle_err;
 933         if (dws->mem_ops.exec_op)
 934                 master->mem_ops = &dws->mem_ops;
 935         master->max_speed_hz = dws->max_freq;
 936         master->flags = SPI_MASTER_GPIO_SS;
 937         master->auto_runtime_pm = true;
 938
 939         /* Get default rx sample delay */
 940         device_property_read_u32(dev, "rx-sample-delay-ns",
 941                                  &dws->def_rx_sample_dly_ns);
 942
 943         if (dws->dma_ops && dws->dma_ops->dma_init) {
 944                 ret = dws->dma_ops->dma_init(dev, dws);
 945                 if (ret) {
 946                         dev_warn(dev, "DMA init failed\n");
 947                 } else {
 948                         master->can_dma = dws->dma_ops->can_dma;
 949                         master->flags |= SPI_CONTROLLER_MUST_TX;
 950                 }
 951         }
 952
 953         ret = spi_register_controller(master);
 954         if (ret) {
 955                 dev_err(&master->dev, "problem registering spi master\n");
 956                 goto err_dma_exit;
 957         }
 958
 959         dw_spi_debugfs_init(dws);
 960         return 0;
 961
 962 err_dma_exit:
 963         if (dws->dma_ops && dws->dma_ops->dma_exit)
 964                 dws->dma_ops->dma_exit(dws);
 965         dw_spi_enable_chip(dws, 0);
 966         free_irq(dws->irq, master);
 967 err_free_master:
 968         spi_controller_put(master);
 969         return ret;
 970 }
 971 EXPORT_SYMBOL_NS_GPL(dw_spi_add_host, SPI_DW_CORE);
 972
 973 void dw_spi_remove_host(struct dw_spi *dws)
 974 {
 975         dw_spi_debugfs_remove(dws);
 976
 977         spi_unregister_controller(dws->master);
 978
 979         if (dws->dma_ops && dws->dma_ops->dma_exit)
 980                 dws->dma_ops->dma_exit(dws);
 981
 982         dw_spi_shutdown_chip(dws);
 983
 984         free_irq(dws->irq, dws->master);
 985 }
 986 EXPORT_SYMBOL_NS_GPL(dw_spi_remove_host, SPI_DW_CORE);
 987
 988 int dw_spi_suspend_host(struct dw_spi *dws)
 989 {
 990         int ret;
 991
 992         ret = spi_controller_suspend(dws->master);
 993         if (ret)
 994                 return ret;
 995
 996         dw_spi_shutdown_chip(dws);
 997         return 0;
 998 }
 999 EXPORT_SYMBOL_NS_GPL(dw_spi_suspend_host, SPI_DW_CORE);
1000
1001 int dw_spi_resume_host(struct dw_spi *dws)
1002 {
1003         dw_spi_hw_init(&dws->master->dev, dws);
1004         return spi_controller_resume(dws->master);
1005 }
1006 EXPORT_SYMBOL_NS_GPL(dw_spi_resume_host, SPI_DW_CORE);
1007
1008 MODULE_AUTHOR("Feng Tang <feng.tang@intel.com>");
1009 MODULE_DESCRIPTION("Driver for DesignWare SPI controller core");
1010 MODULE_LICENSE("GPL v2");