drivers/spi/spi-omap2-mcspi.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * OMAP2 McSPI controller driver
   4  *
   5  * Copyright (C) 2005, 2006 Nokia Corporation
   6  * Author:      Samuel Ortiz <samuel.ortiz@nokia.com> and
   7  *              Juha Yrj�l� <juha.yrjola@nokia.com>
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/module.h>
  13 #include <linux/device.h>
  14 #include <linux/delay.h>
  15 #include <linux/dma-mapping.h>
  16 #include <linux/dmaengine.h>
  17 #include <linux/pinctrl/consumer.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/err.h>
  20 #include <linux/clk.h>
  21 #include <linux/io.h>
  22 #include <linux/slab.h>
  23 #include <linux/pm_runtime.h>
  24 #include <linux/of.h>
  25 #include <linux/of_device.h>
  26 #include <linux/gcd.h>
  27 #include <linux/iopoll.h>
  28
  29 #include <linux/spi/spi.h>
  30 #include <linux/gpio.h>
  31
  32 #include <linux/platform_data/spi-omap2-mcspi.h>
  33
  34 #define OMAP2_MCSPI_MAX_FREQ            48000000
  35 #define OMAP2_MCSPI_MAX_DIVIDER         4096
  36 #define OMAP2_MCSPI_MAX_FIFODEPTH       64
  37 #define OMAP2_MCSPI_MAX_FIFOWCNT        0xFFFF
  38 #define SPI_AUTOSUSPEND_TIMEOUT         2000
  39
  40 #define OMAP2_MCSPI_REVISION            0x00
  41 #define OMAP2_MCSPI_SYSSTATUS           0x14
  42 #define OMAP2_MCSPI_IRQSTATUS           0x18
  43 #define OMAP2_MCSPI_IRQENABLE           0x1c
  44 #define OMAP2_MCSPI_WAKEUPENABLE        0x20
  45 #define OMAP2_MCSPI_SYST                0x24
  46 #define OMAP2_MCSPI_MODULCTRL           0x28
  47 #define OMAP2_MCSPI_XFERLEVEL           0x7c
  48
  49 /* per-channel banks, 0x14 bytes each, first is: */
  50 #define OMAP2_MCSPI_CHCONF0             0x2c
  51 #define OMAP2_MCSPI_CHSTAT0             0x30
  52 #define OMAP2_MCSPI_CHCTRL0             0x34
  53 #define OMAP2_MCSPI_TX0                 0x38
  54 #define OMAP2_MCSPI_RX0                 0x3c
  55
  56 /* per-register bitmasks: */
  57 #define OMAP2_MCSPI_IRQSTATUS_EOW       BIT(17)
  58
  59 #define OMAP2_MCSPI_MODULCTRL_SINGLE    BIT(0)
  60 #define OMAP2_MCSPI_MODULCTRL_MS        BIT(2)
  61 #define OMAP2_MCSPI_MODULCTRL_STEST     BIT(3)
  62
  63 #define OMAP2_MCSPI_CHCONF_PHA          BIT(0)
  64 #define OMAP2_MCSPI_CHCONF_POL          BIT(1)
  65 #define OMAP2_MCSPI_CHCONF_CLKD_MASK    (0x0f << 2)
  66 #define OMAP2_MCSPI_CHCONF_EPOL         BIT(6)
  67 #define OMAP2_MCSPI_CHCONF_WL_MASK      (0x1f << 7)
  68 #define OMAP2_MCSPI_CHCONF_TRM_RX_ONLY  BIT(12)
  69 #define OMAP2_MCSPI_CHCONF_TRM_TX_ONLY  BIT(13)
  70 #define OMAP2_MCSPI_CHCONF_TRM_MASK     (0x03 << 12)
  71 #define OMAP2_MCSPI_CHCONF_DMAW         BIT(14)
  72 #define OMAP2_MCSPI_CHCONF_DMAR         BIT(15)
  73 #define OMAP2_MCSPI_CHCONF_DPE0         BIT(16)
  74 #define OMAP2_MCSPI_CHCONF_DPE1         BIT(17)
  75 #define OMAP2_MCSPI_CHCONF_IS           BIT(18)
  76 #define OMAP2_MCSPI_CHCONF_TURBO        BIT(19)
  77 #define OMAP2_MCSPI_CHCONF_FORCE        BIT(20)
  78 #define OMAP2_MCSPI_CHCONF_FFET         BIT(27)
  79 #define OMAP2_MCSPI_CHCONF_FFER         BIT(28)
  80 #define OMAP2_MCSPI_CHCONF_CLKG         BIT(29)
  81
  82 #define OMAP2_MCSPI_CHSTAT_RXS          BIT(0)
  83 #define OMAP2_MCSPI_CHSTAT_TXS          BIT(1)
  84 #define OMAP2_MCSPI_CHSTAT_EOT          BIT(2)
  85 #define OMAP2_MCSPI_CHSTAT_TXFFE        BIT(3)
  86
  87 #define OMAP2_MCSPI_CHCTRL_EN           BIT(0)
  88 #define OMAP2_MCSPI_CHCTRL_EXTCLK_MASK  (0xff << 8)
  89
  90 #define OMAP2_MCSPI_WAKEUPENABLE_WKEN   BIT(0)
  91
  92 /* We have 2 DMA channels per CS, one for RX and one for TX */
  93 struct omap2_mcspi_dma {
  94         struct dma_chan *dma_tx;
  95         struct dma_chan *dma_rx;
  96
  97         struct completion dma_tx_completion;
  98         struct completion dma_rx_completion;
  99
 100         char dma_rx_ch_name[14];
 101         char dma_tx_ch_name[14];
 102 };
 103
 104 /* use PIO for small transfers, avoiding DMA setup/teardown overhead and
 105  * cache operations; better heuristics consider wordsize and bitrate.
 106  */
 107 #define DMA_MIN_BYTES                   160
 108
 109
 110 /*
 111  * Used for context save and restore, structure members to be updated whenever
 112  * corresponding registers are modified.
 113  */
 114 struct omap2_mcspi_regs {
 115         u32 modulctrl;
 116         u32 wakeupenable;
 117         struct list_head cs;
 118 };
 119
 120 struct omap2_mcspi {
 121         struct completion       txdone;
 122         struct spi_master       *master;
 123         /* Virtual base address of the controller */
 124         void __iomem            *base;
 125         unsigned long           phys;
 126         /* SPI1 has 4 channels, while SPI2 has 2 */
 127         struct omap2_mcspi_dma  *dma_channels;
 128         struct device           *dev;
 129         struct omap2_mcspi_regs ctx;
 130         int                     fifo_depth;
 131         bool                    slave_aborted;
 132         unsigned int            pin_dir:1;
 133         size_t                  max_xfer_len;
 134 };
 135
 136 struct omap2_mcspi_cs {
 137         void __iomem            *base;
 138         unsigned long           phys;
 139         int                     word_len;
 140         u16                     mode;
 141         struct list_head        node;
 142         /* Context save and restore shadow register */
 143         u32                     chconf0, chctrl0;
 144 };
 145
 146 static inline void mcspi_write_reg(struct spi_master *master,
 147                 int idx, u32 val)
 148 {
 149         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
 150
 151         writel_relaxed(val, mcspi->base + idx);
 152 }
 153
 154 static inline u32 mcspi_read_reg(struct spi_master *master, int idx)
 155 {
 156         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
 157
 158         return readl_relaxed(mcspi->base + idx);
 159 }
 160
 161 static inline void mcspi_write_cs_reg(const struct spi_device *spi,
 162                 int idx, u32 val)
 163 {
 164         struct omap2_mcspi_cs   *cs = spi->controller_state;
 165
 166         writel_relaxed(val, cs->base +  idx);
 167 }
 168
 169 static inline u32 mcspi_read_cs_reg(const struct spi_device *spi, int idx)
 170 {
 171         struct omap2_mcspi_cs   *cs = spi->controller_state;
 172
 173         return readl_relaxed(cs->base + idx);
 174 }
 175
 176 static inline u32 mcspi_cached_chconf0(const struct spi_device *spi)
 177 {
 178         struct omap2_mcspi_cs *cs = spi->controller_state;
 179
 180         return cs->chconf0;
 181 }
 182
 183 static inline void mcspi_write_chconf0(const struct spi_device *spi, u32 val)
 184 {
 185         struct omap2_mcspi_cs *cs = spi->controller_state;
 186
 187         cs->chconf0 = val;
 188         mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCONF0, val);
 189         mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCONF0);
 190 }
 191
 192 static inline int mcspi_bytes_per_word(int word_len)
 193 {
 194         if (word_len <= 8)
 195                 return 1;
 196         else if (word_len <= 16)
 197                 return 2;
 198         else /* word_len <= 32 */
 199                 return 4;
 200 }
 201
 202 static void omap2_mcspi_set_dma_req(const struct spi_device *spi,
 203                 int is_read, int enable)
 204 {
 205         u32 l, rw;
 206
 207         l = mcspi_cached_chconf0(spi);
 208
 209         if (is_read) /* 1 is read, 0 write */
 210                 rw = OMAP2_MCSPI_CHCONF_DMAR;
 211         else
 212                 rw = OMAP2_MCSPI_CHCONF_DMAW;
 213
 214         if (enable)
 215                 l |= rw;
 216         else
 217                 l &= ~rw;
 218
 219         mcspi_write_chconf0(spi, l);
 220 }
 221
 222 static void omap2_mcspi_set_enable(const struct spi_device *spi, int enable)
 223 {
 224         struct omap2_mcspi_cs *cs = spi->controller_state;
 225         u32 l;
 226
 227         l = cs->chctrl0;
 228         if (enable)
 229                 l |= OMAP2_MCSPI_CHCTRL_EN;
 230         else
 231                 l &= ~OMAP2_MCSPI_CHCTRL_EN;
 232         cs->chctrl0 = l;
 233         mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
 234         /* Flash post-writes */
 235         mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCTRL0);
 236 }
 237
 238 static void omap2_mcspi_set_cs(struct spi_device *spi, bool enable)
 239 {
 240         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 241         u32 l;
 242
 243         /* The controller handles the inverted chip selects
 244          * using the OMAP2_MCSPI_CHCONF_EPOL bit so revert
 245          * the inversion from the core spi_set_cs function.
 246          */
 247         if (spi->mode & SPI_CS_HIGH)
 248                 enable = !enable;
 249
 250         if (spi->controller_state) {
 251                 int err = pm_runtime_get_sync(mcspi->dev);
 252                 if (err < 0) {
 253                         pm_runtime_put_noidle(mcspi->dev);
 254                         dev_err(mcspi->dev, "failed to get sync: %d\n", err);
 255                         return;
 256                 }
 257
 258                 l = mcspi_cached_chconf0(spi);
 259
 260                 if (enable)
 261                         l &= ~OMAP2_MCSPI_CHCONF_FORCE;
 262                 else
 263                         l |= OMAP2_MCSPI_CHCONF_FORCE;
 264
 265                 mcspi_write_chconf0(spi, l);
 266
 267                 pm_runtime_mark_last_busy(mcspi->dev);
 268                 pm_runtime_put_autosuspend(mcspi->dev);
 269         }
 270 }
 271
 272 static void omap2_mcspi_set_mode(struct spi_master *master)
 273 {
 274         struct omap2_mcspi      *mcspi = spi_master_get_devdata(master);
 275         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
 276         u32 l;
 277
 278         /*
 279          * Choose master or slave mode
 280          */
 281         l = mcspi_read_reg(master, OMAP2_MCSPI_MODULCTRL);
 282         l &= ~(OMAP2_MCSPI_MODULCTRL_STEST);
 283         if (spi_controller_is_slave(master)) {
 284                 l |= (OMAP2_MCSPI_MODULCTRL_MS);
 285         } else {
 286                 l &= ~(OMAP2_MCSPI_MODULCTRL_MS);
 287                 l |= OMAP2_MCSPI_MODULCTRL_SINGLE;
 288         }
 289         mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, l);
 290
 291         ctx->modulctrl = l;
 292 }
 293
 294 static void omap2_mcspi_set_fifo(const struct spi_device *spi,
 295                                 struct spi_transfer *t, int enable)
 296 {
 297         struct spi_master *master = spi->master;
 298         struct omap2_mcspi_cs *cs = spi->controller_state;
 299         struct omap2_mcspi *mcspi;
 300         unsigned int wcnt;
 301         int max_fifo_depth, bytes_per_word;
 302         u32 chconf, xferlevel;
 303
 304         mcspi = spi_master_get_devdata(master);
 305
 306         chconf = mcspi_cached_chconf0(spi);
 307         if (enable) {
 308                 bytes_per_word = mcspi_bytes_per_word(cs->word_len);
 309                 if (t->len % bytes_per_word != 0)
 310                         goto disable_fifo;
 311
 312                 if (t->rx_buf != NULL && t->tx_buf != NULL)
 313                         max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH / 2;
 314                 else
 315                         max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH;
 316
 317                 wcnt = t->len / bytes_per_word;
 318                 if (wcnt > OMAP2_MCSPI_MAX_FIFOWCNT)
 319                         goto disable_fifo;
 320
 321                 xferlevel = wcnt << 16;
 322                 if (t->rx_buf != NULL) {
 323                         chconf |= OMAP2_MCSPI_CHCONF_FFER;
 324                         xferlevel |= (bytes_per_word - 1) << 8;
 325                 }
 326
 327                 if (t->tx_buf != NULL) {
 328                         chconf |= OMAP2_MCSPI_CHCONF_FFET;
 329                         xferlevel |= bytes_per_word - 1;
 330                 }
 331
 332                 mcspi_write_reg(master, OMAP2_MCSPI_XFERLEVEL, xferlevel);
 333                 mcspi_write_chconf0(spi, chconf);
 334                 mcspi->fifo_depth = max_fifo_depth;
 335
 336                 return;
 337         }
 338
 339 disable_fifo:
 340         if (t->rx_buf != NULL)
 341                 chconf &= ~OMAP2_MCSPI_CHCONF_FFER;
 342
 343         if (t->tx_buf != NULL)
 344                 chconf &= ~OMAP2_MCSPI_CHCONF_FFET;
 345
 346         mcspi_write_chconf0(spi, chconf);
 347         mcspi->fifo_depth = 0;
 348 }
 349
 350 static int mcspi_wait_for_reg_bit(void __iomem *reg, unsigned long bit)
 351 {
 352         u32 val;
 353
 354         return readl_poll_timeout(reg, val, val & bit, 1, MSEC_PER_SEC);
 355 }
 356
 357 static int mcspi_wait_for_completion(struct  omap2_mcspi *mcspi,
 358                                      struct completion *x)
 359 {
 360         if (spi_controller_is_slave(mcspi->master)) {
 361                 if (wait_for_completion_interruptible(x) ||
 362                     mcspi->slave_aborted)
 363                         return -EINTR;
 364         } else {
 365                 wait_for_completion(x);
 366         }
 367
 368         return 0;
 369 }
 370
 371 static void omap2_mcspi_rx_callback(void *data)
 372 {
 373         struct spi_device *spi = data;
 374         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 375         struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 376
 377         /* We must disable the DMA RX request */
 378         omap2_mcspi_set_dma_req(spi, 1, 0);
 379
 380         complete(&mcspi_dma->dma_rx_completion);
 381 }
 382
 383 static void omap2_mcspi_tx_callback(void *data)
 384 {
 385         struct spi_device *spi = data;
 386         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 387         struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 388
 389         /* We must disable the DMA TX request */
 390         omap2_mcspi_set_dma_req(spi, 0, 0);
 391
 392         complete(&mcspi_dma->dma_tx_completion);
 393 }
 394
 395 static void omap2_mcspi_tx_dma(struct spi_device *spi,
 396                                 struct spi_transfer *xfer,
 397                                 struct dma_slave_config cfg)
 398 {
 399         struct omap2_mcspi      *mcspi;
 400         struct omap2_mcspi_dma  *mcspi_dma;
 401         struct dma_async_tx_descriptor *tx;
 402
 403         mcspi = spi_master_get_devdata(spi->master);
 404         mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 405
 406         dmaengine_slave_config(mcspi_dma->dma_tx, &cfg);
 407
 408         tx = dmaengine_prep_slave_sg(mcspi_dma->dma_tx, xfer->tx_sg.sgl,
 409                                      xfer->tx_sg.nents,
 410                                      DMA_MEM_TO_DEV,
 411                                      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 412         if (tx) {
 413                 tx->callback = omap2_mcspi_tx_callback;
 414                 tx->callback_param = spi;
 415                 dmaengine_submit(tx);
 416         } else {
 417                 /* FIXME: fall back to PIO? */
 418         }
 419         dma_async_issue_pending(mcspi_dma->dma_tx);
 420         omap2_mcspi_set_dma_req(spi, 0, 1);
 421 }
 422
 423 static unsigned
 424 omap2_mcspi_rx_dma(struct spi_device *spi, struct spi_transfer *xfer,
 425                                 struct dma_slave_config cfg,
 426                                 unsigned es)
 427 {
 428         struct omap2_mcspi      *mcspi;
 429         struct omap2_mcspi_dma  *mcspi_dma;
 430         unsigned int            count, transfer_reduction = 0;
 431         struct scatterlist      *sg_out[2];
 432         int                     nb_sizes = 0, out_mapped_nents[2], ret, x;
 433         size_t                  sizes[2];
 434         u32                     l;
 435         int                     elements = 0;
 436         int                     word_len, element_count;
 437         struct omap2_mcspi_cs   *cs = spi->controller_state;
 438         void __iomem            *chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
 439         struct dma_async_tx_descriptor *tx;
 440
 441         mcspi = spi_master_get_devdata(spi->master);
 442         mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 443         count = xfer->len;
 444
 445         /*
 446          *  In the "End-of-Transfer Procedure" section for DMA RX in OMAP35x TRM
 447          *  it mentions reducing DMA transfer length by one element in master
 448          *  normal mode.
 449          */
 450         if (mcspi->fifo_depth == 0)
 451                 transfer_reduction = es;
 452
 453         word_len = cs->word_len;
 454         l = mcspi_cached_chconf0(spi);
 455
 456         if (word_len <= 8)
 457                 element_count = count;
 458         else if (word_len <= 16)
 459                 element_count = count >> 1;
 460         else /* word_len <= 32 */
 461                 element_count = count >> 2;
 462
 463
 464         dmaengine_slave_config(mcspi_dma->dma_rx, &cfg);
 465
 466         /*
 467          *  Reduce DMA transfer length by one more if McSPI is
 468          *  configured in turbo mode.
 469          */
 470         if ((l & OMAP2_MCSPI_CHCONF_TURBO) && mcspi->fifo_depth == 0)
 471                 transfer_reduction += es;
 472
 473         if (transfer_reduction) {
 474                 /* Split sgl into two. The second sgl won't be used. */
 475                 sizes[0] = count - transfer_reduction;
 476                 sizes[1] = transfer_reduction;
 477                 nb_sizes = 2;
 478         } else {
 479                 /*
 480                  * Don't bother splitting the sgl. This essentially
 481                  * clones the original sgl.
 482                  */
 483                 sizes[0] = count;
 484                 nb_sizes = 1;
 485         }
 486
 487         ret = sg_split(xfer->rx_sg.sgl, xfer->rx_sg.nents, 0, nb_sizes,
 488                        sizes, sg_out, out_mapped_nents, GFP_KERNEL);
 489
 490         if (ret < 0) {
 491                 dev_err(&spi->dev, "sg_split failed\n");
 492                 return 0;
 493         }
 494
 495         tx = dmaengine_prep_slave_sg(mcspi_dma->dma_rx, sg_out[0],
 496                                      out_mapped_nents[0], DMA_DEV_TO_MEM,
 497                                      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 498         if (tx) {
 499                 tx->callback = omap2_mcspi_rx_callback;
 500                 tx->callback_param = spi;
 501                 dmaengine_submit(tx);
 502         } else {
 503                 /* FIXME: fall back to PIO? */
 504         }
 505
 506         dma_async_issue_pending(mcspi_dma->dma_rx);
 507         omap2_mcspi_set_dma_req(spi, 1, 1);
 508
 509         ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_rx_completion);
 510         if (ret || mcspi->slave_aborted) {
 511                 dmaengine_terminate_sync(mcspi_dma->dma_rx);
 512                 omap2_mcspi_set_dma_req(spi, 1, 0);
 513                 return 0;
 514         }
 515
 516         for (x = 0; x < nb_sizes; x++)
 517                 kfree(sg_out[x]);
 518
 519         if (mcspi->fifo_depth > 0)
 520                 return count;
 521
 522         /*
 523          *  Due to the DMA transfer length reduction the missing bytes must
 524          *  be read manually to receive all of the expected data.
 525          */
 526         omap2_mcspi_set_enable(spi, 0);
 527
 528         elements = element_count - 1;
 529
 530         if (l & OMAP2_MCSPI_CHCONF_TURBO) {
 531                 elements--;
 532
 533                 if (!mcspi_wait_for_reg_bit(chstat_reg,
 534                                             OMAP2_MCSPI_CHSTAT_RXS)) {
 535                         u32 w;
 536
 537                         w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
 538                         if (word_len <= 8)
 539                                 ((u8 *)xfer->rx_buf)[elements++] = w;
 540                         else if (word_len <= 16)
 541                                 ((u16 *)xfer->rx_buf)[elements++] = w;
 542                         else /* word_len <= 32 */
 543                                 ((u32 *)xfer->rx_buf)[elements++] = w;
 544                 } else {
 545                         int bytes_per_word = mcspi_bytes_per_word(word_len);
 546                         dev_err(&spi->dev, "DMA RX penultimate word empty\n");
 547                         count -= (bytes_per_word << 1);
 548                         omap2_mcspi_set_enable(spi, 1);
 549                         return count;
 550                 }
 551         }
 552         if (!mcspi_wait_for_reg_bit(chstat_reg, OMAP2_MCSPI_CHSTAT_RXS)) {
 553                 u32 w;
 554
 555                 w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
 556                 if (word_len <= 8)
 557                         ((u8 *)xfer->rx_buf)[elements] = w;
 558                 else if (word_len <= 16)
 559                         ((u16 *)xfer->rx_buf)[elements] = w;
 560                 else /* word_len <= 32 */
 561                         ((u32 *)xfer->rx_buf)[elements] = w;
 562         } else {
 563                 dev_err(&spi->dev, "DMA RX last word empty\n");
 564                 count -= mcspi_bytes_per_word(word_len);
 565         }
 566         omap2_mcspi_set_enable(spi, 1);
 567         return count;
 568 }
 569
 570 static unsigned
 571 omap2_mcspi_txrx_dma(struct spi_device *spi, struct spi_transfer *xfer)
 572 {
 573         struct omap2_mcspi      *mcspi;
 574         struct omap2_mcspi_cs   *cs = spi->controller_state;
 575         struct omap2_mcspi_dma  *mcspi_dma;
 576         unsigned int            count;
 577         u8                      *rx;
 578         const u8                *tx;
 579         struct dma_slave_config cfg;
 580         enum dma_slave_buswidth width;
 581         unsigned es;
 582         void __iomem            *chstat_reg;
 583         void __iomem            *irqstat_reg;
 584         int                     wait_res;
 585
 586         mcspi = spi_master_get_devdata(spi->master);
 587         mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 588
 589         if (cs->word_len <= 8) {
 590                 width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 591                 es = 1;
 592         } else if (cs->word_len <= 16) {
 593                 width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 594                 es = 2;
 595         } else {
 596                 width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 597                 es = 4;
 598         }
 599
 600         count = xfer->len;
 601
 602         memset(&cfg, 0, sizeof(cfg));
 603         cfg.src_addr = cs->phys + OMAP2_MCSPI_RX0;
 604         cfg.dst_addr = cs->phys + OMAP2_MCSPI_TX0;
 605         cfg.src_addr_width = width;
 606         cfg.dst_addr_width = width;
 607         cfg.src_maxburst = 1;
 608         cfg.dst_maxburst = 1;
 609
 610         rx = xfer->rx_buf;
 611         tx = xfer->tx_buf;
 612
 613         mcspi->slave_aborted = false;
 614         reinit_completion(&mcspi_dma->dma_tx_completion);
 615         reinit_completion(&mcspi_dma->dma_rx_completion);
 616         reinit_completion(&mcspi->txdone);
 617         if (tx) {
 618                 /* Enable EOW IRQ to know end of tx in slave mode */
 619                 if (spi_controller_is_slave(spi->master))
 620                         mcspi_write_reg(spi->master,
 621                                         OMAP2_MCSPI_IRQENABLE,
 622                                         OMAP2_MCSPI_IRQSTATUS_EOW);
 623                 omap2_mcspi_tx_dma(spi, xfer, cfg);
 624         }
 625
 626         if (rx != NULL)
 627                 count = omap2_mcspi_rx_dma(spi, xfer, cfg, es);
 628
 629         if (tx != NULL) {
 630                 int ret;
 631
 632                 ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_tx_completion);
 633                 if (ret || mcspi->slave_aborted) {
 634                         dmaengine_terminate_sync(mcspi_dma->dma_tx);
 635                         omap2_mcspi_set_dma_req(spi, 0, 0);
 636                         return 0;
 637                 }
 638
 639                 if (spi_controller_is_slave(mcspi->master)) {
 640                         ret = mcspi_wait_for_completion(mcspi, &mcspi->txdone);
 641                         if (ret || mcspi->slave_aborted)
 642                                 return 0;
 643                 }
 644
 645                 if (mcspi->fifo_depth > 0) {
 646                         irqstat_reg = mcspi->base + OMAP2_MCSPI_IRQSTATUS;
 647
 648                         if (mcspi_wait_for_reg_bit(irqstat_reg,
 649                                                 OMAP2_MCSPI_IRQSTATUS_EOW) < 0)
 650                                 dev_err(&spi->dev, "EOW timed out\n");
 651
 652                         mcspi_write_reg(mcspi->master, OMAP2_MCSPI_IRQSTATUS,
 653                                         OMAP2_MCSPI_IRQSTATUS_EOW);
 654                 }
 655
 656                 /* for TX_ONLY mode, be sure all words have shifted out */
 657                 if (rx == NULL) {
 658                         chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
 659                         if (mcspi->fifo_depth > 0) {
 660                                 wait_res = mcspi_wait_for_reg_bit(chstat_reg,
 661                                                 OMAP2_MCSPI_CHSTAT_TXFFE);
 662                                 if (wait_res < 0)
 663                                         dev_err(&spi->dev, "TXFFE timed out\n");
 664                         } else {
 665                                 wait_res = mcspi_wait_for_reg_bit(chstat_reg,
 666                                                 OMAP2_MCSPI_CHSTAT_TXS);
 667                                 if (wait_res < 0)
 668                                         dev_err(&spi->dev, "TXS timed out\n");
 669                         }
 670                         if (wait_res >= 0 &&
 671                                 (mcspi_wait_for_reg_bit(chstat_reg,
 672                                         OMAP2_MCSPI_CHSTAT_EOT) < 0))
 673                                 dev_err(&spi->dev, "EOT timed out\n");
 674                 }
 675         }
 676         return count;
 677 }
 678
 679 static unsigned
 680 omap2_mcspi_txrx_pio(struct spi_device *spi, struct spi_transfer *xfer)
 681 {
 682         struct omap2_mcspi_cs   *cs = spi->controller_state;
 683         unsigned int            count, c;
 684         u32                     l;
 685         void __iomem            *base = cs->base;
 686         void __iomem            *tx_reg;
 687         void __iomem            *rx_reg;
 688         void __iomem            *chstat_reg;
 689         int                     word_len;
 690
 691         count = xfer->len;
 692         c = count;
 693         word_len = cs->word_len;
 694
 695         l = mcspi_cached_chconf0(spi);
 696
 697         /* We store the pre-calculated register addresses on stack to speed
 698          * up the transfer loop. */
 699         tx_reg          = base + OMAP2_MCSPI_TX0;
 700         rx_reg          = base + OMAP2_MCSPI_RX0;
 701         chstat_reg      = base + OMAP2_MCSPI_CHSTAT0;
 702
 703         if (c < (word_len>>3))
 704                 return 0;
 705
 706         if (word_len <= 8) {
 707                 u8              *rx;
 708                 const u8        *tx;
 709
 710                 rx = xfer->rx_buf;
 711                 tx = xfer->tx_buf;
 712
 713                 do {
 714                         c -= 1;
 715                         if (tx != NULL) {
 716                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 717                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 718                                         dev_err(&spi->dev, "TXS timed out\n");
 719                                         goto out;
 720                                 }
 721                                 dev_vdbg(&spi->dev, "write-%d %02x\n",
 722                                                 word_len, *tx);
 723                                 writel_relaxed(*tx++, tx_reg);
 724                         }
 725                         if (rx != NULL) {
 726                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 727                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 728                                         dev_err(&spi->dev, "RXS timed out\n");
 729                                         goto out;
 730                                 }
 731
 732                                 if (c == 1 && tx == NULL &&
 733                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 734                                         omap2_mcspi_set_enable(spi, 0);
 735                                         *rx++ = readl_relaxed(rx_reg);
 736                                         dev_vdbg(&spi->dev, "read-%d %02x\n",
 737                                                     word_len, *(rx - 1));
 738                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 739                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 740                                                 dev_err(&spi->dev,
 741                                                         "RXS timed out\n");
 742                                                 goto out;
 743                                         }
 744                                         c = 0;
 745                                 } else if (c == 0 && tx == NULL) {
 746                                         omap2_mcspi_set_enable(spi, 0);
 747                                 }
 748
 749                                 *rx++ = readl_relaxed(rx_reg);
 750                                 dev_vdbg(&spi->dev, "read-%d %02x\n",
 751                                                 word_len, *(rx - 1));
 752                         }
 753                 } while (c);
 754         } else if (word_len <= 16) {
 755                 u16             *rx;
 756                 const u16       *tx;
 757
 758                 rx = xfer->rx_buf;
 759                 tx = xfer->tx_buf;
 760                 do {
 761                         c -= 2;
 762                         if (tx != NULL) {
 763                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 764                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 765                                         dev_err(&spi->dev, "TXS timed out\n");
 766                                         goto out;
 767                                 }
 768                                 dev_vdbg(&spi->dev, "write-%d %04x\n",
 769                                                 word_len, *tx);
 770                                 writel_relaxed(*tx++, tx_reg);
 771                         }
 772                         if (rx != NULL) {
 773                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 774                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 775                                         dev_err(&spi->dev, "RXS timed out\n");
 776                                         goto out;
 777                                 }
 778
 779                                 if (c == 2 && tx == NULL &&
 780                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 781                                         omap2_mcspi_set_enable(spi, 0);
 782                                         *rx++ = readl_relaxed(rx_reg);
 783                                         dev_vdbg(&spi->dev, "read-%d %04x\n",
 784                                                     word_len, *(rx - 1));
 785                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 786                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 787                                                 dev_err(&spi->dev,
 788                                                         "RXS timed out\n");
 789                                                 goto out;
 790                                         }
 791                                         c = 0;
 792                                 } else if (c == 0 && tx == NULL) {
 793                                         omap2_mcspi_set_enable(spi, 0);
 794                                 }
 795
 796                                 *rx++ = readl_relaxed(rx_reg);
 797                                 dev_vdbg(&spi->dev, "read-%d %04x\n",
 798                                                 word_len, *(rx - 1));
 799                         }
 800                 } while (c >= 2);
 801         } else if (word_len <= 32) {
 802                 u32             *rx;
 803                 const u32       *tx;
 804
 805                 rx = xfer->rx_buf;
 806                 tx = xfer->tx_buf;
 807                 do {
 808                         c -= 4;
 809                         if (tx != NULL) {
 810                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 811                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 812                                         dev_err(&spi->dev, "TXS timed out\n");
 813                                         goto out;
 814                                 }
 815                                 dev_vdbg(&spi->dev, "write-%d %08x\n",
 816                                                 word_len, *tx);
 817                                 writel_relaxed(*tx++, tx_reg);
 818                         }
 819                         if (rx != NULL) {
 820                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 821                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 822                                         dev_err(&spi->dev, "RXS timed out\n");
 823                                         goto out;
 824                                 }
 825
 826                                 if (c == 4 && tx == NULL &&
 827                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 828                                         omap2_mcspi_set_enable(spi, 0);
 829                                         *rx++ = readl_relaxed(rx_reg);
 830                                         dev_vdbg(&spi->dev, "read-%d %08x\n",
 831                                                     word_len, *(rx - 1));
 832                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 833                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 834                                                 dev_err(&spi->dev,
 835                                                         "RXS timed out\n");
 836                                                 goto out;
 837                                         }
 838                                         c = 0;
 839                                 } else if (c == 0 && tx == NULL) {
 840                                         omap2_mcspi_set_enable(spi, 0);
 841                                 }
 842
 843                                 *rx++ = readl_relaxed(rx_reg);
 844                                 dev_vdbg(&spi->dev, "read-%d %08x\n",
 845                                                 word_len, *(rx - 1));
 846                         }
 847                 } while (c >= 4);
 848         }
 849
 850         /* for TX_ONLY mode, be sure all words have shifted out */
 851         if (xfer->rx_buf == NULL) {
 852                 if (mcspi_wait_for_reg_bit(chstat_reg,
 853                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 854                         dev_err(&spi->dev, "TXS timed out\n");
 855                 } else if (mcspi_wait_for_reg_bit(chstat_reg,
 856                                 OMAP2_MCSPI_CHSTAT_EOT) < 0)
 857                         dev_err(&spi->dev, "EOT timed out\n");
 858
 859                 /* disable chan to purge rx datas received in TX_ONLY transfer,
 860                  * otherwise these rx datas will affect the direct following
 861                  * RX_ONLY transfer.
 862                  */
 863                 omap2_mcspi_set_enable(spi, 0);
 864         }
 865 out:
 866         omap2_mcspi_set_enable(spi, 1);
 867         return count - c;
 868 }
 869
 870 static u32 omap2_mcspi_calc_divisor(u32 speed_hz)
 871 {
 872         u32 div;
 873
 874         for (div = 0; div < 15; div++)
 875                 if (speed_hz >= (OMAP2_MCSPI_MAX_FREQ >> div))
 876                         return div;
 877
 878         return 15;
 879 }
 880
 881 /* called only when no transfer is active to this device */
 882 static int omap2_mcspi_setup_transfer(struct spi_device *spi,
 883                 struct spi_transfer *t)
 884 {
 885         struct omap2_mcspi_cs *cs = spi->controller_state;
 886         struct omap2_mcspi *mcspi;
 887         u32 l = 0, clkd = 0, div, extclk = 0, clkg = 0;
 888         u8 word_len = spi->bits_per_word;
 889         u32 speed_hz = spi->max_speed_hz;
 890
 891         mcspi = spi_master_get_devdata(spi->master);
 892
 893         if (t != NULL && t->bits_per_word)
 894                 word_len = t->bits_per_word;
 895
 896         cs->word_len = word_len;
 897
 898         if (t && t->speed_hz)
 899                 speed_hz = t->speed_hz;
 900
 901         speed_hz = min_t(u32, speed_hz, OMAP2_MCSPI_MAX_FREQ);
 902         if (speed_hz < (OMAP2_MCSPI_MAX_FREQ / OMAP2_MCSPI_MAX_DIVIDER)) {
 903                 clkd = omap2_mcspi_calc_divisor(speed_hz);
 904                 speed_hz = OMAP2_MCSPI_MAX_FREQ >> clkd;
 905                 clkg = 0;
 906         } else {
 907                 div = (OMAP2_MCSPI_MAX_FREQ + speed_hz - 1) / speed_hz;
 908                 speed_hz = OMAP2_MCSPI_MAX_FREQ / div;
 909                 clkd = (div - 1) & 0xf;
 910                 extclk = (div - 1) >> 4;
 911                 clkg = OMAP2_MCSPI_CHCONF_CLKG;
 912         }
 913
 914         l = mcspi_cached_chconf0(spi);
 915
 916         /* standard 4-wire master mode:  SCK, MOSI/out, MISO/in, nCS
 917          * REVISIT: this controller could support SPI_3WIRE mode.
 918          */
 919         if (mcspi->pin_dir == MCSPI_PINDIR_D0_IN_D1_OUT) {
 920                 l &= ~OMAP2_MCSPI_CHCONF_IS;
 921                 l &= ~OMAP2_MCSPI_CHCONF_DPE1;
 922                 l |= OMAP2_MCSPI_CHCONF_DPE0;
 923         } else {
 924                 l |= OMAP2_MCSPI_CHCONF_IS;
 925                 l |= OMAP2_MCSPI_CHCONF_DPE1;
 926                 l &= ~OMAP2_MCSPI_CHCONF_DPE0;
 927         }
 928
 929         /* wordlength */
 930         l &= ~OMAP2_MCSPI_CHCONF_WL_MASK;
 931         l |= (word_len - 1) << 7;
 932
 933         /* set chipselect polarity; manage with FORCE */
 934         if (!(spi->mode & SPI_CS_HIGH))
 935                 l |= OMAP2_MCSPI_CHCONF_EPOL;   /* active-low; normal */
 936         else
 937                 l &= ~OMAP2_MCSPI_CHCONF_EPOL;
 938
 939         /* set clock divisor */
 940         l &= ~OMAP2_MCSPI_CHCONF_CLKD_MASK;
 941         l |= clkd << 2;
 942
 943         /* set clock granularity */
 944         l &= ~OMAP2_MCSPI_CHCONF_CLKG;
 945         l |= clkg;
 946         if (clkg) {
 947                 cs->chctrl0 &= ~OMAP2_MCSPI_CHCTRL_EXTCLK_MASK;
 948                 cs->chctrl0 |= extclk << 8;
 949                 mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
 950         }
 951
 952         /* set SPI mode 0..3 */
 953         if (spi->mode & SPI_CPOL)
 954                 l |= OMAP2_MCSPI_CHCONF_POL;
 955         else
 956                 l &= ~OMAP2_MCSPI_CHCONF_POL;
 957         if (spi->mode & SPI_CPHA)
 958                 l |= OMAP2_MCSPI_CHCONF_PHA;
 959         else
 960                 l &= ~OMAP2_MCSPI_CHCONF_PHA;
 961
 962         mcspi_write_chconf0(spi, l);
 963
 964         cs->mode = spi->mode;
 965
 966         dev_dbg(&spi->dev, "setup: speed %d, sample %s edge, clk %s\n",
 967                         speed_hz,
 968                         (spi->mode & SPI_CPHA) ? "trailing" : "leading",
 969                         (spi->mode & SPI_CPOL) ? "inverted" : "normal");
 970
 971         return 0;
 972 }
 973
 974 /*
 975  * Note that we currently allow DMA only if we get a channel
 976  * for both rx and tx. Otherwise we'll do PIO for both rx and tx.
 977  */
 978 static int omap2_mcspi_request_dma(struct omap2_mcspi *mcspi,
 979                                    struct omap2_mcspi_dma *mcspi_dma)
 980 {
 981         int ret = 0;
 982
 983         mcspi_dma->dma_rx = dma_request_chan(mcspi->dev,
 984                                              mcspi_dma->dma_rx_ch_name);
 985         if (IS_ERR(mcspi_dma->dma_rx)) {
 986                 ret = PTR_ERR(mcspi_dma->dma_rx);
 987                 mcspi_dma->dma_rx = NULL;
 988                 goto no_dma;
 989         }
 990
 991         mcspi_dma->dma_tx = dma_request_chan(mcspi->dev,
 992                                              mcspi_dma->dma_tx_ch_name);
 993         if (IS_ERR(mcspi_dma->dma_tx)) {
 994                 ret = PTR_ERR(mcspi_dma->dma_tx);
 995                 mcspi_dma->dma_tx = NULL;
 996                 dma_release_channel(mcspi_dma->dma_rx);
 997                 mcspi_dma->dma_rx = NULL;
 998         }
 999
1000         init_completion(&mcspi_dma->dma_rx_completion);
1001         init_completion(&mcspi_dma->dma_tx_completion);
1002
1003 no_dma:
1004         return ret;
1005 }
1006
1007 static void omap2_mcspi_release_dma(struct spi_master *master)
1008 {
1009         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1010         struct omap2_mcspi_dma  *mcspi_dma;
1011         int i;
1012
1013         for (i = 0; i < master->num_chipselect; i++) {
1014                 mcspi_dma = &mcspi->dma_channels[i];
1015
1016                 if (mcspi_dma->dma_rx) {
1017                         dma_release_channel(mcspi_dma->dma_rx);
1018                         mcspi_dma->dma_rx = NULL;
1019                 }
1020                 if (mcspi_dma->dma_tx) {
1021                         dma_release_channel(mcspi_dma->dma_tx);
1022                         mcspi_dma->dma_tx = NULL;
1023                 }
1024         }
1025 }
1026
1027 static int omap2_mcspi_setup(struct spi_device *spi)
1028 {
1029         int                     ret;
1030         struct omap2_mcspi      *mcspi = spi_master_get_devdata(spi->master);
1031         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1032         struct omap2_mcspi_cs   *cs = spi->controller_state;
1033
1034         if (!cs) {
1035                 cs = kzalloc(sizeof *cs, GFP_KERNEL);
1036                 if (!cs)
1037                         return -ENOMEM;
1038                 cs->base = mcspi->base + spi->chip_select * 0x14;
1039                 cs->phys = mcspi->phys + spi->chip_select * 0x14;
1040                 cs->mode = 0;
1041                 cs->chconf0 = 0;
1042                 cs->chctrl0 = 0;
1043                 spi->controller_state = cs;
1044                 /* Link this to context save list */
1045                 list_add_tail(&cs->node, &ctx->cs);
1046
1047                 if (gpio_is_valid(spi->cs_gpio)) {
1048                         ret = gpio_request(spi->cs_gpio, dev_name(&spi->dev));
1049                         if (ret) {
1050                                 dev_err(&spi->dev, "failed to request gpio\n");
1051                                 return ret;
1052                         }
1053                         gpio_direction_output(spi->cs_gpio,
1054                                          !(spi->mode & SPI_CS_HIGH));
1055                 }
1056         }
1057
1058         ret = pm_runtime_get_sync(mcspi->dev);
1059         if (ret < 0) {
1060                 pm_runtime_put_noidle(mcspi->dev);
1061
1062                 return ret;
1063         }
1064
1065         ret = omap2_mcspi_setup_transfer(spi, NULL);
1066         pm_runtime_mark_last_busy(mcspi->dev);
1067         pm_runtime_put_autosuspend(mcspi->dev);
1068
1069         return ret;
1070 }
1071
1072 static void omap2_mcspi_cleanup(struct spi_device *spi)
1073 {
1074         struct omap2_mcspi_cs   *cs;
1075
1076         if (spi->controller_state) {
1077                 /* Unlink controller state from context save list */
1078                 cs = spi->controller_state;
1079                 list_del(&cs->node);
1080
1081                 kfree(cs);
1082         }
1083
1084         if (gpio_is_valid(spi->cs_gpio))
1085                 gpio_free(spi->cs_gpio);
1086 }
1087
1088 static irqreturn_t omap2_mcspi_irq_handler(int irq, void *data)
1089 {
1090         struct omap2_mcspi *mcspi = data;
1091         u32 irqstat;
1092
1093         irqstat = mcspi_read_reg(mcspi->master, OMAP2_MCSPI_IRQSTATUS);
1094         if (!irqstat)
1095                 return IRQ_NONE;
1096
1097         /* Disable IRQ and wakeup slave xfer task */
1098         mcspi_write_reg(mcspi->master, OMAP2_MCSPI_IRQENABLE, 0);
1099         if (irqstat & OMAP2_MCSPI_IRQSTATUS_EOW)
1100                 complete(&mcspi->txdone);
1101
1102         return IRQ_HANDLED;
1103 }
1104
1105 static int omap2_mcspi_slave_abort(struct spi_master *master)
1106 {
1107         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1108         struct omap2_mcspi_dma *mcspi_dma = mcspi->dma_channels;
1109
1110         mcspi->slave_aborted = true;
1111         complete(&mcspi_dma->dma_rx_completion);
1112         complete(&mcspi_dma->dma_tx_completion);
1113         complete(&mcspi->txdone);
1114
1115         return 0;
1116 }
1117
1118 static int omap2_mcspi_transfer_one(struct spi_master *master,
1119                                     struct spi_device *spi,
1120                                     struct spi_transfer *t)
1121 {
1122
1123         /* We only enable one channel at a time -- the one whose message is
1124          * -- although this controller would gladly
1125          * arbitrate among multiple channels.  This corresponds to "single
1126          * channel" master mode.  As a side effect, we need to manage the
1127          * chipselect with the FORCE bit ... CS != channel enable.
1128          */
1129
1130         struct omap2_mcspi              *mcspi;
1131         struct omap2_mcspi_dma          *mcspi_dma;
1132         struct omap2_mcspi_cs           *cs;
1133         struct omap2_mcspi_device_config *cd;
1134         int                             par_override = 0;
1135         int                             status = 0;
1136         u32                             chconf;
1137
1138         mcspi = spi_master_get_devdata(master);
1139         mcspi_dma = mcspi->dma_channels + spi->chip_select;
1140         cs = spi->controller_state;
1141         cd = spi->controller_data;
1142
1143         /*
1144          * The slave driver could have changed spi->mode in which case
1145          * it will be different from cs->mode (the current hardware setup).
1146          * If so, set par_override (even though its not a parity issue) so
1147          * omap2_mcspi_setup_transfer will be called to configure the hardware
1148          * with the correct mode on the first iteration of the loop below.
1149          */
1150         if (spi->mode != cs->mode)
1151                 par_override = 1;
1152
1153         omap2_mcspi_set_enable(spi, 0);
1154
1155         if (gpio_is_valid(spi->cs_gpio))
1156                 omap2_mcspi_set_cs(spi, spi->mode & SPI_CS_HIGH);
1157
1158         if (par_override ||
1159             (t->speed_hz != spi->max_speed_hz) ||
1160             (t->bits_per_word != spi->bits_per_word)) {
1161                 par_override = 1;
1162                 status = omap2_mcspi_setup_transfer(spi, t);
1163                 if (status < 0)
1164                         goto out;
1165                 if (t->speed_hz == spi->max_speed_hz &&
1166                     t->bits_per_word == spi->bits_per_word)
1167                         par_override = 0;
1168         }
1169         if (cd && cd->cs_per_word) {
1170                 chconf = mcspi->ctx.modulctrl;
1171                 chconf &= ~OMAP2_MCSPI_MODULCTRL_SINGLE;
1172                 mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, chconf);
1173                 mcspi->ctx.modulctrl =
1174                         mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1175         }
1176
1177         chconf = mcspi_cached_chconf0(spi);
1178         chconf &= ~OMAP2_MCSPI_CHCONF_TRM_MASK;
1179         chconf &= ~OMAP2_MCSPI_CHCONF_TURBO;
1180
1181         if (t->tx_buf == NULL)
1182                 chconf |= OMAP2_MCSPI_CHCONF_TRM_RX_ONLY;
1183         else if (t->rx_buf == NULL)
1184                 chconf |= OMAP2_MCSPI_CHCONF_TRM_TX_ONLY;
1185
1186         if (cd && cd->turbo_mode && t->tx_buf == NULL) {
1187                 /* Turbo mode is for more than one word */
1188                 if (t->len > ((cs->word_len + 7) >> 3))
1189                         chconf |= OMAP2_MCSPI_CHCONF_TURBO;
1190         }
1191
1192         mcspi_write_chconf0(spi, chconf);
1193
1194         if (t->len) {
1195                 unsigned        count;
1196
1197                 if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1198                     master->cur_msg_mapped &&
1199                     master->can_dma(master, spi, t))
1200                         omap2_mcspi_set_fifo(spi, t, 1);
1201
1202                 omap2_mcspi_set_enable(spi, 1);
1203
1204                 /* RX_ONLY mode needs dummy data in TX reg */
1205                 if (t->tx_buf == NULL)
1206                         writel_relaxed(0, cs->base
1207                                         + OMAP2_MCSPI_TX0);
1208
1209                 if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1210                     master->cur_msg_mapped &&
1211                     master->can_dma(master, spi, t))
1212                         count = omap2_mcspi_txrx_dma(spi, t);
1213                 else
1214                         count = omap2_mcspi_txrx_pio(spi, t);
1215
1216                 if (count != t->len) {
1217                         status = -EIO;
1218                         goto out;
1219                 }
1220         }
1221
1222         omap2_mcspi_set_enable(spi, 0);
1223
1224         if (mcspi->fifo_depth > 0)
1225                 omap2_mcspi_set_fifo(spi, t, 0);
1226
1227 out:
1228         /* Restore defaults if they were overriden */
1229         if (par_override) {
1230                 par_override = 0;
1231                 status = omap2_mcspi_setup_transfer(spi, NULL);
1232         }
1233
1234         if (cd && cd->cs_per_word) {
1235                 chconf = mcspi->ctx.modulctrl;
1236                 chconf |= OMAP2_MCSPI_MODULCTRL_SINGLE;
1237                 mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, chconf);
1238                 mcspi->ctx.modulctrl =
1239                         mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1240         }
1241
1242         omap2_mcspi_set_enable(spi, 0);
1243
1244         if (gpio_is_valid(spi->cs_gpio))
1245                 omap2_mcspi_set_cs(spi, !(spi->mode & SPI_CS_HIGH));
1246
1247         if (mcspi->fifo_depth > 0 && t)
1248                 omap2_mcspi_set_fifo(spi, t, 0);
1249
1250         return status;
1251 }
1252
1253 static int omap2_mcspi_prepare_message(struct spi_master *master,
1254                                        struct spi_message *msg)
1255 {
1256         struct omap2_mcspi      *mcspi = spi_master_get_devdata(master);
1257         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1258         struct omap2_mcspi_cs   *cs;
1259
1260         /* Only a single channel can have the FORCE bit enabled
1261          * in its chconf0 register.
1262          * Scan all channels and disable them except the current one.
1263          * A FORCE can remain from a last transfer having cs_change enabled
1264          */
1265         list_for_each_entry(cs, &ctx->cs, node) {
1266                 if (msg->spi->controller_state == cs)
1267                         continue;
1268
1269                 if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE)) {
1270                         cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1271                         writel_relaxed(cs->chconf0,
1272                                         cs->base + OMAP2_MCSPI_CHCONF0);
1273                         readl_relaxed(cs->base + OMAP2_MCSPI_CHCONF0);
1274                 }
1275         }
1276
1277         return 0;
1278 }
1279
1280 static bool omap2_mcspi_can_dma(struct spi_master *master,
1281                                 struct spi_device *spi,
1282                                 struct spi_transfer *xfer)
1283 {
1284         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
1285         struct omap2_mcspi_dma *mcspi_dma =
1286                 &mcspi->dma_channels[spi->chip_select];
1287
1288         if (!mcspi_dma->dma_rx || !mcspi_dma->dma_tx)
1289                 return false;
1290
1291         if (spi_controller_is_slave(master))
1292                 return true;
1293
1294         master->dma_rx = mcspi_dma->dma_rx;
1295         master->dma_tx = mcspi_dma->dma_tx;
1296
1297         return (xfer->len >= DMA_MIN_BYTES);
1298 }
1299
1300 static size_t omap2_mcspi_max_xfer_size(struct spi_device *spi)
1301 {
1302         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
1303         struct omap2_mcspi_dma *mcspi_dma =
1304                 &mcspi->dma_channels[spi->chip_select];
1305
1306         if (mcspi->max_xfer_len && mcspi_dma->dma_rx)
1307                 return mcspi->max_xfer_len;
1308
1309         return SIZE_MAX;
1310 }
1311
1312 static int omap2_mcspi_controller_setup(struct omap2_mcspi *mcspi)
1313 {
1314         struct spi_master       *master = mcspi->master;
1315         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1316         int                     ret = 0;
1317
1318         ret = pm_runtime_get_sync(mcspi->dev);
1319         if (ret < 0) {
1320                 pm_runtime_put_noidle(mcspi->dev);
1321
1322                 return ret;
1323         }
1324
1325         mcspi_write_reg(master, OMAP2_MCSPI_WAKEUPENABLE,
1326                         OMAP2_MCSPI_WAKEUPENABLE_WKEN);
1327         ctx->wakeupenable = OMAP2_MCSPI_WAKEUPENABLE_WKEN;
1328
1329         omap2_mcspi_set_mode(master);
1330         pm_runtime_mark_last_busy(mcspi->dev);
1331         pm_runtime_put_autosuspend(mcspi->dev);
1332         return 0;
1333 }
1334
1335 /*
1336  * When SPI wake up from off-mode, CS is in activate state. If it was in
1337  * inactive state when driver was suspend, then force it to inactive state at
1338  * wake up.
1339  */
1340 static int omap_mcspi_runtime_resume(struct device *dev)
1341 {
1342         struct spi_master *master = dev_get_drvdata(dev);
1343         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1344         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1345         struct omap2_mcspi_cs *cs;
1346
1347         /* McSPI: context restore */
1348         mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, ctx->modulctrl);
1349         mcspi_write_reg(master, OMAP2_MCSPI_WAKEUPENABLE, ctx->wakeupenable);
1350
1351         list_for_each_entry(cs, &ctx->cs, node) {
1352                 /*
1353                  * We need to toggle CS state for OMAP take this
1354                  * change in account.
1355                  */
1356                 if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE) == 0) {
1357                         cs->chconf0 |= OMAP2_MCSPI_CHCONF_FORCE;
1358                         writel_relaxed(cs->chconf0,
1359                                        cs->base + OMAP2_MCSPI_CHCONF0);
1360                         cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1361                         writel_relaxed(cs->chconf0,
1362                                        cs->base + OMAP2_MCSPI_CHCONF0);
1363                 } else {
1364                         writel_relaxed(cs->chconf0,
1365                                        cs->base + OMAP2_MCSPI_CHCONF0);
1366                 }
1367         }
1368
1369         return 0;
1370 }
1371
1372 static struct omap2_mcspi_platform_config omap2_pdata = {
1373         .regs_offset = 0,
1374 };
1375
1376 static struct omap2_mcspi_platform_config omap4_pdata = {
1377         .regs_offset = OMAP4_MCSPI_REG_OFFSET,
1378 };
1379
1380 static struct omap2_mcspi_platform_config am654_pdata = {
1381         .regs_offset = OMAP4_MCSPI_REG_OFFSET,
1382         .max_xfer_len = SZ_4K - 1,
1383 };
1384
1385 static const struct of_device_id omap_mcspi_of_match[] = {
1386         {
1387                 .compatible = "ti,omap2-mcspi",
1388                 .data = &omap2_pdata,
1389         },
1390         {
1391                 .compatible = "ti,omap4-mcspi",
1392                 .data = &omap4_pdata,
1393         },
1394         {
1395                 .compatible = "ti,am654-mcspi",
1396                 .data = &am654_pdata,
1397         },
1398         { },
1399 };
1400 MODULE_DEVICE_TABLE(of, omap_mcspi_of_match);
1401
1402 static int omap2_mcspi_probe(struct platform_device *pdev)
1403 {
1404         struct spi_master       *master;
1405         const struct omap2_mcspi_platform_config *pdata;
1406         struct omap2_mcspi      *mcspi;
1407         struct resource         *r;
1408         int                     status = 0, i;
1409         u32                     regs_offset = 0;
1410         struct device_node      *node = pdev->dev.of_node;
1411         const struct of_device_id *match;
1412
1413         if (of_property_read_bool(node, "spi-slave"))
1414                 master = spi_alloc_slave(&pdev->dev, sizeof(*mcspi));
1415         else
1416                 master = spi_alloc_master(&pdev->dev, sizeof(*mcspi));
1417         if (!master)
1418                 return -ENOMEM;
1419
1420         /* the spi->mode bits understood by this driver: */
1421         master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
1422         master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
1423         master->setup = omap2_mcspi_setup;
1424         master->auto_runtime_pm = true;
1425         master->prepare_message = omap2_mcspi_prepare_message;
1426         master->can_dma = omap2_mcspi_can_dma;
1427         master->transfer_one = omap2_mcspi_transfer_one;
1428         master->set_cs = omap2_mcspi_set_cs;
1429         master->cleanup = omap2_mcspi_cleanup;
1430         master->slave_abort = omap2_mcspi_slave_abort;
1431         master->dev.of_node = node;
1432         master->max_speed_hz = OMAP2_MCSPI_MAX_FREQ;
1433         master->min_speed_hz = OMAP2_MCSPI_MAX_FREQ >> 15;
1434
1435         platform_set_drvdata(pdev, master);
1436
1437         mcspi = spi_master_get_devdata(master);
1438         mcspi->master = master;
1439
1440         match = of_match_device(omap_mcspi_of_match, &pdev->dev);
1441         if (match) {
1442                 u32 num_cs = 1; /* default number of chipselect */
1443                 pdata = match->data;
1444
1445                 of_property_read_u32(node, "ti,spi-num-cs", &num_cs);
1446                 master->num_chipselect = num_cs;
1447                 if (of_get_property(node, "ti,pindir-d0-out-d1-in", NULL))
1448                         mcspi->pin_dir = MCSPI_PINDIR_D0_OUT_D1_IN;
1449         } else {
1450                 pdata = dev_get_platdata(&pdev->dev);
1451                 master->num_chipselect = pdata->num_cs;
1452                 mcspi->pin_dir = pdata->pin_dir;
1453         }
1454         regs_offset = pdata->regs_offset;
1455         if (pdata->max_xfer_len) {
1456                 mcspi->max_xfer_len = pdata->max_xfer_len;
1457                 master->max_transfer_size = omap2_mcspi_max_xfer_size;
1458         }
1459
1460         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1461         mcspi->base = devm_ioremap_resource(&pdev->dev, r);
1462         if (IS_ERR(mcspi->base)) {
1463                 status = PTR_ERR(mcspi->base);
1464                 goto free_master;
1465         }
1466         mcspi->phys = r->start + regs_offset;
1467         mcspi->base += regs_offset;
1468
1469         mcspi->dev = &pdev->dev;
1470
1471         INIT_LIST_HEAD(&mcspi->ctx.cs);
1472
1473         mcspi->dma_channels = devm_kcalloc(&pdev->dev, master->num_chipselect,
1474                                            sizeof(struct omap2_mcspi_dma),
1475                                            GFP_KERNEL);
1476         if (mcspi->dma_channels == NULL) {
1477                 status = -ENOMEM;
1478                 goto free_master;
1479         }
1480
1481         for (i = 0; i < master->num_chipselect; i++) {
1482                 sprintf(mcspi->dma_channels[i].dma_rx_ch_name, "rx%d", i);
1483                 sprintf(mcspi->dma_channels[i].dma_tx_ch_name, "tx%d", i);
1484
1485                 status = omap2_mcspi_request_dma(mcspi,
1486                                                  &mcspi->dma_channels[i]);
1487                 if (status == -EPROBE_DEFER)
1488                         goto free_master;
1489         }
1490
1491         status = platform_get_irq(pdev, 0);
1492         if (status == -EPROBE_DEFER)
1493                 goto free_master;
1494         if (status < 0) {
1495                 dev_err(&pdev->dev, "no irq resource found\n");
1496                 goto free_master;
1497         }
1498         init_completion(&mcspi->txdone);
1499         status = devm_request_irq(&pdev->dev, status,
1500                                   omap2_mcspi_irq_handler, 0, pdev->name,
1501                                   mcspi);
1502         if (status) {
1503                 dev_err(&pdev->dev, "Cannot request IRQ");
1504                 goto free_master;
1505         }
1506
1507         pm_runtime_use_autosuspend(&pdev->dev);
1508         pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT);
1509         pm_runtime_enable(&pdev->dev);
1510
1511         status = omap2_mcspi_controller_setup(mcspi);
1512         if (status < 0)
1513                 goto disable_pm;
1514
1515         status = devm_spi_register_controller(&pdev->dev, master);
1516         if (status < 0)
1517                 goto disable_pm;
1518
1519         return status;
1520
1521 disable_pm:
1522         pm_runtime_dont_use_autosuspend(&pdev->dev);
1523         pm_runtime_put_sync(&pdev->dev);
1524         pm_runtime_disable(&pdev->dev);
1525 free_master:
1526         omap2_mcspi_release_dma(master);
1527         spi_master_put(master);
1528         return status;
1529 }
1530
1531 static int omap2_mcspi_remove(struct platform_device *pdev)
1532 {
1533         struct spi_master *master = platform_get_drvdata(pdev);
1534         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1535
1536         omap2_mcspi_release_dma(master);
1537
1538         pm_runtime_dont_use_autosuspend(mcspi->dev);
1539         pm_runtime_put_sync(mcspi->dev);
1540         pm_runtime_disable(&pdev->dev);
1541
1542         return 0;
1543 }
1544
1545 /* work with hotplug and coldplug */
1546 MODULE_ALIAS("platform:omap2_mcspi");
1547
1548 static int __maybe_unused omap2_mcspi_suspend(struct device *dev)
1549 {
1550         struct spi_master *master = dev_get_drvdata(dev);
1551         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1552         int error;
1553
1554         error = pinctrl_pm_select_sleep_state(dev);
1555         if (error)
1556                 dev_warn(mcspi->dev, "%s: failed to set pins: %i\n",
1557                          __func__, error);
1558
1559         error = spi_master_suspend(master);
1560         if (error)
1561                 dev_warn(mcspi->dev, "%s: master suspend failed: %i\n",
1562                          __func__, error);
1563
1564         return pm_runtime_force_suspend(dev);
1565 }
1566
1567 static int __maybe_unused omap2_mcspi_resume(struct device *dev)
1568 {
1569         struct spi_master *master = dev_get_drvdata(dev);
1570         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1571         int error;
1572
1573         error = pinctrl_pm_select_default_state(dev);
1574         if (error)
1575                 dev_warn(mcspi->dev, "%s: failed to set pins: %i\n",
1576                          __func__, error);
1577
1578         error = spi_master_resume(master);
1579         if (error)
1580                 dev_warn(mcspi->dev, "%s: master resume failed: %i\n",
1581                          __func__, error);
1582
1583         return pm_runtime_force_resume(dev);
1584 }
1585
1586 static const struct dev_pm_ops omap2_mcspi_pm_ops = {
1587         SET_SYSTEM_SLEEP_PM_OPS(omap2_mcspi_suspend,
1588                                 omap2_mcspi_resume)
1589         .runtime_resume = omap_mcspi_runtime_resume,
1590 };
1591
1592 static struct platform_driver omap2_mcspi_driver = {
1593         .driver = {
1594                 .name =         "omap2_mcspi",
1595                 .pm =           &omap2_mcspi_pm_ops,
1596                 .of_match_table = omap_mcspi_of_match,
1597         },
1598         .probe =        omap2_mcspi_probe,
1599         .remove =       omap2_mcspi_remove,
1600 };
1601
1602 module_platform_driver(omap2_mcspi_driver);
1603 MODULE_LICENSE("GPL");