drivers/spi/spi-omap2-mcspi.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * OMAP2 McSPI controller driver
   4  *
   5  * Copyright (C) 2005, 2006 Nokia Corporation
   6  * Author:      Samuel Ortiz <samuel.ortiz@nokia.com> and
   7  *              Juha Yrj�l� <juha.yrjola@nokia.com>
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/module.h>
  13 #include <linux/device.h>
  14 #include <linux/delay.h>
  15 #include <linux/dma-mapping.h>
  16 #include <linux/dmaengine.h>
  17 #include <linux/pinctrl/consumer.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/err.h>
  20 #include <linux/clk.h>
  21 #include <linux/io.h>
  22 #include <linux/slab.h>
  23 #include <linux/pm_runtime.h>
  24 #include <linux/of.h>
  25 #include <linux/of_device.h>
  26 #include <linux/gcd.h>
  27
  28 #include <linux/spi/spi.h>
  29
  30 #include <linux/platform_data/spi-omap2-mcspi.h>
  31
  32 #define OMAP2_MCSPI_MAX_FREQ            48000000
  33 #define OMAP2_MCSPI_MAX_DIVIDER         4096
  34 #define OMAP2_MCSPI_MAX_FIFODEPTH       64
  35 #define OMAP2_MCSPI_MAX_FIFOWCNT        0xFFFF
  36 #define SPI_AUTOSUSPEND_TIMEOUT         2000
  37
  38 #define OMAP2_MCSPI_REVISION            0x00
  39 #define OMAP2_MCSPI_SYSSTATUS           0x14
  40 #define OMAP2_MCSPI_IRQSTATUS           0x18
  41 #define OMAP2_MCSPI_IRQENABLE           0x1c
  42 #define OMAP2_MCSPI_WAKEUPENABLE        0x20
  43 #define OMAP2_MCSPI_SYST                0x24
  44 #define OMAP2_MCSPI_MODULCTRL           0x28
  45 #define OMAP2_MCSPI_XFERLEVEL           0x7c
  46
  47 /* per-channel banks, 0x14 bytes each, first is: */
  48 #define OMAP2_MCSPI_CHCONF0             0x2c
  49 #define OMAP2_MCSPI_CHSTAT0             0x30
  50 #define OMAP2_MCSPI_CHCTRL0             0x34
  51 #define OMAP2_MCSPI_TX0                 0x38
  52 #define OMAP2_MCSPI_RX0                 0x3c
  53
  54 /* per-register bitmasks: */
  55 #define OMAP2_MCSPI_IRQSTATUS_EOW       BIT(17)
  56
  57 #define OMAP2_MCSPI_MODULCTRL_SINGLE    BIT(0)
  58 #define OMAP2_MCSPI_MODULCTRL_MS        BIT(2)
  59 #define OMAP2_MCSPI_MODULCTRL_STEST     BIT(3)
  60
  61 #define OMAP2_MCSPI_CHCONF_PHA          BIT(0)
  62 #define OMAP2_MCSPI_CHCONF_POL          BIT(1)
  63 #define OMAP2_MCSPI_CHCONF_CLKD_MASK    (0x0f << 2)
  64 #define OMAP2_MCSPI_CHCONF_EPOL         BIT(6)
  65 #define OMAP2_MCSPI_CHCONF_WL_MASK      (0x1f << 7)
  66 #define OMAP2_MCSPI_CHCONF_TRM_RX_ONLY  BIT(12)
  67 #define OMAP2_MCSPI_CHCONF_TRM_TX_ONLY  BIT(13)
  68 #define OMAP2_MCSPI_CHCONF_TRM_MASK     (0x03 << 12)
  69 #define OMAP2_MCSPI_CHCONF_DMAW         BIT(14)
  70 #define OMAP2_MCSPI_CHCONF_DMAR         BIT(15)
  71 #define OMAP2_MCSPI_CHCONF_DPE0         BIT(16)
  72 #define OMAP2_MCSPI_CHCONF_DPE1         BIT(17)
  73 #define OMAP2_MCSPI_CHCONF_IS           BIT(18)
  74 #define OMAP2_MCSPI_CHCONF_TURBO        BIT(19)
  75 #define OMAP2_MCSPI_CHCONF_FORCE        BIT(20)
  76 #define OMAP2_MCSPI_CHCONF_FFET         BIT(27)
  77 #define OMAP2_MCSPI_CHCONF_FFER         BIT(28)
  78 #define OMAP2_MCSPI_CHCONF_CLKG         BIT(29)
  79
  80 #define OMAP2_MCSPI_CHSTAT_RXS          BIT(0)
  81 #define OMAP2_MCSPI_CHSTAT_TXS          BIT(1)
  82 #define OMAP2_MCSPI_CHSTAT_EOT          BIT(2)
  83 #define OMAP2_MCSPI_CHSTAT_TXFFE        BIT(3)
  84
  85 #define OMAP2_MCSPI_CHCTRL_EN           BIT(0)
  86 #define OMAP2_MCSPI_CHCTRL_EXTCLK_MASK  (0xff << 8)
  87
  88 #define OMAP2_MCSPI_WAKEUPENABLE_WKEN   BIT(0)
  89
  90 /* We have 2 DMA channels per CS, one for RX and one for TX */
  91 struct omap2_mcspi_dma {
  92         struct dma_chan *dma_tx;
  93         struct dma_chan *dma_rx;
  94
  95         struct completion dma_tx_completion;
  96         struct completion dma_rx_completion;
  97
  98         char dma_rx_ch_name[14];
  99         char dma_tx_ch_name[14];
 100 };
 101
 102 /* use PIO for small transfers, avoiding DMA setup/teardown overhead and
 103  * cache operations; better heuristics consider wordsize and bitrate.
 104  */
 105 #define DMA_MIN_BYTES                   160
 106
 107
 108 /*
 109  * Used for context save and restore, structure members to be updated whenever
 110  * corresponding registers are modified.
 111  */
 112 struct omap2_mcspi_regs {
 113         u32 modulctrl;
 114         u32 wakeupenable;
 115         struct list_head cs;
 116 };
 117
 118 struct omap2_mcspi {
 119         struct completion       txdone;
 120         struct spi_master       *master;
 121         /* Virtual base address of the controller */
 122         void __iomem            *base;
 123         unsigned long           phys;
 124         /* SPI1 has 4 channels, while SPI2 has 2 */
 125         struct omap2_mcspi_dma  *dma_channels;
 126         struct device           *dev;
 127         struct omap2_mcspi_regs ctx;
 128         int                     fifo_depth;
 129         bool                    slave_aborted;
 130         unsigned int            pin_dir:1;
 131         size_t                  max_xfer_len;
 132 };
 133
 134 struct omap2_mcspi_cs {
 135         void __iomem            *base;
 136         unsigned long           phys;
 137         int                     word_len;
 138         u16                     mode;
 139         struct list_head        node;
 140         /* Context save and restore shadow register */
 141         u32                     chconf0, chctrl0;
 142 };
 143
 144 static inline void mcspi_write_reg(struct spi_master *master,
 145                 int idx, u32 val)
 146 {
 147         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
 148
 149         writel_relaxed(val, mcspi->base + idx);
 150 }
 151
 152 static inline u32 mcspi_read_reg(struct spi_master *master, int idx)
 153 {
 154         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
 155
 156         return readl_relaxed(mcspi->base + idx);
 157 }
 158
 159 static inline void mcspi_write_cs_reg(const struct spi_device *spi,
 160                 int idx, u32 val)
 161 {
 162         struct omap2_mcspi_cs   *cs = spi->controller_state;
 163
 164         writel_relaxed(val, cs->base +  idx);
 165 }
 166
 167 static inline u32 mcspi_read_cs_reg(const struct spi_device *spi, int idx)
 168 {
 169         struct omap2_mcspi_cs   *cs = spi->controller_state;
 170
 171         return readl_relaxed(cs->base + idx);
 172 }
 173
 174 static inline u32 mcspi_cached_chconf0(const struct spi_device *spi)
 175 {
 176         struct omap2_mcspi_cs *cs = spi->controller_state;
 177
 178         return cs->chconf0;
 179 }
 180
 181 static inline void mcspi_write_chconf0(const struct spi_device *spi, u32 val)
 182 {
 183         struct omap2_mcspi_cs *cs = spi->controller_state;
 184
 185         cs->chconf0 = val;
 186         mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCONF0, val);
 187         mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCONF0);
 188 }
 189
 190 static inline int mcspi_bytes_per_word(int word_len)
 191 {
 192         if (word_len <= 8)
 193                 return 1;
 194         else if (word_len <= 16)
 195                 return 2;
 196         else /* word_len <= 32 */
 197                 return 4;
 198 }
 199
 200 static void omap2_mcspi_set_dma_req(const struct spi_device *spi,
 201                 int is_read, int enable)
 202 {
 203         u32 l, rw;
 204
 205         l = mcspi_cached_chconf0(spi);
 206
 207         if (is_read) /* 1 is read, 0 write */
 208                 rw = OMAP2_MCSPI_CHCONF_DMAR;
 209         else
 210                 rw = OMAP2_MCSPI_CHCONF_DMAW;
 211
 212         if (enable)
 213                 l |= rw;
 214         else
 215                 l &= ~rw;
 216
 217         mcspi_write_chconf0(spi, l);
 218 }
 219
 220 static void omap2_mcspi_set_enable(const struct spi_device *spi, int enable)
 221 {
 222         struct omap2_mcspi_cs *cs = spi->controller_state;
 223         u32 l;
 224
 225         l = cs->chctrl0;
 226         if (enable)
 227                 l |= OMAP2_MCSPI_CHCTRL_EN;
 228         else
 229                 l &= ~OMAP2_MCSPI_CHCTRL_EN;
 230         cs->chctrl0 = l;
 231         mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
 232         /* Flash post-writes */
 233         mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCTRL0);
 234 }
 235
 236 static void omap2_mcspi_set_cs(struct spi_device *spi, bool enable)
 237 {
 238         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 239         u32 l;
 240
 241         /* The controller handles the inverted chip selects
 242          * using the OMAP2_MCSPI_CHCONF_EPOL bit so revert
 243          * the inversion from the core spi_set_cs function.
 244          */
 245         if (spi->mode & SPI_CS_HIGH)
 246                 enable = !enable;
 247
 248         if (spi->controller_state) {
 249                 int err = pm_runtime_get_sync(mcspi->dev);
 250                 if (err < 0) {
 251                         pm_runtime_put_noidle(mcspi->dev);
 252                         dev_err(mcspi->dev, "failed to get sync: %d\n", err);
 253                         return;
 254                 }
 255
 256                 l = mcspi_cached_chconf0(spi);
 257
 258                 if (enable)
 259                         l &= ~OMAP2_MCSPI_CHCONF_FORCE;
 260                 else
 261                         l |= OMAP2_MCSPI_CHCONF_FORCE;
 262
 263                 mcspi_write_chconf0(spi, l);
 264
 265                 pm_runtime_mark_last_busy(mcspi->dev);
 266                 pm_runtime_put_autosuspend(mcspi->dev);
 267         }
 268 }
 269
 270 static void omap2_mcspi_set_mode(struct spi_master *master)
 271 {
 272         struct omap2_mcspi      *mcspi = spi_master_get_devdata(master);
 273         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
 274         u32 l;
 275
 276         /*
 277          * Choose master or slave mode
 278          */
 279         l = mcspi_read_reg(master, OMAP2_MCSPI_MODULCTRL);
 280         l &= ~(OMAP2_MCSPI_MODULCTRL_STEST);
 281         if (spi_controller_is_slave(master)) {
 282                 l |= (OMAP2_MCSPI_MODULCTRL_MS);
 283         } else {
 284                 l &= ~(OMAP2_MCSPI_MODULCTRL_MS);
 285                 l |= OMAP2_MCSPI_MODULCTRL_SINGLE;
 286         }
 287         mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, l);
 288
 289         ctx->modulctrl = l;
 290 }
 291
 292 static void omap2_mcspi_set_fifo(const struct spi_device *spi,
 293                                 struct spi_transfer *t, int enable)
 294 {
 295         struct spi_master *master = spi->master;
 296         struct omap2_mcspi_cs *cs = spi->controller_state;
 297         struct omap2_mcspi *mcspi;
 298         unsigned int wcnt;
 299         int max_fifo_depth, bytes_per_word;
 300         u32 chconf, xferlevel;
 301
 302         mcspi = spi_master_get_devdata(master);
 303
 304         chconf = mcspi_cached_chconf0(spi);
 305         if (enable) {
 306                 bytes_per_word = mcspi_bytes_per_word(cs->word_len);
 307                 if (t->len % bytes_per_word != 0)
 308                         goto disable_fifo;
 309
 310                 if (t->rx_buf != NULL && t->tx_buf != NULL)
 311                         max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH / 2;
 312                 else
 313                         max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH;
 314
 315                 wcnt = t->len / bytes_per_word;
 316                 if (wcnt > OMAP2_MCSPI_MAX_FIFOWCNT)
 317                         goto disable_fifo;
 318
 319                 xferlevel = wcnt << 16;
 320                 if (t->rx_buf != NULL) {
 321                         chconf |= OMAP2_MCSPI_CHCONF_FFER;
 322                         xferlevel |= (bytes_per_word - 1) << 8;
 323                 }
 324
 325                 if (t->tx_buf != NULL) {
 326                         chconf |= OMAP2_MCSPI_CHCONF_FFET;
 327                         xferlevel |= bytes_per_word - 1;
 328                 }
 329
 330                 mcspi_write_reg(master, OMAP2_MCSPI_XFERLEVEL, xferlevel);
 331                 mcspi_write_chconf0(spi, chconf);
 332                 mcspi->fifo_depth = max_fifo_depth;
 333
 334                 return;
 335         }
 336
 337 disable_fifo:
 338         if (t->rx_buf != NULL)
 339                 chconf &= ~OMAP2_MCSPI_CHCONF_FFER;
 340
 341         if (t->tx_buf != NULL)
 342                 chconf &= ~OMAP2_MCSPI_CHCONF_FFET;
 343
 344         mcspi_write_chconf0(spi, chconf);
 345         mcspi->fifo_depth = 0;
 346 }
 347
 348 static int mcspi_wait_for_reg_bit(void __iomem *reg, unsigned long bit)
 349 {
 350         unsigned long timeout;
 351
 352         timeout = jiffies + msecs_to_jiffies(1000);
 353         while (!(readl_relaxed(reg) & bit)) {
 354                 if (time_after(jiffies, timeout)) {
 355                         if (!(readl_relaxed(reg) & bit))
 356                                 return -ETIMEDOUT;
 357                         else
 358                                 return 0;
 359                 }
 360                 cpu_relax();
 361         }
 362         return 0;
 363 }
 364
 365 static int mcspi_wait_for_completion(struct  omap2_mcspi *mcspi,
 366                                      struct completion *x)
 367 {
 368         if (spi_controller_is_slave(mcspi->master)) {
 369                 if (wait_for_completion_interruptible(x) ||
 370                     mcspi->slave_aborted)
 371                         return -EINTR;
 372         } else {
 373                 wait_for_completion(x);
 374         }
 375
 376         return 0;
 377 }
 378
 379 static void omap2_mcspi_rx_callback(void *data)
 380 {
 381         struct spi_device *spi = data;
 382         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 383         struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 384
 385         /* We must disable the DMA RX request */
 386         omap2_mcspi_set_dma_req(spi, 1, 0);
 387
 388         complete(&mcspi_dma->dma_rx_completion);
 389 }
 390
 391 static void omap2_mcspi_tx_callback(void *data)
 392 {
 393         struct spi_device *spi = data;
 394         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 395         struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 396
 397         /* We must disable the DMA TX request */
 398         omap2_mcspi_set_dma_req(spi, 0, 0);
 399
 400         complete(&mcspi_dma->dma_tx_completion);
 401 }
 402
 403 static void omap2_mcspi_tx_dma(struct spi_device *spi,
 404                                 struct spi_transfer *xfer,
 405                                 struct dma_slave_config cfg)
 406 {
 407         struct omap2_mcspi      *mcspi;
 408         struct omap2_mcspi_dma  *mcspi_dma;
 409         struct dma_async_tx_descriptor *tx;
 410
 411         mcspi = spi_master_get_devdata(spi->master);
 412         mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 413
 414         dmaengine_slave_config(mcspi_dma->dma_tx, &cfg);
 415
 416         tx = dmaengine_prep_slave_sg(mcspi_dma->dma_tx, xfer->tx_sg.sgl,
 417                                      xfer->tx_sg.nents,
 418                                      DMA_MEM_TO_DEV,
 419                                      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 420         if (tx) {
 421                 tx->callback = omap2_mcspi_tx_callback;
 422                 tx->callback_param = spi;
 423                 dmaengine_submit(tx);
 424         } else {
 425                 /* FIXME: fall back to PIO? */
 426         }
 427         dma_async_issue_pending(mcspi_dma->dma_tx);
 428         omap2_mcspi_set_dma_req(spi, 0, 1);
 429 }
 430
 431 static unsigned
 432 omap2_mcspi_rx_dma(struct spi_device *spi, struct spi_transfer *xfer,
 433                                 struct dma_slave_config cfg,
 434                                 unsigned es)
 435 {
 436         struct omap2_mcspi      *mcspi;
 437         struct omap2_mcspi_dma  *mcspi_dma;
 438         unsigned int            count, transfer_reduction = 0;
 439         struct scatterlist      *sg_out[2];
 440         int                     nb_sizes = 0, out_mapped_nents[2], ret, x;
 441         size_t                  sizes[2];
 442         u32                     l;
 443         int                     elements = 0;
 444         int                     word_len, element_count;
 445         struct omap2_mcspi_cs   *cs = spi->controller_state;
 446         void __iomem            *chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
 447         struct dma_async_tx_descriptor *tx;
 448
 449         mcspi = spi_master_get_devdata(spi->master);
 450         mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 451         count = xfer->len;
 452
 453         /*
 454          *  In the "End-of-Transfer Procedure" section for DMA RX in OMAP35x TRM
 455          *  it mentions reducing DMA transfer length by one element in master
 456          *  normal mode.
 457          */
 458         if (mcspi->fifo_depth == 0)
 459                 transfer_reduction = es;
 460
 461         word_len = cs->word_len;
 462         l = mcspi_cached_chconf0(spi);
 463
 464         if (word_len <= 8)
 465                 element_count = count;
 466         else if (word_len <= 16)
 467                 element_count = count >> 1;
 468         else /* word_len <= 32 */
 469                 element_count = count >> 2;
 470
 471
 472         dmaengine_slave_config(mcspi_dma->dma_rx, &cfg);
 473
 474         /*
 475          *  Reduce DMA transfer length by one more if McSPI is
 476          *  configured in turbo mode.
 477          */
 478         if ((l & OMAP2_MCSPI_CHCONF_TURBO) && mcspi->fifo_depth == 0)
 479                 transfer_reduction += es;
 480
 481         if (transfer_reduction) {
 482                 /* Split sgl into two. The second sgl won't be used. */
 483                 sizes[0] = count - transfer_reduction;
 484                 sizes[1] = transfer_reduction;
 485                 nb_sizes = 2;
 486         } else {
 487                 /*
 488                  * Don't bother splitting the sgl. This essentially
 489                  * clones the original sgl.
 490                  */
 491                 sizes[0] = count;
 492                 nb_sizes = 1;
 493         }
 494
 495         ret = sg_split(xfer->rx_sg.sgl, xfer->rx_sg.nents, 0, nb_sizes,
 496                        sizes, sg_out, out_mapped_nents, GFP_KERNEL);
 497
 498         if (ret < 0) {
 499                 dev_err(&spi->dev, "sg_split failed\n");
 500                 return 0;
 501         }
 502
 503         tx = dmaengine_prep_slave_sg(mcspi_dma->dma_rx, sg_out[0],
 504                                      out_mapped_nents[0], DMA_DEV_TO_MEM,
 505                                      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 506         if (tx) {
 507                 tx->callback = omap2_mcspi_rx_callback;
 508                 tx->callback_param = spi;
 509                 dmaengine_submit(tx);
 510         } else {
 511                 /* FIXME: fall back to PIO? */
 512         }
 513
 514         dma_async_issue_pending(mcspi_dma->dma_rx);
 515         omap2_mcspi_set_dma_req(spi, 1, 1);
 516
 517         ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_rx_completion);
 518         if (ret || mcspi->slave_aborted) {
 519                 dmaengine_terminate_sync(mcspi_dma->dma_rx);
 520                 omap2_mcspi_set_dma_req(spi, 1, 0);
 521                 return 0;
 522         }
 523
 524         for (x = 0; x < nb_sizes; x++)
 525                 kfree(sg_out[x]);
 526
 527         if (mcspi->fifo_depth > 0)
 528                 return count;
 529
 530         /*
 531          *  Due to the DMA transfer length reduction the missing bytes must
 532          *  be read manually to receive all of the expected data.
 533          */
 534         omap2_mcspi_set_enable(spi, 0);
 535
 536         elements = element_count - 1;
 537
 538         if (l & OMAP2_MCSPI_CHCONF_TURBO) {
 539                 elements--;
 540
 541                 if (!mcspi_wait_for_reg_bit(chstat_reg,
 542                                             OMAP2_MCSPI_CHSTAT_RXS)) {
 543                         u32 w;
 544
 545                         w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
 546                         if (word_len <= 8)
 547                                 ((u8 *)xfer->rx_buf)[elements++] = w;
 548                         else if (word_len <= 16)
 549                                 ((u16 *)xfer->rx_buf)[elements++] = w;
 550                         else /* word_len <= 32 */
 551                                 ((u32 *)xfer->rx_buf)[elements++] = w;
 552                 } else {
 553                         int bytes_per_word = mcspi_bytes_per_word(word_len);
 554                         dev_err(&spi->dev, "DMA RX penultimate word empty\n");
 555                         count -= (bytes_per_word << 1);
 556                         omap2_mcspi_set_enable(spi, 1);
 557                         return count;
 558                 }
 559         }
 560         if (!mcspi_wait_for_reg_bit(chstat_reg, OMAP2_MCSPI_CHSTAT_RXS)) {
 561                 u32 w;
 562
 563                 w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
 564                 if (word_len <= 8)
 565                         ((u8 *)xfer->rx_buf)[elements] = w;
 566                 else if (word_len <= 16)
 567                         ((u16 *)xfer->rx_buf)[elements] = w;
 568                 else /* word_len <= 32 */
 569                         ((u32 *)xfer->rx_buf)[elements] = w;
 570         } else {
 571                 dev_err(&spi->dev, "DMA RX last word empty\n");
 572                 count -= mcspi_bytes_per_word(word_len);
 573         }
 574         omap2_mcspi_set_enable(spi, 1);
 575         return count;
 576 }
 577
 578 static unsigned
 579 omap2_mcspi_txrx_dma(struct spi_device *spi, struct spi_transfer *xfer)
 580 {
 581         struct omap2_mcspi      *mcspi;
 582         struct omap2_mcspi_cs   *cs = spi->controller_state;
 583         struct omap2_mcspi_dma  *mcspi_dma;
 584         unsigned int            count;
 585         u8                      *rx;
 586         const u8                *tx;
 587         struct dma_slave_config cfg;
 588         enum dma_slave_buswidth width;
 589         unsigned es;
 590         void __iomem            *chstat_reg;
 591         void __iomem            *irqstat_reg;
 592         int                     wait_res;
 593
 594         mcspi = spi_master_get_devdata(spi->master);
 595         mcspi_dma = &mcspi->dma_channels[spi->chip_select];
 596
 597         if (cs->word_len <= 8) {
 598                 width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 599                 es = 1;
 600         } else if (cs->word_len <= 16) {
 601                 width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 602                 es = 2;
 603         } else {
 604                 width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 605                 es = 4;
 606         }
 607
 608         count = xfer->len;
 609
 610         memset(&cfg, 0, sizeof(cfg));
 611         cfg.src_addr = cs->phys + OMAP2_MCSPI_RX0;
 612         cfg.dst_addr = cs->phys + OMAP2_MCSPI_TX0;
 613         cfg.src_addr_width = width;
 614         cfg.dst_addr_width = width;
 615         cfg.src_maxburst = 1;
 616         cfg.dst_maxburst = 1;
 617
 618         rx = xfer->rx_buf;
 619         tx = xfer->tx_buf;
 620
 621         mcspi->slave_aborted = false;
 622         reinit_completion(&mcspi_dma->dma_tx_completion);
 623         reinit_completion(&mcspi_dma->dma_rx_completion);
 624         reinit_completion(&mcspi->txdone);
 625         if (tx) {
 626                 /* Enable EOW IRQ to know end of tx in slave mode */
 627                 if (spi_controller_is_slave(spi->master))
 628                         mcspi_write_reg(spi->master,
 629                                         OMAP2_MCSPI_IRQENABLE,
 630                                         OMAP2_MCSPI_IRQSTATUS_EOW);
 631                 omap2_mcspi_tx_dma(spi, xfer, cfg);
 632         }
 633
 634         if (rx != NULL)
 635                 count = omap2_mcspi_rx_dma(spi, xfer, cfg, es);
 636
 637         if (tx != NULL) {
 638                 int ret;
 639
 640                 ret = mcspi_wait_for_completion(mcspi, &mcspi_dma->dma_tx_completion);
 641                 if (ret || mcspi->slave_aborted) {
 642                         dmaengine_terminate_sync(mcspi_dma->dma_tx);
 643                         omap2_mcspi_set_dma_req(spi, 0, 0);
 644                         return 0;
 645                 }
 646
 647                 if (spi_controller_is_slave(mcspi->master)) {
 648                         ret = mcspi_wait_for_completion(mcspi, &mcspi->txdone);
 649                         if (ret || mcspi->slave_aborted)
 650                                 return 0;
 651                 }
 652
 653                 if (mcspi->fifo_depth > 0) {
 654                         irqstat_reg = mcspi->base + OMAP2_MCSPI_IRQSTATUS;
 655
 656                         if (mcspi_wait_for_reg_bit(irqstat_reg,
 657                                                 OMAP2_MCSPI_IRQSTATUS_EOW) < 0)
 658                                 dev_err(&spi->dev, "EOW timed out\n");
 659
 660                         mcspi_write_reg(mcspi->master, OMAP2_MCSPI_IRQSTATUS,
 661                                         OMAP2_MCSPI_IRQSTATUS_EOW);
 662                 }
 663
 664                 /* for TX_ONLY mode, be sure all words have shifted out */
 665                 if (rx == NULL) {
 666                         chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
 667                         if (mcspi->fifo_depth > 0) {
 668                                 wait_res = mcspi_wait_for_reg_bit(chstat_reg,
 669                                                 OMAP2_MCSPI_CHSTAT_TXFFE);
 670                                 if (wait_res < 0)
 671                                         dev_err(&spi->dev, "TXFFE timed out\n");
 672                         } else {
 673                                 wait_res = mcspi_wait_for_reg_bit(chstat_reg,
 674                                                 OMAP2_MCSPI_CHSTAT_TXS);
 675                                 if (wait_res < 0)
 676                                         dev_err(&spi->dev, "TXS timed out\n");
 677                         }
 678                         if (wait_res >= 0 &&
 679                                 (mcspi_wait_for_reg_bit(chstat_reg,
 680                                         OMAP2_MCSPI_CHSTAT_EOT) < 0))
 681                                 dev_err(&spi->dev, "EOT timed out\n");
 682                 }
 683         }
 684         return count;
 685 }
 686
 687 static unsigned
 688 omap2_mcspi_txrx_pio(struct spi_device *spi, struct spi_transfer *xfer)
 689 {
 690         struct omap2_mcspi_cs   *cs = spi->controller_state;
 691         unsigned int            count, c;
 692         u32                     l;
 693         void __iomem            *base = cs->base;
 694         void __iomem            *tx_reg;
 695         void __iomem            *rx_reg;
 696         void __iomem            *chstat_reg;
 697         int                     word_len;
 698
 699         count = xfer->len;
 700         c = count;
 701         word_len = cs->word_len;
 702
 703         l = mcspi_cached_chconf0(spi);
 704
 705         /* We store the pre-calculated register addresses on stack to speed
 706          * up the transfer loop. */
 707         tx_reg          = base + OMAP2_MCSPI_TX0;
 708         rx_reg          = base + OMAP2_MCSPI_RX0;
 709         chstat_reg      = base + OMAP2_MCSPI_CHSTAT0;
 710
 711         if (c < (word_len>>3))
 712                 return 0;
 713
 714         if (word_len <= 8) {
 715                 u8              *rx;
 716                 const u8        *tx;
 717
 718                 rx = xfer->rx_buf;
 719                 tx = xfer->tx_buf;
 720
 721                 do {
 722                         c -= 1;
 723                         if (tx != NULL) {
 724                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 725                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 726                                         dev_err(&spi->dev, "TXS timed out\n");
 727                                         goto out;
 728                                 }
 729                                 dev_vdbg(&spi->dev, "write-%d %02x\n",
 730                                                 word_len, *tx);
 731                                 writel_relaxed(*tx++, tx_reg);
 732                         }
 733                         if (rx != NULL) {
 734                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 735                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 736                                         dev_err(&spi->dev, "RXS timed out\n");
 737                                         goto out;
 738                                 }
 739
 740                                 if (c == 1 && tx == NULL &&
 741                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 742                                         omap2_mcspi_set_enable(spi, 0);
 743                                         *rx++ = readl_relaxed(rx_reg);
 744                                         dev_vdbg(&spi->dev, "read-%d %02x\n",
 745                                                     word_len, *(rx - 1));
 746                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 747                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 748                                                 dev_err(&spi->dev,
 749                                                         "RXS timed out\n");
 750                                                 goto out;
 751                                         }
 752                                         c = 0;
 753                                 } else if (c == 0 && tx == NULL) {
 754                                         omap2_mcspi_set_enable(spi, 0);
 755                                 }
 756
 757                                 *rx++ = readl_relaxed(rx_reg);
 758                                 dev_vdbg(&spi->dev, "read-%d %02x\n",
 759                                                 word_len, *(rx - 1));
 760                         }
 761                 } while (c);
 762         } else if (word_len <= 16) {
 763                 u16             *rx;
 764                 const u16       *tx;
 765
 766                 rx = xfer->rx_buf;
 767                 tx = xfer->tx_buf;
 768                 do {
 769                         c -= 2;
 770                         if (tx != NULL) {
 771                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 772                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 773                                         dev_err(&spi->dev, "TXS timed out\n");
 774                                         goto out;
 775                                 }
 776                                 dev_vdbg(&spi->dev, "write-%d %04x\n",
 777                                                 word_len, *tx);
 778                                 writel_relaxed(*tx++, tx_reg);
 779                         }
 780                         if (rx != NULL) {
 781                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 782                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 783                                         dev_err(&spi->dev, "RXS timed out\n");
 784                                         goto out;
 785                                 }
 786
 787                                 if (c == 2 && tx == NULL &&
 788                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 789                                         omap2_mcspi_set_enable(spi, 0);
 790                                         *rx++ = readl_relaxed(rx_reg);
 791                                         dev_vdbg(&spi->dev, "read-%d %04x\n",
 792                                                     word_len, *(rx - 1));
 793                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 794                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 795                                                 dev_err(&spi->dev,
 796                                                         "RXS timed out\n");
 797                                                 goto out;
 798                                         }
 799                                         c = 0;
 800                                 } else if (c == 0 && tx == NULL) {
 801                                         omap2_mcspi_set_enable(spi, 0);
 802                                 }
 803
 804                                 *rx++ = readl_relaxed(rx_reg);
 805                                 dev_vdbg(&spi->dev, "read-%d %04x\n",
 806                                                 word_len, *(rx - 1));
 807                         }
 808                 } while (c >= 2);
 809         } else if (word_len <= 32) {
 810                 u32             *rx;
 811                 const u32       *tx;
 812
 813                 rx = xfer->rx_buf;
 814                 tx = xfer->tx_buf;
 815                 do {
 816                         c -= 4;
 817                         if (tx != NULL) {
 818                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 819                                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 820                                         dev_err(&spi->dev, "TXS timed out\n");
 821                                         goto out;
 822                                 }
 823                                 dev_vdbg(&spi->dev, "write-%d %08x\n",
 824                                                 word_len, *tx);
 825                                 writel_relaxed(*tx++, tx_reg);
 826                         }
 827                         if (rx != NULL) {
 828                                 if (mcspi_wait_for_reg_bit(chstat_reg,
 829                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 830                                         dev_err(&spi->dev, "RXS timed out\n");
 831                                         goto out;
 832                                 }
 833
 834                                 if (c == 4 && tx == NULL &&
 835                                     (l & OMAP2_MCSPI_CHCONF_TURBO)) {
 836                                         omap2_mcspi_set_enable(spi, 0);
 837                                         *rx++ = readl_relaxed(rx_reg);
 838                                         dev_vdbg(&spi->dev, "read-%d %08x\n",
 839                                                     word_len, *(rx - 1));
 840                                         if (mcspi_wait_for_reg_bit(chstat_reg,
 841                                                 OMAP2_MCSPI_CHSTAT_RXS) < 0) {
 842                                                 dev_err(&spi->dev,
 843                                                         "RXS timed out\n");
 844                                                 goto out;
 845                                         }
 846                                         c = 0;
 847                                 } else if (c == 0 && tx == NULL) {
 848                                         omap2_mcspi_set_enable(spi, 0);
 849                                 }
 850
 851                                 *rx++ = readl_relaxed(rx_reg);
 852                                 dev_vdbg(&spi->dev, "read-%d %08x\n",
 853                                                 word_len, *(rx - 1));
 854                         }
 855                 } while (c >= 4);
 856         }
 857
 858         /* for TX_ONLY mode, be sure all words have shifted out */
 859         if (xfer->rx_buf == NULL) {
 860                 if (mcspi_wait_for_reg_bit(chstat_reg,
 861                                 OMAP2_MCSPI_CHSTAT_TXS) < 0) {
 862                         dev_err(&spi->dev, "TXS timed out\n");
 863                 } else if (mcspi_wait_for_reg_bit(chstat_reg,
 864                                 OMAP2_MCSPI_CHSTAT_EOT) < 0)
 865                         dev_err(&spi->dev, "EOT timed out\n");
 866
 867                 /* disable chan to purge rx datas received in TX_ONLY transfer,
 868                  * otherwise these rx datas will affect the direct following
 869                  * RX_ONLY transfer.
 870                  */
 871                 omap2_mcspi_set_enable(spi, 0);
 872         }
 873 out:
 874         omap2_mcspi_set_enable(spi, 1);
 875         return count - c;
 876 }
 877
 878 static u32 omap2_mcspi_calc_divisor(u32 speed_hz)
 879 {
 880         u32 div;
 881
 882         for (div = 0; div < 15; div++)
 883                 if (speed_hz >= (OMAP2_MCSPI_MAX_FREQ >> div))
 884                         return div;
 885
 886         return 15;
 887 }
 888
 889 /* called only when no transfer is active to this device */
 890 static int omap2_mcspi_setup_transfer(struct spi_device *spi,
 891                 struct spi_transfer *t)
 892 {
 893         struct omap2_mcspi_cs *cs = spi->controller_state;
 894         struct omap2_mcspi *mcspi;
 895         u32 l = 0, clkd = 0, div, extclk = 0, clkg = 0;
 896         u8 word_len = spi->bits_per_word;
 897         u32 speed_hz = spi->max_speed_hz;
 898
 899         mcspi = spi_master_get_devdata(spi->master);
 900
 901         if (t != NULL && t->bits_per_word)
 902                 word_len = t->bits_per_word;
 903
 904         cs->word_len = word_len;
 905
 906         if (t && t->speed_hz)
 907                 speed_hz = t->speed_hz;
 908
 909         speed_hz = min_t(u32, speed_hz, OMAP2_MCSPI_MAX_FREQ);
 910         if (speed_hz < (OMAP2_MCSPI_MAX_FREQ / OMAP2_MCSPI_MAX_DIVIDER)) {
 911                 clkd = omap2_mcspi_calc_divisor(speed_hz);
 912                 speed_hz = OMAP2_MCSPI_MAX_FREQ >> clkd;
 913                 clkg = 0;
 914         } else {
 915                 div = (OMAP2_MCSPI_MAX_FREQ + speed_hz - 1) / speed_hz;
 916                 speed_hz = OMAP2_MCSPI_MAX_FREQ / div;
 917                 clkd = (div - 1) & 0xf;
 918                 extclk = (div - 1) >> 4;
 919                 clkg = OMAP2_MCSPI_CHCONF_CLKG;
 920         }
 921
 922         l = mcspi_cached_chconf0(spi);
 923
 924         /* standard 4-wire master mode:  SCK, MOSI/out, MISO/in, nCS
 925          * REVISIT: this controller could support SPI_3WIRE mode.
 926          */
 927         if (mcspi->pin_dir == MCSPI_PINDIR_D0_IN_D1_OUT) {
 928                 l &= ~OMAP2_MCSPI_CHCONF_IS;
 929                 l &= ~OMAP2_MCSPI_CHCONF_DPE1;
 930                 l |= OMAP2_MCSPI_CHCONF_DPE0;
 931         } else {
 932                 l |= OMAP2_MCSPI_CHCONF_IS;
 933                 l |= OMAP2_MCSPI_CHCONF_DPE1;
 934                 l &= ~OMAP2_MCSPI_CHCONF_DPE0;
 935         }
 936
 937         /* wordlength */
 938         l &= ~OMAP2_MCSPI_CHCONF_WL_MASK;
 939         l |= (word_len - 1) << 7;
 940
 941         /* set chipselect polarity; manage with FORCE */
 942         if (!(spi->mode & SPI_CS_HIGH))
 943                 l |= OMAP2_MCSPI_CHCONF_EPOL;   /* active-low; normal */
 944         else
 945                 l &= ~OMAP2_MCSPI_CHCONF_EPOL;
 946
 947         /* set clock divisor */
 948         l &= ~OMAP2_MCSPI_CHCONF_CLKD_MASK;
 949         l |= clkd << 2;
 950
 951         /* set clock granularity */
 952         l &= ~OMAP2_MCSPI_CHCONF_CLKG;
 953         l |= clkg;
 954         if (clkg) {
 955                 cs->chctrl0 &= ~OMAP2_MCSPI_CHCTRL_EXTCLK_MASK;
 956                 cs->chctrl0 |= extclk << 8;
 957                 mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, cs->chctrl0);
 958         }
 959
 960         /* set SPI mode 0..3 */
 961         if (spi->mode & SPI_CPOL)
 962                 l |= OMAP2_MCSPI_CHCONF_POL;
 963         else
 964                 l &= ~OMAP2_MCSPI_CHCONF_POL;
 965         if (spi->mode & SPI_CPHA)
 966                 l |= OMAP2_MCSPI_CHCONF_PHA;
 967         else
 968                 l &= ~OMAP2_MCSPI_CHCONF_PHA;
 969
 970         mcspi_write_chconf0(spi, l);
 971
 972         cs->mode = spi->mode;
 973
 974         dev_dbg(&spi->dev, "setup: speed %d, sample %s edge, clk %s\n",
 975                         speed_hz,
 976                         (spi->mode & SPI_CPHA) ? "trailing" : "leading",
 977                         (spi->mode & SPI_CPOL) ? "inverted" : "normal");
 978
 979         return 0;
 980 }
 981
 982 /*
 983  * Note that we currently allow DMA only if we get a channel
 984  * for both rx and tx. Otherwise we'll do PIO for both rx and tx.
 985  */
 986 static int omap2_mcspi_request_dma(struct omap2_mcspi *mcspi,
 987                                    struct omap2_mcspi_dma *mcspi_dma)
 988 {
 989         int ret = 0;
 990
 991         mcspi_dma->dma_rx = dma_request_chan(mcspi->dev,
 992                                              mcspi_dma->dma_rx_ch_name);
 993         if (IS_ERR(mcspi_dma->dma_rx)) {
 994                 ret = PTR_ERR(mcspi_dma->dma_rx);
 995                 mcspi_dma->dma_rx = NULL;
 996                 goto no_dma;
 997         }
 998
 999         mcspi_dma->dma_tx = dma_request_chan(mcspi->dev,
1000                                              mcspi_dma->dma_tx_ch_name);
1001         if (IS_ERR(mcspi_dma->dma_tx)) {
1002                 ret = PTR_ERR(mcspi_dma->dma_tx);
1003                 mcspi_dma->dma_tx = NULL;
1004                 dma_release_channel(mcspi_dma->dma_rx);
1005                 mcspi_dma->dma_rx = NULL;
1006         }
1007
1008         init_completion(&mcspi_dma->dma_rx_completion);
1009         init_completion(&mcspi_dma->dma_tx_completion);
1010
1011 no_dma:
1012         return ret;
1013 }
1014
1015 static void omap2_mcspi_release_dma(struct spi_master *master)
1016 {
1017         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1018         struct omap2_mcspi_dma  *mcspi_dma;
1019         int i;
1020
1021         for (i = 0; i < master->num_chipselect; i++) {
1022                 mcspi_dma = &mcspi->dma_channels[i];
1023
1024                 if (mcspi_dma->dma_rx) {
1025                         dma_release_channel(mcspi_dma->dma_rx);
1026                         mcspi_dma->dma_rx = NULL;
1027                 }
1028                 if (mcspi_dma->dma_tx) {
1029                         dma_release_channel(mcspi_dma->dma_tx);
1030                         mcspi_dma->dma_tx = NULL;
1031                 }
1032         }
1033 }
1034
1035 static int omap2_mcspi_setup(struct spi_device *spi)
1036 {
1037         int                     ret;
1038         struct omap2_mcspi      *mcspi = spi_master_get_devdata(spi->master);
1039         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1040         struct omap2_mcspi_cs   *cs = spi->controller_state;
1041
1042         if (!cs) {
1043                 cs = kzalloc(sizeof *cs, GFP_KERNEL);
1044                 if (!cs)
1045                         return -ENOMEM;
1046                 cs->base = mcspi->base + spi->chip_select * 0x14;
1047                 cs->phys = mcspi->phys + spi->chip_select * 0x14;
1048                 cs->mode = 0;
1049                 cs->chconf0 = 0;
1050                 cs->chctrl0 = 0;
1051                 spi->controller_state = cs;
1052                 /* Link this to context save list */
1053                 list_add_tail(&cs->node, &ctx->cs);
1054         }
1055
1056         ret = pm_runtime_get_sync(mcspi->dev);
1057         if (ret < 0) {
1058                 pm_runtime_put_noidle(mcspi->dev);
1059
1060                 return ret;
1061         }
1062
1063         ret = omap2_mcspi_setup_transfer(spi, NULL);
1064         pm_runtime_mark_last_busy(mcspi->dev);
1065         pm_runtime_put_autosuspend(mcspi->dev);
1066
1067         return ret;
1068 }
1069
1070 static void omap2_mcspi_cleanup(struct spi_device *spi)
1071 {
1072         struct omap2_mcspi_cs   *cs;
1073
1074         if (spi->controller_state) {
1075                 /* Unlink controller state from context save list */
1076                 cs = spi->controller_state;
1077                 list_del(&cs->node);
1078
1079                 kfree(cs);
1080         }
1081 }
1082
1083 static irqreturn_t omap2_mcspi_irq_handler(int irq, void *data)
1084 {
1085         struct omap2_mcspi *mcspi = data;
1086         u32 irqstat;
1087
1088         irqstat = mcspi_read_reg(mcspi->master, OMAP2_MCSPI_IRQSTATUS);
1089         if (!irqstat)
1090                 return IRQ_NONE;
1091
1092         /* Disable IRQ and wakeup slave xfer task */
1093         mcspi_write_reg(mcspi->master, OMAP2_MCSPI_IRQENABLE, 0);
1094         if (irqstat & OMAP2_MCSPI_IRQSTATUS_EOW)
1095                 complete(&mcspi->txdone);
1096
1097         return IRQ_HANDLED;
1098 }
1099
1100 static int omap2_mcspi_slave_abort(struct spi_master *master)
1101 {
1102         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1103         struct omap2_mcspi_dma *mcspi_dma = mcspi->dma_channels;
1104
1105         mcspi->slave_aborted = true;
1106         complete(&mcspi_dma->dma_rx_completion);
1107         complete(&mcspi_dma->dma_tx_completion);
1108         complete(&mcspi->txdone);
1109
1110         return 0;
1111 }
1112
1113 static int omap2_mcspi_transfer_one(struct spi_master *master,
1114                                     struct spi_device *spi,
1115                                     struct spi_transfer *t)
1116 {
1117
1118         /* We only enable one channel at a time -- the one whose message is
1119          * -- although this controller would gladly
1120          * arbitrate among multiple channels.  This corresponds to "single
1121          * channel" master mode.  As a side effect, we need to manage the
1122          * chipselect with the FORCE bit ... CS != channel enable.
1123          */
1124
1125         struct omap2_mcspi              *mcspi;
1126         struct omap2_mcspi_dma          *mcspi_dma;
1127         struct omap2_mcspi_cs           *cs;
1128         struct omap2_mcspi_device_config *cd;
1129         int                             par_override = 0;
1130         int                             status = 0;
1131         u32                             chconf;
1132
1133         mcspi = spi_master_get_devdata(master);
1134         mcspi_dma = mcspi->dma_channels + spi->chip_select;
1135         cs = spi->controller_state;
1136         cd = spi->controller_data;
1137
1138         /*
1139          * The slave driver could have changed spi->mode in which case
1140          * it will be different from cs->mode (the current hardware setup).
1141          * If so, set par_override (even though its not a parity issue) so
1142          * omap2_mcspi_setup_transfer will be called to configure the hardware
1143          * with the correct mode on the first iteration of the loop below.
1144          */
1145         if (spi->mode != cs->mode)
1146                 par_override = 1;
1147
1148         omap2_mcspi_set_enable(spi, 0);
1149
1150         if (spi->cs_gpiod)
1151                 omap2_mcspi_set_cs(spi, spi->mode & SPI_CS_HIGH);
1152
1153         if (par_override ||
1154             (t->speed_hz != spi->max_speed_hz) ||
1155             (t->bits_per_word != spi->bits_per_word)) {
1156                 par_override = 1;
1157                 status = omap2_mcspi_setup_transfer(spi, t);
1158                 if (status < 0)
1159                         goto out;
1160                 if (t->speed_hz == spi->max_speed_hz &&
1161                     t->bits_per_word == spi->bits_per_word)
1162                         par_override = 0;
1163         }
1164         if (cd && cd->cs_per_word) {
1165                 chconf = mcspi->ctx.modulctrl;
1166                 chconf &= ~OMAP2_MCSPI_MODULCTRL_SINGLE;
1167                 mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, chconf);
1168                 mcspi->ctx.modulctrl =
1169                         mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1170         }
1171
1172         chconf = mcspi_cached_chconf0(spi);
1173         chconf &= ~OMAP2_MCSPI_CHCONF_TRM_MASK;
1174         chconf &= ~OMAP2_MCSPI_CHCONF_TURBO;
1175
1176         if (t->tx_buf == NULL)
1177                 chconf |= OMAP2_MCSPI_CHCONF_TRM_RX_ONLY;
1178         else if (t->rx_buf == NULL)
1179                 chconf |= OMAP2_MCSPI_CHCONF_TRM_TX_ONLY;
1180
1181         if (cd && cd->turbo_mode && t->tx_buf == NULL) {
1182                 /* Turbo mode is for more than one word */
1183                 if (t->len > ((cs->word_len + 7) >> 3))
1184                         chconf |= OMAP2_MCSPI_CHCONF_TURBO;
1185         }
1186
1187         mcspi_write_chconf0(spi, chconf);
1188
1189         if (t->len) {
1190                 unsigned        count;
1191
1192                 if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1193                     master->cur_msg_mapped &&
1194                     master->can_dma(master, spi, t))
1195                         omap2_mcspi_set_fifo(spi, t, 1);
1196
1197                 omap2_mcspi_set_enable(spi, 1);
1198
1199                 /* RX_ONLY mode needs dummy data in TX reg */
1200                 if (t->tx_buf == NULL)
1201                         writel_relaxed(0, cs->base
1202                                         + OMAP2_MCSPI_TX0);
1203
1204                 if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1205                     master->cur_msg_mapped &&
1206                     master->can_dma(master, spi, t))
1207                         count = omap2_mcspi_txrx_dma(spi, t);
1208                 else
1209                         count = omap2_mcspi_txrx_pio(spi, t);
1210
1211                 if (count != t->len) {
1212                         status = -EIO;
1213                         goto out;
1214                 }
1215         }
1216
1217         omap2_mcspi_set_enable(spi, 0);
1218
1219         if (mcspi->fifo_depth > 0)
1220                 omap2_mcspi_set_fifo(spi, t, 0);
1221
1222 out:
1223         /* Restore defaults if they were overriden */
1224         if (par_override) {
1225                 par_override = 0;
1226                 status = omap2_mcspi_setup_transfer(spi, NULL);
1227         }
1228
1229         if (cd && cd->cs_per_word) {
1230                 chconf = mcspi->ctx.modulctrl;
1231                 chconf |= OMAP2_MCSPI_MODULCTRL_SINGLE;
1232                 mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, chconf);
1233                 mcspi->ctx.modulctrl =
1234                         mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1235         }
1236
1237         omap2_mcspi_set_enable(spi, 0);
1238
1239         if (spi->cs_gpiod)
1240                 omap2_mcspi_set_cs(spi, !(spi->mode & SPI_CS_HIGH));
1241
1242         if (mcspi->fifo_depth > 0 && t)
1243                 omap2_mcspi_set_fifo(spi, t, 0);
1244
1245         return status;
1246 }
1247
1248 static int omap2_mcspi_prepare_message(struct spi_master *master,
1249                                        struct spi_message *msg)
1250 {
1251         struct omap2_mcspi      *mcspi = spi_master_get_devdata(master);
1252         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1253         struct omap2_mcspi_cs   *cs;
1254
1255         /* Only a single channel can have the FORCE bit enabled
1256          * in its chconf0 register.
1257          * Scan all channels and disable them except the current one.
1258          * A FORCE can remain from a last transfer having cs_change enabled
1259          */
1260         list_for_each_entry(cs, &ctx->cs, node) {
1261                 if (msg->spi->controller_state == cs)
1262                         continue;
1263
1264                 if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE)) {
1265                         cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1266                         writel_relaxed(cs->chconf0,
1267                                         cs->base + OMAP2_MCSPI_CHCONF0);
1268                         readl_relaxed(cs->base + OMAP2_MCSPI_CHCONF0);
1269                 }
1270         }
1271
1272         return 0;
1273 }
1274
1275 static bool omap2_mcspi_can_dma(struct spi_master *master,
1276                                 struct spi_device *spi,
1277                                 struct spi_transfer *xfer)
1278 {
1279         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
1280         struct omap2_mcspi_dma *mcspi_dma =
1281                 &mcspi->dma_channels[spi->chip_select];
1282
1283         if (!mcspi_dma->dma_rx || !mcspi_dma->dma_tx)
1284                 return false;
1285
1286         if (spi_controller_is_slave(master))
1287                 return true;
1288
1289         master->dma_rx = mcspi_dma->dma_rx;
1290         master->dma_tx = mcspi_dma->dma_tx;
1291
1292         return (xfer->len >= DMA_MIN_BYTES);
1293 }
1294
1295 static size_t omap2_mcspi_max_xfer_size(struct spi_device *spi)
1296 {
1297         struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
1298         struct omap2_mcspi_dma *mcspi_dma =
1299                 &mcspi->dma_channels[spi->chip_select];
1300
1301         if (mcspi->max_xfer_len && mcspi_dma->dma_rx)
1302                 return mcspi->max_xfer_len;
1303
1304         return SIZE_MAX;
1305 }
1306
1307 static int omap2_mcspi_controller_setup(struct omap2_mcspi *mcspi)
1308 {
1309         struct spi_master       *master = mcspi->master;
1310         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1311         int                     ret = 0;
1312
1313         ret = pm_runtime_get_sync(mcspi->dev);
1314         if (ret < 0) {
1315                 pm_runtime_put_noidle(mcspi->dev);
1316
1317                 return ret;
1318         }
1319
1320         mcspi_write_reg(master, OMAP2_MCSPI_WAKEUPENABLE,
1321                         OMAP2_MCSPI_WAKEUPENABLE_WKEN);
1322         ctx->wakeupenable = OMAP2_MCSPI_WAKEUPENABLE_WKEN;
1323
1324         omap2_mcspi_set_mode(master);
1325         pm_runtime_mark_last_busy(mcspi->dev);
1326         pm_runtime_put_autosuspend(mcspi->dev);
1327         return 0;
1328 }
1329
1330 static int omap_mcspi_runtime_suspend(struct device *dev)
1331 {
1332         int error;
1333
1334         error = pinctrl_pm_select_idle_state(dev);
1335         if (error)
1336                 dev_warn(dev, "%s: failed to set pins: %i\n", __func__, error);
1337
1338         return 0;
1339 }
1340
1341 /*
1342  * When SPI wake up from off-mode, CS is in activate state. If it was in
1343  * inactive state when driver was suspend, then force it to inactive state at
1344  * wake up.
1345  */
1346 static int omap_mcspi_runtime_resume(struct device *dev)
1347 {
1348         struct spi_master *master = dev_get_drvdata(dev);
1349         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1350         struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1351         struct omap2_mcspi_cs *cs;
1352         int error;
1353
1354         error = pinctrl_pm_select_default_state(dev);
1355         if (error)
1356                 dev_warn(dev, "%s: failed to set pins: %i\n", __func__, error);
1357
1358         /* McSPI: context restore */
1359         mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, ctx->modulctrl);
1360         mcspi_write_reg(master, OMAP2_MCSPI_WAKEUPENABLE, ctx->wakeupenable);
1361
1362         list_for_each_entry(cs, &ctx->cs, node) {
1363                 /*
1364                  * We need to toggle CS state for OMAP take this
1365                  * change in account.
1366                  */
1367                 if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE) == 0) {
1368                         cs->chconf0 |= OMAP2_MCSPI_CHCONF_FORCE;
1369                         writel_relaxed(cs->chconf0,
1370                                        cs->base + OMAP2_MCSPI_CHCONF0);
1371                         cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1372                         writel_relaxed(cs->chconf0,
1373                                        cs->base + OMAP2_MCSPI_CHCONF0);
1374                 } else {
1375                         writel_relaxed(cs->chconf0,
1376                                        cs->base + OMAP2_MCSPI_CHCONF0);
1377                 }
1378         }
1379
1380         return 0;
1381 }
1382
1383 static struct omap2_mcspi_platform_config omap2_pdata = {
1384         .regs_offset = 0,
1385 };
1386
1387 static struct omap2_mcspi_platform_config omap4_pdata = {
1388         .regs_offset = OMAP4_MCSPI_REG_OFFSET,
1389 };
1390
1391 static struct omap2_mcspi_platform_config am654_pdata = {
1392         .regs_offset = OMAP4_MCSPI_REG_OFFSET,
1393         .max_xfer_len = SZ_4K - 1,
1394 };
1395
1396 static const struct of_device_id omap_mcspi_of_match[] = {
1397         {
1398                 .compatible = "ti,omap2-mcspi",
1399                 .data = &omap2_pdata,
1400         },
1401         {
1402                 .compatible = "ti,omap4-mcspi",
1403                 .data = &omap4_pdata,
1404         },
1405         {
1406                 .compatible = "ti,am654-mcspi",
1407                 .data = &am654_pdata,
1408         },
1409         { },
1410 };
1411 MODULE_DEVICE_TABLE(of, omap_mcspi_of_match);
1412
1413 static int omap2_mcspi_probe(struct platform_device *pdev)
1414 {
1415         struct spi_master       *master;
1416         const struct omap2_mcspi_platform_config *pdata;
1417         struct omap2_mcspi      *mcspi;
1418         struct resource         *r;
1419         int                     status = 0, i;
1420         u32                     regs_offset = 0;
1421         struct device_node      *node = pdev->dev.of_node;
1422         const struct of_device_id *match;
1423
1424         if (of_property_read_bool(node, "spi-slave"))
1425                 master = spi_alloc_slave(&pdev->dev, sizeof(*mcspi));
1426         else
1427                 master = spi_alloc_master(&pdev->dev, sizeof(*mcspi));
1428         if (!master)
1429                 return -ENOMEM;
1430
1431         /* the spi->mode bits understood by this driver: */
1432         master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
1433         master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
1434         master->setup = omap2_mcspi_setup;
1435         master->auto_runtime_pm = true;
1436         master->prepare_message = omap2_mcspi_prepare_message;
1437         master->can_dma = omap2_mcspi_can_dma;
1438         master->transfer_one = omap2_mcspi_transfer_one;
1439         master->set_cs = omap2_mcspi_set_cs;
1440         master->cleanup = omap2_mcspi_cleanup;
1441         master->slave_abort = omap2_mcspi_slave_abort;
1442         master->dev.of_node = node;
1443         master->max_speed_hz = OMAP2_MCSPI_MAX_FREQ;
1444         master->min_speed_hz = OMAP2_MCSPI_MAX_FREQ >> 15;
1445         master->use_gpio_descriptors = true;
1446
1447         platform_set_drvdata(pdev, master);
1448
1449         mcspi = spi_master_get_devdata(master);
1450         mcspi->master = master;
1451
1452         match = of_match_device(omap_mcspi_of_match, &pdev->dev);
1453         if (match) {
1454                 u32 num_cs = 1; /* default number of chipselect */
1455                 pdata = match->data;
1456
1457                 of_property_read_u32(node, "ti,spi-num-cs", &num_cs);
1458                 master->num_chipselect = num_cs;
1459                 if (of_get_property(node, "ti,pindir-d0-out-d1-in", NULL))
1460                         mcspi->pin_dir = MCSPI_PINDIR_D0_OUT_D1_IN;
1461         } else {
1462                 pdata = dev_get_platdata(&pdev->dev);
1463                 master->num_chipselect = pdata->num_cs;
1464                 mcspi->pin_dir = pdata->pin_dir;
1465         }
1466         regs_offset = pdata->regs_offset;
1467         if (pdata->max_xfer_len) {
1468                 mcspi->max_xfer_len = pdata->max_xfer_len;
1469                 master->max_transfer_size = omap2_mcspi_max_xfer_size;
1470         }
1471
1472         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1473         mcspi->base = devm_ioremap_resource(&pdev->dev, r);
1474         if (IS_ERR(mcspi->base)) {
1475                 status = PTR_ERR(mcspi->base);
1476                 goto free_master;
1477         }
1478         mcspi->phys = r->start + regs_offset;
1479         mcspi->base += regs_offset;
1480
1481         mcspi->dev = &pdev->dev;
1482
1483         INIT_LIST_HEAD(&mcspi->ctx.cs);
1484
1485         mcspi->dma_channels = devm_kcalloc(&pdev->dev, master->num_chipselect,
1486                                            sizeof(struct omap2_mcspi_dma),
1487                                            GFP_KERNEL);
1488         if (mcspi->dma_channels == NULL) {
1489                 status = -ENOMEM;
1490                 goto free_master;
1491         }
1492
1493         for (i = 0; i < master->num_chipselect; i++) {
1494                 sprintf(mcspi->dma_channels[i].dma_rx_ch_name, "rx%d", i);
1495                 sprintf(mcspi->dma_channels[i].dma_tx_ch_name, "tx%d", i);
1496
1497                 status = omap2_mcspi_request_dma(mcspi,
1498                                                  &mcspi->dma_channels[i]);
1499                 if (status == -EPROBE_DEFER)
1500                         goto free_master;
1501         }
1502
1503         status = platform_get_irq(pdev, 0);
1504         if (status == -EPROBE_DEFER)
1505                 goto free_master;
1506         if (status < 0) {
1507                 dev_err(&pdev->dev, "no irq resource found\n");
1508                 goto free_master;
1509         }
1510         init_completion(&mcspi->txdone);
1511         status = devm_request_irq(&pdev->dev, status,
1512                                   omap2_mcspi_irq_handler, 0, pdev->name,
1513                                   mcspi);
1514         if (status) {
1515                 dev_err(&pdev->dev, "Cannot request IRQ");
1516                 goto free_master;
1517         }
1518
1519         pm_runtime_use_autosuspend(&pdev->dev);
1520         pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT);
1521         pm_runtime_enable(&pdev->dev);
1522
1523         status = omap2_mcspi_controller_setup(mcspi);
1524         if (status < 0)
1525                 goto disable_pm;
1526
1527         status = devm_spi_register_controller(&pdev->dev, master);
1528         if (status < 0)
1529                 goto disable_pm;
1530
1531         return status;
1532
1533 disable_pm:
1534         pm_runtime_dont_use_autosuspend(&pdev->dev);
1535         pm_runtime_put_sync(&pdev->dev);
1536         pm_runtime_disable(&pdev->dev);
1537 free_master:
1538         omap2_mcspi_release_dma(master);
1539         spi_master_put(master);
1540         return status;
1541 }
1542
1543 static int omap2_mcspi_remove(struct platform_device *pdev)
1544 {
1545         struct spi_master *master = platform_get_drvdata(pdev);
1546         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1547
1548         omap2_mcspi_release_dma(master);
1549
1550         pm_runtime_dont_use_autosuspend(mcspi->dev);
1551         pm_runtime_put_sync(mcspi->dev);
1552         pm_runtime_disable(&pdev->dev);
1553
1554         return 0;
1555 }
1556
1557 /* work with hotplug and coldplug */
1558 MODULE_ALIAS("platform:omap2_mcspi");
1559
1560 static int __maybe_unused omap2_mcspi_suspend(struct device *dev)
1561 {
1562         struct spi_master *master = dev_get_drvdata(dev);
1563         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1564         int error;
1565
1566         error = pinctrl_pm_select_sleep_state(dev);
1567         if (error)
1568                 dev_warn(mcspi->dev, "%s: failed to set pins: %i\n",
1569                          __func__, error);
1570
1571         error = spi_master_suspend(master);
1572         if (error)
1573                 dev_warn(mcspi->dev, "%s: master suspend failed: %i\n",
1574                          __func__, error);
1575
1576         return pm_runtime_force_suspend(dev);
1577 }
1578
1579 static int __maybe_unused omap2_mcspi_resume(struct device *dev)
1580 {
1581         struct spi_master *master = dev_get_drvdata(dev);
1582         struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
1583         int error;
1584
1585         error = spi_master_resume(master);
1586         if (error)
1587                 dev_warn(mcspi->dev, "%s: master resume failed: %i\n",
1588                          __func__, error);
1589
1590         return pm_runtime_force_resume(dev);
1591 }
1592
1593 static const struct dev_pm_ops omap2_mcspi_pm_ops = {
1594         SET_SYSTEM_SLEEP_PM_OPS(omap2_mcspi_suspend,
1595                                 omap2_mcspi_resume)
1596         .runtime_suspend        = omap_mcspi_runtime_suspend,
1597         .runtime_resume         = omap_mcspi_runtime_resume,
1598 };
1599
1600 static struct platform_driver omap2_mcspi_driver = {
1601         .driver = {
1602                 .name =         "omap2_mcspi",
1603                 .pm =           &omap2_mcspi_pm_ops,
1604                 .of_match_table = omap_mcspi_of_match,
1605         },
1606         .probe =        omap2_mcspi_probe,
1607         .remove =       omap2_mcspi_remove,
1608 };
1609
1610 module_platform_driver(omap2_mcspi_driver);
1611 MODULE_LICENSE("GPL");