drivers/mtd/nand/raw/sunxi_nand.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
   4  *
   5  * Derived from:
   6  *      https://github.com/yuq/sunxi-nfc-mtd
   7  *      Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
   8  *
   9  *      https://github.com/hno/Allwinner-Info
  10  *      Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  11  *
  12  *      Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
  13  *      Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
  14  */
  15
  16 #include <linux/dma-mapping.h>
  17 #include <linux/slab.h>
  18 #include <linux/module.h>
  19 #include <linux/moduleparam.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/of.h>
  22 #include <linux/of_device.h>
  23 #include <linux/mtd/mtd.h>
  24 #include <linux/mtd/rawnand.h>
  25 #include <linux/mtd/partitions.h>
  26 #include <linux/clk.h>
  27 #include <linux/delay.h>
  28 #include <linux/dmaengine.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/iopoll.h>
  31 #include <linux/reset.h>
  32
  33 #define NFC_REG_CTL             0x0000
  34 #define NFC_REG_ST              0x0004
  35 #define NFC_REG_INT             0x0008
  36 #define NFC_REG_TIMING_CTL      0x000C
  37 #define NFC_REG_TIMING_CFG      0x0010
  38 #define NFC_REG_ADDR_LOW        0x0014
  39 #define NFC_REG_ADDR_HIGH       0x0018
  40 #define NFC_REG_SECTOR_NUM      0x001C
  41 #define NFC_REG_CNT             0x0020
  42 #define NFC_REG_CMD             0x0024
  43 #define NFC_REG_RCMD_SET        0x0028
  44 #define NFC_REG_WCMD_SET        0x002C
  45 #define NFC_REG_A10_IO_DATA     0x0030
  46 #define NFC_REG_A23_IO_DATA     0x0300
  47 #define NFC_REG_ECC_CTL         0x0034
  48 #define NFC_REG_ECC_ST          0x0038
  49 #define NFC_REG_DEBUG           0x003C
  50 #define NFC_REG_ECC_ERR_CNT(x)  ((0x0040 + (x)) & ~0x3)
  51 #define NFC_REG_USER_DATA(x)    (0x0050 + ((x) * 4))
  52 #define NFC_REG_SPARE_AREA      0x00A0
  53 #define NFC_REG_PAT_ID          0x00A4
  54 #define NFC_REG_MDMA_ADDR       0x00C0
  55 #define NFC_REG_MDMA_CNT        0x00C4
  56 #define NFC_RAM0_BASE           0x0400
  57 #define NFC_RAM1_BASE           0x0800
  58
  59 /* define bit use in NFC_CTL */
  60 #define NFC_EN                  BIT(0)
  61 #define NFC_RESET               BIT(1)
  62 #define NFC_BUS_WIDTH_MSK       BIT(2)
  63 #define NFC_BUS_WIDTH_8         (0 << 2)
  64 #define NFC_BUS_WIDTH_16        (1 << 2)
  65 #define NFC_RB_SEL_MSK          BIT(3)
  66 #define NFC_RB_SEL(x)           ((x) << 3)
  67 #define NFC_CE_SEL_MSK          GENMASK(26, 24)
  68 #define NFC_CE_SEL(x)           ((x) << 24)
  69 #define NFC_CE_CTL              BIT(6)
  70 #define NFC_PAGE_SHIFT_MSK      GENMASK(11, 8)
  71 #define NFC_PAGE_SHIFT(x)       (((x) < 10 ? 0 : (x) - 10) << 8)
  72 #define NFC_SAM                 BIT(12)
  73 #define NFC_RAM_METHOD          BIT(14)
  74 #define NFC_DMA_TYPE_NORMAL     BIT(15)
  75 #define NFC_DEBUG_CTL           BIT(31)
  76
  77 /* define bit use in NFC_ST */
  78 #define NFC_RB_B2R              BIT(0)
  79 #define NFC_CMD_INT_FLAG        BIT(1)
  80 #define NFC_DMA_INT_FLAG        BIT(2)
  81 #define NFC_CMD_FIFO_STATUS     BIT(3)
  82 #define NFC_STA                 BIT(4)
  83 #define NFC_NATCH_INT_FLAG      BIT(5)
  84 #define NFC_RB_STATE(x)         BIT(x + 8)
  85
  86 /* define bit use in NFC_INT */
  87 #define NFC_B2R_INT_ENABLE      BIT(0)
  88 #define NFC_CMD_INT_ENABLE      BIT(1)
  89 #define NFC_DMA_INT_ENABLE      BIT(2)
  90 #define NFC_INT_MASK            (NFC_B2R_INT_ENABLE | \
  91                                  NFC_CMD_INT_ENABLE | \
  92                                  NFC_DMA_INT_ENABLE)
  93
  94 /* define bit use in NFC_TIMING_CTL */
  95 #define NFC_TIMING_CTL_EDO      BIT(8)
  96
  97 /* define NFC_TIMING_CFG register layout */
  98 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)             \
  99         (((tWB) & 0x3) | (((tADL) & 0x3) << 2) |                \
 100         (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |         \
 101         (((tCAD) & 0x7) << 8))
 102
 103 /* define bit use in NFC_CMD */
 104 #define NFC_CMD_LOW_BYTE_MSK    GENMASK(7, 0)
 105 #define NFC_CMD_HIGH_BYTE_MSK   GENMASK(15, 8)
 106 #define NFC_CMD(x)              (x)
 107 #define NFC_ADR_NUM_MSK         GENMASK(18, 16)
 108 #define NFC_ADR_NUM(x)          (((x) - 1) << 16)
 109 #define NFC_SEND_ADR            BIT(19)
 110 #define NFC_ACCESS_DIR          BIT(20)
 111 #define NFC_DATA_TRANS          BIT(21)
 112 #define NFC_SEND_CMD1           BIT(22)
 113 #define NFC_WAIT_FLAG           BIT(23)
 114 #define NFC_SEND_CMD2           BIT(24)
 115 #define NFC_SEQ                 BIT(25)
 116 #define NFC_DATA_SWAP_METHOD    BIT(26)
 117 #define NFC_ROW_AUTO_INC        BIT(27)
 118 #define NFC_SEND_CMD3           BIT(28)
 119 #define NFC_SEND_CMD4           BIT(29)
 120 #define NFC_CMD_TYPE_MSK        GENMASK(31, 30)
 121 #define NFC_NORMAL_OP           (0 << 30)
 122 #define NFC_ECC_OP              (1 << 30)
 123 #define NFC_PAGE_OP             (2U << 30)
 124
 125 /* define bit use in NFC_RCMD_SET */
 126 #define NFC_READ_CMD_MSK        GENMASK(7, 0)
 127 #define NFC_RND_READ_CMD0_MSK   GENMASK(15, 8)
 128 #define NFC_RND_READ_CMD1_MSK   GENMASK(23, 16)
 129
 130 /* define bit use in NFC_WCMD_SET */
 131 #define NFC_PROGRAM_CMD_MSK     GENMASK(7, 0)
 132 #define NFC_RND_WRITE_CMD_MSK   GENMASK(15, 8)
 133 #define NFC_READ_CMD0_MSK       GENMASK(23, 16)
 134 #define NFC_READ_CMD1_MSK       GENMASK(31, 24)
 135
 136 /* define bit use in NFC_ECC_CTL */
 137 #define NFC_ECC_EN              BIT(0)
 138 #define NFC_ECC_PIPELINE        BIT(3)
 139 #define NFC_ECC_EXCEPTION       BIT(4)
 140 #define NFC_ECC_BLOCK_SIZE_MSK  BIT(5)
 141 #define NFC_ECC_BLOCK_512       BIT(5)
 142 #define NFC_RANDOM_EN           BIT(9)
 143 #define NFC_RANDOM_DIRECTION    BIT(10)
 144 #define NFC_ECC_MODE_MSK        GENMASK(15, 12)
 145 #define NFC_ECC_MODE(x)         ((x) << 12)
 146 #define NFC_RANDOM_SEED_MSK     GENMASK(30, 16)
 147 #define NFC_RANDOM_SEED(x)      ((x) << 16)
 148
 149 /* define bit use in NFC_ECC_ST */
 150 #define NFC_ECC_ERR(x)          BIT(x)
 151 #define NFC_ECC_ERR_MSK         GENMASK(15, 0)
 152 #define NFC_ECC_PAT_FOUND(x)    BIT(x + 16)
 153 #define NFC_ECC_ERR_CNT(b, x)   (((x) >> (((b) % 4) * 8)) & 0xff)
 154
 155 #define NFC_DEFAULT_TIMEOUT_MS  1000
 156
 157 #define NFC_SRAM_SIZE           1024
 158
 159 #define NFC_MAX_CS              7
 160
 161 /**
 162  * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select
 163  *
 164  * @cs: the NAND CS id used to communicate with a NAND Chip
 165  * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC
 166  */
 167 struct sunxi_nand_chip_sel {
 168         u8 cs;
 169         s8 rb;
 170 };
 171
 172 /**
 173  * struct sunxi_nand_hw_ecc - stores information related to HW ECC support
 174  *
 175  * @mode: the sunxi ECC mode field deduced from ECC requirements
 176  */
 177 struct sunxi_nand_hw_ecc {
 178         int mode;
 179 };
 180
 181 /**
 182  * struct sunxi_nand_chip - stores NAND chip device related information
 183  *
 184  * @node: used to store NAND chips into a list
 185  * @nand: base NAND chip structure
 186  * @ecc: ECC controller structure
 187  * @clk_rate: clk_rate required for this NAND chip
 188  * @timing_cfg: TIMING_CFG register value for this NAND chip
 189  * @timing_ctl: TIMING_CTL register value for this NAND chip
 190  * @nsels: number of CS lines required by the NAND chip
 191  * @sels: array of CS lines descriptions
 192  */
 193 struct sunxi_nand_chip {
 194         struct list_head node;
 195         struct nand_chip nand;
 196         struct sunxi_nand_hw_ecc *ecc;
 197         unsigned long clk_rate;
 198         u32 timing_cfg;
 199         u32 timing_ctl;
 200         int nsels;
 201         struct sunxi_nand_chip_sel sels[];
 202 };
 203
 204 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 205 {
 206         return container_of(nand, struct sunxi_nand_chip, nand);
 207 }
 208
 209 /*
 210  * NAND Controller capabilities structure: stores NAND controller capabilities
 211  * for distinction between compatible strings.
 212  *
 213  * @has_mdma:           Use mbus dma mode, otherwise general dma
 214  *                      through MBUS on A23/A33 needs extra configuration.
 215  * @reg_io_data:        I/O data register
 216  * @dma_maxburst:       DMA maxburst
 217  */
 218 struct sunxi_nfc_caps {
 219         bool has_mdma;
 220         unsigned int reg_io_data;
 221         unsigned int dma_maxburst;
 222 };
 223
 224 /**
 225  * struct sunxi_nfc - stores sunxi NAND controller information
 226  *
 227  * @controller: base controller structure
 228  * @dev: parent device (used to print error messages)
 229  * @regs: NAND controller registers
 230  * @ahb_clk: NAND controller AHB clock
 231  * @mod_clk: NAND controller mod clock
 232  * @reset: NAND controller reset line
 233  * @assigned_cs: bitmask describing already assigned CS lines
 234  * @clk_rate: NAND controller current clock rate
 235  * @chips: a list containing all the NAND chips attached to this NAND
 236  *         controller
 237  * @complete: a completion object used to wait for NAND controller events
 238  * @dmac: the DMA channel attached to the NAND controller
 239  * @caps: NAND Controller capabilities
 240  */
 241 struct sunxi_nfc {
 242         struct nand_controller controller;
 243         struct device *dev;
 244         void __iomem *regs;
 245         struct clk *ahb_clk;
 246         struct clk *mod_clk;
 247         struct reset_control *reset;
 248         unsigned long assigned_cs;
 249         unsigned long clk_rate;
 250         struct list_head chips;
 251         struct completion complete;
 252         struct dma_chan *dmac;
 253         const struct sunxi_nfc_caps *caps;
 254 };
 255
 256 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl)
 257 {
 258         return container_of(ctrl, struct sunxi_nfc, controller);
 259 }
 260
 261 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 262 {
 263         struct sunxi_nfc *nfc = dev_id;
 264         u32 st = readl(nfc->regs + NFC_REG_ST);
 265         u32 ien = readl(nfc->regs + NFC_REG_INT);
 266
 267         if (!(ien & st))
 268                 return IRQ_NONE;
 269
 270         if ((ien & st) == ien)
 271                 complete(&nfc->complete);
 272
 273         writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 274         writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 275
 276         return IRQ_HANDLED;
 277 }
 278
 279 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 280                                  bool use_polling, unsigned int timeout_ms)
 281 {
 282         int ret;
 283
 284         if (events & ~NFC_INT_MASK)
 285                 return -EINVAL;
 286
 287         if (!timeout_ms)
 288                 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 289
 290         if (!use_polling) {
 291                 init_completion(&nfc->complete);
 292
 293                 writel(events, nfc->regs + NFC_REG_INT);
 294
 295                 ret = wait_for_completion_timeout(&nfc->complete,
 296                                                 msecs_to_jiffies(timeout_ms));
 297                 if (!ret)
 298                         ret = -ETIMEDOUT;
 299                 else
 300                         ret = 0;
 301
 302                 writel(0, nfc->regs + NFC_REG_INT);
 303         } else {
 304                 u32 status;
 305
 306                 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 307                                          (status & events) == events, 1,
 308                                          timeout_ms * 1000);
 309         }
 310
 311         writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 312
 313         if (ret)
 314                 dev_err(nfc->dev, "wait interrupt timedout\n");
 315
 316         return ret;
 317 }
 318
 319 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 320 {
 321         u32 status;
 322         int ret;
 323
 324         ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 325                                  !(status & NFC_CMD_FIFO_STATUS), 1,
 326                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 327         if (ret)
 328                 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 329
 330         return ret;
 331 }
 332
 333 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 334 {
 335         u32 ctl;
 336         int ret;
 337
 338         writel(0, nfc->regs + NFC_REG_ECC_CTL);
 339         writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 340
 341         ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 342                                  !(ctl & NFC_RESET), 1,
 343                                  NFC_DEFAULT_TIMEOUT_MS * 1000);
 344         if (ret)
 345                 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 346
 347         return ret;
 348 }
 349
 350 static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf,
 351                                     int chunksize, int nchunks,
 352                                     enum dma_data_direction ddir,
 353                                     struct scatterlist *sg)
 354 {
 355         struct dma_async_tx_descriptor *dmad;
 356         enum dma_transfer_direction tdir;
 357         dma_cookie_t dmat;
 358         int ret;
 359
 360         if (ddir == DMA_FROM_DEVICE)
 361                 tdir = DMA_DEV_TO_MEM;
 362         else
 363                 tdir = DMA_MEM_TO_DEV;
 364
 365         sg_init_one(sg, buf, nchunks * chunksize);
 366         ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 367         if (!ret)
 368                 return -ENOMEM;
 369
 370         if (!nfc->caps->has_mdma) {
 371                 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 372                 if (!dmad) {
 373                         ret = -EINVAL;
 374                         goto err_unmap_buf;
 375                 }
 376         }
 377
 378         writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 379                nfc->regs + NFC_REG_CTL);
 380         writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 381         writel(chunksize, nfc->regs + NFC_REG_CNT);
 382
 383         if (nfc->caps->has_mdma) {
 384                 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_DMA_TYPE_NORMAL,
 385                        nfc->regs + NFC_REG_CTL);
 386                 writel(chunksize * nchunks, nfc->regs + NFC_REG_MDMA_CNT);
 387                 writel(sg_dma_address(sg), nfc->regs + NFC_REG_MDMA_ADDR);
 388         } else {
 389                 dmat = dmaengine_submit(dmad);
 390
 391                 ret = dma_submit_error(dmat);
 392                 if (ret)
 393                         goto err_clr_dma_flag;
 394         }
 395
 396         return 0;
 397
 398 err_clr_dma_flag:
 399         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 400                nfc->regs + NFC_REG_CTL);
 401
 402 err_unmap_buf:
 403         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 404         return ret;
 405 }
 406
 407 static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc,
 408                                      enum dma_data_direction ddir,
 409                                      struct scatterlist *sg)
 410 {
 411         dma_unmap_sg(nfc->dev, sg, 1, ddir);
 412         writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 413                nfc->regs + NFC_REG_CTL);
 414 }
 415
 416 static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs)
 417 {
 418         struct mtd_info *mtd = nand_to_mtd(nand);
 419         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 420         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 421         struct sunxi_nand_chip_sel *sel;
 422         u32 ctl;
 423
 424         if (cs > 0 && cs >= sunxi_nand->nsels)
 425                 return;
 426
 427         ctl = readl(nfc->regs + NFC_REG_CTL) &
 428               ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 429
 430         sel = &sunxi_nand->sels[cs];
 431         ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift);
 432         if (sel->rb >= 0)
 433                 ctl |= NFC_RB_SEL(sel->rb);
 434
 435         writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 436
 437         if (nfc->clk_rate != sunxi_nand->clk_rate) {
 438                 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 439                 nfc->clk_rate = sunxi_nand->clk_rate;
 440         }
 441
 442         writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 443         writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 444         writel(ctl, nfc->regs + NFC_REG_CTL);
 445 }
 446
 447 static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len)
 448 {
 449         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 450         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 451         int ret;
 452         int cnt;
 453         int offs = 0;
 454         u32 tmp;
 455
 456         while (len > offs) {
 457                 bool poll = false;
 458
 459                 cnt = min(len - offs, NFC_SRAM_SIZE);
 460
 461                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 462                 if (ret)
 463                         break;
 464
 465                 writel(cnt, nfc->regs + NFC_REG_CNT);
 466                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 467                 writel(tmp, nfc->regs + NFC_REG_CMD);
 468
 469                 /* Arbitrary limit for polling mode */
 470                 if (cnt < 64)
 471                         poll = true;
 472
 473                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 474                 if (ret)
 475                         break;
 476
 477                 if (buf)
 478                         memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 479                                       cnt);
 480                 offs += cnt;
 481         }
 482 }
 483
 484 static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf,
 485                                 int len)
 486 {
 487         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 488         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 489         int ret;
 490         int cnt;
 491         int offs = 0;
 492         u32 tmp;
 493
 494         while (len > offs) {
 495                 bool poll = false;
 496
 497                 cnt = min(len - offs, NFC_SRAM_SIZE);
 498
 499                 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 500                 if (ret)
 501                         break;
 502
 503                 writel(cnt, nfc->regs + NFC_REG_CNT);
 504                 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 505                 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 506                       NFC_ACCESS_DIR;
 507                 writel(tmp, nfc->regs + NFC_REG_CMD);
 508
 509                 /* Arbitrary limit for polling mode */
 510                 if (cnt < 64)
 511                         poll = true;
 512
 513                 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 514                 if (ret)
 515                         break;
 516
 517                 offs += cnt;
 518         }
 519 }
 520
 521 /* These seed values have been extracted from Allwinner's BSP */
 522 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 523         0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 524         0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 525         0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 526         0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 527         0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 528         0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 529         0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 530         0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 531         0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 532         0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 533         0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 534         0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 535         0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 536         0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 537         0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 538         0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 539 };
 540
 541 /*
 542  * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 543  * have been generated using
 544  * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 545  * the randomizer engine does internally before de/scrambling OOB data.
 546  *
 547  * Those tables are statically defined to avoid calculating randomizer state
 548  * at runtime.
 549  */
 550 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 551         0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 552         0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 553         0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 554         0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 555         0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 556         0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 557         0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 558         0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 559         0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 560         0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 561         0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 562         0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 563         0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 564         0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 565         0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 566         0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 567 };
 568
 569 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 570         0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 571         0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 572         0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 573         0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 574         0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 575         0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 576         0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 577         0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 578         0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 579         0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 580         0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 581         0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 582         0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 583         0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 584         0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 585         0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 586 };
 587
 588 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 589 {
 590         state &= 0x7fff;
 591
 592         /*
 593          * This loop is just a simple implementation of a Fibonacci LFSR using
 594          * the x16 + x15 + 1 polynomial.
 595          */
 596         while (count--)
 597                 state = ((state >> 1) |
 598                          (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 599
 600         return state;
 601 }
 602
 603 static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page,
 604                                       bool ecc)
 605 {
 606         struct mtd_info *mtd = nand_to_mtd(nand);
 607         const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 608         int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 609
 610         if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 611                 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 612
 613         if (ecc) {
 614                 if (mtd->ecc_step_size == 512)
 615                         seeds = sunxi_nfc_randomizer_ecc512_seeds;
 616                 else
 617                         seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 618         }
 619
 620         return seeds[page % mod];
 621 }
 622
 623 static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page,
 624                                         bool ecc)
 625 {
 626         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 627         u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 628         u16 state;
 629
 630         if (!(nand->options & NAND_NEED_SCRAMBLING))
 631                 return;
 632
 633         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 634         state = sunxi_nfc_randomizer_state(nand, page, ecc);
 635         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 636         writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 637 }
 638
 639 static void sunxi_nfc_randomizer_enable(struct nand_chip *nand)
 640 {
 641         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 642
 643         if (!(nand->options & NAND_NEED_SCRAMBLING))
 644                 return;
 645
 646         writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 647                nfc->regs + NFC_REG_ECC_CTL);
 648 }
 649
 650 static void sunxi_nfc_randomizer_disable(struct nand_chip *nand)
 651 {
 652         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 653
 654         if (!(nand->options & NAND_NEED_SCRAMBLING))
 655                 return;
 656
 657         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 658                nfc->regs + NFC_REG_ECC_CTL);
 659 }
 660
 661 static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm)
 662 {
 663         u16 state = sunxi_nfc_randomizer_state(nand, page, true);
 664
 665         bbm[0] ^= state;
 666         bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 667 }
 668
 669 static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand,
 670                                            const uint8_t *buf, int len,
 671                                            bool ecc, int page)
 672 {
 673         sunxi_nfc_randomizer_config(nand, page, ecc);
 674         sunxi_nfc_randomizer_enable(nand);
 675         sunxi_nfc_write_buf(nand, buf, len);
 676         sunxi_nfc_randomizer_disable(nand);
 677 }
 678
 679 static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf,
 680                                           int len, bool ecc, int page)
 681 {
 682         sunxi_nfc_randomizer_config(nand, page, ecc);
 683         sunxi_nfc_randomizer_enable(nand);
 684         sunxi_nfc_read_buf(nand, buf, len);
 685         sunxi_nfc_randomizer_disable(nand);
 686 }
 687
 688 static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand)
 689 {
 690         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 691         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 692         u32 ecc_ctl;
 693
 694         ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 695         ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
 696                      NFC_ECC_BLOCK_SIZE_MSK);
 697         ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(sunxi_nand->ecc->mode) |
 698                    NFC_ECC_EXCEPTION | NFC_ECC_PIPELINE;
 699
 700         if (nand->ecc.size == 512)
 701                 ecc_ctl |= NFC_ECC_BLOCK_512;
 702
 703         writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 704 }
 705
 706 static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand)
 707 {
 708         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 709
 710         writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
 711                nfc->regs + NFC_REG_ECC_CTL);
 712 }
 713
 714 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 715 {
 716         buf[0] = user_data;
 717         buf[1] = user_data >> 8;
 718         buf[2] = user_data >> 16;
 719         buf[3] = user_data >> 24;
 720 }
 721
 722 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 723 {
 724         return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 725 }
 726
 727 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob,
 728                                                 int step, bool bbm, int page)
 729 {
 730         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 731
 732         sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 733                                    oob);
 734
 735         /* De-randomize the Bad Block Marker. */
 736         if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 737                 sunxi_nfc_randomize_bbm(nand, page, oob);
 738 }
 739
 740 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,
 741                                                 const u8 *oob, int step,
 742                                                 bool bbm, int page)
 743 {
 744         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 745         u8 user_data[4];
 746
 747         /* Randomize the Bad Block Marker. */
 748         if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 749                 memcpy(user_data, oob, sizeof(user_data));
 750                 sunxi_nfc_randomize_bbm(nand, page, user_data);
 751                 oob = user_data;
 752         }
 753
 754         writel(sunxi_nfc_buf_to_user_data(oob),
 755                nfc->regs + NFC_REG_USER_DATA(step));
 756 }
 757
 758 static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand,
 759                                           unsigned int *max_bitflips, int ret)
 760 {
 761         struct mtd_info *mtd = nand_to_mtd(nand);
 762
 763         if (ret < 0) {
 764                 mtd->ecc_stats.failed++;
 765         } else {
 766                 mtd->ecc_stats.corrected += ret;
 767                 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 768         }
 769 }
 770
 771 static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob,
 772                                     int step, u32 status, bool *erased)
 773 {
 774         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 775         struct nand_ecc_ctrl *ecc = &nand->ecc;
 776         u32 tmp;
 777
 778         *erased = false;
 779
 780         if (status & NFC_ECC_ERR(step))
 781                 return -EBADMSG;
 782
 783         if (status & NFC_ECC_PAT_FOUND(step)) {
 784                 u8 pattern;
 785
 786                 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 787                         pattern = 0x0;
 788                 } else {
 789                         pattern = 0xff;
 790                         *erased = true;
 791                 }
 792
 793                 if (data)
 794                         memset(data, pattern, ecc->size);
 795
 796                 if (oob)
 797                         memset(oob, pattern, ecc->bytes + 4);
 798
 799                 return 0;
 800         }
 801
 802         tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 803
 804         return NFC_ECC_ERR_CNT(step, tmp);
 805 }
 806
 807 static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand,
 808                                        u8 *data, int data_off,
 809                                        u8 *oob, int oob_off,
 810                                        int *cur_off,
 811                                        unsigned int *max_bitflips,
 812                                        bool bbm, bool oob_required, int page)
 813 {
 814         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 815         struct nand_ecc_ctrl *ecc = &nand->ecc;
 816         int raw_mode = 0;
 817         bool erased;
 818         int ret;
 819
 820         if (*cur_off != data_off)
 821                 nand_change_read_column_op(nand, data_off, NULL, 0, false);
 822
 823         sunxi_nfc_randomizer_read_buf(nand, NULL, ecc->size, false, page);
 824
 825         if (data_off + ecc->size != oob_off)
 826                 nand_change_read_column_op(nand, oob_off, NULL, 0, false);
 827
 828         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 829         if (ret)
 830                 return ret;
 831
 832         sunxi_nfc_randomizer_enable(nand);
 833         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 834                nfc->regs + NFC_REG_CMD);
 835
 836         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
 837         sunxi_nfc_randomizer_disable(nand);
 838         if (ret)
 839                 return ret;
 840
 841         *cur_off = oob_off + ecc->bytes + 4;
 842
 843         ret = sunxi_nfc_hw_ecc_correct(nand, data, oob_required ? oob : NULL, 0,
 844                                        readl(nfc->regs + NFC_REG_ECC_ST),
 845                                        &erased);
 846         if (erased)
 847                 return 1;
 848
 849         if (ret < 0) {
 850                 /*
 851                  * Re-read the data with the randomizer disabled to identify
 852                  * bitflips in erased pages.
 853                  */
 854                 if (nand->options & NAND_NEED_SCRAMBLING)
 855                         nand_change_read_column_op(nand, data_off, data,
 856                                                    ecc->size, false);
 857                 else
 858                         memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 859                                       ecc->size);
 860
 861                 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
 862                                            false);
 863
 864                 ret = nand_check_erased_ecc_chunk(data, ecc->size,
 865                                                   oob, ecc->bytes + 4,
 866                                                   NULL, 0, ecc->strength);
 867                 if (ret >= 0)
 868                         raw_mode = 1;
 869         } else {
 870                 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
 871
 872                 if (oob_required) {
 873                         nand_change_read_column_op(nand, oob_off, NULL, 0,
 874                                                    false);
 875                         sunxi_nfc_randomizer_read_buf(nand, oob, ecc->bytes + 4,
 876                                                       true, page);
 877
 878                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, 0,
 879                                                             bbm, page);
 880                 }
 881         }
 882
 883         sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret);
 884
 885         return raw_mode;
 886 }
 887
 888 static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand,
 889                                             u8 *oob, int *cur_off,
 890                                             bool randomize, int page)
 891 {
 892         struct mtd_info *mtd = nand_to_mtd(nand);
 893         struct nand_ecc_ctrl *ecc = &nand->ecc;
 894         int offset = ((ecc->bytes + 4) * ecc->steps);
 895         int len = mtd->oobsize - offset;
 896
 897         if (len <= 0)
 898                 return;
 899
 900         if (!cur_off || *cur_off != offset)
 901                 nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
 902                                            false);
 903
 904         if (!randomize)
 905                 sunxi_nfc_read_buf(nand, oob + offset, len);
 906         else
 907                 sunxi_nfc_randomizer_read_buf(nand, oob + offset, len,
 908                                               false, page);
 909
 910         if (cur_off)
 911                 *cur_off = mtd->oobsize + mtd->writesize;
 912 }
 913
 914 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf,
 915                                             int oob_required, int page,
 916                                             int nchunks)
 917 {
 918         bool randomized = nand->options & NAND_NEED_SCRAMBLING;
 919         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 920         struct mtd_info *mtd = nand_to_mtd(nand);
 921         struct nand_ecc_ctrl *ecc = &nand->ecc;
 922         unsigned int max_bitflips = 0;
 923         int ret, i, raw_mode = 0;
 924         struct scatterlist sg;
 925         u32 status, wait;
 926
 927         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 928         if (ret)
 929                 return ret;
 930
 931         ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks,
 932                                        DMA_FROM_DEVICE, &sg);
 933         if (ret)
 934                 return ret;
 935
 936         sunxi_nfc_hw_ecc_enable(nand);
 937         sunxi_nfc_randomizer_config(nand, page, false);
 938         sunxi_nfc_randomizer_enable(nand);
 939
 940         writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
 941                NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
 942
 943         wait = NFC_CMD_INT_FLAG;
 944
 945         if (nfc->caps->has_mdma)
 946                 wait |= NFC_DMA_INT_FLAG;
 947         else
 948                 dma_async_issue_pending(nfc->dmac);
 949
 950         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
 951                nfc->regs + NFC_REG_CMD);
 952
 953         ret = sunxi_nfc_wait_events(nfc, wait, false, 0);
 954         if (ret && !nfc->caps->has_mdma)
 955                 dmaengine_terminate_all(nfc->dmac);
 956
 957         sunxi_nfc_randomizer_disable(nand);
 958         sunxi_nfc_hw_ecc_disable(nand);
 959
 960         sunxi_nfc_dma_op_cleanup(nfc, DMA_FROM_DEVICE, &sg);
 961
 962         if (ret)
 963                 return ret;
 964
 965         status = readl(nfc->regs + NFC_REG_ECC_ST);
 966
 967         for (i = 0; i < nchunks; i++) {
 968                 int data_off = i * ecc->size;
 969                 int oob_off = i * (ecc->bytes + 4);
 970                 u8 *data = buf + data_off;
 971                 u8 *oob = nand->oob_poi + oob_off;
 972                 bool erased;
 973
 974                 ret = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL,
 975                                                oob_required ? oob : NULL,
 976                                                i, status, &erased);
 977
 978                 /* ECC errors are handled in the second loop. */
 979                 if (ret < 0)
 980                         continue;
 981
 982                 if (oob_required && !erased) {
 983                         /* TODO: use DMA to retrieve OOB */
 984                         nand_change_read_column_op(nand,
 985                                                    mtd->writesize + oob_off,
 986                                                    oob, ecc->bytes + 4, false);
 987
 988                         sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i,
 989                                                             !i, page);
 990                 }
 991
 992                 if (erased)
 993                         raw_mode = 1;
 994
 995                 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
 996         }
 997
 998         if (status & NFC_ECC_ERR_MSK) {
 999                 for (i = 0; i < nchunks; i++) {
1000                         int data_off = i * ecc->size;
1001                         int oob_off = i * (ecc->bytes + 4);
1002                         u8 *data = buf + data_off;
1003                         u8 *oob = nand->oob_poi + oob_off;
1004
1005                         if (!(status & NFC_ECC_ERR(i)))
1006                                 continue;
1007
1008                         /*
1009                          * Re-read the data with the randomizer disabled to
1010                          * identify bitflips in erased pages.
1011                          * TODO: use DMA to read page in raw mode
1012                          */
1013                         if (randomized)
1014                                 nand_change_read_column_op(nand, data_off,
1015                                                            data, ecc->size,
1016                                                            false);
1017
1018                         /* TODO: use DMA to retrieve OOB */
1019                         nand_change_read_column_op(nand,
1020                                                    mtd->writesize + oob_off,
1021                                                    oob, ecc->bytes + 4, false);
1022
1023                         ret = nand_check_erased_ecc_chunk(data, ecc->size,
1024                                                           oob, ecc->bytes + 4,
1025                                                           NULL, 0,
1026                                                           ecc->strength);
1027                         if (ret >= 0)
1028                                 raw_mode = 1;
1029
1030                         sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
1031                 }
1032         }
1033
1034         if (oob_required)
1035                 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi,
1036                                                 NULL, !raw_mode,
1037                                                 page);
1038
1039         return max_bitflips;
1040 }
1041
1042 static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand,
1043                                         const u8 *data, int data_off,
1044                                         const u8 *oob, int oob_off,
1045                                         int *cur_off, bool bbm,
1046                                         int page)
1047 {
1048         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1049         struct nand_ecc_ctrl *ecc = &nand->ecc;
1050         int ret;
1051
1052         if (data_off != *cur_off)
1053                 nand_change_write_column_op(nand, data_off, NULL, 0, false);
1054
1055         sunxi_nfc_randomizer_write_buf(nand, data, ecc->size, false, page);
1056
1057         if (data_off + ecc->size != oob_off)
1058                 nand_change_write_column_op(nand, oob_off, NULL, 0, false);
1059
1060         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1061         if (ret)
1062                 return ret;
1063
1064         sunxi_nfc_randomizer_enable(nand);
1065         sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page);
1066
1067         writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1068                NFC_ACCESS_DIR | NFC_ECC_OP,
1069                nfc->regs + NFC_REG_CMD);
1070
1071         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1072         sunxi_nfc_randomizer_disable(nand);
1073         if (ret)
1074                 return ret;
1075
1076         *cur_off = oob_off + ecc->bytes + 4;
1077
1078         return 0;
1079 }
1080
1081 static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand,
1082                                              u8 *oob, int *cur_off,
1083                                              int page)
1084 {
1085         struct mtd_info *mtd = nand_to_mtd(nand);
1086         struct nand_ecc_ctrl *ecc = &nand->ecc;
1087         int offset = ((ecc->bytes + 4) * ecc->steps);
1088         int len = mtd->oobsize - offset;
1089
1090         if (len <= 0)
1091                 return;
1092
1093         if (!cur_off || *cur_off != offset)
1094                 nand_change_write_column_op(nand, offset + mtd->writesize,
1095                                             NULL, 0, false);
1096
1097         sunxi_nfc_randomizer_write_buf(nand, oob + offset, len, false, page);
1098
1099         if (cur_off)
1100                 *cur_off = mtd->oobsize + mtd->writesize;
1101 }
1102
1103 static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf,
1104                                       int oob_required, int page)
1105 {
1106         struct mtd_info *mtd = nand_to_mtd(nand);
1107         struct nand_ecc_ctrl *ecc = &nand->ecc;
1108         unsigned int max_bitflips = 0;
1109         int ret, i, cur_off = 0;
1110         bool raw_mode = false;
1111
1112         sunxi_nfc_select_chip(nand, nand->cur_cs);
1113
1114         nand_read_page_op(nand, page, 0, NULL, 0);
1115
1116         sunxi_nfc_hw_ecc_enable(nand);
1117
1118         for (i = 0; i < ecc->steps; i++) {
1119                 int data_off = i * ecc->size;
1120                 int oob_off = i * (ecc->bytes + 4);
1121                 u8 *data = buf + data_off;
1122                 u8 *oob = nand->oob_poi + oob_off;
1123
1124                 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob,
1125                                                   oob_off + mtd->writesize,
1126                                                   &cur_off, &max_bitflips,
1127                                                   !i, oob_required, page);
1128                 if (ret < 0)
1129                         return ret;
1130                 else if (ret)
1131                         raw_mode = true;
1132         }
1133
1134         if (oob_required)
1135                 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, &cur_off,
1136                                                 !raw_mode, page);
1137
1138         sunxi_nfc_hw_ecc_disable(nand);
1139
1140         return max_bitflips;
1141 }
1142
1143 static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf,
1144                                           int oob_required, int page)
1145 {
1146         int ret;
1147
1148         sunxi_nfc_select_chip(nand, nand->cur_cs);
1149
1150         nand_read_page_op(nand, page, 0, NULL, 0);
1151
1152         ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page,
1153                                                nand->ecc.steps);
1154         if (ret >= 0)
1155                 return ret;
1156
1157         /* Fallback to PIO mode */
1158         return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page);
1159 }
1160
1161 static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand,
1162                                          u32 data_offs, u32 readlen,
1163                                          u8 *bufpoi, int page)
1164 {
1165         struct mtd_info *mtd = nand_to_mtd(nand);
1166         struct nand_ecc_ctrl *ecc = &nand->ecc;
1167         int ret, i, cur_off = 0;
1168         unsigned int max_bitflips = 0;
1169
1170         sunxi_nfc_select_chip(nand, nand->cur_cs);
1171
1172         nand_read_page_op(nand, page, 0, NULL, 0);
1173
1174         sunxi_nfc_hw_ecc_enable(nand);
1175
1176         for (i = data_offs / ecc->size;
1177              i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1178                 int data_off = i * ecc->size;
1179                 int oob_off = i * (ecc->bytes + 4);
1180                 u8 *data = bufpoi + data_off;
1181                 u8 *oob = nand->oob_poi + oob_off;
1182
1183                 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off,
1184                                                   oob,
1185                                                   oob_off + mtd->writesize,
1186                                                   &cur_off, &max_bitflips, !i,
1187                                                   false, page);
1188                 if (ret < 0)
1189                         return ret;
1190         }
1191
1192         sunxi_nfc_hw_ecc_disable(nand);
1193
1194         return max_bitflips;
1195 }
1196
1197 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand,
1198                                              u32 data_offs, u32 readlen,
1199                                              u8 *buf, int page)
1200 {
1201         int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size);
1202         int ret;
1203
1204         sunxi_nfc_select_chip(nand, nand->cur_cs);
1205
1206         nand_read_page_op(nand, page, 0, NULL, 0);
1207
1208         ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, false, page, nchunks);
1209         if (ret >= 0)
1210                 return ret;
1211
1212         /* Fallback to PIO mode */
1213         return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen,
1214                                              buf, page);
1215 }
1216
1217 static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand,
1218                                        const uint8_t *buf, int oob_required,
1219                                        int page)
1220 {
1221         struct mtd_info *mtd = nand_to_mtd(nand);
1222         struct nand_ecc_ctrl *ecc = &nand->ecc;
1223         int ret, i, cur_off = 0;
1224
1225         sunxi_nfc_select_chip(nand, nand->cur_cs);
1226
1227         nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1228
1229         sunxi_nfc_hw_ecc_enable(nand);
1230
1231         for (i = 0; i < ecc->steps; i++) {
1232                 int data_off = i * ecc->size;
1233                 int oob_off = i * (ecc->bytes + 4);
1234                 const u8 *data = buf + data_off;
1235                 const u8 *oob = nand->oob_poi + oob_off;
1236
1237                 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1238                                                    oob_off + mtd->writesize,
1239                                                    &cur_off, !i, page);
1240                 if (ret)
1241                         return ret;
1242         }
1243
1244         if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1245                 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1246                                                  &cur_off, page);
1247
1248         sunxi_nfc_hw_ecc_disable(nand);
1249
1250         return nand_prog_page_end_op(nand);
1251 }
1252
1253 static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand,
1254                                           u32 data_offs, u32 data_len,
1255                                           const u8 *buf, int oob_required,
1256                                           int page)
1257 {
1258         struct mtd_info *mtd = nand_to_mtd(nand);
1259         struct nand_ecc_ctrl *ecc = &nand->ecc;
1260         int ret, i, cur_off = 0;
1261
1262         sunxi_nfc_select_chip(nand, nand->cur_cs);
1263
1264         nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1265
1266         sunxi_nfc_hw_ecc_enable(nand);
1267
1268         for (i = data_offs / ecc->size;
1269              i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1270                 int data_off = i * ecc->size;
1271                 int oob_off = i * (ecc->bytes + 4);
1272                 const u8 *data = buf + data_off;
1273                 const u8 *oob = nand->oob_poi + oob_off;
1274
1275                 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1276                                                    oob_off + mtd->writesize,
1277                                                    &cur_off, !i, page);
1278                 if (ret)
1279                         return ret;
1280         }
1281
1282         sunxi_nfc_hw_ecc_disable(nand);
1283
1284         return nand_prog_page_end_op(nand);
1285 }
1286
1287 static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand,
1288                                            const u8 *buf,
1289                                            int oob_required,
1290                                            int page)
1291 {
1292         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1293         struct nand_ecc_ctrl *ecc = &nand->ecc;
1294         struct scatterlist sg;
1295         u32 wait;
1296         int ret, i;
1297
1298         sunxi_nfc_select_chip(nand, nand->cur_cs);
1299
1300         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1301         if (ret)
1302                 return ret;
1303
1304         ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps,
1305                                        DMA_TO_DEVICE, &sg);
1306         if (ret)
1307                 goto pio_fallback;
1308
1309         for (i = 0; i < ecc->steps; i++) {
1310                 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1311
1312                 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page);
1313         }
1314
1315         nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1316
1317         sunxi_nfc_hw_ecc_enable(nand);
1318         sunxi_nfc_randomizer_config(nand, page, false);
1319         sunxi_nfc_randomizer_enable(nand);
1320
1321         writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1322                nfc->regs + NFC_REG_WCMD_SET);
1323
1324         wait = NFC_CMD_INT_FLAG;
1325
1326         if (nfc->caps->has_mdma)
1327                 wait |= NFC_DMA_INT_FLAG;
1328         else
1329                 dma_async_issue_pending(nfc->dmac);
1330
1331         writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1332                NFC_DATA_TRANS | NFC_ACCESS_DIR,
1333                nfc->regs + NFC_REG_CMD);
1334
1335         ret = sunxi_nfc_wait_events(nfc, wait, false, 0);
1336         if (ret && !nfc->caps->has_mdma)
1337                 dmaengine_terminate_all(nfc->dmac);
1338
1339         sunxi_nfc_randomizer_disable(nand);
1340         sunxi_nfc_hw_ecc_disable(nand);
1341
1342         sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg);
1343
1344         if (ret)
1345                 return ret;
1346
1347         if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1348                 /* TODO: use DMA to transfer extra OOB bytes ? */
1349                 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1350                                                  NULL, page);
1351
1352         return nand_prog_page_end_op(nand);
1353
1354 pio_fallback:
1355         return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page);
1356 }
1357
1358 static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page)
1359 {
1360         u8 *buf = nand_get_data_buf(nand);
1361
1362         return nand->ecc.read_page(nand, buf, 1, page);
1363 }
1364
1365 static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page)
1366 {
1367         struct mtd_info *mtd = nand_to_mtd(nand);
1368         u8 *buf = nand_get_data_buf(nand);
1369         int ret;
1370
1371         memset(buf, 0xff, mtd->writesize);
1372         ret = nand->ecc.write_page(nand, buf, 1, page);
1373         if (ret)
1374                 return ret;
1375
1376         /* Send command to program the OOB data */
1377         return nand_prog_page_end_op(nand);
1378 }
1379
1380 static const s32 tWB_lut[] = {6, 12, 16, 20};
1381 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1382
1383 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1384                 u32 clk_period)
1385 {
1386         u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1387         int i;
1388
1389         for (i = 0; i < lut_size; i++) {
1390                 if (clk_cycles <= lut[i])
1391                         return i;
1392         }
1393
1394         /* Doesn't fit */
1395         return -EINVAL;
1396 }
1397
1398 #define sunxi_nand_lookup_timing(l, p, c) \
1399                         _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1400
1401 static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
1402                                      const struct nand_interface_config *conf)
1403 {
1404         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1405         struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1406         const struct nand_sdr_timings *timings;
1407         u32 min_clk_period = 0;
1408         s32 tWB, tADL, tWHR, tRHW, tCAD;
1409         long real_clk_rate;
1410
1411         timings = nand_get_sdr_timings(conf);
1412         if (IS_ERR(timings))
1413                 return -ENOTSUPP;
1414
1415         /* T1 <=> tCLS */
1416         if (timings->tCLS_min > min_clk_period)
1417                 min_clk_period = timings->tCLS_min;
1418
1419         /* T2 <=> tCLH */
1420         if (timings->tCLH_min > min_clk_period)
1421                 min_clk_period = timings->tCLH_min;
1422
1423         /* T3 <=> tCS */
1424         if (timings->tCS_min > min_clk_period)
1425                 min_clk_period = timings->tCS_min;
1426
1427         /* T4 <=> tCH */
1428         if (timings->tCH_min > min_clk_period)
1429                 min_clk_period = timings->tCH_min;
1430
1431         /* T5 <=> tWP */
1432         if (timings->tWP_min > min_clk_period)
1433                 min_clk_period = timings->tWP_min;
1434
1435         /* T6 <=> tWH */
1436         if (timings->tWH_min > min_clk_period)
1437                 min_clk_period = timings->tWH_min;
1438
1439         /* T7 <=> tALS */
1440         if (timings->tALS_min > min_clk_period)
1441                 min_clk_period = timings->tALS_min;
1442
1443         /* T8 <=> tDS */
1444         if (timings->tDS_min > min_clk_period)
1445                 min_clk_period = timings->tDS_min;
1446
1447         /* T9 <=> tDH */
1448         if (timings->tDH_min > min_clk_period)
1449                 min_clk_period = timings->tDH_min;
1450
1451         /* T10 <=> tRR */
1452         if (timings->tRR_min > (min_clk_period * 3))
1453                 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1454
1455         /* T11 <=> tALH */
1456         if (timings->tALH_min > min_clk_period)
1457                 min_clk_period = timings->tALH_min;
1458
1459         /* T12 <=> tRP */
1460         if (timings->tRP_min > min_clk_period)
1461                 min_clk_period = timings->tRP_min;
1462
1463         /* T13 <=> tREH */
1464         if (timings->tREH_min > min_clk_period)
1465                 min_clk_period = timings->tREH_min;
1466
1467         /* T14 <=> tRC */
1468         if (timings->tRC_min > (min_clk_period * 2))
1469                 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1470
1471         /* T15 <=> tWC */
1472         if (timings->tWC_min > (min_clk_period * 2))
1473                 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1474
1475         /* T16 - T19 + tCAD */
1476         if (timings->tWB_max > (min_clk_period * 20))
1477                 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1478
1479         if (timings->tADL_min > (min_clk_period * 32))
1480                 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1481
1482         if (timings->tWHR_min > (min_clk_period * 32))
1483                 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1484
1485         if (timings->tRHW_min > (min_clk_period * 20))
1486                 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1487
1488         /*
1489          * In non-EDO, tREA should be less than tRP to guarantee that the
1490          * controller does not sample the IO lines too early. Unfortunately,
1491          * the sunxi NAND controller does not allow us to have different
1492          * values for tRP and tREH (tRP = tREH = tRW / 2).
1493          *
1494          * We have 2 options to overcome this limitation:
1495          *
1496          * 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint
1497          * 2/ Use EDO mode (only works if timings->tRLOH > 0)
1498          */
1499         if (timings->tREA_max > min_clk_period && !timings->tRLOH_min)
1500                 min_clk_period = timings->tREA_max;
1501
1502         tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1503                                         min_clk_period);
1504         if (tWB < 0) {
1505                 dev_err(nfc->dev, "unsupported tWB\n");
1506                 return tWB;
1507         }
1508
1509         tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1510         if (tADL > 3) {
1511                 dev_err(nfc->dev, "unsupported tADL\n");
1512                 return -EINVAL;
1513         }
1514
1515         tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1516         if (tWHR > 3) {
1517                 dev_err(nfc->dev, "unsupported tWHR\n");
1518                 return -EINVAL;
1519         }
1520
1521         tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1522                                         min_clk_period);
1523         if (tRHW < 0) {
1524                 dev_err(nfc->dev, "unsupported tRHW\n");
1525                 return tRHW;
1526         }
1527
1528         if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1529                 return 0;
1530
1531         /*
1532          * TODO: according to ONFI specs this value only applies for DDR NAND,
1533          * but Allwinner seems to set this to 0x7. Mimic them for now.
1534          */
1535         tCAD = 0x7;
1536
1537         /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1538         sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1539
1540         /* Convert min_clk_period from picoseconds to nanoseconds */
1541         min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1542
1543         /*
1544          * Unlike what is stated in Allwinner datasheet, the clk_rate should
1545          * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1546          * This new formula was verified with a scope and validated by
1547          * Allwinner engineers.
1548          */
1549         sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period;
1550         real_clk_rate = clk_round_rate(nfc->mod_clk, sunxi_nand->clk_rate);
1551         if (real_clk_rate <= 0) {
1552                 dev_err(nfc->dev, "Unable to round clk %lu\n",
1553                         sunxi_nand->clk_rate);
1554                 return -EINVAL;
1555         }
1556
1557         sunxi_nand->timing_ctl = 0;
1558
1559         /*
1560          * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1561          * output cycle timings shall be used if the host drives tRC less than
1562          * 30 ns. We should also use EDO mode if tREA is bigger than tRP.
1563          */
1564         min_clk_period = NSEC_PER_SEC / real_clk_rate;
1565         if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max)
1566                 sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO;
1567
1568         return 0;
1569 }
1570
1571 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1572                                     struct mtd_oob_region *oobregion)
1573 {
1574         struct nand_chip *nand = mtd_to_nand(mtd);
1575         struct nand_ecc_ctrl *ecc = &nand->ecc;
1576
1577         if (section >= ecc->steps)
1578                 return -ERANGE;
1579
1580         oobregion->offset = section * (ecc->bytes + 4) + 4;
1581         oobregion->length = ecc->bytes;
1582
1583         return 0;
1584 }
1585
1586 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1587                                      struct mtd_oob_region *oobregion)
1588 {
1589         struct nand_chip *nand = mtd_to_nand(mtd);
1590         struct nand_ecc_ctrl *ecc = &nand->ecc;
1591
1592         if (section > ecc->steps)
1593                 return -ERANGE;
1594
1595         /*
1596          * The first 2 bytes are used for BB markers, hence we
1597          * only have 2 bytes available in the first user data
1598          * section.
1599          */
1600         if (!section && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) {
1601                 oobregion->offset = 2;
1602                 oobregion->length = 2;
1603
1604                 return 0;
1605         }
1606
1607         oobregion->offset = section * (ecc->bytes + 4);
1608
1609         if (section < ecc->steps)
1610                 oobregion->length = 4;
1611         else
1612                 oobregion->offset = mtd->oobsize - oobregion->offset;
1613
1614         return 0;
1615 }
1616
1617 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1618         .ecc = sunxi_nand_ooblayout_ecc,
1619         .free = sunxi_nand_ooblayout_free,
1620 };
1621
1622 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct sunxi_nand_chip *sunxi_nand)
1623 {
1624         kfree(sunxi_nand->ecc);
1625 }
1626
1627 static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,
1628                                        struct nand_ecc_ctrl *ecc,
1629                                        struct device_node *np)
1630 {
1631         static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1632         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1633         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1634         struct mtd_info *mtd = nand_to_mtd(nand);
1635         struct nand_device *nanddev = mtd_to_nanddev(mtd);
1636         int nsectors;
1637         int ret;
1638         int i;
1639
1640         if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) {
1641                 int bytes;
1642
1643                 ecc->size = 1024;
1644                 nsectors = mtd->writesize / ecc->size;
1645
1646                 /* Reserve 2 bytes for the BBM */
1647                 bytes = (mtd->oobsize - 2) / nsectors;
1648
1649                 /* 4 non-ECC bytes are added before each ECC bytes section */
1650                 bytes -= 4;
1651
1652                 /* and bytes has to be even. */
1653                 if (bytes % 2)
1654                         bytes--;
1655
1656                 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1657
1658                 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1659                         if (strengths[i] > ecc->strength)
1660                                 break;
1661                 }
1662
1663                 if (!i)
1664                         ecc->strength = 0;
1665                 else
1666                         ecc->strength = strengths[i - 1];
1667         }
1668
1669         if (ecc->size != 512 && ecc->size != 1024)
1670                 return -EINVAL;
1671
1672         sunxi_nand->ecc = kzalloc(sizeof(*sunxi_nand->ecc), GFP_KERNEL);
1673         if (!sunxi_nand->ecc)
1674                 return -ENOMEM;
1675
1676         /* Prefer 1k ECC chunk over 512 ones */
1677         if (ecc->size == 512 && mtd->writesize > 512) {
1678                 ecc->size = 1024;
1679                 ecc->strength *= 2;
1680         }
1681
1682         /* Add ECC info retrieval from DT */
1683         for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1684                 if (ecc->strength <= strengths[i]) {
1685                         /*
1686                          * Update ecc->strength value with the actual strength
1687                          * that will be used by the ECC engine.
1688                          */
1689                         ecc->strength = strengths[i];
1690                         break;
1691                 }
1692         }
1693
1694         if (i >= ARRAY_SIZE(strengths)) {
1695                 dev_err(nfc->dev, "unsupported strength\n");
1696                 ret = -ENOTSUPP;
1697                 goto err;
1698         }
1699
1700         sunxi_nand->ecc->mode = i;
1701
1702         /* HW ECC always request ECC bytes for 1024 bytes blocks */
1703         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1704
1705         /* HW ECC always work with even numbers of ECC bytes */
1706         ecc->bytes = ALIGN(ecc->bytes, 2);
1707
1708         nsectors = mtd->writesize / ecc->size;
1709
1710         if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1711                 ret = -EINVAL;
1712                 goto err;
1713         }
1714
1715         ecc->read_oob = sunxi_nfc_hw_ecc_read_oob;
1716         ecc->write_oob = sunxi_nfc_hw_ecc_write_oob;
1717         mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1718
1719         if (nfc->dmac || nfc->caps->has_mdma) {
1720                 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1721                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1722                 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1723                 nand->options |= NAND_USES_DMA;
1724         } else {
1725                 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1726                 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1727                 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1728         }
1729
1730         /* TODO: support DMA for raw accesses and subpage write */
1731         ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1732         ecc->read_oob_raw = nand_read_oob_std;
1733         ecc->write_oob_raw = nand_write_oob_std;
1734
1735         return 0;
1736
1737 err:
1738         kfree(sunxi_nand->ecc);
1739
1740         return ret;
1741 }
1742
1743 static void sunxi_nand_ecc_cleanup(struct sunxi_nand_chip *sunxi_nand)
1744 {
1745         struct nand_ecc_ctrl *ecc = &sunxi_nand->nand.ecc;
1746
1747         switch (ecc->engine_type) {
1748         case NAND_ECC_ENGINE_TYPE_ON_HOST:
1749                 sunxi_nand_hw_ecc_ctrl_cleanup(sunxi_nand);
1750                 break;
1751         case NAND_ECC_ENGINE_TYPE_NONE:
1752         default:
1753                 break;
1754         }
1755 }
1756
1757 static int sunxi_nand_attach_chip(struct nand_chip *nand)
1758 {
1759         const struct nand_ecc_props *requirements =
1760                 nanddev_get_ecc_requirements(&nand->base);
1761         struct nand_ecc_ctrl *ecc = &nand->ecc;
1762         struct device_node *np = nand_get_flash_node(nand);
1763         int ret;
1764
1765         if (nand->bbt_options & NAND_BBT_USE_FLASH)
1766                 nand->bbt_options |= NAND_BBT_NO_OOB;
1767
1768         if (nand->options & NAND_NEED_SCRAMBLING)
1769                 nand->options |= NAND_NO_SUBPAGE_WRITE;
1770
1771         nand->options |= NAND_SUBPAGE_READ;
1772
1773         if (!ecc->size) {
1774                 ecc->size = requirements->step_size;
1775                 ecc->strength = requirements->strength;
1776         }
1777
1778         if (!ecc->size || !ecc->strength)
1779                 return -EINVAL;
1780
1781         switch (ecc->engine_type) {
1782         case NAND_ECC_ENGINE_TYPE_ON_HOST:
1783                 ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np);
1784                 if (ret)
1785                         return ret;
1786                 break;
1787         case NAND_ECC_ENGINE_TYPE_NONE:
1788         case NAND_ECC_ENGINE_TYPE_SOFT:
1789                 break;
1790         default:
1791                 return -EINVAL;
1792         }
1793
1794         return 0;
1795 }
1796
1797 static int sunxi_nfc_exec_subop(struct nand_chip *nand,
1798                                 const struct nand_subop *subop)
1799 {
1800         struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1801         u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { };
1802         unsigned int i, j, remaining, start;
1803         void *inbuf = NULL;
1804         int ret;
1805
1806         for (i = 0; i < subop->ninstrs; i++) {
1807                 const struct nand_op_instr *instr = &subop->instrs[i];
1808
1809                 switch (instr->type) {
1810                 case NAND_OP_CMD_INSTR:
1811                         if (cmd & NFC_SEND_CMD1) {
1812                                 if (WARN_ON(cmd & NFC_SEND_CMD2))
1813                                         return -EINVAL;
1814
1815                                 cmd |= NFC_SEND_CMD2;
1816                                 extcmd |= instr->ctx.cmd.opcode;
1817                         } else {
1818                                 cmd |= NFC_SEND_CMD1 |
1819                                        NFC_CMD(instr->ctx.cmd.opcode);
1820                         }
1821                         break;
1822
1823                 case NAND_OP_ADDR_INSTR:
1824                         remaining = nand_subop_get_num_addr_cyc(subop, i);
1825                         start = nand_subop_get_addr_start_off(subop, i);
1826                         for (j = 0; j < 8 && j + start < remaining; j++) {
1827                                 u32 addr = instr->ctx.addr.addrs[j + start];
1828
1829                                 addrs[j / 4] |= addr << (j % 4) * 8;
1830                         }
1831
1832                         if (j)
1833                                 cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j);
1834
1835                         break;
1836
1837                 case NAND_OP_DATA_IN_INSTR:
1838                 case NAND_OP_DATA_OUT_INSTR:
1839                         start = nand_subop_get_data_start_off(subop, i);
1840                         remaining = nand_subop_get_data_len(subop, i);
1841                         cnt = min_t(u32, remaining, NFC_SRAM_SIZE);
1842                         cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
1843
1844                         if (instr->type == NAND_OP_DATA_OUT_INSTR) {
1845                                 cmd |= NFC_ACCESS_DIR;
1846                                 memcpy_toio(nfc->regs + NFC_RAM0_BASE,
1847                                             instr->ctx.data.buf.out + start,
1848                                             cnt);
1849                         } else {
1850                                 inbuf = instr->ctx.data.buf.in + start;
1851                         }
1852
1853                         break;
1854
1855                 case NAND_OP_WAITRDY_INSTR:
1856                         cmd |= NFC_WAIT_FLAG;
1857                         break;
1858                 }
1859         }
1860
1861         ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1862         if (ret)
1863                 return ret;
1864
1865         if (cmd & NFC_SEND_ADR) {
1866                 writel(addrs[0], nfc->regs + NFC_REG_ADDR_LOW);
1867                 writel(addrs[1], nfc->regs + NFC_REG_ADDR_HIGH);
1868         }
1869
1870         if (cmd & NFC_SEND_CMD2)
1871                 writel(extcmd,
1872                        nfc->regs +
1873                        (cmd & NFC_ACCESS_DIR ?
1874                         NFC_REG_WCMD_SET : NFC_REG_RCMD_SET));
1875
1876         if (cmd & NFC_DATA_TRANS)
1877                 writel(cnt, nfc->regs + NFC_REG_CNT);
1878
1879         writel(cmd, nfc->regs + NFC_REG_CMD);
1880
1881         ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG,
1882                                     !(cmd & NFC_WAIT_FLAG) && cnt < 64,
1883                                     0);
1884         if (ret)
1885                 return ret;
1886
1887         if (inbuf)
1888                 memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt);
1889
1890         return 0;
1891 }
1892
1893 static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand,
1894                                   const struct nand_subop *subop)
1895 {
1896         return nand_soft_waitrdy(nand,
1897                                  subop->instrs[0].ctx.waitrdy.timeout_ms);
1898 }
1899
1900 static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER(
1901         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1902                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1903                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1904                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1905                                NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
1906                                NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1907         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1908                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1909                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1910                                NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1911                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1912                                NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
1913 );
1914
1915 static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER(
1916         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1917                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1918                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1919                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1920                                NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1921         NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1922                                NAND_OP_PARSER_PAT_CMD_ELEM(true),
1923                                NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1924                                NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1925                                NAND_OP_PARSER_PAT_CMD_ELEM(true)),
1926         NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy,
1927                                NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
1928 );
1929
1930 static int sunxi_nfc_exec_op(struct nand_chip *nand,
1931                              const struct nand_operation *op, bool check_only)
1932 {
1933         struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1934         const struct nand_op_parser *parser;
1935
1936         if (!check_only)
1937                 sunxi_nfc_select_chip(nand, op->cs);
1938
1939         if (sunxi_nand->sels[op->cs].rb >= 0)
1940                 parser = &sunxi_nfc_op_parser;
1941         else
1942                 parser = &sunxi_nfc_norb_op_parser;
1943
1944         return nand_op_parser_exec_op(nand, parser, op, check_only);
1945 }
1946
1947 static const struct nand_controller_ops sunxi_nand_controller_ops = {
1948         .attach_chip = sunxi_nand_attach_chip,
1949         .setup_interface = sunxi_nfc_setup_interface,
1950         .exec_op = sunxi_nfc_exec_op,
1951 };
1952
1953 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1954                                 struct device_node *np)
1955 {
1956         struct sunxi_nand_chip *sunxi_nand;
1957         struct mtd_info *mtd;
1958         struct nand_chip *nand;
1959         int nsels;
1960         int ret;
1961         int i;
1962         u32 tmp;
1963
1964         if (!of_get_property(np, "reg", &nsels))
1965                 return -EINVAL;
1966
1967         nsels /= sizeof(u32);
1968         if (!nsels) {
1969                 dev_err(dev, "invalid reg property size\n");
1970                 return -EINVAL;
1971         }
1972
1973         sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels),
1974                                   GFP_KERNEL);
1975         if (!sunxi_nand) {
1976                 dev_err(dev, "could not allocate chip\n");
1977                 return -ENOMEM;
1978         }
1979
1980         sunxi_nand->nsels = nsels;
1981
1982         for (i = 0; i < nsels; i++) {
1983                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1984                 if (ret) {
1985                         dev_err(dev, "could not retrieve reg property: %d\n",
1986                                 ret);
1987                         return ret;
1988                 }
1989
1990                 if (tmp > NFC_MAX_CS) {
1991                         dev_err(dev,
1992                                 "invalid reg value: %u (max CS = 7)\n",
1993                                 tmp);
1994                         return -EINVAL;
1995                 }
1996
1997                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1998                         dev_err(dev, "CS %d already assigned\n", tmp);
1999                         return -EINVAL;
2000                 }
2001
2002                 sunxi_nand->sels[i].cs = tmp;
2003
2004                 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2005                     tmp < 2)
2006                         sunxi_nand->sels[i].rb = tmp;
2007                 else
2008                         sunxi_nand->sels[i].rb = -1;
2009         }
2010
2011         nand = &sunxi_nand->nand;
2012         /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2013         nand->controller = &nfc->controller;
2014         nand->controller->ops = &sunxi_nand_controller_ops;
2015
2016         /*
2017          * Set the ECC mode to the default value in case nothing is specified
2018          * in the DT.
2019          */
2020         nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2021         nand_set_flash_node(nand, np);
2022
2023         mtd = nand_to_mtd(nand);
2024         mtd->dev.parent = dev;
2025
2026         ret = nand_scan(nand, nsels);
2027         if (ret)
2028                 return ret;
2029
2030         ret = mtd_device_register(mtd, NULL, 0);
2031         if (ret) {
2032                 dev_err(dev, "failed to register mtd device: %d\n", ret);
2033                 nand_cleanup(nand);
2034                 return ret;
2035         }
2036
2037         list_add_tail(&sunxi_nand->node, &nfc->chips);
2038
2039         return 0;
2040 }
2041
2042 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2043 {
2044         struct device_node *np = dev->of_node;
2045         struct device_node *nand_np;
2046         int nchips = of_get_child_count(np);
2047         int ret;
2048
2049         if (nchips > 8) {
2050                 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2051                 return -EINVAL;
2052         }
2053
2054         for_each_child_of_node(np, nand_np) {
2055                 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2056                 if (ret) {
2057                         of_node_put(nand_np);
2058                         return ret;
2059                 }
2060         }
2061
2062         return 0;
2063 }
2064
2065 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2066 {
2067         struct sunxi_nand_chip *sunxi_nand;
2068         struct nand_chip *chip;
2069         int ret;
2070
2071         while (!list_empty(&nfc->chips)) {
2072                 sunxi_nand = list_first_entry(&nfc->chips,
2073                                               struct sunxi_nand_chip,
2074                                               node);
2075                 chip = &sunxi_nand->nand;
2076                 ret = mtd_device_unregister(nand_to_mtd(chip));
2077                 WARN_ON(ret);
2078                 nand_cleanup(chip);
2079                 sunxi_nand_ecc_cleanup(sunxi_nand);
2080                 list_del(&sunxi_nand->node);
2081         }
2082 }
2083
2084 static int sunxi_nfc_dma_init(struct sunxi_nfc *nfc, struct resource *r)
2085 {
2086         int ret;
2087
2088         if (nfc->caps->has_mdma)
2089                 return 0;
2090
2091         nfc->dmac = dma_request_chan(nfc->dev, "rxtx");
2092         if (IS_ERR(nfc->dmac)) {
2093                 ret = PTR_ERR(nfc->dmac);
2094                 if (ret == -EPROBE_DEFER)
2095                         return ret;
2096
2097                 /* Ignore errors to fall back to PIO mode */
2098                 dev_warn(nfc->dev, "failed to request rxtx DMA channel: %d\n", ret);
2099                 nfc->dmac = NULL;
2100         } else {
2101                 struct dma_slave_config dmac_cfg = { };
2102
2103                 dmac_cfg.src_addr = r->start + nfc->caps->reg_io_data;
2104                 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2105                 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2106                 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2107                 dmac_cfg.src_maxburst = nfc->caps->dma_maxburst;
2108                 dmac_cfg.dst_maxburst = nfc->caps->dma_maxburst;
2109                 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2110         }
2111         return 0;
2112 }
2113
2114 static int sunxi_nfc_probe(struct platform_device *pdev)
2115 {
2116         struct device *dev = &pdev->dev;
2117         struct resource *r;
2118         struct sunxi_nfc *nfc;
2119         int irq;
2120         int ret;
2121
2122         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2123         if (!nfc)
2124                 return -ENOMEM;
2125
2126         nfc->dev = dev;
2127         nand_controller_init(&nfc->controller);
2128         INIT_LIST_HEAD(&nfc->chips);
2129
2130         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2131         nfc->regs = devm_ioremap_resource(dev, r);
2132         if (IS_ERR(nfc->regs))
2133                 return PTR_ERR(nfc->regs);
2134
2135         irq = platform_get_irq(pdev, 0);
2136         if (irq < 0)
2137                 return irq;
2138
2139         nfc->ahb_clk = devm_clk_get(dev, "ahb");
2140         if (IS_ERR(nfc->ahb_clk)) {
2141                 dev_err(dev, "failed to retrieve ahb clk\n");
2142                 return PTR_ERR(nfc->ahb_clk);
2143         }
2144
2145         ret = clk_prepare_enable(nfc->ahb_clk);
2146         if (ret)
2147                 return ret;
2148
2149         nfc->mod_clk = devm_clk_get(dev, "mod");
2150         if (IS_ERR(nfc->mod_clk)) {
2151                 dev_err(dev, "failed to retrieve mod clk\n");
2152                 ret = PTR_ERR(nfc->mod_clk);
2153                 goto out_ahb_clk_unprepare;
2154         }
2155
2156         ret = clk_prepare_enable(nfc->mod_clk);
2157         if (ret)
2158                 goto out_ahb_clk_unprepare;
2159
2160         nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
2161         if (IS_ERR(nfc->reset)) {
2162                 ret = PTR_ERR(nfc->reset);
2163                 goto out_mod_clk_unprepare;
2164         }
2165
2166         ret = reset_control_deassert(nfc->reset);
2167         if (ret) {
2168                 dev_err(dev, "reset err %d\n", ret);
2169                 goto out_mod_clk_unprepare;
2170         }
2171
2172         nfc->caps = of_device_get_match_data(&pdev->dev);
2173         if (!nfc->caps) {
2174                 ret = -EINVAL;
2175                 goto out_ahb_reset_reassert;
2176         }
2177
2178         ret = sunxi_nfc_rst(nfc);
2179         if (ret)
2180                 goto out_ahb_reset_reassert;
2181
2182         writel(0, nfc->regs + NFC_REG_INT);
2183         ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2184                                0, "sunxi-nand", nfc);
2185         if (ret)
2186                 goto out_ahb_reset_reassert;
2187
2188         ret = sunxi_nfc_dma_init(nfc, r);
2189
2190         if (ret)
2191                 goto out_ahb_reset_reassert;
2192
2193         platform_set_drvdata(pdev, nfc);
2194
2195         ret = sunxi_nand_chips_init(dev, nfc);
2196         if (ret) {
2197                 dev_err(dev, "failed to init nand chips\n");
2198                 goto out_release_dmac;
2199         }
2200
2201         return 0;
2202
2203 out_release_dmac:
2204         if (nfc->dmac)
2205                 dma_release_channel(nfc->dmac);
2206 out_ahb_reset_reassert:
2207         reset_control_assert(nfc->reset);
2208 out_mod_clk_unprepare:
2209         clk_disable_unprepare(nfc->mod_clk);
2210 out_ahb_clk_unprepare:
2211         clk_disable_unprepare(nfc->ahb_clk);
2212
2213         return ret;
2214 }
2215
2216 static int sunxi_nfc_remove(struct platform_device *pdev)
2217 {
2218         struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2219
2220         sunxi_nand_chips_cleanup(nfc);
2221
2222         reset_control_assert(nfc->reset);
2223
2224         if (nfc->dmac)
2225                 dma_release_channel(nfc->dmac);
2226         clk_disable_unprepare(nfc->mod_clk);
2227         clk_disable_unprepare(nfc->ahb_clk);
2228
2229         return 0;
2230 }
2231
2232 static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = {
2233         .reg_io_data = NFC_REG_A10_IO_DATA,
2234         .dma_maxburst = 4,
2235 };
2236
2237 static const struct sunxi_nfc_caps sunxi_nfc_a23_caps = {
2238         .has_mdma = true,
2239         .reg_io_data = NFC_REG_A23_IO_DATA,
2240         .dma_maxburst = 8,
2241 };
2242
2243 static const struct of_device_id sunxi_nfc_ids[] = {
2244         {
2245                 .compatible = "allwinner,sun4i-a10-nand",
2246                 .data = &sunxi_nfc_a10_caps,
2247         },
2248         {
2249                 .compatible = "allwinner,sun8i-a23-nand-controller",
2250                 .data = &sunxi_nfc_a23_caps,
2251         },
2252         { /* sentinel */ }
2253 };
2254 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2255
2256 static struct platform_driver sunxi_nfc_driver = {
2257         .driver = {
2258                 .name = "sunxi_nand",
2259                 .of_match_table = sunxi_nfc_ids,
2260         },
2261         .probe = sunxi_nfc_probe,
2262         .remove = sunxi_nfc_remove,
2263 };
2264 module_platform_driver(sunxi_nfc_driver);
2265
2266 MODULE_LICENSE("GPL");
2267 MODULE_AUTHOR("Boris BREZILLON");
2268 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2269 MODULE_ALIAS("platform:sunxi_nand");