drivers/mtd/nand/raw/nand_micron.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2017 Free Electrons
   4  * Copyright (C) 2017 NextThing Co
   5  *
   6  * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
   7  */
   8
   9 #include <linux/slab.h>
  10
  11 #include "internals.h"
  12
  13 /*
  14  * Special Micron status bit 3 indicates that the block has been
  15  * corrected by on-die ECC and should be rewritten.
  16  */
  17 #define NAND_ECC_STATUS_WRITE_RECOMMENDED       BIT(3)
  18
  19 /*
  20  * On chips with 8-bit ECC and additional bit can be used to distinguish
  21  * cases where a errors were corrected without needing a rewrite
  22  *
  23  * Bit 4 Bit 3 Bit 0 Description
  24  * ----- ----- ----- -----------
  25  * 0     0     0     No Errors
  26  * 0     0     1     Multiple uncorrected errors
  27  * 0     1     0     4 - 6 errors corrected, recommend rewrite
  28  * 0     1     1     Reserved
  29  * 1     0     0     1 - 3 errors corrected
  30  * 1     0     1     Reserved
  31  * 1     1     0     7 - 8 errors corrected, recommend rewrite
  32  */
  33 #define NAND_ECC_STATUS_MASK            (BIT(4) | BIT(3) | BIT(0))
  34 #define NAND_ECC_STATUS_UNCORRECTABLE   BIT(0)
  35 #define NAND_ECC_STATUS_4_6_CORRECTED   BIT(3)
  36 #define NAND_ECC_STATUS_1_3_CORRECTED   BIT(4)
  37 #define NAND_ECC_STATUS_7_8_CORRECTED   (BIT(4) | BIT(3))
  38
  39 struct nand_onfi_vendor_micron {
  40         u8 two_plane_read;
  41         u8 read_cache;
  42         u8 read_unique_id;
  43         u8 dq_imped;
  44         u8 dq_imped_num_settings;
  45         u8 dq_imped_feat_addr;
  46         u8 rb_pulldown_strength;
  47         u8 rb_pulldown_strength_feat_addr;
  48         u8 rb_pulldown_strength_num_settings;
  49         u8 otp_mode;
  50         u8 otp_page_start;
  51         u8 otp_data_prot_addr;
  52         u8 otp_num_pages;
  53         u8 otp_feat_addr;
  54         u8 read_retry_options;
  55         u8 reserved[72];
  56         u8 param_revision;
  57 } __packed;
  58
  59 struct micron_on_die_ecc {
  60         bool forced;
  61         bool enabled;
  62         void *rawbuf;
  63 };
  64
  65 struct micron_nand {
  66         struct micron_on_die_ecc ecc;
  67 };
  68
  69 static int micron_nand_setup_read_retry(struct nand_chip *chip, int retry_mode)
  70 {
  71         u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode};
  72
  73         return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature);
  74 }
  75
  76 /*
  77  * Configure chip properties from Micron vendor-specific ONFI table
  78  */
  79 static int micron_nand_onfi_init(struct nand_chip *chip)
  80 {
  81         struct nand_parameters *p = &chip->parameters;
  82
  83         if (p->onfi) {
  84                 struct nand_onfi_vendor_micron *micron = (void *)p->onfi->vendor;
  85
  86                 chip->read_retries = micron->read_retry_options;
  87                 chip->ops.setup_read_retry = micron_nand_setup_read_retry;
  88         }
  89
  90         if (p->supports_set_get_features) {
  91                 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->set_feature_list);
  92                 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->set_feature_list);
  93                 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->get_feature_list);
  94                 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->get_feature_list);
  95         }
  96
  97         return 0;
  98 }
  99
 100 static int micron_nand_on_die_4_ooblayout_ecc(struct mtd_info *mtd,
 101                                               int section,
 102                                               struct mtd_oob_region *oobregion)
 103 {
 104         if (section >= 4)
 105                 return -ERANGE;
 106
 107         oobregion->offset = (section * 16) + 8;
 108         oobregion->length = 8;
 109
 110         return 0;
 111 }
 112
 113 static int micron_nand_on_die_4_ooblayout_free(struct mtd_info *mtd,
 114                                                int section,
 115                                                struct mtd_oob_region *oobregion)
 116 {
 117         if (section >= 4)
 118                 return -ERANGE;
 119
 120         oobregion->offset = (section * 16) + 2;
 121         oobregion->length = 6;
 122
 123         return 0;
 124 }
 125
 126 static const struct mtd_ooblayout_ops micron_nand_on_die_4_ooblayout_ops = {
 127         .ecc = micron_nand_on_die_4_ooblayout_ecc,
 128         .free = micron_nand_on_die_4_ooblayout_free,
 129 };
 130
 131 static int micron_nand_on_die_8_ooblayout_ecc(struct mtd_info *mtd,
 132                                               int section,
 133                                               struct mtd_oob_region *oobregion)
 134 {
 135         struct nand_chip *chip = mtd_to_nand(mtd);
 136
 137         if (section)
 138                 return -ERANGE;
 139
 140         oobregion->offset = mtd->oobsize - chip->ecc.total;
 141         oobregion->length = chip->ecc.total;
 142
 143         return 0;
 144 }
 145
 146 static int micron_nand_on_die_8_ooblayout_free(struct mtd_info *mtd,
 147                                                int section,
 148                                                struct mtd_oob_region *oobregion)
 149 {
 150         struct nand_chip *chip = mtd_to_nand(mtd);
 151
 152         if (section)
 153                 return -ERANGE;
 154
 155         oobregion->offset = 2;
 156         oobregion->length = mtd->oobsize - chip->ecc.total - 2;
 157
 158         return 0;
 159 }
 160
 161 static const struct mtd_ooblayout_ops micron_nand_on_die_8_ooblayout_ops = {
 162         .ecc = micron_nand_on_die_8_ooblayout_ecc,
 163         .free = micron_nand_on_die_8_ooblayout_free,
 164 };
 165
 166 static int micron_nand_on_die_ecc_setup(struct nand_chip *chip, bool enable)
 167 {
 168         struct micron_nand *micron = nand_get_manufacturer_data(chip);
 169         u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = { 0, };
 170         int ret;
 171
 172         if (micron->ecc.forced)
 173                 return 0;
 174
 175         if (micron->ecc.enabled == enable)
 176                 return 0;
 177
 178         if (enable)
 179                 feature[0] |= ONFI_FEATURE_ON_DIE_ECC_EN;
 180
 181         ret = nand_set_features(chip, ONFI_FEATURE_ON_DIE_ECC, feature);
 182         if (!ret)
 183                 micron->ecc.enabled = enable;
 184
 185         return ret;
 186 }
 187
 188 static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status,
 189                                            void *buf, int page,
 190                                            int oob_required)
 191 {
 192         struct micron_nand *micron = nand_get_manufacturer_data(chip);
 193         struct mtd_info *mtd = nand_to_mtd(chip);
 194         unsigned int step, max_bitflips = 0;
 195         bool use_datain = false;
 196         int ret;
 197
 198         if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) {
 199                 if (status & NAND_STATUS_FAIL)
 200                         mtd->ecc_stats.failed++;
 201
 202                 return 0;
 203         }
 204
 205         /*
 206          * The internal ECC doesn't tell us the number of bitflips that have
 207          * been corrected, but tells us if it recommends to rewrite the block.
 208          * If it's the case, we need to read the page in raw mode and compare
 209          * its content to the corrected version to extract the actual number of
 210          * bitflips.
 211          * But before we do that, we must make sure we have all OOB bytes read
 212          * in non-raw mode, even if the user did not request those bytes.
 213          */
 214         if (!oob_required) {
 215                 /*
 216                  * We first check which operation is supported by the controller
 217                  * before running it. This trick makes it possible to support
 218                  * all controllers, even the most constraints, without almost
 219                  * any performance hit.
 220                  *
 221                  * TODO: could be enhanced to avoid repeating the same check
 222                  * over and over in the fast path.
 223                  */
 224                 if (!nand_has_exec_op(chip) ||
 225                     !nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false,
 226                                        true))
 227                         use_datain = true;
 228
 229                 if (use_datain)
 230                         ret = nand_read_data_op(chip, chip->oob_poi,
 231                                                 mtd->oobsize, false, false);
 232                 else
 233                         ret = nand_change_read_column_op(chip, mtd->writesize,
 234                                                          chip->oob_poi,
 235                                                          mtd->oobsize, false);
 236                 if (ret)
 237                         return ret;
 238         }
 239
 240         micron_nand_on_die_ecc_setup(chip, false);
 241
 242         ret = nand_read_page_op(chip, page, 0, micron->ecc.rawbuf,
 243                                 mtd->writesize + mtd->oobsize);
 244         if (ret)
 245                 return ret;
 246
 247         for (step = 0; step < chip->ecc.steps; step++) {
 248                 unsigned int offs, i, nbitflips = 0;
 249                 u8 *rawbuf, *corrbuf;
 250
 251                 offs = step * chip->ecc.size;
 252                 rawbuf = micron->ecc.rawbuf + offs;
 253                 corrbuf = buf + offs;
 254
 255                 for (i = 0; i < chip->ecc.size; i++)
 256                         nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);
 257
 258                 offs = (step * 16) + 4;
 259                 rawbuf = micron->ecc.rawbuf + mtd->writesize + offs;
 260                 corrbuf = chip->oob_poi + offs;
 261
 262                 for (i = 0; i < chip->ecc.bytes + 4; i++)
 263                         nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);
 264
 265                 if (WARN_ON(nbitflips > chip->ecc.strength))
 266                         return -EINVAL;
 267
 268                 max_bitflips = max(nbitflips, max_bitflips);
 269                 mtd->ecc_stats.corrected += nbitflips;
 270         }
 271
 272         return max_bitflips;
 273 }
 274
 275 static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status)
 276 {
 277         struct mtd_info *mtd = nand_to_mtd(chip);
 278
 279         /*
 280          * With 8/512 we have more information but still don't know precisely
 281          * how many bit-flips were seen.
 282          */
 283         switch (status & NAND_ECC_STATUS_MASK) {
 284         case NAND_ECC_STATUS_UNCORRECTABLE:
 285                 mtd->ecc_stats.failed++;
 286                 return 0;
 287         case NAND_ECC_STATUS_1_3_CORRECTED:
 288                 mtd->ecc_stats.corrected += 3;
 289                 return 3;
 290         case NAND_ECC_STATUS_4_6_CORRECTED:
 291                 mtd->ecc_stats.corrected += 6;
 292                 /* rewrite recommended */
 293                 return 6;
 294         case NAND_ECC_STATUS_7_8_CORRECTED:
 295                 mtd->ecc_stats.corrected += 8;
 296                 /* rewrite recommended */
 297                 return 8;
 298         default:
 299                 return 0;
 300         }
 301 }
 302
 303 static int
 304 micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf,
 305                                  int oob_required, int page)
 306 {
 307         struct mtd_info *mtd = nand_to_mtd(chip);
 308         bool use_datain = false;
 309         u8 status;
 310         int ret, max_bitflips = 0;
 311
 312         ret = micron_nand_on_die_ecc_setup(chip, true);
 313         if (ret)
 314                 return ret;
 315
 316         ret = nand_read_page_op(chip, page, 0, NULL, 0);
 317         if (ret)
 318                 goto out;
 319
 320         ret = nand_status_op(chip, &status);
 321         if (ret)
 322                 goto out;
 323
 324         /*
 325          * We first check which operation is supported by the controller before
 326          * running it. This trick makes it possible to support all controllers,
 327          * even the most constraints, without almost any performance hit.
 328          *
 329          * TODO: could be enhanced to avoid repeating the same check over and
 330          * over in the fast path.
 331          */
 332         if (!nand_has_exec_op(chip) ||
 333             !nand_read_data_op(chip, buf, mtd->writesize, false, true))
 334                 use_datain = true;
 335
 336         if (use_datain) {
 337                 ret = nand_exit_status_op(chip);
 338                 if (ret)
 339                         goto out;
 340
 341                 ret = nand_read_data_op(chip, buf, mtd->writesize, false,
 342                                         false);
 343                 if (!ret && oob_required)
 344                         ret = nand_read_data_op(chip, chip->oob_poi,
 345                                                 mtd->oobsize, false, false);
 346         } else {
 347                 ret = nand_change_read_column_op(chip, 0, buf, mtd->writesize,
 348                                                  false);
 349                 if (!ret && oob_required)
 350                         ret = nand_change_read_column_op(chip, mtd->writesize,
 351                                                          chip->oob_poi,
 352                                                          mtd->oobsize, false);
 353         }
 354
 355         if (chip->ecc.strength == 4)
 356                 max_bitflips = micron_nand_on_die_ecc_status_4(chip, status,
 357                                                                buf, page,
 358                                                                oob_required);
 359         else
 360                 max_bitflips = micron_nand_on_die_ecc_status_8(chip, status);
 361
 362 out:
 363         micron_nand_on_die_ecc_setup(chip, false);
 364
 365         return ret ? ret : max_bitflips;
 366 }
 367
 368 static int
 369 micron_nand_write_page_on_die_ecc(struct nand_chip *chip, const uint8_t *buf,
 370                                   int oob_required, int page)
 371 {
 372         int ret;
 373
 374         ret = micron_nand_on_die_ecc_setup(chip, true);
 375         if (ret)
 376                 return ret;
 377
 378         ret = nand_write_page_raw(chip, buf, oob_required, page);
 379         micron_nand_on_die_ecc_setup(chip, false);
 380
 381         return ret;
 382 }
 383
 384 enum {
 385         /* The NAND flash doesn't support on-die ECC */
 386         MICRON_ON_DIE_UNSUPPORTED,
 387
 388         /*
 389          * The NAND flash supports on-die ECC and it can be
 390          * enabled/disabled by a set features command.
 391          */
 392         MICRON_ON_DIE_SUPPORTED,
 393
 394         /*
 395          * The NAND flash supports on-die ECC, and it cannot be
 396          * disabled.
 397          */
 398         MICRON_ON_DIE_MANDATORY,
 399 };
 400
 401 #define MICRON_ID_INTERNAL_ECC_MASK     GENMASK(1, 0)
 402 #define MICRON_ID_ECC_ENABLED           BIT(7)
 403
 404 /*
 405  * Try to detect if the NAND support on-die ECC. To do this, we enable
 406  * the feature, and read back if it has been enabled as expected. We
 407  * also check if it can be disabled, because some Micron NANDs do not
 408  * allow disabling the on-die ECC and we don't support such NANDs for
 409  * now.
 410  *
 411  * This function also has the side effect of disabling on-die ECC if
 412  * it had been left enabled by the firmware/bootloader.
 413  */
 414 static int micron_supports_on_die_ecc(struct nand_chip *chip)
 415 {
 416         const struct nand_ecc_props *requirements =
 417                 nanddev_get_ecc_requirements(&chip->base);
 418         u8 id[5];
 419         int ret;
 420
 421         if (!chip->parameters.onfi)
 422                 return MICRON_ON_DIE_UNSUPPORTED;
 423
 424         if (nanddev_bits_per_cell(&chip->base) != 1)
 425                 return MICRON_ON_DIE_UNSUPPORTED;
 426
 427         /*
 428          * We only support on-die ECC of 4/512 or 8/512
 429          */
 430         if  (requirements->strength != 4 && requirements->strength != 8)
 431                 return MICRON_ON_DIE_UNSUPPORTED;
 432
 433         /* 0x2 means on-die ECC is available. */
 434         if (chip->id.len != 5 ||
 435             (chip->id.data[4] & MICRON_ID_INTERNAL_ECC_MASK) != 0x2)
 436                 return MICRON_ON_DIE_UNSUPPORTED;
 437
 438         /*
 439          * It seems that there are devices which do not support ECC officially.
 440          * At least the MT29F2G08ABAGA / MT29F2G08ABBGA devices supports
 441          * enabling the ECC feature but don't reflect that to the READ_ID table.
 442          * So we have to guarantee that we disable the ECC feature directly
 443          * after we did the READ_ID table command. Later we can evaluate the
 444          * ECC_ENABLE support.
 445          */
 446         ret = micron_nand_on_die_ecc_setup(chip, true);
 447         if (ret)
 448                 return MICRON_ON_DIE_UNSUPPORTED;
 449
 450         ret = nand_readid_op(chip, 0, id, sizeof(id));
 451         if (ret)
 452                 return MICRON_ON_DIE_UNSUPPORTED;
 453
 454         ret = micron_nand_on_die_ecc_setup(chip, false);
 455         if (ret)
 456                 return MICRON_ON_DIE_UNSUPPORTED;
 457
 458         if (!(id[4] & MICRON_ID_ECC_ENABLED))
 459                 return MICRON_ON_DIE_UNSUPPORTED;
 460
 461         ret = nand_readid_op(chip, 0, id, sizeof(id));
 462         if (ret)
 463                 return MICRON_ON_DIE_UNSUPPORTED;
 464
 465         if (id[4] & MICRON_ID_ECC_ENABLED)
 466                 return MICRON_ON_DIE_MANDATORY;
 467
 468         /*
 469          * We only support on-die ECC of 4/512 or 8/512
 470          */
 471         if  (requirements->strength != 4 && requirements->strength != 8)
 472                 return MICRON_ON_DIE_UNSUPPORTED;
 473
 474         return MICRON_ON_DIE_SUPPORTED;
 475 }
 476
 477 static int micron_nand_init(struct nand_chip *chip)
 478 {
 479         struct nand_device *base = &chip->base;
 480         const struct nand_ecc_props *requirements =
 481                 nanddev_get_ecc_requirements(base);
 482         struct mtd_info *mtd = nand_to_mtd(chip);
 483         struct micron_nand *micron;
 484         int ondie;
 485         int ret;
 486
 487         micron = kzalloc(sizeof(*micron), GFP_KERNEL);
 488         if (!micron)
 489                 return -ENOMEM;
 490
 491         nand_set_manufacturer_data(chip, micron);
 492
 493         ret = micron_nand_onfi_init(chip);
 494         if (ret)
 495                 goto err_free_manuf_data;
 496
 497         chip->options |= NAND_BBM_FIRSTPAGE;
 498
 499         if (mtd->writesize == 2048)
 500                 chip->options |= NAND_BBM_SECONDPAGE;
 501
 502         ondie = micron_supports_on_die_ecc(chip);
 503
 504         if (ondie == MICRON_ON_DIE_MANDATORY &&
 505             chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_DIE) {
 506                 pr_err("On-die ECC forcefully enabled, not supported\n");
 507                 ret = -EINVAL;
 508                 goto err_free_manuf_data;
 509         }
 510
 511         if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE) {
 512                 if (ondie == MICRON_ON_DIE_UNSUPPORTED) {
 513                         pr_err("On-die ECC selected but not supported\n");
 514                         ret = -EINVAL;
 515                         goto err_free_manuf_data;
 516                 }
 517
 518                 if (ondie == MICRON_ON_DIE_MANDATORY) {
 519                         micron->ecc.forced = true;
 520                         micron->ecc.enabled = true;
 521                 }
 522
 523                 /*
 524                  * In case of 4bit on-die ECC, we need a buffer to store a
 525                  * page dumped in raw mode so that we can compare its content
 526                  * to the same page after ECC correction happened and extract
 527                  * the real number of bitflips from this comparison.
 528                  * That's not needed for 8-bit ECC, because the status expose
 529                  * a better approximation of the number of bitflips in a page.
 530                  */
 531                 if (requirements->strength == 4) {
 532                         micron->ecc.rawbuf = kmalloc(mtd->writesize +
 533                                                      mtd->oobsize,
 534                                                      GFP_KERNEL);
 535                         if (!micron->ecc.rawbuf) {
 536                                 ret = -ENOMEM;
 537                                 goto err_free_manuf_data;
 538                         }
 539                 }
 540
 541                 if (requirements->strength == 4)
 542                         mtd_set_ooblayout(mtd,
 543                                           &micron_nand_on_die_4_ooblayout_ops);
 544                 else
 545                         mtd_set_ooblayout(mtd,
 546                                           &micron_nand_on_die_8_ooblayout_ops);
 547
 548                 chip->ecc.bytes = requirements->strength * 2;
 549                 chip->ecc.size = 512;
 550                 chip->ecc.strength = requirements->strength;
 551                 chip->ecc.algo = NAND_ECC_ALGO_BCH;
 552                 chip->ecc.read_page = micron_nand_read_page_on_die_ecc;
 553                 chip->ecc.write_page = micron_nand_write_page_on_die_ecc;
 554
 555                 if (ondie == MICRON_ON_DIE_MANDATORY) {
 556                         chip->ecc.read_page_raw = nand_read_page_raw_notsupp;
 557                         chip->ecc.write_page_raw = nand_write_page_raw_notsupp;
 558                 } else {
 559                         if (!chip->ecc.read_page_raw)
 560                                 chip->ecc.read_page_raw = nand_read_page_raw;
 561                         if (!chip->ecc.write_page_raw)
 562                                 chip->ecc.write_page_raw = nand_write_page_raw;
 563                 }
 564         }
 565
 566         return 0;
 567
 568 err_free_manuf_data:
 569         kfree(micron->ecc.rawbuf);
 570         kfree(micron);
 571
 572         return ret;
 573 }
 574
 575 static void micron_nand_cleanup(struct nand_chip *chip)
 576 {
 577         struct micron_nand *micron = nand_get_manufacturer_data(chip);
 578
 579         kfree(micron->ecc.rawbuf);
 580         kfree(micron);
 581 }
 582
 583 static void micron_fixup_onfi_param_page(struct nand_chip *chip,
 584                                          struct nand_onfi_params *p)
 585 {
 586         /*
 587          * MT29F1G08ABAFAWP-ITE:F and possibly others report 00 00 for the
 588          * revision number field of the ONFI parameter page. Assume ONFI
 589          * version 1.0 if the revision number is 00 00.
 590          */
 591         if (le16_to_cpu(p->revision) == 0)
 592                 p->revision = cpu_to_le16(ONFI_VERSION_1_0);
 593 }
 594
 595 const struct nand_manufacturer_ops micron_nand_manuf_ops = {
 596         .init = micron_nand_init,
 597         .cleanup = micron_nand_cleanup,
 598         .fixup_onfi_param_page = micron_fixup_onfi_param_page,
 599 };