drivers/mtd/nand/ecc.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Generic Error-Correcting Code (ECC) engine
   4  *
   5  * Copyright (C) 2019 Macronix
   6  * Author:
   7  *     Miquèl RAYNAL <miquel.raynal@bootlin.com>
   8  *
   9  *
  10  * This file describes the abstraction of any NAND ECC engine. It has been
  11  * designed to fit most cases, including parallel NANDs and SPI-NANDs.
  12  *
  13  * There are three main situations where instantiating this ECC engine makes
  14  * sense:
  15  *   - external: The ECC engine is outside the NAND pipeline, typically this
  16  *               is a software ECC engine, or an hardware engine that is
  17  *               outside the NAND controller pipeline.
  18  *   - pipelined: The ECC engine is inside the NAND pipeline, ie. on the
  19  *                controller's side. This is the case of most of the raw NAND
  20  *                controllers. In the pipeline case, the ECC bytes are
  21  *                generated/data corrected on the fly when a page is
  22  *                written/read.
  23  *   - ondie: The ECC engine is inside the NAND pipeline, on the chip's side.
  24  *            Some NAND chips can correct themselves the data.
  25  *
  26  * Besides the initial setup and final cleanups, the interfaces are rather
  27  * simple:
  28  *   - prepare: Prepare an I/O request. Enable/disable the ECC engine based on
  29  *              the I/O request type. In case of software correction or external
  30  *              engine, this step may involve to derive the ECC bytes and place
  31  *              them in the OOB area before a write.
  32  *   - finish: Finish an I/O request. Correct the data in case of a read
  33  *             request and report the number of corrected bits/uncorrectable
  34  *             errors. Most likely empty for write operations, unless you have
  35  *             hardware specific stuff to do, like shutting down the engine to
  36  *             save power.
  37  *
  38  * The I/O request should be enclosed in a prepare()/finish() pair of calls
  39  * and will behave differently depending on the requested I/O type:
  40  *   - raw: Correction disabled
  41  *   - ecc: Correction enabled
  42  *
  43  * The request direction is impacting the logic as well:
  44  *   - read: Load data from the NAND chip
  45  *   - write: Store data in the NAND chip
  46  *
  47  * Mixing all this combinations together gives the following behavior.
  48  * Those are just examples, drivers are free to add custom steps in their
  49  * prepare/finish hook.
  50  *
  51  * [external ECC engine]
  52  *   - external + prepare + raw + read: do nothing
  53  *   - external + finish  + raw + read: do nothing
  54  *   - external + prepare + raw + write: do nothing
  55  *   - external + finish  + raw + write: do nothing
  56  *   - external + prepare + ecc + read: do nothing
  57  *   - external + finish  + ecc + read: calculate expected ECC bytes, extract
  58  *                                      ECC bytes from OOB buffer, correct
  59  *                                      and report any bitflip/error
  60  *   - external + prepare + ecc + write: calculate ECC bytes and store them at
  61  *                                       the right place in the OOB buffer based
  62  *                                       on the OOB layout
  63  *   - external + finish  + ecc + write: do nothing
  64  *
  65  * [pipelined ECC engine]
  66  *   - pipelined + prepare + raw + read: disable the controller's ECC engine if
  67  *                                       activated
  68  *   - pipelined + finish  + raw + read: do nothing
  69  *   - pipelined + prepare + raw + write: disable the controller's ECC engine if
  70  *                                        activated
  71  *   - pipelined + finish  + raw + write: do nothing
  72  *   - pipelined + prepare + ecc + read: enable the controller's ECC engine if
  73  *                                       deactivated
  74  *   - pipelined + finish  + ecc + read: check the status, report any
  75  *                                       error/bitflip
  76  *   - pipelined + prepare + ecc + write: enable the controller's ECC engine if
  77  *                                        deactivated
  78  *   - pipelined + finish  + ecc + write: do nothing
  79  *
  80  * [ondie ECC engine]
  81  *   - ondie + prepare + raw + read: send commands to disable the on-chip ECC
  82  *                                   engine if activated
  83  *   - ondie + finish  + raw + read: do nothing
  84  *   - ondie + prepare + raw + write: send commands to disable the on-chip ECC
  85  *                                    engine if activated
  86  *   - ondie + finish  + raw + write: do nothing
  87  *   - ondie + prepare + ecc + read: send commands to enable the on-chip ECC
  88  *                                   engine if deactivated
  89  *   - ondie + finish  + ecc + read: send commands to check the status, report
  90  *                                   any error/bitflip
  91  *   - ondie + prepare + ecc + write: send commands to enable the on-chip ECC
  92  *                                    engine if deactivated
  93  *   - ondie + finish  + ecc + write: do nothing
  94  */
  95
  96 #include <linux/module.h>
  97 #include <linux/mtd/nand.h>
  98 #include <linux/slab.h>
  99
 100 /**
 101  * nand_ecc_init_ctx - Init the ECC engine context
 102  * @nand: the NAND device
 103  *
 104  * On success, the caller is responsible of calling @nand_ecc_cleanup_ctx().
 105  */
 106 int nand_ecc_init_ctx(struct nand_device *nand)
 107 {
 108         if (!nand->ecc.engine || !nand->ecc.engine->ops->init_ctx)
 109                 return 0;
 110
 111         return nand->ecc.engine->ops->init_ctx(nand);
 112 }
 113 EXPORT_SYMBOL(nand_ecc_init_ctx);
 114
 115 /**
 116  * nand_ecc_cleanup_ctx - Cleanup the ECC engine context
 117  * @nand: the NAND device
 118  */
 119 void nand_ecc_cleanup_ctx(struct nand_device *nand)
 120 {
 121         if (nand->ecc.engine && nand->ecc.engine->ops->cleanup_ctx)
 122                 nand->ecc.engine->ops->cleanup_ctx(nand);
 123 }
 124 EXPORT_SYMBOL(nand_ecc_cleanup_ctx);
 125
 126 /**
 127  * nand_ecc_prepare_io_req - Prepare an I/O request
 128  * @nand: the NAND device
 129  * @req: the I/O request
 130  */
 131 int nand_ecc_prepare_io_req(struct nand_device *nand,
 132                             struct nand_page_io_req *req)
 133 {
 134         if (!nand->ecc.engine || !nand->ecc.engine->ops->prepare_io_req)
 135                 return 0;
 136
 137         return nand->ecc.engine->ops->prepare_io_req(nand, req);
 138 }
 139 EXPORT_SYMBOL(nand_ecc_prepare_io_req);
 140
 141 /**
 142  * nand_ecc_finish_io_req - Finish an I/O request
 143  * @nand: the NAND device
 144  * @req: the I/O request
 145  */
 146 int nand_ecc_finish_io_req(struct nand_device *nand,
 147                            struct nand_page_io_req *req)
 148 {
 149         if (!nand->ecc.engine || !nand->ecc.engine->ops->finish_io_req)
 150                 return 0;
 151
 152         return nand->ecc.engine->ops->finish_io_req(nand, req);
 153 }
 154 EXPORT_SYMBOL(nand_ecc_finish_io_req);
 155
 156 /* Define default OOB placement schemes for large and small page devices */
 157 static int nand_ooblayout_ecc_sp(struct mtd_info *mtd, int section,
 158                                  struct mtd_oob_region *oobregion)
 159 {
 160         struct nand_device *nand = mtd_to_nanddev(mtd);
 161         unsigned int total_ecc_bytes = nand->ecc.ctx.total;
 162
 163         if (section > 1)
 164                 return -ERANGE;
 165
 166         if (!section) {
 167                 oobregion->offset = 0;
 168                 if (mtd->oobsize == 16)
 169                         oobregion->length = 4;
 170                 else
 171                         oobregion->length = 3;
 172         } else {
 173                 if (mtd->oobsize == 8)
 174                         return -ERANGE;
 175
 176                 oobregion->offset = 6;
 177                 oobregion->length = total_ecc_bytes - 4;
 178         }
 179
 180         return 0;
 181 }
 182
 183 static int nand_ooblayout_free_sp(struct mtd_info *mtd, int section,
 184                                   struct mtd_oob_region *oobregion)
 185 {
 186         if (section > 1)
 187                 return -ERANGE;
 188
 189         if (mtd->oobsize == 16) {
 190                 if (section)
 191                         return -ERANGE;
 192
 193                 oobregion->length = 8;
 194                 oobregion->offset = 8;
 195         } else {
 196                 oobregion->length = 2;
 197                 if (!section)
 198                         oobregion->offset = 3;
 199                 else
 200                         oobregion->offset = 6;
 201         }
 202
 203         return 0;
 204 }
 205
 206 static const struct mtd_ooblayout_ops nand_ooblayout_sp_ops = {
 207         .ecc = nand_ooblayout_ecc_sp,
 208         .free = nand_ooblayout_free_sp,
 209 };
 210
 211 const struct mtd_ooblayout_ops *nand_get_small_page_ooblayout(void)
 212 {
 213         return &nand_ooblayout_sp_ops;
 214 }
 215 EXPORT_SYMBOL_GPL(nand_get_small_page_ooblayout);
 216
 217 static int nand_ooblayout_ecc_lp(struct mtd_info *mtd, int section,
 218                                  struct mtd_oob_region *oobregion)
 219 {
 220         struct nand_device *nand = mtd_to_nanddev(mtd);
 221         unsigned int total_ecc_bytes = nand->ecc.ctx.total;
 222
 223         if (section || !total_ecc_bytes)
 224                 return -ERANGE;
 225
 226         oobregion->length = total_ecc_bytes;
 227         oobregion->offset = mtd->oobsize - oobregion->length;
 228
 229         return 0;
 230 }
 231
 232 static int nand_ooblayout_free_lp(struct mtd_info *mtd, int section,
 233                                   struct mtd_oob_region *oobregion)
 234 {
 235         struct nand_device *nand = mtd_to_nanddev(mtd);
 236         unsigned int total_ecc_bytes = nand->ecc.ctx.total;
 237
 238         if (section)
 239                 return -ERANGE;
 240
 241         oobregion->length = mtd->oobsize - total_ecc_bytes - 2;
 242         oobregion->offset = 2;
 243
 244         return 0;
 245 }
 246
 247 static const struct mtd_ooblayout_ops nand_ooblayout_lp_ops = {
 248         .ecc = nand_ooblayout_ecc_lp,
 249         .free = nand_ooblayout_free_lp,
 250 };
 251
 252 const struct mtd_ooblayout_ops *nand_get_large_page_ooblayout(void)
 253 {
 254         return &nand_ooblayout_lp_ops;
 255 }
 256 EXPORT_SYMBOL_GPL(nand_get_large_page_ooblayout);
 257
 258 /*
 259  * Support the old "large page" layout used for 1-bit Hamming ECC where ECC
 260  * are placed at a fixed offset.
 261  */
 262 static int nand_ooblayout_ecc_lp_hamming(struct mtd_info *mtd, int section,
 263                                          struct mtd_oob_region *oobregion)
 264 {
 265         struct nand_device *nand = mtd_to_nanddev(mtd);
 266         unsigned int total_ecc_bytes = nand->ecc.ctx.total;
 267
 268         if (section)
 269                 return -ERANGE;
 270
 271         switch (mtd->oobsize) {
 272         case 64:
 273                 oobregion->offset = 40;
 274                 break;
 275         case 128:
 276                 oobregion->offset = 80;
 277                 break;
 278         default:
 279                 return -EINVAL;
 280         }
 281
 282         oobregion->length = total_ecc_bytes;
 283         if (oobregion->offset + oobregion->length > mtd->oobsize)
 284                 return -ERANGE;
 285
 286         return 0;
 287 }
 288
 289 static int nand_ooblayout_free_lp_hamming(struct mtd_info *mtd, int section,
 290                                           struct mtd_oob_region *oobregion)
 291 {
 292         struct nand_device *nand = mtd_to_nanddev(mtd);
 293         unsigned int total_ecc_bytes = nand->ecc.ctx.total;
 294         int ecc_offset = 0;
 295
 296         if (section < 0 || section > 1)
 297                 return -ERANGE;
 298
 299         switch (mtd->oobsize) {
 300         case 64:
 301                 ecc_offset = 40;
 302                 break;
 303         case 128:
 304                 ecc_offset = 80;
 305                 break;
 306         default:
 307                 return -EINVAL;
 308         }
 309
 310         if (section == 0) {
 311                 oobregion->offset = 2;
 312                 oobregion->length = ecc_offset - 2;
 313         } else {
 314                 oobregion->offset = ecc_offset + total_ecc_bytes;
 315                 oobregion->length = mtd->oobsize - oobregion->offset;
 316         }
 317
 318         return 0;
 319 }
 320
 321 static const struct mtd_ooblayout_ops nand_ooblayout_lp_hamming_ops = {
 322         .ecc = nand_ooblayout_ecc_lp_hamming,
 323         .free = nand_ooblayout_free_lp_hamming,
 324 };
 325
 326 const struct mtd_ooblayout_ops *nand_get_large_page_hamming_ooblayout(void)
 327 {
 328         return &nand_ooblayout_lp_hamming_ops;
 329 }
 330 EXPORT_SYMBOL_GPL(nand_get_large_page_hamming_ooblayout);
 331
 332 static enum nand_ecc_engine_type
 333 of_get_nand_ecc_engine_type(struct device_node *np)
 334 {
 335         struct device_node *eng_np;
 336
 337         if (of_property_read_bool(np, "nand-no-ecc-engine"))
 338                 return NAND_ECC_ENGINE_TYPE_NONE;
 339
 340         if (of_property_read_bool(np, "nand-use-soft-ecc-engine"))
 341                 return NAND_ECC_ENGINE_TYPE_SOFT;
 342
 343         eng_np = of_parse_phandle(np, "nand-ecc-engine", 0);
 344         of_node_put(eng_np);
 345
 346         if (eng_np) {
 347                 if (eng_np == np)
 348                         return NAND_ECC_ENGINE_TYPE_ON_DIE;
 349                 else
 350                         return NAND_ECC_ENGINE_TYPE_ON_HOST;
 351         }
 352
 353         return NAND_ECC_ENGINE_TYPE_INVALID;
 354 }
 355
 356 static const char * const nand_ecc_placement[] = {
 357         [NAND_ECC_PLACEMENT_OOB] = "oob",
 358         [NAND_ECC_PLACEMENT_INTERLEAVED] = "interleaved",
 359 };
 360
 361 static enum nand_ecc_placement of_get_nand_ecc_placement(struct device_node *np)
 362 {
 363         enum nand_ecc_placement placement;
 364         const char *pm;
 365         int err;
 366
 367         err = of_property_read_string(np, "nand-ecc-placement", &pm);
 368         if (!err) {
 369                 for (placement = NAND_ECC_PLACEMENT_OOB;
 370                      placement < ARRAY_SIZE(nand_ecc_placement); placement++) {
 371                         if (!strcasecmp(pm, nand_ecc_placement[placement]))
 372                                 return placement;
 373                 }
 374         }
 375
 376         return NAND_ECC_PLACEMENT_UNKNOWN;
 377 }
 378
 379 static const char * const nand_ecc_algos[] = {
 380         [NAND_ECC_ALGO_HAMMING] = "hamming",
 381         [NAND_ECC_ALGO_BCH] = "bch",
 382         [NAND_ECC_ALGO_RS] = "rs",
 383 };
 384
 385 static enum nand_ecc_algo of_get_nand_ecc_algo(struct device_node *np)
 386 {
 387         enum nand_ecc_algo ecc_algo;
 388         const char *pm;
 389         int err;
 390
 391         err = of_property_read_string(np, "nand-ecc-algo", &pm);
 392         if (!err) {
 393                 for (ecc_algo = NAND_ECC_ALGO_HAMMING;
 394                      ecc_algo < ARRAY_SIZE(nand_ecc_algos);
 395                      ecc_algo++) {
 396                         if (!strcasecmp(pm, nand_ecc_algos[ecc_algo]))
 397                                 return ecc_algo;
 398                 }
 399         }
 400
 401         return NAND_ECC_ALGO_UNKNOWN;
 402 }
 403
 404 static int of_get_nand_ecc_step_size(struct device_node *np)
 405 {
 406         int ret;
 407         u32 val;
 408
 409         ret = of_property_read_u32(np, "nand-ecc-step-size", &val);
 410         return ret ? ret : val;
 411 }
 412
 413 static int of_get_nand_ecc_strength(struct device_node *np)
 414 {
 415         int ret;
 416         u32 val;
 417
 418         ret = of_property_read_u32(np, "nand-ecc-strength", &val);
 419         return ret ? ret : val;
 420 }
 421
 422 void of_get_nand_ecc_user_config(struct nand_device *nand)
 423 {
 424         struct device_node *dn = nanddev_get_of_node(nand);
 425         int strength, size;
 426
 427         nand->ecc.user_conf.engine_type = of_get_nand_ecc_engine_type(dn);
 428         nand->ecc.user_conf.algo = of_get_nand_ecc_algo(dn);
 429         nand->ecc.user_conf.placement = of_get_nand_ecc_placement(dn);
 430
 431         strength = of_get_nand_ecc_strength(dn);
 432         if (strength >= 0)
 433                 nand->ecc.user_conf.strength = strength;
 434
 435         size = of_get_nand_ecc_step_size(dn);
 436         if (size >= 0)
 437                 nand->ecc.user_conf.step_size = size;
 438
 439         if (of_property_read_bool(dn, "nand-ecc-maximize"))
 440                 nand->ecc.user_conf.flags |= NAND_ECC_MAXIMIZE_STRENGTH;
 441 }
 442 EXPORT_SYMBOL(of_get_nand_ecc_user_config);
 443
 444 /**
 445  * nand_ecc_is_strong_enough - Check if the chip configuration meets the
 446  *                             datasheet requirements.
 447  *
 448  * @nand: Device to check
 449  *
 450  * If our configuration corrects A bits per B bytes and the minimum
 451  * required correction level is X bits per Y bytes, then we must ensure
 452  * both of the following are true:
 453  *
 454  * (1) A / B >= X / Y
 455  * (2) A >= X
 456  *
 457  * Requirement (1) ensures we can correct for the required bitflip density.
 458  * Requirement (2) ensures we can correct even when all bitflips are clumped
 459  * in the same sector.
 460  */
 461 bool nand_ecc_is_strong_enough(struct nand_device *nand)
 462 {
 463         const struct nand_ecc_props *reqs = nanddev_get_ecc_requirements(nand);
 464         const struct nand_ecc_props *conf = nanddev_get_ecc_conf(nand);
 465         struct mtd_info *mtd = nanddev_to_mtd(nand);
 466         int corr, ds_corr;
 467
 468         if (conf->step_size == 0 || reqs->step_size == 0)
 469                 /* Not enough information */
 470                 return true;
 471
 472         /*
 473          * We get the number of corrected bits per page to compare
 474          * the correction density.
 475          */
 476         corr = (mtd->writesize * conf->strength) / conf->step_size;
 477         ds_corr = (mtd->writesize * reqs->strength) / reqs->step_size;
 478
 479         return corr >= ds_corr && conf->strength >= reqs->strength;
 480 }
 481 EXPORT_SYMBOL(nand_ecc_is_strong_enough);
 482
 483 /* ECC engine driver internal helpers */
 484 int nand_ecc_init_req_tweaking(struct nand_ecc_req_tweak_ctx *ctx,
 485                                struct nand_device *nand)
 486 {
 487         unsigned int total_buffer_size;
 488
 489         ctx->nand = nand;
 490
 491         /* Let the user decide the exact length of each buffer */
 492         if (!ctx->page_buffer_size)
 493                 ctx->page_buffer_size = nanddev_page_size(nand);
 494         if (!ctx->oob_buffer_size)
 495                 ctx->oob_buffer_size = nanddev_per_page_oobsize(nand);
 496
 497         total_buffer_size = ctx->page_buffer_size + ctx->oob_buffer_size;
 498
 499         ctx->spare_databuf = kzalloc(total_buffer_size, GFP_KERNEL);
 500         if (!ctx->spare_databuf)
 501                 return -ENOMEM;
 502
 503         ctx->spare_oobbuf = ctx->spare_databuf + ctx->page_buffer_size;
 504
 505         return 0;
 506 }
 507 EXPORT_SYMBOL_GPL(nand_ecc_init_req_tweaking);
 508
 509 void nand_ecc_cleanup_req_tweaking(struct nand_ecc_req_tweak_ctx *ctx)
 510 {
 511         kfree(ctx->spare_databuf);
 512 }
 513 EXPORT_SYMBOL_GPL(nand_ecc_cleanup_req_tweaking);
 514
 515 /*
 516  * Ensure data and OOB area is fully read/written otherwise the correction might
 517  * not work as expected.
 518  */
 519 void nand_ecc_tweak_req(struct nand_ecc_req_tweak_ctx *ctx,
 520                         struct nand_page_io_req *req)
 521 {
 522         struct nand_device *nand = ctx->nand;
 523         struct nand_page_io_req *orig, *tweak;
 524
 525         /* Save the original request */
 526         ctx->orig_req = *req;
 527         ctx->bounce_data = false;
 528         ctx->bounce_oob = false;
 529         orig = &ctx->orig_req;
 530         tweak = req;
 531
 532         /* Ensure the request covers the entire page */
 533         if (orig->datalen < nanddev_page_size(nand)) {
 534                 ctx->bounce_data = true;
 535                 tweak->dataoffs = 0;
 536                 tweak->datalen = nanddev_page_size(nand);
 537                 tweak->databuf.in = ctx->spare_databuf;
 538                 memset(tweak->databuf.in, 0xFF, ctx->page_buffer_size);
 539         }
 540
 541         if (orig->ooblen < nanddev_per_page_oobsize(nand)) {
 542                 ctx->bounce_oob = true;
 543                 tweak->ooboffs = 0;
 544                 tweak->ooblen = nanddev_per_page_oobsize(nand);
 545                 tweak->oobbuf.in = ctx->spare_oobbuf;
 546                 memset(tweak->oobbuf.in, 0xFF, ctx->oob_buffer_size);
 547         }
 548
 549         /* Copy the data that must be writen in the bounce buffers, if needed */
 550         if (orig->type == NAND_PAGE_WRITE) {
 551                 if (ctx->bounce_data)
 552                         memcpy((void *)tweak->databuf.out + orig->dataoffs,
 553                                orig->databuf.out, orig->datalen);
 554
 555                 if (ctx->bounce_oob)
 556                         memcpy((void *)tweak->oobbuf.out + orig->ooboffs,
 557                                orig->oobbuf.out, orig->ooblen);
 558         }
 559 }
 560 EXPORT_SYMBOL_GPL(nand_ecc_tweak_req);
 561
 562 void nand_ecc_restore_req(struct nand_ecc_req_tweak_ctx *ctx,
 563                           struct nand_page_io_req *req)
 564 {
 565         struct nand_page_io_req *orig, *tweak;
 566
 567         orig = &ctx->orig_req;
 568         tweak = req;
 569
 570         /* Restore the data read from the bounce buffers, if needed */
 571         if (orig->type == NAND_PAGE_READ) {
 572                 if (ctx->bounce_data)
 573                         memcpy(orig->databuf.in,
 574                                tweak->databuf.in + orig->dataoffs,
 575                                orig->datalen);
 576
 577                 if (ctx->bounce_oob)
 578                         memcpy(orig->oobbuf.in,
 579                                tweak->oobbuf.in + orig->ooboffs,
 580                                orig->ooblen);
 581         }
 582
 583         /* Ensure the original request is restored */
 584         *req = *orig;
 585 }
 586 EXPORT_SYMBOL_GPL(nand_ecc_restore_req);
 587
 588 struct nand_ecc_engine *nand_ecc_get_sw_engine(struct nand_device *nand)
 589 {
 590         unsigned int algo = nand->ecc.user_conf.algo;
 591
 592         if (algo == NAND_ECC_ALGO_UNKNOWN)
 593                 algo = nand->ecc.defaults.algo;
 594
 595         switch (algo) {
 596         case NAND_ECC_ALGO_HAMMING:
 597                 return nand_ecc_sw_hamming_get_engine();
 598         case NAND_ECC_ALGO_BCH:
 599                 return nand_ecc_sw_bch_get_engine();
 600         default:
 601                 break;
 602         }
 603
 604         return NULL;
 605 }
 606 EXPORT_SYMBOL(nand_ecc_get_sw_engine);
 607
 608 struct nand_ecc_engine *nand_ecc_get_on_die_hw_engine(struct nand_device *nand)
 609 {
 610         return nand->ecc.ondie_engine;
 611 }
 612 EXPORT_SYMBOL(nand_ecc_get_on_die_hw_engine);
 613
 614 MODULE_LICENSE("GPL");
 615 MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>");
 616 MODULE_DESCRIPTION("Generic ECC engine");