1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2017 Free Electrons 4 * Copyright (C) 2017 NextThing Co 5 * 6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com> 7 */ 8 9 #include <linux/slab.h> 10 11 #include "internals.h" 12 13 /* 14 * Special Micron status bit 3 indicates that the block has been 15 * corrected by on-die ECC and should be rewritten. 16 */ 17 #define NAND_ECC_STATUS_WRITE_RECOMMENDED BIT(3) 18 19 /* 20 * On chips with 8-bit ECC and additional bit can be used to distinguish 21 * cases where a errors were corrected without needing a rewrite 22 * 23 * Bit 4 Bit 3 Bit 0 Description 24 * ----- ----- ----- ----------- 25 * 0 0 0 No Errors 26 * 0 0 1 Multiple uncorrected errors 27 * 0 1 0 4 - 6 errors corrected, recommend rewrite 28 * 0 1 1 Reserved 29 * 1 0 0 1 - 3 errors corrected 30 * 1 0 1 Reserved 31 * 1 1 0 7 - 8 errors corrected, recommend rewrite 32 */ 33 #define NAND_ECC_STATUS_MASK (BIT(4) | BIT(3) | BIT(0)) 34 #define NAND_ECC_STATUS_UNCORRECTABLE BIT(0) 35 #define NAND_ECC_STATUS_4_6_CORRECTED BIT(3) 36 #define NAND_ECC_STATUS_1_3_CORRECTED BIT(4) 37 #define NAND_ECC_STATUS_7_8_CORRECTED (BIT(4) | BIT(3)) 38 39 struct nand_onfi_vendor_micron { 40 u8 two_plane_read; 41 u8 read_cache; 42 u8 read_unique_id; 43 u8 dq_imped; 44 u8 dq_imped_num_settings; 45 u8 dq_imped_feat_addr; 46 u8 rb_pulldown_strength; 47 u8 rb_pulldown_strength_feat_addr; 48 u8 rb_pulldown_strength_num_settings; 49 u8 otp_mode; 50 u8 otp_page_start; 51 u8 otp_data_prot_addr; 52 u8 otp_num_pages; 53 u8 otp_feat_addr; 54 u8 read_retry_options; 55 u8 reserved[72]; 56 u8 param_revision; 57 } __packed; 58 59 struct micron_on_die_ecc { 60 bool forced; 61 bool enabled; 62 void *rawbuf; 63 }; 64 65 struct micron_nand { 66 struct micron_on_die_ecc ecc; 67 }; 68 69 static int micron_nand_setup_read_retry(struct nand_chip *chip, int retry_mode) 70 { 71 u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode}; 72 73 return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature); 74 } 75 76 /* 77 * Configure chip properties from Micron vendor-specific ONFI table 78 */ 79 static int micron_nand_onfi_init(struct nand_chip *chip) 80 { 81 struct nand_parameters *p = &chip->parameters; 82 83 if (p->onfi) { 84 struct nand_onfi_vendor_micron *micron = (void *)p->onfi->vendor; 85 86 chip->read_retries = micron->read_retry_options; 87 chip->setup_read_retry = micron_nand_setup_read_retry; 88 } 89 90 if (p->supports_set_get_features) { 91 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->set_feature_list); 92 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->set_feature_list); 93 set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->get_feature_list); 94 set_bit(ONFI_FEATURE_ON_DIE_ECC, p->get_feature_list); 95 } 96 97 return 0; 98 } 99 100 static int micron_nand_on_die_4_ooblayout_ecc(struct mtd_info *mtd, 101 int section, 102 struct mtd_oob_region *oobregion) 103 { 104 if (section >= 4) 105 return -ERANGE; 106 107 oobregion->offset = (section * 16) + 8; 108 oobregion->length = 8; 109 110 return 0; 111 } 112 113 static int micron_nand_on_die_4_ooblayout_free(struct mtd_info *mtd, 114 int section, 115 struct mtd_oob_region *oobregion) 116 { 117 if (section >= 4) 118 return -ERANGE; 119 120 oobregion->offset = (section * 16) + 2; 121 oobregion->length = 6; 122 123 return 0; 124 } 125 126 static const struct mtd_ooblayout_ops micron_nand_on_die_4_ooblayout_ops = { 127 .ecc = micron_nand_on_die_4_ooblayout_ecc, 128 .free = micron_nand_on_die_4_ooblayout_free, 129 }; 130 131 static int micron_nand_on_die_8_ooblayout_ecc(struct mtd_info *mtd, 132 int section, 133 struct mtd_oob_region *oobregion) 134 { 135 struct nand_chip *chip = mtd_to_nand(mtd); 136 137 if (section) 138 return -ERANGE; 139 140 oobregion->offset = mtd->oobsize - chip->ecc.total; 141 oobregion->length = chip->ecc.total; 142 143 return 0; 144 } 145 146 static int micron_nand_on_die_8_ooblayout_free(struct mtd_info *mtd, 147 int section, 148 struct mtd_oob_region *oobregion) 149 { 150 struct nand_chip *chip = mtd_to_nand(mtd); 151 152 if (section) 153 return -ERANGE; 154 155 oobregion->offset = 2; 156 oobregion->length = mtd->oobsize - chip->ecc.total - 2; 157 158 return 0; 159 } 160 161 static const struct mtd_ooblayout_ops micron_nand_on_die_8_ooblayout_ops = { 162 .ecc = micron_nand_on_die_8_ooblayout_ecc, 163 .free = micron_nand_on_die_8_ooblayout_free, 164 }; 165 166 static int micron_nand_on_die_ecc_setup(struct nand_chip *chip, bool enable) 167 { 168 struct micron_nand *micron = nand_get_manufacturer_data(chip); 169 u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = { 0, }; 170 int ret; 171 172 if (micron->ecc.forced) 173 return 0; 174 175 if (micron->ecc.enabled == enable) 176 return 0; 177 178 if (enable) 179 feature[0] |= ONFI_FEATURE_ON_DIE_ECC_EN; 180 181 ret = nand_set_features(chip, ONFI_FEATURE_ON_DIE_ECC, feature); 182 if (!ret) 183 micron->ecc.enabled = enable; 184 185 return ret; 186 } 187 188 static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status, 189 void *buf, int page, 190 int oob_required) 191 { 192 struct micron_nand *micron = nand_get_manufacturer_data(chip); 193 struct mtd_info *mtd = nand_to_mtd(chip); 194 unsigned int step, max_bitflips = 0; 195 int ret; 196 197 if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) { 198 if (status & NAND_STATUS_FAIL) 199 mtd->ecc_stats.failed++; 200 201 return 0; 202 } 203 204 /* 205 * The internal ECC doesn't tell us the number of bitflips that have 206 * been corrected, but tells us if it recommends to rewrite the block. 207 * If it's the case, we need to read the page in raw mode and compare 208 * its content to the corrected version to extract the actual number of 209 * bitflips. 210 * But before we do that, we must make sure we have all OOB bytes read 211 * in non-raw mode, even if the user did not request those bytes. 212 */ 213 if (!oob_required) { 214 ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, 215 false); 216 if (ret) 217 return ret; 218 } 219 220 micron_nand_on_die_ecc_setup(chip, false); 221 222 ret = nand_read_page_op(chip, page, 0, micron->ecc.rawbuf, 223 mtd->writesize + mtd->oobsize); 224 if (ret) 225 return ret; 226 227 for (step = 0; step < chip->ecc.steps; step++) { 228 unsigned int offs, i, nbitflips = 0; 229 u8 *rawbuf, *corrbuf; 230 231 offs = step * chip->ecc.size; 232 rawbuf = micron->ecc.rawbuf + offs; 233 corrbuf = buf + offs; 234 235 for (i = 0; i < chip->ecc.size; i++) 236 nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]); 237 238 offs = (step * 16) + 4; 239 rawbuf = micron->ecc.rawbuf + mtd->writesize + offs; 240 corrbuf = chip->oob_poi + offs; 241 242 for (i = 0; i < chip->ecc.bytes + 4; i++) 243 nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]); 244 245 if (WARN_ON(nbitflips > chip->ecc.strength)) 246 return -EINVAL; 247 248 max_bitflips = max(nbitflips, max_bitflips); 249 mtd->ecc_stats.corrected += nbitflips; 250 } 251 252 return max_bitflips; 253 } 254 255 static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status) 256 { 257 struct mtd_info *mtd = nand_to_mtd(chip); 258 259 /* 260 * With 8/512 we have more information but still don't know precisely 261 * how many bit-flips were seen. 262 */ 263 switch (status & NAND_ECC_STATUS_MASK) { 264 case NAND_ECC_STATUS_UNCORRECTABLE: 265 mtd->ecc_stats.failed++; 266 return 0; 267 case NAND_ECC_STATUS_1_3_CORRECTED: 268 mtd->ecc_stats.corrected += 3; 269 return 3; 270 case NAND_ECC_STATUS_4_6_CORRECTED: 271 mtd->ecc_stats.corrected += 6; 272 /* rewrite recommended */ 273 return 6; 274 case NAND_ECC_STATUS_7_8_CORRECTED: 275 mtd->ecc_stats.corrected += 8; 276 /* rewrite recommended */ 277 return 8; 278 default: 279 return 0; 280 } 281 } 282 283 static int 284 micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf, 285 int oob_required, int page) 286 { 287 struct mtd_info *mtd = nand_to_mtd(chip); 288 u8 status; 289 int ret, max_bitflips = 0; 290 291 ret = micron_nand_on_die_ecc_setup(chip, true); 292 if (ret) 293 return ret; 294 295 ret = nand_read_page_op(chip, page, 0, NULL, 0); 296 if (ret) 297 goto out; 298 299 ret = nand_status_op(chip, &status); 300 if (ret) 301 goto out; 302 303 ret = nand_exit_status_op(chip); 304 if (ret) 305 goto out; 306 307 ret = nand_read_data_op(chip, buf, mtd->writesize, false); 308 if (!ret && oob_required) 309 ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, 310 false); 311 312 if (chip->ecc.strength == 4) 313 max_bitflips = micron_nand_on_die_ecc_status_4(chip, status, 314 buf, page, 315 oob_required); 316 else 317 max_bitflips = micron_nand_on_die_ecc_status_8(chip, status); 318 319 out: 320 micron_nand_on_die_ecc_setup(chip, false); 321 322 return ret ? ret : max_bitflips; 323 } 324 325 static int 326 micron_nand_write_page_on_die_ecc(struct nand_chip *chip, const uint8_t *buf, 327 int oob_required, int page) 328 { 329 int ret; 330 331 ret = micron_nand_on_die_ecc_setup(chip, true); 332 if (ret) 333 return ret; 334 335 ret = nand_write_page_raw(chip, buf, oob_required, page); 336 micron_nand_on_die_ecc_setup(chip, false); 337 338 return ret; 339 } 340 341 enum { 342 /* The NAND flash doesn't support on-die ECC */ 343 MICRON_ON_DIE_UNSUPPORTED, 344 345 /* 346 * The NAND flash supports on-die ECC and it can be 347 * enabled/disabled by a set features command. 348 */ 349 MICRON_ON_DIE_SUPPORTED, 350 351 /* 352 * The NAND flash supports on-die ECC, and it cannot be 353 * disabled. 354 */ 355 MICRON_ON_DIE_MANDATORY, 356 }; 357 358 #define MICRON_ID_INTERNAL_ECC_MASK GENMASK(1, 0) 359 #define MICRON_ID_ECC_ENABLED BIT(7) 360 361 /* 362 * Try to detect if the NAND support on-die ECC. To do this, we enable 363 * the feature, and read back if it has been enabled as expected. We 364 * also check if it can be disabled, because some Micron NANDs do not 365 * allow disabling the on-die ECC and we don't support such NANDs for 366 * now. 367 * 368 * This function also has the side effect of disabling on-die ECC if 369 * it had been left enabled by the firmware/bootloader. 370 */ 371 static int micron_supports_on_die_ecc(struct nand_chip *chip) 372 { 373 u8 id[5]; 374 int ret; 375 376 if (!chip->parameters.onfi) 377 return MICRON_ON_DIE_UNSUPPORTED; 378 379 if (nanddev_bits_per_cell(&chip->base) != 1) 380 return MICRON_ON_DIE_UNSUPPORTED; 381 382 /* 383 * We only support on-die ECC of 4/512 or 8/512 384 */ 385 if (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8) 386 return MICRON_ON_DIE_UNSUPPORTED; 387 388 /* 0x2 means on-die ECC is available. */ 389 if (chip->id.len != 5 || 390 (chip->id.data[4] & MICRON_ID_INTERNAL_ECC_MASK) != 0x2) 391 return MICRON_ON_DIE_UNSUPPORTED; 392 393 /* 394 * It seems that there are devices which do not support ECC officially. 395 * At least the MT29F2G08ABAGA / MT29F2G08ABBGA devices supports 396 * enabling the ECC feature but don't reflect that to the READ_ID table. 397 * So we have to guarantee that we disable the ECC feature directly 398 * after we did the READ_ID table command. Later we can evaluate the 399 * ECC_ENABLE support. 400 */ 401 ret = micron_nand_on_die_ecc_setup(chip, true); 402 if (ret) 403 return MICRON_ON_DIE_UNSUPPORTED; 404 405 ret = nand_readid_op(chip, 0, id, sizeof(id)); 406 if (ret) 407 return MICRON_ON_DIE_UNSUPPORTED; 408 409 ret = micron_nand_on_die_ecc_setup(chip, false); 410 if (ret) 411 return MICRON_ON_DIE_UNSUPPORTED; 412 413 if (!(id[4] & MICRON_ID_ECC_ENABLED)) 414 return MICRON_ON_DIE_UNSUPPORTED; 415 416 ret = nand_readid_op(chip, 0, id, sizeof(id)); 417 if (ret) 418 return MICRON_ON_DIE_UNSUPPORTED; 419 420 if (id[4] & MICRON_ID_ECC_ENABLED) 421 return MICRON_ON_DIE_MANDATORY; 422 423 /* 424 * We only support on-die ECC of 4/512 or 8/512 425 */ 426 if (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8) 427 return MICRON_ON_DIE_UNSUPPORTED; 428 429 return MICRON_ON_DIE_SUPPORTED; 430 } 431 432 static int micron_nand_init(struct nand_chip *chip) 433 { 434 struct mtd_info *mtd = nand_to_mtd(chip); 435 struct micron_nand *micron; 436 int ondie; 437 int ret; 438 439 micron = kzalloc(sizeof(*micron), GFP_KERNEL); 440 if (!micron) 441 return -ENOMEM; 442 443 nand_set_manufacturer_data(chip, micron); 444 445 ret = micron_nand_onfi_init(chip); 446 if (ret) 447 goto err_free_manuf_data; 448 449 if (mtd->writesize == 2048) 450 chip->options |= NAND_BBM_FIRSTPAGE | NAND_BBM_SECONDPAGE; 451 452 ondie = micron_supports_on_die_ecc(chip); 453 454 if (ondie == MICRON_ON_DIE_MANDATORY && 455 chip->ecc.mode != NAND_ECC_ON_DIE) { 456 pr_err("On-die ECC forcefully enabled, not supported\n"); 457 ret = -EINVAL; 458 goto err_free_manuf_data; 459 } 460 461 if (chip->ecc.mode == NAND_ECC_ON_DIE) { 462 if (ondie == MICRON_ON_DIE_UNSUPPORTED) { 463 pr_err("On-die ECC selected but not supported\n"); 464 ret = -EINVAL; 465 goto err_free_manuf_data; 466 } 467 468 if (ondie == MICRON_ON_DIE_MANDATORY) { 469 micron->ecc.forced = true; 470 micron->ecc.enabled = true; 471 } 472 473 /* 474 * In case of 4bit on-die ECC, we need a buffer to store a 475 * page dumped in raw mode so that we can compare its content 476 * to the same page after ECC correction happened and extract 477 * the real number of bitflips from this comparison. 478 * That's not needed for 8-bit ECC, because the status expose 479 * a better approximation of the number of bitflips in a page. 480 */ 481 if (chip->base.eccreq.strength == 4) { 482 micron->ecc.rawbuf = kmalloc(mtd->writesize + 483 mtd->oobsize, 484 GFP_KERNEL); 485 if (!micron->ecc.rawbuf) { 486 ret = -ENOMEM; 487 goto err_free_manuf_data; 488 } 489 } 490 491 if (chip->base.eccreq.strength == 4) 492 mtd_set_ooblayout(mtd, 493 µn_nand_on_die_4_ooblayout_ops); 494 else 495 mtd_set_ooblayout(mtd, 496 µn_nand_on_die_8_ooblayout_ops); 497 498 chip->ecc.bytes = chip->base.eccreq.strength * 2; 499 chip->ecc.size = 512; 500 chip->ecc.strength = chip->base.eccreq.strength; 501 chip->ecc.algo = NAND_ECC_BCH; 502 chip->ecc.read_page = micron_nand_read_page_on_die_ecc; 503 chip->ecc.write_page = micron_nand_write_page_on_die_ecc; 504 505 if (ondie == MICRON_ON_DIE_MANDATORY) { 506 chip->ecc.read_page_raw = nand_read_page_raw_notsupp; 507 chip->ecc.write_page_raw = nand_write_page_raw_notsupp; 508 } else { 509 chip->ecc.read_page_raw = nand_read_page_raw; 510 chip->ecc.write_page_raw = nand_write_page_raw; 511 } 512 } 513 514 return 0; 515 516 err_free_manuf_data: 517 kfree(micron->ecc.rawbuf); 518 kfree(micron); 519 520 return ret; 521 } 522 523 static void micron_nand_cleanup(struct nand_chip *chip) 524 { 525 struct micron_nand *micron = nand_get_manufacturer_data(chip); 526 527 kfree(micron->ecc.rawbuf); 528 kfree(micron); 529 } 530 531 static void micron_fixup_onfi_param_page(struct nand_chip *chip, 532 struct nand_onfi_params *p) 533 { 534 /* 535 * MT29F1G08ABAFAWP-ITE:F and possibly others report 00 00 for the 536 * revision number field of the ONFI parameter page. Assume ONFI 537 * version 1.0 if the revision number is 00 00. 538 */ 539 if (le16_to_cpu(p->revision) == 0) 540 p->revision = cpu_to_le16(ONFI_VERSION_1_0); 541 } 542 543 const struct nand_manufacturer_ops micron_nand_manuf_ops = { 544 .init = micron_nand_init, 545 .cleanup = micron_nand_cleanup, 546 .fixup_onfi_param_page = micron_fixup_onfi_param_page, 547 }; 548