1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2017 Free Electrons
4  * Copyright (C) 2017 NextThing Co
5  *
6  * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
7  */
8 
9 #include <linux/slab.h>
10 
11 #include "internals.h"
12 
13 /*
14  * Special Micron status bit 3 indicates that the block has been
15  * corrected by on-die ECC and should be rewritten.
16  */
17 #define NAND_ECC_STATUS_WRITE_RECOMMENDED	BIT(3)
18 
19 /*
20  * On chips with 8-bit ECC and additional bit can be used to distinguish
21  * cases where a errors were corrected without needing a rewrite
22  *
23  * Bit 4 Bit 3 Bit 0 Description
24  * ----- ----- ----- -----------
25  * 0     0     0     No Errors
26  * 0     0     1     Multiple uncorrected errors
27  * 0     1     0     4 - 6 errors corrected, recommend rewrite
28  * 0     1     1     Reserved
29  * 1     0     0     1 - 3 errors corrected
30  * 1     0     1     Reserved
31  * 1     1     0     7 - 8 errors corrected, recommend rewrite
32  */
33 #define NAND_ECC_STATUS_MASK		(BIT(4) | BIT(3) | BIT(0))
34 #define NAND_ECC_STATUS_UNCORRECTABLE	BIT(0)
35 #define NAND_ECC_STATUS_4_6_CORRECTED	BIT(3)
36 #define NAND_ECC_STATUS_1_3_CORRECTED	BIT(4)
37 #define NAND_ECC_STATUS_7_8_CORRECTED	(BIT(4) | BIT(3))
38 
39 struct nand_onfi_vendor_micron {
40 	u8 two_plane_read;
41 	u8 read_cache;
42 	u8 read_unique_id;
43 	u8 dq_imped;
44 	u8 dq_imped_num_settings;
45 	u8 dq_imped_feat_addr;
46 	u8 rb_pulldown_strength;
47 	u8 rb_pulldown_strength_feat_addr;
48 	u8 rb_pulldown_strength_num_settings;
49 	u8 otp_mode;
50 	u8 otp_page_start;
51 	u8 otp_data_prot_addr;
52 	u8 otp_num_pages;
53 	u8 otp_feat_addr;
54 	u8 read_retry_options;
55 	u8 reserved[72];
56 	u8 param_revision;
57 } __packed;
58 
59 struct micron_on_die_ecc {
60 	bool forced;
61 	bool enabled;
62 	void *rawbuf;
63 };
64 
65 struct micron_nand {
66 	struct micron_on_die_ecc ecc;
67 };
68 
micron_nand_setup_read_retry(struct nand_chip * chip,int retry_mode)69 static int micron_nand_setup_read_retry(struct nand_chip *chip, int retry_mode)
70 {
71 	u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode};
72 
73 	return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature);
74 }
75 
76 /*
77  * Configure chip properties from Micron vendor-specific ONFI table
78  */
micron_nand_onfi_init(struct nand_chip * chip)79 static int micron_nand_onfi_init(struct nand_chip *chip)
80 {
81 	struct nand_parameters *p = &chip->parameters;
82 
83 	if (p->onfi) {
84 		struct nand_onfi_vendor_micron *micron = (void *)p->onfi->vendor;
85 
86 		chip->read_retries = micron->read_retry_options;
87 		chip->ops.setup_read_retry = micron_nand_setup_read_retry;
88 	}
89 
90 	if (p->supports_set_get_features) {
91 		set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->set_feature_list);
92 		set_bit(ONFI_FEATURE_ON_DIE_ECC, p->set_feature_list);
93 		set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->get_feature_list);
94 		set_bit(ONFI_FEATURE_ON_DIE_ECC, p->get_feature_list);
95 	}
96 
97 	return 0;
98 }
99 
micron_nand_on_die_4_ooblayout_ecc(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)100 static int micron_nand_on_die_4_ooblayout_ecc(struct mtd_info *mtd,
101 					      int section,
102 					      struct mtd_oob_region *oobregion)
103 {
104 	if (section >= 4)
105 		return -ERANGE;
106 
107 	oobregion->offset = (section * 16) + 8;
108 	oobregion->length = 8;
109 
110 	return 0;
111 }
112 
micron_nand_on_die_4_ooblayout_free(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)113 static int micron_nand_on_die_4_ooblayout_free(struct mtd_info *mtd,
114 					       int section,
115 					       struct mtd_oob_region *oobregion)
116 {
117 	if (section >= 4)
118 		return -ERANGE;
119 
120 	oobregion->offset = (section * 16) + 2;
121 	oobregion->length = 6;
122 
123 	return 0;
124 }
125 
126 static const struct mtd_ooblayout_ops micron_nand_on_die_4_ooblayout_ops = {
127 	.ecc = micron_nand_on_die_4_ooblayout_ecc,
128 	.free = micron_nand_on_die_4_ooblayout_free,
129 };
130 
micron_nand_on_die_8_ooblayout_ecc(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)131 static int micron_nand_on_die_8_ooblayout_ecc(struct mtd_info *mtd,
132 					      int section,
133 					      struct mtd_oob_region *oobregion)
134 {
135 	struct nand_chip *chip = mtd_to_nand(mtd);
136 
137 	if (section)
138 		return -ERANGE;
139 
140 	oobregion->offset = mtd->oobsize - chip->ecc.total;
141 	oobregion->length = chip->ecc.total;
142 
143 	return 0;
144 }
145 
micron_nand_on_die_8_ooblayout_free(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)146 static int micron_nand_on_die_8_ooblayout_free(struct mtd_info *mtd,
147 					       int section,
148 					       struct mtd_oob_region *oobregion)
149 {
150 	struct nand_chip *chip = mtd_to_nand(mtd);
151 
152 	if (section)
153 		return -ERANGE;
154 
155 	oobregion->offset = 2;
156 	oobregion->length = mtd->oobsize - chip->ecc.total - 2;
157 
158 	return 0;
159 }
160 
161 static const struct mtd_ooblayout_ops micron_nand_on_die_8_ooblayout_ops = {
162 	.ecc = micron_nand_on_die_8_ooblayout_ecc,
163 	.free = micron_nand_on_die_8_ooblayout_free,
164 };
165 
micron_nand_on_die_ecc_setup(struct nand_chip * chip,bool enable)166 static int micron_nand_on_die_ecc_setup(struct nand_chip *chip, bool enable)
167 {
168 	struct micron_nand *micron = nand_get_manufacturer_data(chip);
169 	u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = { 0, };
170 	int ret;
171 
172 	if (micron->ecc.forced)
173 		return 0;
174 
175 	if (micron->ecc.enabled == enable)
176 		return 0;
177 
178 	if (enable)
179 		feature[0] |= ONFI_FEATURE_ON_DIE_ECC_EN;
180 
181 	ret = nand_set_features(chip, ONFI_FEATURE_ON_DIE_ECC, feature);
182 	if (!ret)
183 		micron->ecc.enabled = enable;
184 
185 	return ret;
186 }
187 
micron_nand_on_die_ecc_status_4(struct nand_chip * chip,u8 status,void * buf,int page,int oob_required)188 static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status,
189 					   void *buf, int page,
190 					   int oob_required)
191 {
192 	struct micron_nand *micron = nand_get_manufacturer_data(chip);
193 	struct mtd_info *mtd = nand_to_mtd(chip);
194 	unsigned int step, max_bitflips = 0;
195 	bool use_datain = false;
196 	int ret;
197 
198 	if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) {
199 		if (status & NAND_STATUS_FAIL)
200 			mtd->ecc_stats.failed++;
201 
202 		return 0;
203 	}
204 
205 	/*
206 	 * The internal ECC doesn't tell us the number of bitflips that have
207 	 * been corrected, but tells us if it recommends to rewrite the block.
208 	 * If it's the case, we need to read the page in raw mode and compare
209 	 * its content to the corrected version to extract the actual number of
210 	 * bitflips.
211 	 * But before we do that, we must make sure we have all OOB bytes read
212 	 * in non-raw mode, even if the user did not request those bytes.
213 	 */
214 	if (!oob_required) {
215 		/*
216 		 * We first check which operation is supported by the controller
217 		 * before running it. This trick makes it possible to support
218 		 * all controllers, even the most constraints, without almost
219 		 * any performance hit.
220 		 *
221 		 * TODO: could be enhanced to avoid repeating the same check
222 		 * over and over in the fast path.
223 		 */
224 		if (!nand_has_exec_op(chip) ||
225 		    !nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false,
226 				       true))
227 			use_datain = true;
228 
229 		if (use_datain)
230 			ret = nand_read_data_op(chip, chip->oob_poi,
231 						mtd->oobsize, false, false);
232 		else
233 			ret = nand_change_read_column_op(chip, mtd->writesize,
234 							 chip->oob_poi,
235 							 mtd->oobsize, false);
236 		if (ret)
237 			return ret;
238 	}
239 
240 	micron_nand_on_die_ecc_setup(chip, false);
241 
242 	ret = nand_read_page_op(chip, page, 0, micron->ecc.rawbuf,
243 				mtd->writesize + mtd->oobsize);
244 	if (ret)
245 		return ret;
246 
247 	for (step = 0; step < chip->ecc.steps; step++) {
248 		unsigned int offs, i, nbitflips = 0;
249 		u8 *rawbuf, *corrbuf;
250 
251 		offs = step * chip->ecc.size;
252 		rawbuf = micron->ecc.rawbuf + offs;
253 		corrbuf = buf + offs;
254 
255 		for (i = 0; i < chip->ecc.size; i++)
256 			nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);
257 
258 		offs = (step * 16) + 4;
259 		rawbuf = micron->ecc.rawbuf + mtd->writesize + offs;
260 		corrbuf = chip->oob_poi + offs;
261 
262 		for (i = 0; i < chip->ecc.bytes + 4; i++)
263 			nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);
264 
265 		if (WARN_ON(nbitflips > chip->ecc.strength))
266 			return -EINVAL;
267 
268 		max_bitflips = max(nbitflips, max_bitflips);
269 		mtd->ecc_stats.corrected += nbitflips;
270 	}
271 
272 	return max_bitflips;
273 }
274 
micron_nand_on_die_ecc_status_8(struct nand_chip * chip,u8 status)275 static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status)
276 {
277 	struct mtd_info *mtd = nand_to_mtd(chip);
278 
279 	/*
280 	 * With 8/512 we have more information but still don't know precisely
281 	 * how many bit-flips were seen.
282 	 */
283 	switch (status & NAND_ECC_STATUS_MASK) {
284 	case NAND_ECC_STATUS_UNCORRECTABLE:
285 		mtd->ecc_stats.failed++;
286 		return 0;
287 	case NAND_ECC_STATUS_1_3_CORRECTED:
288 		mtd->ecc_stats.corrected += 3;
289 		return 3;
290 	case NAND_ECC_STATUS_4_6_CORRECTED:
291 		mtd->ecc_stats.corrected += 6;
292 		/* rewrite recommended */
293 		return 6;
294 	case NAND_ECC_STATUS_7_8_CORRECTED:
295 		mtd->ecc_stats.corrected += 8;
296 		/* rewrite recommended */
297 		return 8;
298 	default:
299 		return 0;
300 	}
301 }
302 
303 static int
micron_nand_read_page_on_die_ecc(struct nand_chip * chip,uint8_t * buf,int oob_required,int page)304 micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf,
305 				 int oob_required, int page)
306 {
307 	struct mtd_info *mtd = nand_to_mtd(chip);
308 	bool use_datain = false;
309 	u8 status;
310 	int ret, max_bitflips = 0;
311 
312 	ret = micron_nand_on_die_ecc_setup(chip, true);
313 	if (ret)
314 		return ret;
315 
316 	ret = nand_read_page_op(chip, page, 0, NULL, 0);
317 	if (ret)
318 		goto out;
319 
320 	ret = nand_status_op(chip, &status);
321 	if (ret)
322 		goto out;
323 
324 	/*
325 	 * We first check which operation is supported by the controller before
326 	 * running it. This trick makes it possible to support all controllers,
327 	 * even the most constraints, without almost any performance hit.
328 	 *
329 	 * TODO: could be enhanced to avoid repeating the same check over and
330 	 * over in the fast path.
331 	 */
332 	if (!nand_has_exec_op(chip) ||
333 	    !nand_read_data_op(chip, buf, mtd->writesize, false, true))
334 		use_datain = true;
335 
336 	if (use_datain) {
337 		ret = nand_exit_status_op(chip);
338 		if (ret)
339 			goto out;
340 
341 		ret = nand_read_data_op(chip, buf, mtd->writesize, false,
342 					false);
343 		if (!ret && oob_required)
344 			ret = nand_read_data_op(chip, chip->oob_poi,
345 						mtd->oobsize, false, false);
346 	} else {
347 		ret = nand_change_read_column_op(chip, 0, buf, mtd->writesize,
348 						 false);
349 		if (!ret && oob_required)
350 			ret = nand_change_read_column_op(chip, mtd->writesize,
351 							 chip->oob_poi,
352 							 mtd->oobsize, false);
353 	}
354 
355 	if (chip->ecc.strength == 4)
356 		max_bitflips = micron_nand_on_die_ecc_status_4(chip, status,
357 							       buf, page,
358 							       oob_required);
359 	else
360 		max_bitflips = micron_nand_on_die_ecc_status_8(chip, status);
361 
362 out:
363 	micron_nand_on_die_ecc_setup(chip, false);
364 
365 	return ret ? ret : max_bitflips;
366 }
367 
368 static int
micron_nand_write_page_on_die_ecc(struct nand_chip * chip,const uint8_t * buf,int oob_required,int page)369 micron_nand_write_page_on_die_ecc(struct nand_chip *chip, const uint8_t *buf,
370 				  int oob_required, int page)
371 {
372 	int ret;
373 
374 	ret = micron_nand_on_die_ecc_setup(chip, true);
375 	if (ret)
376 		return ret;
377 
378 	ret = nand_write_page_raw(chip, buf, oob_required, page);
379 	micron_nand_on_die_ecc_setup(chip, false);
380 
381 	return ret;
382 }
383 
384 enum {
385 	/* The NAND flash doesn't support on-die ECC */
386 	MICRON_ON_DIE_UNSUPPORTED,
387 
388 	/*
389 	 * The NAND flash supports on-die ECC and it can be
390 	 * enabled/disabled by a set features command.
391 	 */
392 	MICRON_ON_DIE_SUPPORTED,
393 
394 	/*
395 	 * The NAND flash supports on-die ECC, and it cannot be
396 	 * disabled.
397 	 */
398 	MICRON_ON_DIE_MANDATORY,
399 };
400 
401 #define MICRON_ID_INTERNAL_ECC_MASK	GENMASK(1, 0)
402 #define MICRON_ID_ECC_ENABLED		BIT(7)
403 
404 /*
405  * Try to detect if the NAND support on-die ECC. To do this, we enable
406  * the feature, and read back if it has been enabled as expected. We
407  * also check if it can be disabled, because some Micron NANDs do not
408  * allow disabling the on-die ECC and we don't support such NANDs for
409  * now.
410  *
411  * This function also has the side effect of disabling on-die ECC if
412  * it had been left enabled by the firmware/bootloader.
413  */
micron_supports_on_die_ecc(struct nand_chip * chip)414 static int micron_supports_on_die_ecc(struct nand_chip *chip)
415 {
416 	const struct nand_ecc_props *requirements =
417 		nanddev_get_ecc_requirements(&chip->base);
418 	u8 id[5];
419 	int ret;
420 
421 	if (!chip->parameters.onfi)
422 		return MICRON_ON_DIE_UNSUPPORTED;
423 
424 	if (nanddev_bits_per_cell(&chip->base) != 1)
425 		return MICRON_ON_DIE_UNSUPPORTED;
426 
427 	/*
428 	 * We only support on-die ECC of 4/512 or 8/512
429 	 */
430 	if  (requirements->strength != 4 && requirements->strength != 8)
431 		return MICRON_ON_DIE_UNSUPPORTED;
432 
433 	/* 0x2 means on-die ECC is available. */
434 	if (chip->id.len != 5 ||
435 	    (chip->id.data[4] & MICRON_ID_INTERNAL_ECC_MASK) != 0x2)
436 		return MICRON_ON_DIE_UNSUPPORTED;
437 
438 	/*
439 	 * It seems that there are devices which do not support ECC officially.
440 	 * At least the MT29F2G08ABAGA / MT29F2G08ABBGA devices supports
441 	 * enabling the ECC feature but don't reflect that to the READ_ID table.
442 	 * So we have to guarantee that we disable the ECC feature directly
443 	 * after we did the READ_ID table command. Later we can evaluate the
444 	 * ECC_ENABLE support.
445 	 */
446 	ret = micron_nand_on_die_ecc_setup(chip, true);
447 	if (ret)
448 		return MICRON_ON_DIE_UNSUPPORTED;
449 
450 	ret = nand_readid_op(chip, 0, id, sizeof(id));
451 	if (ret)
452 		return MICRON_ON_DIE_UNSUPPORTED;
453 
454 	ret = micron_nand_on_die_ecc_setup(chip, false);
455 	if (ret)
456 		return MICRON_ON_DIE_UNSUPPORTED;
457 
458 	if (!(id[4] & MICRON_ID_ECC_ENABLED))
459 		return MICRON_ON_DIE_UNSUPPORTED;
460 
461 	ret = nand_readid_op(chip, 0, id, sizeof(id));
462 	if (ret)
463 		return MICRON_ON_DIE_UNSUPPORTED;
464 
465 	if (id[4] & MICRON_ID_ECC_ENABLED)
466 		return MICRON_ON_DIE_MANDATORY;
467 
468 	/*
469 	 * We only support on-die ECC of 4/512 or 8/512
470 	 */
471 	if  (requirements->strength != 4 && requirements->strength != 8)
472 		return MICRON_ON_DIE_UNSUPPORTED;
473 
474 	return MICRON_ON_DIE_SUPPORTED;
475 }
476 
micron_nand_init(struct nand_chip * chip)477 static int micron_nand_init(struct nand_chip *chip)
478 {
479 	struct nand_device *base = &chip->base;
480 	const struct nand_ecc_props *requirements =
481 		nanddev_get_ecc_requirements(base);
482 	struct mtd_info *mtd = nand_to_mtd(chip);
483 	struct micron_nand *micron;
484 	int ondie;
485 	int ret;
486 
487 	micron = kzalloc(sizeof(*micron), GFP_KERNEL);
488 	if (!micron)
489 		return -ENOMEM;
490 
491 	nand_set_manufacturer_data(chip, micron);
492 
493 	ret = micron_nand_onfi_init(chip);
494 	if (ret)
495 		goto err_free_manuf_data;
496 
497 	chip->options |= NAND_BBM_FIRSTPAGE;
498 
499 	if (mtd->writesize == 2048)
500 		chip->options |= NAND_BBM_SECONDPAGE;
501 
502 	ondie = micron_supports_on_die_ecc(chip);
503 
504 	if (ondie == MICRON_ON_DIE_MANDATORY &&
505 	    chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_DIE) {
506 		pr_err("On-die ECC forcefully enabled, not supported\n");
507 		ret = -EINVAL;
508 		goto err_free_manuf_data;
509 	}
510 
511 	if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE) {
512 		if (ondie == MICRON_ON_DIE_UNSUPPORTED) {
513 			pr_err("On-die ECC selected but not supported\n");
514 			ret = -EINVAL;
515 			goto err_free_manuf_data;
516 		}
517 
518 		if (ondie == MICRON_ON_DIE_MANDATORY) {
519 			micron->ecc.forced = true;
520 			micron->ecc.enabled = true;
521 		}
522 
523 		/*
524 		 * In case of 4bit on-die ECC, we need a buffer to store a
525 		 * page dumped in raw mode so that we can compare its content
526 		 * to the same page after ECC correction happened and extract
527 		 * the real number of bitflips from this comparison.
528 		 * That's not needed for 8-bit ECC, because the status expose
529 		 * a better approximation of the number of bitflips in a page.
530 		 */
531 		if (requirements->strength == 4) {
532 			micron->ecc.rawbuf = kmalloc(mtd->writesize +
533 						     mtd->oobsize,
534 						     GFP_KERNEL);
535 			if (!micron->ecc.rawbuf) {
536 				ret = -ENOMEM;
537 				goto err_free_manuf_data;
538 			}
539 		}
540 
541 		if (requirements->strength == 4)
542 			mtd_set_ooblayout(mtd,
543 					  &micron_nand_on_die_4_ooblayout_ops);
544 		else
545 			mtd_set_ooblayout(mtd,
546 					  &micron_nand_on_die_8_ooblayout_ops);
547 
548 		chip->ecc.bytes = requirements->strength * 2;
549 		chip->ecc.size = 512;
550 		chip->ecc.strength = requirements->strength;
551 		chip->ecc.algo = NAND_ECC_ALGO_BCH;
552 		chip->ecc.read_page = micron_nand_read_page_on_die_ecc;
553 		chip->ecc.write_page = micron_nand_write_page_on_die_ecc;
554 
555 		if (ondie == MICRON_ON_DIE_MANDATORY) {
556 			chip->ecc.read_page_raw = nand_read_page_raw_notsupp;
557 			chip->ecc.write_page_raw = nand_write_page_raw_notsupp;
558 		} else {
559 			if (!chip->ecc.read_page_raw)
560 				chip->ecc.read_page_raw = nand_read_page_raw;
561 			if (!chip->ecc.write_page_raw)
562 				chip->ecc.write_page_raw = nand_write_page_raw;
563 		}
564 	}
565 
566 	return 0;
567 
568 err_free_manuf_data:
569 	kfree(micron->ecc.rawbuf);
570 	kfree(micron);
571 
572 	return ret;
573 }
574 
micron_nand_cleanup(struct nand_chip * chip)575 static void micron_nand_cleanup(struct nand_chip *chip)
576 {
577 	struct micron_nand *micron = nand_get_manufacturer_data(chip);
578 
579 	kfree(micron->ecc.rawbuf);
580 	kfree(micron);
581 }
582 
micron_fixup_onfi_param_page(struct nand_chip * chip,struct nand_onfi_params * p)583 static void micron_fixup_onfi_param_page(struct nand_chip *chip,
584 					 struct nand_onfi_params *p)
585 {
586 	/*
587 	 * MT29F1G08ABAFAWP-ITE:F and possibly others report 00 00 for the
588 	 * revision number field of the ONFI parameter page. Assume ONFI
589 	 * version 1.0 if the revision number is 00 00.
590 	 */
591 	if (le16_to_cpu(p->revision) == 0)
592 		p->revision = cpu_to_le16(ONFI_VERSION_1_0);
593 }
594 
595 const struct nand_manufacturer_ops micron_nand_manuf_ops = {
596 	.init = micron_nand_init,
597 	.cleanup = micron_nand_cleanup,
598 	.fixup_onfi_param_page = micron_fixup_onfi_param_page,
599 };
600