1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) Marvell International Ltd. and its affiliates
4 */
5
6 #include <common.h>
7 #include <i2c.h>
8 #include <spl.h>
9 #include <asm/io.h>
10 #include <asm/arch/cpu.h>
11 #include <asm/arch/soc.h>
12 #include <linux/delay.h>
13
14 #include "ddr3_hw_training.h"
15
16 /*
17 * Debug
18 */
19 #define DEBUG_PBS_FULL_C(s, d, l) \
20 DEBUG_PBS_FULL_S(s); DEBUG_PBS_FULL_D(d, l); DEBUG_PBS_FULL_S("\n")
21 #define DEBUG_PBS_C(s, d, l) \
22 DEBUG_PBS_S(s); DEBUG_PBS_D(d, l); DEBUG_PBS_S("\n")
23
24 #ifdef MV_DEBUG_PBS
25 #define DEBUG_PBS_S(s) puts(s)
26 #define DEBUG_PBS_D(d, l) printf("%x", d)
27 #else
28 #define DEBUG_PBS_S(s)
29 #define DEBUG_PBS_D(d, l)
30 #endif
31
32 #ifdef MV_DEBUG_FULL_PBS
33 #define DEBUG_PBS_FULL_S(s) puts(s)
34 #define DEBUG_PBS_FULL_D(d, l) printf("%x", d)
35 #else
36 #define DEBUG_PBS_FULL_S(s)
37 #define DEBUG_PBS_FULL_D(d, l)
38 #endif
39
40 #if defined(MV88F78X60) || defined(MV88F672X)
41
42 /* Temp array for skew data storage */
43 static u32 skew_array[(MAX_PUP_NUM) * DQ_NUM] = { 0 };
44
45 /* PBS locked dq (per pup) */
46 extern u32 pbs_locked_dq[MAX_PUP_NUM][DQ_NUM];
47 extern u32 pbs_locked_dm[MAX_PUP_NUM];
48 extern u32 pbs_locked_value[MAX_PUP_NUM][DQ_NUM];
49
50 #if defined(MV88F672X)
51 extern u32 pbs_pattern[2][LEN_16BIT_PBS_PATTERN];
52 extern u32 pbs_pattern_32b[2][LEN_PBS_PATTERN];
53 #else
54 extern u32 pbs_pattern_32b[2][LEN_PBS_PATTERN];
55 extern u32 pbs_pattern_64b[2][LEN_PBS_PATTERN];
56 #endif
57
58 extern u32 pbs_dq_mapping[PUP_NUM_64BIT + 1][DQ_NUM];
59
60 static int ddr3_tx_shift_dqs_adll_step_before_fail(MV_DRAM_INFO *dram_info,
61 u32 cur_pup, u32 pbs_pattern_idx, u32 ecc);
62 static int ddr3_rx_shift_dqs_to_first_fail(MV_DRAM_INFO *dram_info, u32 cur_pup,
63 u32 pbs_pattern_idx, u32 ecc);
64 static int ddr3_pbs_per_bit(MV_DRAM_INFO *dram_info, int *start_over, int is_tx,
65 u32 *pcur_pup, u32 pbs_pattern_idx, u32 ecc);
66 static int ddr3_set_pbs_results(MV_DRAM_INFO *dram_info, int is_tx);
67 static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay);
68
69 /*
70 * Name: ddr3_pbs_tx
71 * Desc: Execute the PBS TX phase.
72 * Args: dram_info ddr3 training information struct
73 * Notes:
74 * Returns: MV_OK if success, other error code if fail.
75 */
ddr3_pbs_tx(MV_DRAM_INFO * dram_info)76 int ddr3_pbs_tx(MV_DRAM_INFO *dram_info)
77 {
78 /* Array of Deskew results */
79
80 /*
81 * Array to hold the total sum of skew from all iterations
82 * (for average purpose)
83 */
84 u32 skew_sum_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
85
86 /*
87 * Array to hold the total average skew from both patterns
88 * (for average purpose)
89 */
90 u32 pattern_skew_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
91
92 u32 pbs_rep_time = 0; /* counts number of loop in case of fail */
93 /* bit array for unlock pups - used to repeat on the RX operation */
94 u32 cur_pup;
95 u32 max_pup;
96 u32 pbs_retry;
97 u32 pup, dq, pups, cur_max_pup, valid_pup, reg;
98 u32 pattern_idx;
99 u32 ecc;
100 /* indicates whether we need to start the loop again */
101 int start_over;
102
103 DEBUG_PBS_S("DDR3 - PBS TX - Starting PBS TX procedure\n");
104
105 pups = dram_info->num_of_total_pups;
106 max_pup = dram_info->num_of_total_pups;
107
108 /* Enable SW override */
109 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
110 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
111 /* [0] = 1 - Enable SW override */
112 /* 0x15B8 - Training SW 2 Register */
113 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
114 DEBUG_PBS_S("DDR3 - PBS RX - SW Override Enabled\n");
115
116 reg = 1 << REG_DRAM_TRAINING_AUTO_OFFS;
117 reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
118
119 /* Running twice for 2 different patterns. each patterns - 3 times */
120 for (pattern_idx = 0; pattern_idx < COUNT_PBS_PATTERN; pattern_idx++) {
121 DEBUG_PBS_C("DDR3 - PBS TX - Working with pattern - ",
122 pattern_idx, 1);
123
124 /* Reset sum array */
125 for (pup = 0; pup < pups; pup++) {
126 for (dq = 0; dq < DQ_NUM; dq++)
127 skew_sum_array[pup][dq] = 0;
128 }
129
130 /*
131 * Perform PBS several of times (3 for each pattern).
132 * At the end, we'll use the average
133 */
134 /* If there is ECC, do each PBS again with mux change */
135 for (pbs_retry = 0; pbs_retry < COUNT_PBS_REPEAT; pbs_retry++) {
136 for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
137
138 /*
139 * This parameter stores the current PUP
140 * num - ecc mode dependent - 4-8 / 1 pups
141 */
142 cur_max_pup = (1 - ecc) *
143 dram_info->num_of_std_pups + ecc;
144
145 if (ecc) {
146 /* Only 1 pup in this case */
147 valid_pup = 0x1;
148 } else if (cur_max_pup > 4) {
149 /* 64 bit - 8 pups */
150 valid_pup = 0xFF;
151 } else if (cur_max_pup == 4) {
152 /* 32 bit - 4 pups */
153 valid_pup = 0xF;
154 } else {
155 /* 16 bit - 2 pups */
156 valid_pup = 0x3;
157 }
158
159 /* ECC Support - Switch ECC Mux on ecc=1 */
160 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
161 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
162 reg |= (dram_info->ecc_ena * ecc <<
163 REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
164 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
165
166 if (ecc)
167 DEBUG_PBS_S("DDR3 - PBS Tx - ECC Mux Enabled\n");
168 else
169 DEBUG_PBS_S("DDR3 - PBS Tx - ECC Mux Disabled\n");
170
171 /* Init iteration values */
172 /* Clear the locked DQs */
173 for (pup = 0; pup < cur_max_pup; pup++) {
174 for (dq = 0; dq < DQ_NUM; dq++) {
175 pbs_locked_dq[
176 pup + ecc *
177 (max_pup - 1)][dq] =
178 0;
179 }
180 }
181
182 pbs_rep_time = 0;
183 cur_pup = valid_pup;
184 start_over = 0;
185
186 /*
187 * Run loop On current Pattern and current
188 * pattern iteration (just to cover the false
189 * fail problem)
190 */
191 do {
192 DEBUG_PBS_S("DDR3 - PBS Tx - Pbs Rep Loop is ");
193 DEBUG_PBS_D(pbs_rep_time, 1);
194 DEBUG_PBS_S(", for Retry No.");
195 DEBUG_PBS_D(pbs_retry, 1);
196 DEBUG_PBS_S("\n");
197
198 /* Set all PBS values to MIN (0) */
199 DEBUG_PBS_S("DDR3 - PBS Tx - Set all PBS values to MIN\n");
200
201 for (dq = 0; dq < DQ_NUM; dq++) {
202 ddr3_write_pup_reg(
203 PUP_PBS_TX +
204 pbs_dq_mapping[pup *
205 (1 - ecc) +
206 ecc * ECC_PUP]
207 [dq], CS0, (1 - ecc) *
208 PUP_BC + ecc * ECC_PUP, 0,
209 0);
210 }
211
212 /*
213 * Shift DQ ADLL right, One step before
214 * fail
215 */
216 DEBUG_PBS_S("DDR3 - PBS Tx - ADLL shift right one phase before fail\n");
217
218 if (MV_OK != ddr3_tx_shift_dqs_adll_step_before_fail
219 (dram_info, cur_pup, pattern_idx,
220 ecc))
221 return MV_DDR3_TRAINING_ERR_PBS_ADLL_SHR_1PHASE;
222
223 /* PBS For each bit */
224 DEBUG_PBS_S("DDR3 - PBS Tx - perform PBS for each bit\n");
225
226 /*
227 * In this stage - start_over = 0
228 */
229 if (MV_OK != ddr3_pbs_per_bit(
230 dram_info, &start_over, 1,
231 &cur_pup, pattern_idx, ecc))
232 return MV_DDR3_TRAINING_ERR_PBS_TX_PER_BIT;
233
234 } while ((start_over == 1) &&
235 (++pbs_rep_time < COUNT_PBS_STARTOVER));
236
237 if (pbs_rep_time == COUNT_PBS_STARTOVER &&
238 start_over == 1) {
239 DEBUG_PBS_S("DDR3 - PBS Tx - FAIL - Adll reach max value\n");
240 return MV_DDR3_TRAINING_ERR_PBS_TX_MAX_VAL;
241 }
242
243 DEBUG_PBS_FULL_C("DDR3 - PBS TX - values for iteration - ",
244 pbs_retry, 1);
245 for (pup = 0; pup < cur_max_pup; pup++) {
246 /*
247 * To minimize delay elements, inc
248 * from pbs value the min pbs val
249 */
250 DEBUG_PBS_S("DDR3 - PBS - PUP");
251 DEBUG_PBS_D((pup + (ecc * ECC_PUP)), 1);
252 DEBUG_PBS_S(": ");
253
254 for (dq = 0; dq < DQ_NUM; dq++) {
255 /* Set skew value for all dq */
256 /*
257 * Bit# Deskew <- Bit# Deskew -
258 * last / first failing bit
259 * Deskew For all bits (per PUP)
260 * (minimize delay elements)
261 */
262 DEBUG_PBS_S("DQ");
263 DEBUG_PBS_D(dq, 1);
264 DEBUG_PBS_S("-");
265 DEBUG_PBS_D(skew_array
266 [((pup) * DQ_NUM) +
267 dq], 2);
268 DEBUG_PBS_S(", ");
269 }
270 DEBUG_PBS_S("\n");
271 }
272
273 /*
274 * Collect the results we got on this trial
275 * of PBS
276 */
277 for (pup = 0; pup < cur_max_pup; pup++) {
278 for (dq = 0; dq < DQ_NUM; dq++) {
279 skew_sum_array[pup + (ecc * (max_pup - 1))]
280 [dq] += skew_array
281 [((pup) * DQ_NUM) + dq];
282 }
283 }
284
285 /* ECC Support - Disable ECC MUX */
286 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
287 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
288 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
289 }
290 }
291
292 DEBUG_PBS_C("DDR3 - PBS TX - values for current pattern - ",
293 pattern_idx, 1);
294 for (pup = 0; pup < max_pup; pup++) {
295 /*
296 * To minimize delay elements, inc from pbs value the
297 * min pbs val
298 */
299 DEBUG_PBS_S("DDR3 - PBS - PUP");
300 DEBUG_PBS_D(pup, 1);
301 DEBUG_PBS_S(": ");
302
303 for (dq = 0; dq < DQ_NUM; dq++) {
304 /* set skew value for all dq */
305 /* Bit# Deskew <- Bit# Deskew - last / first failing bit Deskew For all bits (per PUP) (minimize delay elements) */
306 DEBUG_PBS_S("DQ");
307 DEBUG_PBS_D(dq, 1);
308 DEBUG_PBS_S("-");
309 DEBUG_PBS_D(skew_sum_array[pup][dq] /
310 COUNT_PBS_REPEAT, 2);
311 DEBUG_PBS_S(", ");
312 }
313 DEBUG_PBS_S("\n");
314 }
315
316 /*
317 * Calculate the average skew for current pattern for each
318 * pup and each bit
319 */
320 DEBUG_PBS_C("DDR3 - PBS TX - Average for pattern - ",
321 pattern_idx, 1);
322
323 for (pup = 0; pup < max_pup; pup++) {
324 /*
325 * FOR ECC only :: found min and max value for current
326 * pattern skew array
327 */
328 /* Loop for all dqs */
329 for (dq = 0; dq < DQ_NUM; dq++) {
330 pattern_skew_array[pup][dq] +=
331 (skew_sum_array[pup][dq] /
332 COUNT_PBS_REPEAT);
333 }
334 }
335 }
336
337 /* Calculate the average skew */
338 for (pup = 0; pup < max_pup; pup++) {
339 for (dq = 0; dq < DQ_NUM; dq++)
340 skew_array[((pup) * DQ_NUM) + dq] =
341 pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN;
342 }
343
344 DEBUG_PBS_S("DDR3 - PBS TX - Average for all patterns:\n");
345 for (pup = 0; pup < max_pup; pup++) {
346 /*
347 * To minimize delay elements, inc from pbs value the min
348 * pbs val
349 */
350 DEBUG_PBS_S("DDR3 - PBS - PUP");
351 DEBUG_PBS_D(pup, 1);
352 DEBUG_PBS_S(": ");
353
354 for (dq = 0; dq < DQ_NUM; dq++) {
355 /* Set skew value for all dq */
356 /*
357 * Bit# Deskew <- Bit# Deskew - last / first
358 * failing bit Deskew For all bits (per PUP)
359 * (minimize delay elements)
360 */
361 DEBUG_PBS_S("DQ");
362 DEBUG_PBS_D(dq, 1);
363 DEBUG_PBS_S("-");
364 DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2);
365 DEBUG_PBS_S(", ");
366 }
367 DEBUG_PBS_S("\n");
368 }
369
370 /* Return ADLL to default value */
371 for (pup = 0; pup < max_pup; pup++) {
372 if (pup == (max_pup - 1) && dram_info->ecc_ena)
373 pup = ECC_PUP;
374 ddr3_pbs_write_pup_dqs_reg(CS0, pup, INIT_WL_DELAY);
375 }
376
377 /* Set averaged PBS results */
378 ddr3_set_pbs_results(dram_info, 1);
379
380 /* Disable SW override - Must be in a different stage */
381 /* [0]=0 - Enable SW override */
382 reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
383 reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
384 /* 0x15B8 - Training SW 2 Register */
385 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
386
387 reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
388 (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
389 reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
390
391 DEBUG_PBS_S("DDR3 - PBS Tx - PBS TX ended successfuly\n");
392
393 return MV_OK;
394 }
395
396 /*
397 * Name: ddr3_tx_shift_dqs_adll_step_before_fail
398 * Desc: Execute the Tx shift DQ phase.
399 * Args: dram_info ddr3 training information struct
400 * cur_pup bit array of the function active pups.
401 * pbs_pattern_idx Index of PBS pattern
402 * Notes:
403 * Returns: MV_OK if success, other error code if fail.
404 */
ddr3_tx_shift_dqs_adll_step_before_fail(MV_DRAM_INFO * dram_info,u32 cur_pup,u32 pbs_pattern_idx,u32 ecc)405 static int ddr3_tx_shift_dqs_adll_step_before_fail(MV_DRAM_INFO *dram_info,
406 u32 cur_pup,
407 u32 pbs_pattern_idx, u32 ecc)
408 {
409 u32 unlock_pup; /* bit array of unlock pups */
410 u32 new_lockup_pup; /* bit array of compare failed pups */
411 u32 adll_val = 4; /* INIT_WL_DELAY */
412 u32 cur_max_pup, pup;
413 u32 dqs_dly_set[MAX_PUP_NUM] = { 0 };
414 u32 *pattern_ptr;
415
416 /* Choose pattern */
417 switch (dram_info->ddr_width) {
418 #if defined(MV88F672X)
419 case 16:
420 pattern_ptr = (u32 *)&pbs_pattern[pbs_pattern_idx];
421 break;
422 #endif
423 case 32:
424 pattern_ptr = (u32 *)&pbs_pattern_32b[pbs_pattern_idx];
425 break;
426 #if defined(MV88F78X60)
427 case 64:
428 pattern_ptr = (u32 *)&pbs_pattern_64b[pbs_pattern_idx];
429 break;
430 #endif
431 default:
432 return MV_FAIL;
433 }
434
435 /* Set current pup number */
436 if (cur_pup == 0x1) /* Ecc mode */
437 cur_max_pup = 1;
438 else
439 cur_max_pup = dram_info->num_of_std_pups;
440
441 unlock_pup = cur_pup; /* '1' for each unlocked pup */
442
443 /* Loop on all ADLL Vaules */
444 do {
445 /* Loop until found first fail */
446 adll_val++;
447
448 /*
449 * Increment (Move to right - ADLL) DQ TX delay
450 * (broadcast to all Data PUPs)
451 */
452 for (pup = 0; pup < cur_max_pup; pup++)
453 ddr3_pbs_write_pup_dqs_reg(CS0,
454 pup * (1 - ecc) +
455 ECC_PUP * ecc, adll_val);
456
457 /*
458 * Write and Read, compare results (read was already verified)
459 */
460 /* 0 - all locked */
461 new_lockup_pup = 0;
462
463 if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup,
464 &new_lockup_pup,
465 pattern_ptr, LEN_PBS_PATTERN,
466 SDRAM_PBS_TX_OFFS, 1, 0,
467 NULL,
468 0))
469 return MV_FAIL;
470
471 unlock_pup &= ~new_lockup_pup;
472
473 DEBUG_PBS_FULL_S("Shift DQS by 2 steps for PUPs: ");
474 DEBUG_PBS_FULL_D(unlock_pup, 2);
475 DEBUG_PBS_FULL_C(", Set ADLL value = ", adll_val, 2);
476
477 /* If any PUP failed there is '1' to mark the PUP */
478 if (new_lockup_pup != 0) {
479 /*
480 * Decrement (Move Back to Left two steps - ADLL)
481 * DQ TX delay for current failed pups and save
482 */
483 for (pup = 0; pup < cur_max_pup; pup++) {
484 if (((new_lockup_pup >> pup) & 0x1) &&
485 dqs_dly_set[pup] == 0)
486 dqs_dly_set[pup] = adll_val - 1;
487 }
488 }
489 } while ((unlock_pup != 0) && (adll_val != ADLL_MAX));
490
491 if (unlock_pup != 0) {
492 DEBUG_PBS_FULL_S("DDR3 - PBS Tx - Shift DQ - Adll value reached maximum\n");
493
494 for (pup = 0; pup < cur_max_pup; pup++) {
495 if (((unlock_pup >> pup) & 0x1) &&
496 dqs_dly_set[pup] == 0)
497 dqs_dly_set[pup] = adll_val - 1;
498 }
499 }
500
501 DEBUG_PBS_FULL_C("PBS TX one step before fail last pups locked Adll ",
502 adll_val - 2, 2);
503
504 /* Set the PUP DQS DLY Values */
505 for (pup = 0; pup < cur_max_pup; pup++)
506 ddr3_pbs_write_pup_dqs_reg(CS0, pup * (1 - ecc) + ECC_PUP * ecc,
507 dqs_dly_set[pup]);
508
509 /* Found one phase before fail */
510 return MV_OK;
511 }
512
513 /*
514 * Name: ddr3_pbs_rx
515 * Desc: Execute the PBS RX phase.
516 * Args: dram_info ddr3 training information struct
517 * Notes:
518 * Returns: MV_OK if success, other error code if fail.
519 */
ddr3_pbs_rx(MV_DRAM_INFO * dram_info)520 int ddr3_pbs_rx(MV_DRAM_INFO *dram_info)
521 {
522 /*
523 * Array to hold the total sum of skew from all iterations
524 * (for average purpose)
525 */
526 u32 skew_sum_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
527
528 /*
529 * Array to hold the total average skew from both patterns
530 * (for average purpose)
531 */
532 u32 pattern_skew_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
533
534 u32 pbs_rep_time = 0; /* counts number of loop in case of fail */
535 /* bit array for unlock pups - used to repeat on the RX operation */
536 u32 cur_pup;
537 u32 max_pup;
538 u32 pbs_retry;
539 u32 pup, dq, pups, cur_max_pup, valid_pup, reg;
540 u32 pattern_idx;
541 u32 ecc;
542 /* indicates whether we need to start the loop again */
543 int start_over;
544 int status;
545
546 DEBUG_PBS_S("DDR3 - PBS RX - Starting PBS RX procedure\n");
547
548 pups = dram_info->num_of_total_pups;
549 max_pup = dram_info->num_of_total_pups;
550
551 /* Enable SW override */
552 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
553 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
554 /* [0] = 1 - Enable SW override */
555 /* 0x15B8 - Training SW 2 Register */
556 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
557 DEBUG_PBS_FULL_S("DDR3 - PBS RX - SW Override Enabled\n");
558
559 reg = 1 << REG_DRAM_TRAINING_AUTO_OFFS;
560 reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
561
562 /* Running twice for 2 different patterns. each patterns - 3 times */
563 for (pattern_idx = 0; pattern_idx < COUNT_PBS_PATTERN; pattern_idx++) {
564 DEBUG_PBS_FULL_C("DDR3 - PBS RX - Working with pattern - ",
565 pattern_idx, 1);
566
567 /* Reset sum array */
568 for (pup = 0; pup < pups; pup++) {
569 for (dq = 0; dq < DQ_NUM; dq++)
570 skew_sum_array[pup][dq] = 0;
571 }
572
573 /*
574 * Perform PBS several of times (3 for each pattern).
575 * At the end, we'll use the average
576 */
577 /* If there is ECC, do each PBS again with mux change */
578 for (pbs_retry = 0; pbs_retry < COUNT_PBS_REPEAT; pbs_retry++) {
579 for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
580 /*
581 * This parameter stores the current PUP
582 * num - ecc mode dependent - 4-8 / 1 pups
583 */
584 cur_max_pup = (1 - ecc) *
585 dram_info->num_of_std_pups + ecc;
586
587 if (ecc) {
588 /* Only 1 pup in this case */
589 valid_pup = 0x1;
590 } else if (cur_max_pup > 4) {
591 /* 64 bit - 8 pups */
592 valid_pup = 0xFF;
593 } else if (cur_max_pup == 4) {
594 /* 32 bit - 4 pups */
595 valid_pup = 0xF;
596 } else {
597 /* 16 bit - 2 pups */
598 valid_pup = 0x3;
599 }
600
601 /* ECC Support - Switch ECC Mux on ecc=1 */
602 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
603 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
604 reg |= (dram_info->ecc_ena * ecc <<
605 REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
606 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
607
608 if (ecc)
609 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - ECC Mux Enabled\n");
610 else
611 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - ECC Mux Disabled\n");
612
613 /* Init iteration values */
614 /* Clear the locked DQs */
615 for (pup = 0; pup < cur_max_pup; pup++) {
616 for (dq = 0; dq < DQ_NUM; dq++) {
617 pbs_locked_dq[
618 pup + ecc * (max_pup - 1)][dq] =
619 0;
620 }
621 }
622
623 pbs_rep_time = 0;
624 cur_pup = valid_pup;
625 start_over = 0;
626
627 /*
628 * Run loop On current Pattern and current
629 * pattern iteration (just to cover the false
630 * fail problem
631 */
632 do {
633 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Pbs Rep Loop is ");
634 DEBUG_PBS_FULL_D(pbs_rep_time, 1);
635 DEBUG_PBS_FULL_S(", for Retry No.");
636 DEBUG_PBS_FULL_D(pbs_retry, 1);
637 DEBUG_PBS_FULL_S("\n");
638
639 /* Set all PBS values to MAX (31) */
640 for (pup = 0; pup < cur_max_pup; pup++) {
641 for (dq = 0; dq < DQ_NUM; dq++)
642 ddr3_write_pup_reg(
643 PUP_PBS_RX +
644 pbs_dq_mapping[
645 pup * (1 - ecc)
646 + ecc * ECC_PUP]
647 [dq], CS0,
648 pup + ecc * ECC_PUP,
649 0, MAX_PBS);
650 }
651
652 /* Set all DQS PBS values to MIN (0) */
653 for (pup = 0; pup < cur_max_pup; pup++) {
654 ddr3_write_pup_reg(PUP_PBS_RX +
655 DQ_NUM, CS0,
656 pup +
657 ecc *
658 ECC_PUP, 0,
659 0);
660 }
661
662 /* Shift DQS, To first Fail */
663 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Shift RX DQS to first fail\n");
664
665 status = ddr3_rx_shift_dqs_to_first_fail
666 (dram_info, cur_pup,
667 pattern_idx, ecc);
668 if (MV_OK != status) {
669 DEBUG_PBS_S("DDR3 - PBS Rx - ddr3_rx_shift_dqs_to_first_fail failed.\n");
670 DEBUG_PBS_D(status, 8);
671 DEBUG_PBS_S("\nDDR3 - PBS Rx - SKIP.\n");
672
673 /* Reset read FIFO */
674 reg = reg_read(REG_DRAM_TRAINING_ADDR);
675 /* Start Auto Read Leveling procedure */
676 reg |= (1 << REG_DRAM_TRAINING_RL_OFFS);
677 /* 0x15B0 - Training Register */
678 reg_write(REG_DRAM_TRAINING_ADDR, reg);
679
680 reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
681 reg |= ((1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS)
682 + (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS));
683 /* [0] = 1 - Enable SW override, [4] = 1 - FIFO reset */
684 /* 0x15B8 - Training SW 2 Register */
685 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
686
687 do {
688 reg = (reg_read(REG_DRAM_TRAINING_2_ADDR))
689 & (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
690 } while (reg); /* Wait for '0' */
691
692 reg = reg_read(REG_DRAM_TRAINING_ADDR);
693 /* Clear Auto Read Leveling procedure */
694 reg &= ~(1 << REG_DRAM_TRAINING_RL_OFFS);
695 /* 0x15B0 - Training Register */
696 reg_write(REG_DRAM_TRAINING_ADDR, reg);
697
698 /* Set ADLL to 15 */
699 for (pup = 0; pup < max_pup;
700 pup++) {
701 ddr3_write_pup_reg
702 (PUP_DQS_RD, CS0,
703 pup +
704 (ecc * ECC_PUP), 0,
705 15);
706 }
707
708 /* Set all PBS values to MIN (0) */
709 for (pup = 0; pup < cur_max_pup;
710 pup++) {
711 for (dq = 0;
712 dq < DQ_NUM; dq++)
713 ddr3_write_pup_reg
714 (PUP_PBS_RX +
715 pbs_dq_mapping
716 [pup * (1 - ecc) +
717 ecc * ECC_PUP]
718 [dq], CS0,
719 pup + ecc * ECC_PUP,
720 0, MIN_PBS);
721 }
722
723 return MV_OK;
724 }
725
726 /* PBS For each bit */
727 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - perform PBS for each bit\n");
728 /* in this stage - start_over = 0; */
729 if (MV_OK != ddr3_pbs_per_bit(
730 dram_info, &start_over,
731 0, &cur_pup,
732 pattern_idx, ecc)) {
733 DEBUG_PBS_S("DDR3 - PBS Rx - ddr3_pbs_per_bit failed.");
734 return MV_DDR3_TRAINING_ERR_PBS_RX_PER_BIT;
735 }
736
737 } while ((start_over == 1) &&
738 (++pbs_rep_time < COUNT_PBS_STARTOVER));
739
740 if (pbs_rep_time == COUNT_PBS_STARTOVER &&
741 start_over == 1) {
742 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - FAIL - Algorithm failed doing RX PBS\n");
743 return MV_DDR3_TRAINING_ERR_PBS_RX_MAX_VAL;
744 }
745
746 /* Return DQS ADLL to default value - 15 */
747 /* Set all DQS PBS values to MIN (0) */
748 for (pup = 0; pup < cur_max_pup; pup++)
749 ddr3_write_pup_reg(PUP_DQS_RD, CS0,
750 pup + ecc * ECC_PUP,
751 0, INIT_RL_DELAY);
752
753 DEBUG_PBS_FULL_C("DDR3 - PBS RX - values for iteration - ",
754 pbs_retry, 1);
755 for (pup = 0; pup < cur_max_pup; pup++) {
756 /*
757 * To minimize delay elements, inc from
758 * pbs value the min pbs val
759 */
760 DEBUG_PBS_FULL_S("DDR3 - PBS - PUP");
761 DEBUG_PBS_FULL_D((pup +
762 (ecc * ECC_PUP)), 1);
763 DEBUG_PBS_FULL_S(": ");
764
765 for (dq = 0; dq < DQ_NUM; dq++) {
766 /* Set skew value for all dq */
767 /*
768 * Bit# Deskew <- Bit# Deskew -
769 * last / first failing bit
770 * Deskew For all bits (per PUP)
771 * (minimize delay elements)
772 */
773 DEBUG_PBS_FULL_S("DQ");
774 DEBUG_PBS_FULL_D(dq, 1);
775 DEBUG_PBS_FULL_S("-");
776 DEBUG_PBS_FULL_D(skew_array
777 [((pup) *
778 DQ_NUM) +
779 dq], 2);
780 DEBUG_PBS_FULL_S(", ");
781 }
782 DEBUG_PBS_FULL_S("\n");
783 }
784
785 /*
786 * Collect the results we got on this trial
787 * of PBS
788 */
789 for (pup = 0; pup < cur_max_pup; pup++) {
790 for (dq = 0; dq < DQ_NUM; dq++) {
791 skew_sum_array
792 [pup + (ecc * (max_pup - 1))]
793 [dq] +=
794 skew_array[((pup) * DQ_NUM) + dq];
795 }
796 }
797
798 /* ECC Support - Disable ECC MUX */
799 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
800 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
801 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
802 }
803 }
804
805 /*
806 * Calculate the average skew for current pattern for each
807 * pup and each bit
808 */
809 DEBUG_PBS_FULL_C("DDR3 - PBS RX - Average for pattern - ",
810 pattern_idx, 1);
811 for (pup = 0; pup < max_pup; pup++) {
812 /*
813 * FOR ECC only :: found min and max value for
814 * current pattern skew array
815 */
816 /* Loop for all dqs */
817 for (dq = 0; dq < DQ_NUM; dq++) {
818 pattern_skew_array[pup][dq] +=
819 (skew_sum_array[pup][dq] /
820 COUNT_PBS_REPEAT);
821 }
822 }
823
824 DEBUG_PBS_C("DDR3 - PBS RX - values for current pattern - ",
825 pattern_idx, 1);
826 for (pup = 0; pup < max_pup; pup++) {
827 /*
828 * To minimize delay elements, inc from pbs value the
829 * min pbs val
830 */
831 DEBUG_PBS_S("DDR3 - PBS RX - PUP");
832 DEBUG_PBS_D(pup, 1);
833 DEBUG_PBS_S(": ");
834
835 for (dq = 0; dq < DQ_NUM; dq++) {
836 /* Set skew value for all dq */
837 /*
838 * Bit# Deskew <- Bit# Deskew - last / first
839 * failing bit Deskew For all bits (per PUP)
840 * (minimize delay elements)
841 */
842 DEBUG_PBS_S("DQ");
843 DEBUG_PBS_D(dq, 1);
844 DEBUG_PBS_S("-");
845 DEBUG_PBS_D(skew_sum_array[pup][dq] /
846 COUNT_PBS_REPEAT, 2);
847 DEBUG_PBS_S(", ");
848 }
849 DEBUG_PBS_S("\n");
850 }
851 }
852
853 /* Calculate the average skew */
854 for (pup = 0; pup < max_pup; pup++) {
855 for (dq = 0; dq < DQ_NUM; dq++)
856 skew_array[((pup) * DQ_NUM) + dq] =
857 pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN;
858 }
859
860 DEBUG_PBS_S("DDR3 - PBS RX - Average for all patterns:\n");
861 for (pup = 0; pup < max_pup; pup++) {
862 /*
863 * To minimize delay elements, inc from pbs value the
864 * min pbs val
865 */
866 DEBUG_PBS_S("DDR3 - PBS - PUP");
867 DEBUG_PBS_D(pup, 1);
868 DEBUG_PBS_S(": ");
869
870 for (dq = 0; dq < DQ_NUM; dq++) {
871 /* Set skew value for all dq */
872 /*
873 * Bit# Deskew <- Bit# Deskew - last / first
874 * failing bit Deskew For all bits (per PUP)
875 * (minimize delay elements)
876 */
877 DEBUG_PBS_S("DQ");
878 DEBUG_PBS_D(dq, 1);
879 DEBUG_PBS_S("-");
880 DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2);
881 DEBUG_PBS_S(", ");
882 }
883 DEBUG_PBS_S("\n");
884 }
885
886 /* Return ADLL to default value */
887 ddr3_write_pup_reg(PUP_DQS_RD, CS0, PUP_BC, 0, INIT_RL_DELAY);
888
889 /* Set averaged PBS results */
890 ddr3_set_pbs_results(dram_info, 0);
891
892 /* Disable SW override - Must be in a different stage */
893 /* [0]=0 - Enable SW override */
894 reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
895 reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
896 /* 0x15B8 - Training SW 2 Register */
897 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
898
899 reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
900 (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
901 reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
902
903 DEBUG_PBS_FULL_S("DDR3 - PBS RX - ended successfuly\n");
904
905 return MV_OK;
906 }
907
908 /*
909 * Name: ddr3_rx_shift_dqs_to_first_fail
910 * Desc: Execute the Rx shift DQ phase.
911 * Args: dram_info ddr3 training information struct
912 * cur_pup bit array of the function active pups.
913 * pbs_pattern_idx Index of PBS pattern
914 * Notes:
915 * Returns: MV_OK if success, other error code if fail.
916 */
ddr3_rx_shift_dqs_to_first_fail(MV_DRAM_INFO * dram_info,u32 cur_pup,u32 pbs_pattern_idx,u32 ecc)917 static int ddr3_rx_shift_dqs_to_first_fail(MV_DRAM_INFO *dram_info, u32 cur_pup,
918 u32 pbs_pattern_idx, u32 ecc)
919 {
920 u32 unlock_pup; /* bit array of unlock pups */
921 u32 new_lockup_pup; /* bit array of compare failed pups */
922 u32 adll_val = MAX_DELAY;
923 u32 dqs_deskew_val = 0; /* current value of DQS PBS deskew */
924 u32 cur_max_pup, pup, pass_pup;
925 u32 *pattern_ptr;
926
927 /* Choose pattern */
928 switch (dram_info->ddr_width) {
929 #if defined(MV88F672X)
930 case 16:
931 pattern_ptr = (u32 *)&pbs_pattern[pbs_pattern_idx];
932 break;
933 #endif
934 case 32:
935 pattern_ptr = (u32 *)&pbs_pattern_32b[pbs_pattern_idx];
936 break;
937 #if defined(MV88F78X60)
938 case 64:
939 pattern_ptr = (u32 *)&pbs_pattern_64b[pbs_pattern_idx];
940 break;
941 #endif
942 default:
943 return MV_FAIL;
944 }
945
946 /* Set current pup number */
947 if (cur_pup == 0x1) /* Ecc mode */
948 cur_max_pup = 1;
949 else
950 cur_max_pup = dram_info->num_of_std_pups;
951
952 unlock_pup = cur_pup; /* '1' for each unlocked pup */
953
954 DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Starting...\n");
955
956 /* Set DQS ADLL to MAX */
957 DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Set DQS ADLL to Max for all PUPs\n");
958 for (pup = 0; pup < cur_max_pup; pup++)
959 ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP, 0,
960 MAX_DELAY);
961
962 /* Loop on all ADLL Vaules */
963 do {
964 /* Loop until found fail for all pups */
965 new_lockup_pup = 0;
966 if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup,
967 &new_lockup_pup,
968 pattern_ptr, LEN_PBS_PATTERN,
969 SDRAM_PBS_I_OFFS +
970 pbs_pattern_idx * SDRAM_PBS_NEXT_OFFS,
971 0, 0, NULL, 0)) {
972 DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP(ddr3_sdram_compare)\n");
973 return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
974 }
975
976 if ((new_lockup_pup != 0) && (dqs_deskew_val <= 1)) {
977 /* Fail on start with first deskew value */
978 /* Decrement DQS ADLL */
979 --adll_val;
980 if (adll_val == ADLL_MIN) {
981 DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - fail on start with first deskew value\n");
982 return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
983 }
984 ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP,
985 0, adll_val);
986 continue;
987 }
988
989 /* Update all new locked pups */
990 unlock_pup &= ~new_lockup_pup;
991
992 if ((unlock_pup == 0) || (dqs_deskew_val == MAX_PBS)) {
993 if (dqs_deskew_val == MAX_PBS) {
994 /*
995 * Reach max value of dqs deskew or get fail
996 * for all pups
997 */
998 DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - DQS deskew reached maximum value\n");
999 }
1000 break;
1001 }
1002
1003 DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Inc DQS deskew for PUPs: ");
1004 DEBUG_PBS_FULL_D(unlock_pup, 2);
1005 DEBUG_PBS_FULL_C(", deskew = ", dqs_deskew_val, 2);
1006
1007 /* Increment DQS deskew elements - Only for unlocked pups */
1008 dqs_deskew_val++;
1009 for (pup = 0; pup < cur_max_pup; pup++) {
1010 if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1011 ddr3_write_pup_reg(PUP_PBS_RX + DQS_DQ_NUM, CS0,
1012 pup + ecc * ECC_PUP, 0,
1013 dqs_deskew_val);
1014 }
1015 }
1016 } while (1);
1017
1018 DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - ADLL shift one step before fail\n");
1019 /* Continue to ADLL shift one step before fail */
1020 unlock_pup = cur_pup;
1021 do {
1022 /* Loop until pass compare for all pups */
1023 new_lockup_pup = 0;
1024 /* Read and compare results */
1025 if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup, &new_lockup_pup,
1026 pattern_ptr, LEN_PBS_PATTERN,
1027 SDRAM_PBS_I_OFFS +
1028 pbs_pattern_idx * SDRAM_PBS_NEXT_OFFS,
1029 1, 0, NULL, 0)) {
1030 DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP(ddr3_sdram_compare)\n");
1031 return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
1032 }
1033
1034 /*
1035 * Get mask for pup which passed so their adll will be
1036 * changed to 2 steps before fails
1037 */
1038 pass_pup = unlock_pup & ~new_lockup_pup;
1039
1040 DEBUG_PBS_FULL_S("Shift DQS by 2 steps for PUPs: ");
1041 DEBUG_PBS_FULL_D(pass_pup, 2);
1042 DEBUG_PBS_FULL_C(", Set ADLL value = ", (adll_val - 2), 2);
1043
1044 /* Only for pass pups */
1045 for (pup = 0; pup < cur_max_pup; pup++) {
1046 if (IS_PUP_ACTIVE(pass_pup, pup) == 1) {
1047 ddr3_write_pup_reg(PUP_DQS_RD, CS0,
1048 pup + ecc * ECC_PUP, 0,
1049 (adll_val - 2));
1050 }
1051 }
1052
1053 /* Locked pups that compare success */
1054 unlock_pup &= new_lockup_pup;
1055
1056 if (unlock_pup == 0) {
1057 /* All pups locked */
1058 break;
1059 }
1060
1061 /* Found error */
1062 if (adll_val == 0) {
1063 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Shift DQS - Adll reach min value\n");
1064 return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_MAX_VAL;
1065 }
1066
1067 /*
1068 * Decrement (Move Back to Left one phase - ADLL) dqs RX delay
1069 */
1070 adll_val--;
1071 for (pup = 0; pup < cur_max_pup; pup++) {
1072 if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1073 ddr3_write_pup_reg(PUP_DQS_RD, CS0,
1074 pup + ecc * ECC_PUP, 0,
1075 adll_val);
1076 }
1077 }
1078 } while (1);
1079
1080 return MV_OK;
1081 }
1082
1083 /*
1084 * lock_pups() extracted from ddr3_pbs_per_bit(). This just got too
1085 * much indented making it hard to read / edit.
1086 */
lock_pups(u32 pup,u32 * pup_locked,u8 * unlock_pup_dq_array,u32 pbs_curr_val,u32 start_pbs,u32 ecc,int is_tx)1087 static void lock_pups(u32 pup, u32 *pup_locked, u8 *unlock_pup_dq_array,
1088 u32 pbs_curr_val, u32 start_pbs, u32 ecc, int is_tx)
1089 {
1090 u32 dq;
1091 int idx;
1092
1093 /* Lock PBS value for all remaining PUPs bits */
1094 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Lock PBS value for all remaining PUPs bits, pup ");
1095 DEBUG_PBS_FULL_D(pup, 1);
1096 DEBUG_PBS_FULL_C(" pbs value ", pbs_curr_val, 2);
1097
1098 idx = pup * (1 - ecc) + ecc * ECC_PUP;
1099 *pup_locked &= ~(1 << pup);
1100
1101 for (dq = 0; dq < DQ_NUM; dq++) {
1102 if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup) == 1) {
1103 int offs;
1104
1105 /* Lock current dq */
1106 unlock_pup_dq_array[dq] &= ~(1 << pup);
1107 skew_array[(pup * DQ_NUM) + dq] = pbs_curr_val;
1108
1109 if (is_tx == 1)
1110 offs = PUP_PBS_TX;
1111 else
1112 offs = PUP_PBS_RX;
1113
1114 ddr3_write_pup_reg(offs +
1115 pbs_dq_mapping[idx][dq], CS0,
1116 idx, 0, start_pbs);
1117 }
1118 }
1119 }
1120
1121 /*
1122 * Name: ddr3_pbs_per_bit
1123 * Desc: Execute the Per Bit Skew phase.
1124 * Args: start_over Return whether need to start over the algorithm
1125 * is_tx Indicate whether Rx or Tx
1126 * pcur_pup bit array of the function active pups. return the
1127 * pups that need to repeat on the PBS
1128 * pbs_pattern_idx Index of PBS pattern
1129 *
1130 * Notes: Current implementation supports double activation of this function.
1131 * i.e. in order to activate this function (using start_over) more than
1132 * twice, the implementation should change.
1133 * imlementation limitation are marked using
1134 * ' CHIP-ONLY! - Implementation Limitation '
1135 * Returns: MV_OK if success, other error code if fail.
1136 */
ddr3_pbs_per_bit(MV_DRAM_INFO * dram_info,int * start_over,int is_tx,u32 * pcur_pup,u32 pbs_pattern_idx,u32 ecc)1137 static int ddr3_pbs_per_bit(MV_DRAM_INFO *dram_info, int *start_over, int is_tx,
1138 u32 *pcur_pup, u32 pbs_pattern_idx, u32 ecc)
1139 {
1140 /*
1141 * Bit array to indicate if we already get fail on bit per pup & dq bit
1142 */
1143 u8 unlock_pup_dq_array[DQ_NUM] = {
1144 *pcur_pup, *pcur_pup, *pcur_pup, *pcur_pup, *pcur_pup,
1145 *pcur_pup, *pcur_pup, *pcur_pup
1146 };
1147
1148 u8 cmp_unlock_pup_dq_array[COUNT_PBS_COMP_RETRY_NUM][DQ_NUM];
1149 u32 pup, dq;
1150 /* value of pbs is according to RX or TX */
1151 u32 start_pbs, last_pbs;
1152 u32 pbs_curr_val;
1153 /* bit array that indicates all dq of the pup locked */
1154 u32 pup_locked;
1155 u32 first_fail[MAX_PUP_NUM] = { 0 }; /* count first fail per pup */
1156 /* indicates whether we get first fail per pup */
1157 int first_failed[MAX_PUP_NUM] = { 0 };
1158 /* bit array that indicates pup already get fail */
1159 u32 sum_pup_fail;
1160 /* use to calculate diff between curr pbs to first fail pbs */
1161 u32 calc_pbs_diff;
1162 u32 pbs_cmp_retry;
1163 u32 max_pup;
1164
1165 /* Set init values for retry array - 8 retry */
1166 for (pbs_cmp_retry = 0; pbs_cmp_retry < COUNT_PBS_COMP_RETRY_NUM;
1167 pbs_cmp_retry++) {
1168 for (dq = 0; dq < DQ_NUM; dq++)
1169 cmp_unlock_pup_dq_array[pbs_cmp_retry][dq] = *pcur_pup;
1170 }
1171
1172 memset(&skew_array, 0, MAX_PUP_NUM * DQ_NUM * sizeof(u32));
1173
1174 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Started\n");
1175
1176 /* The pbs value depends if rx or tx */
1177 if (is_tx == 1) {
1178 start_pbs = MIN_PBS;
1179 last_pbs = MAX_PBS;
1180 } else {
1181 start_pbs = MAX_PBS;
1182 last_pbs = MIN_PBS;
1183 }
1184
1185 pbs_curr_val = start_pbs;
1186 pup_locked = *pcur_pup;
1187
1188 /* Set current pup number */
1189 if (pup_locked == 0x1) /* Ecc mode */
1190 max_pup = 1;
1191 else
1192 max_pup = dram_info->num_of_std_pups;
1193
1194 do {
1195 /* Increment/ decrement PBS for un-lock bits only */
1196 if (is_tx == 1)
1197 pbs_curr_val++;
1198 else
1199 pbs_curr_val--;
1200
1201 /* Set Current PBS delay */
1202 for (dq = 0; dq < DQ_NUM; dq++) {
1203 /* Check DQ bits to see if locked in all pups */
1204 if (unlock_pup_dq_array[dq] == 0) {
1205 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - All pups are locked for DQ ");
1206 DEBUG_PBS_FULL_D(dq, 1);
1207 DEBUG_PBS_FULL_S("\n");
1208 continue;
1209 }
1210
1211 for (pup = 0; pup < max_pup; pup++) {
1212 int idx;
1213
1214 idx = pup * (1 - ecc) + ecc * ECC_PUP;
1215
1216 if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup)
1217 == 0)
1218 continue;
1219
1220 if (is_tx == 1)
1221 ddr3_write_pup_reg(
1222 PUP_PBS_TX + pbs_dq_mapping[idx][dq],
1223 CS0, idx, 0, pbs_curr_val);
1224 else
1225 ddr3_write_pup_reg(
1226 PUP_PBS_RX + pbs_dq_mapping[idx][dq],
1227 CS0, idx, 0, pbs_curr_val);
1228 }
1229 }
1230
1231 /*
1232 * Write Read and compare results - run the test
1233 * DDR_PBS_COMP_RETRY_NUM times
1234 */
1235 /* Run number of read and write to verify */
1236 for (pbs_cmp_retry = 0;
1237 pbs_cmp_retry < COUNT_PBS_COMP_RETRY_NUM;
1238 pbs_cmp_retry++) {
1239
1240 if (MV_OK !=
1241 ddr3_sdram_pbs_compare(dram_info, pup_locked, is_tx,
1242 pbs_pattern_idx,
1243 pbs_curr_val, start_pbs,
1244 skew_array,
1245 cmp_unlock_pup_dq_array
1246 [pbs_cmp_retry], ecc))
1247 return MV_FAIL;
1248
1249 for (pup = 0; pup < max_pup; pup++) {
1250 for (dq = 0; dq < DQ_NUM; dq++) {
1251 if ((IS_PUP_ACTIVE(unlock_pup_dq_array[dq],
1252 pup) == 1)
1253 && (IS_PUP_ACTIVE(cmp_unlock_pup_dq_array
1254 [pbs_cmp_retry][dq],
1255 pup) == 0)) {
1256 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - PbsCurrVal: ");
1257 DEBUG_PBS_FULL_D(pbs_curr_val, 2);
1258 DEBUG_PBS_FULL_S(" PUP: ");
1259 DEBUG_PBS_FULL_D(pup, 1);
1260 DEBUG_PBS_FULL_S(" DQ: ");
1261 DEBUG_PBS_FULL_D(dq, 1);
1262 DEBUG_PBS_FULL_S(" - failed\n");
1263 }
1264 }
1265 }
1266
1267 for (dq = 0; dq < DQ_NUM; dq++) {
1268 unlock_pup_dq_array[dq] &=
1269 cmp_unlock_pup_dq_array[pbs_cmp_retry][dq];
1270 }
1271 }
1272
1273 pup_locked = 0;
1274 sum_pup_fail = *pcur_pup;
1275
1276 /* Check which DQ is failed */
1277 for (dq = 0; dq < DQ_NUM; dq++) {
1278 /* Summarize the locked pup */
1279 pup_locked |= unlock_pup_dq_array[dq];
1280
1281 /* Check if get fail */
1282 sum_pup_fail &= unlock_pup_dq_array[dq];
1283 }
1284
1285 /* If all PUPS are locked in all DQ - Break */
1286 if (pup_locked == 0) {
1287 /* All pups are locked */
1288 *start_over = 0;
1289 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - All bit in all pups are successfully locked\n");
1290 break;
1291 }
1292
1293 /* PBS deskew elements reach max ? */
1294 if (pbs_curr_val == last_pbs) {
1295 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - PBS deskew elements reach max\n");
1296 /* CHIP-ONLY! - Implementation Limitation */
1297 *start_over = (sum_pup_fail != 0) && (!(*start_over));
1298 *pcur_pup = pup_locked;
1299
1300 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - StartOver: ");
1301 DEBUG_PBS_FULL_D(*start_over, 1);
1302 DEBUG_PBS_FULL_S(" pup_locked: ");
1303 DEBUG_PBS_FULL_D(pup_locked, 2);
1304 DEBUG_PBS_FULL_S(" sum_pup_fail: ");
1305 DEBUG_PBS_FULL_D(sum_pup_fail, 2);
1306 DEBUG_PBS_FULL_S("\n");
1307
1308 /* Lock PBS value for all remaining bits */
1309 for (pup = 0; pup < max_pup; pup++) {
1310 /* Check if current pup already received error */
1311 if (IS_PUP_ACTIVE(pup_locked, pup) == 1) {
1312 /* Valid pup for current function */
1313 if (IS_PUP_ACTIVE(sum_pup_fail, pup) ==
1314 1 && (*start_over == 1)) {
1315 DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - skipping lock of pup (first loop of pbs)",
1316 pup, 1);
1317 continue;
1318 } else
1319 if (IS_PUP_ACTIVE(sum_pup_fail, pup)
1320 == 1) {
1321 DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - Locking pup %d (even though it wasn't supposed to be locked)",
1322 pup, 1);
1323 }
1324
1325 /* Already got fail on the PUP */
1326 /* Lock PBS value for all remaining bits */
1327 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Locking remaning DQs for pup - ");
1328 DEBUG_PBS_FULL_D(pup, 1);
1329 DEBUG_PBS_FULL_S(": ");
1330
1331 for (dq = 0; dq < DQ_NUM; dq++) {
1332 if (IS_PUP_ACTIVE
1333 (unlock_pup_dq_array[dq],
1334 pup) == 1) {
1335 DEBUG_PBS_FULL_D(dq, 1);
1336 DEBUG_PBS_FULL_S(",");
1337 /* set current PBS */
1338 skew_array[((pup) *
1339 DQ_NUM) +
1340 dq] =
1341 pbs_curr_val;
1342 }
1343 }
1344
1345 if (*start_over == 1) {
1346 /*
1347 * Reset this pup bit - when
1348 * restart the PBS, ignore this
1349 * pup
1350 */
1351 *pcur_pup &= ~(1 << pup);
1352 }
1353 DEBUG_PBS_FULL_S("\n");
1354 } else {
1355 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Pup ");
1356 DEBUG_PBS_FULL_D(pup, 1);
1357 DEBUG_PBS_FULL_C(" is not set in puplocked - ",
1358 pup_locked, 1);
1359 }
1360 }
1361
1362 /* Need to start the PBS again */
1363 if (*start_over == 1) {
1364 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - false fail - returning to start\n");
1365 return MV_OK;
1366 }
1367 break;
1368 }
1369
1370 /* Diff Check */
1371 for (pup = 0; pup < max_pup; pup++) {
1372 if (IS_PUP_ACTIVE(pup_locked, pup) == 1) {
1373 /* pup is not locked */
1374 if (first_failed[pup] == 0) {
1375 /* No first fail until now */
1376 if (IS_PUP_ACTIVE(sum_pup_fail, pup) ==
1377 0) {
1378 /* Get first fail */
1379 DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - First fail in pup ",
1380 pup, 1);
1381 first_failed[pup] = 1;
1382 first_fail[pup] = pbs_curr_val;
1383 }
1384 } else {
1385 /* Already got first fail */
1386 if (is_tx == 1) {
1387 /* TX - inc pbs */
1388 calc_pbs_diff = pbs_curr_val -
1389 first_fail[pup];
1390 } else {
1391 /* RX - dec pbs */
1392 calc_pbs_diff = first_fail[pup] -
1393 pbs_curr_val;
1394 }
1395
1396 if (calc_pbs_diff >= PBS_DIFF_LIMIT) {
1397 lock_pups(pup, &pup_locked,
1398 unlock_pup_dq_array,
1399 pbs_curr_val,
1400 start_pbs, ecc, is_tx);
1401 }
1402 }
1403 }
1404 }
1405 } while (1);
1406
1407 return MV_OK;
1408 }
1409
1410 /*
1411 * Name: ddr3_set_pbs_results
1412 * Desc: Set to HW the PBS phase results.
1413 * Args: is_tx Indicates whether to set Tx or RX results
1414 * Notes:
1415 * Returns: MV_OK if success, other error code if fail.
1416 */
ddr3_set_pbs_results(MV_DRAM_INFO * dram_info,int is_tx)1417 static int ddr3_set_pbs_results(MV_DRAM_INFO *dram_info, int is_tx)
1418 {
1419 u32 pup, phys_pup, dq;
1420 u32 max_pup; /* number of valid pups */
1421 u32 pbs_min; /* minimal pbs val per pup */
1422 u32 pbs_max; /* maximum pbs val per pup */
1423 u32 val[9];
1424
1425 max_pup = dram_info->num_of_total_pups;
1426 DEBUG_PBS_FULL_S("DDR3 - PBS - ddr3_set_pbs_results:\n");
1427
1428 /* Loop for all dqs & pups */
1429 for (pup = 0; pup < max_pup; pup++) {
1430 if (pup == (max_pup - 1) && dram_info->ecc_ena)
1431 phys_pup = ECC_PUP;
1432 else
1433 phys_pup = pup;
1434
1435 /*
1436 * To minimize delay elements, inc from pbs value the min
1437 * pbs val
1438 */
1439 pbs_min = MAX_PBS;
1440 pbs_max = 0;
1441 for (dq = 0; dq < DQ_NUM; dq++) {
1442 if (pbs_min > skew_array[(pup * DQ_NUM) + dq])
1443 pbs_min = skew_array[(pup * DQ_NUM) + dq];
1444
1445 if (pbs_max < skew_array[(pup * DQ_NUM) + dq])
1446 pbs_max = skew_array[(pup * DQ_NUM) + dq];
1447 }
1448
1449 pbs_max -= pbs_min;
1450
1451 DEBUG_PBS_FULL_S("DDR3 - PBS - PUP");
1452 DEBUG_PBS_FULL_D(phys_pup, 1);
1453 DEBUG_PBS_FULL_S(": Min Val = ");
1454 DEBUG_PBS_FULL_D(pbs_min, 2);
1455 DEBUG_PBS_FULL_C(", Max Val = ", pbs_max, 2);
1456
1457 val[pup] = 0;
1458
1459 for (dq = 0; dq < DQ_NUM; dq++) {
1460 int idx;
1461 int offs;
1462
1463 /* Set skew value for all dq */
1464 /*
1465 * Bit# Deskew <- Bit# Deskew - last / first
1466 * failing bit Deskew For all bits (per PUP)
1467 * (minimize delay elements)
1468 */
1469
1470 DEBUG_PBS_FULL_S("DQ");
1471 DEBUG_PBS_FULL_D(dq, 1);
1472 DEBUG_PBS_FULL_S("-");
1473 DEBUG_PBS_FULL_D((skew_array[(pup * DQ_NUM) + dq] -
1474 pbs_min), 2);
1475 DEBUG_PBS_FULL_S(", ");
1476
1477 idx = (pup * DQ_NUM) + dq;
1478
1479 if (is_tx == 1)
1480 offs = PUP_PBS_TX;
1481 else
1482 offs = PUP_PBS_RX;
1483
1484 ddr3_write_pup_reg(offs + pbs_dq_mapping[phys_pup][dq],
1485 CS0, phys_pup, 0,
1486 skew_array[idx] - pbs_min);
1487
1488 if (is_tx == 1)
1489 val[pup] += skew_array[idx] - pbs_min;
1490 }
1491
1492 DEBUG_PBS_FULL_S("\n");
1493
1494 /* Set the DQS the half of the Max PBS of the DQs */
1495 if (is_tx == 1) {
1496 ddr3_write_pup_reg(PUP_PBS_TX + 8, CS0, phys_pup, 0,
1497 pbs_max / 2);
1498 ddr3_write_pup_reg(PUP_PBS_TX + 0xa, CS0, phys_pup, 0,
1499 val[pup] / 8);
1500 } else
1501 ddr3_write_pup_reg(PUP_PBS_RX + 8, CS0, phys_pup, 0,
1502 pbs_max / 2);
1503 }
1504
1505 return MV_OK;
1506 }
1507
ddr3_pbs_write_pup_dqs_reg(u32 cs,u32 pup,u32 dqs_delay)1508 static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay)
1509 {
1510 u32 reg, delay;
1511
1512 reg = (ddr3_read_pup_reg(PUP_WL_MODE, cs, pup) & 0x3FF);
1513 delay = reg & PUP_DELAY_MASK;
1514 reg |= ((dqs_delay + delay) << REG_PHY_DQS_REF_DLY_OFFS);
1515 reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
1516 reg |= (pup << REG_PHY_PUP_OFFS);
1517 reg |= ((0x4 * cs + PUP_WL_MODE) << REG_PHY_CS_OFFS);
1518
1519 reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg); /* 0x16A0 */
1520 do {
1521 reg = reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR) &
1522 REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
1523 } while (reg); /* Wait for '0' to mark the end of the transaction */
1524
1525 udelay(10);
1526 }
1527
1528 /*
1529 * Set training patterns
1530 */
ddr3_load_pbs_patterns(MV_DRAM_INFO * dram_info)1531 int ddr3_load_pbs_patterns(MV_DRAM_INFO *dram_info)
1532 {
1533 u32 cs, cs_count, cs_tmp;
1534 u32 sdram_addr;
1535 u32 *pattern_ptr0, *pattern_ptr1;
1536
1537 /* Choose pattern */
1538 switch (dram_info->ddr_width) {
1539 #if defined(MV88F672X)
1540 case 16:
1541 pattern_ptr0 = (u32 *)&pbs_pattern[0];
1542 pattern_ptr1 = (u32 *)&pbs_pattern[1];
1543 break;
1544 #endif
1545 case 32:
1546 pattern_ptr0 = (u32 *)&pbs_pattern_32b[0];
1547 pattern_ptr1 = (u32 *)&pbs_pattern_32b[1];
1548 break;
1549 #if defined(MV88F78X60)
1550 case 64:
1551 pattern_ptr0 = (u32 *)&pbs_pattern_64b[0];
1552 pattern_ptr1 = (u32 *)&pbs_pattern_64b[1];
1553 break;
1554 #endif
1555 default:
1556 return MV_FAIL;
1557 }
1558
1559 /* Loop for each CS */
1560 for (cs = 0; cs < MAX_CS; cs++) {
1561 if (dram_info->cs_ena & (1 << cs)) {
1562 cs_count = 0;
1563 for (cs_tmp = 0; cs_tmp < cs; cs_tmp++) {
1564 if (dram_info->cs_ena & (1 << cs_tmp))
1565 cs_count++;
1566 }
1567
1568 /* Init PBS I pattern */
1569 sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1570 SDRAM_PBS_I_OFFS);
1571 if (MV_OK !=
1572 ddr3_sdram_compare(dram_info, (u32) NULL, NULL,
1573 pattern_ptr0, LEN_STD_PATTERN,
1574 sdram_addr, 1, 0, NULL,
1575 0))
1576 return MV_FAIL;
1577
1578 /* Init PBS II pattern */
1579 sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1580 SDRAM_PBS_II_OFFS);
1581 if (MV_OK !=
1582 ddr3_sdram_compare(dram_info, (u32) NULL, NULL,
1583 pattern_ptr1, LEN_STD_PATTERN,
1584 sdram_addr, 1, 0, NULL,
1585 0))
1586 return MV_FAIL;
1587 }
1588 }
1589
1590 return MV_OK;
1591 }
1592 #endif
1593