1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) Marvell International Ltd. and its affiliates
4 */
5
6 #include <common.h>
7 #include <i2c.h>
8 #include <log.h>
9 #include <spl.h>
10 #include <asm/io.h>
11 #include <asm/arch/cpu.h>
12 #include <asm/arch/soc.h>
13 #include <linux/delay.h>
14
15 #include "ddr3_init.h"
16 #include "ddr3_hw_training.h"
17 #include "xor.h"
18
19 #ifdef MV88F78X60
20 #include "ddr3_patterns_64bit.h"
21 #else
22 #include "ddr3_patterns_16bit.h"
23 #if defined(MV88F672X)
24 #include "ddr3_patterns_16bit.h"
25 #endif
26 #endif
27
28 /*
29 * Debug
30 */
31
32 #define DEBUG_MAIN_C(s, d, l) \
33 DEBUG_MAIN_S(s); DEBUG_MAIN_D(d, l); DEBUG_MAIN_S("\n")
34 #define DEBUG_MAIN_FULL_C(s, d, l) \
35 DEBUG_MAIN_FULL_S(s); DEBUG_MAIN_FULL_D(d, l); DEBUG_MAIN_FULL_S("\n")
36
37 #ifdef MV_DEBUG_MAIN
38 #define DEBUG_MAIN_S(s) puts(s)
39 #define DEBUG_MAIN_D(d, l) printf("%x", d)
40 #else
41 #define DEBUG_MAIN_S(s)
42 #define DEBUG_MAIN_D(d, l)
43 #endif
44
45 #ifdef MV_DEBUG_MAIN_FULL
46 #define DEBUG_MAIN_FULL_S(s) puts(s)
47 #define DEBUG_MAIN_FULL_D(d, l) printf("%x", d)
48 #else
49 #define DEBUG_MAIN_FULL_S(s)
50 #define DEBUG_MAIN_FULL_D(d, l)
51 #endif
52
53 #ifdef MV_DEBUG_SUSPEND_RESUME
54 #define DEBUG_SUSPEND_RESUME_S(s) puts(s)
55 #define DEBUG_SUSPEND_RESUME_D(d, l) printf("%x", d)
56 #else
57 #define DEBUG_SUSPEND_RESUME_S(s)
58 #define DEBUG_SUSPEND_RESUME_D(d, l)
59 #endif
60
61 static u32 ddr3_sw_wl_rl_debug;
62 static u32 ddr3_run_pbs = 1;
63
ddr3_print_version(void)64 void ddr3_print_version(void)
65 {
66 puts("DDR3 Training Sequence - Ver 5.7.");
67 }
68
ddr3_set_sw_wl_rl_debug(u32 val)69 void ddr3_set_sw_wl_rl_debug(u32 val)
70 {
71 ddr3_sw_wl_rl_debug = val;
72 }
73
ddr3_set_pbs(u32 val)74 void ddr3_set_pbs(u32 val)
75 {
76 ddr3_run_pbs = val;
77 }
78
ddr3_hw_training(u32 target_freq,u32 ddr_width,int xor_bypass,u32 scrub_offs,u32 scrub_size,int dqs_clk_aligned,int debug_mode,int reg_dimm_skip_wl)79 int ddr3_hw_training(u32 target_freq, u32 ddr_width, int xor_bypass,
80 u32 scrub_offs, u32 scrub_size, int dqs_clk_aligned,
81 int debug_mode, int reg_dimm_skip_wl)
82 {
83 /* A370 has no PBS mechanism */
84 __maybe_unused u32 first_loop_flag = 0;
85 u32 freq, reg;
86 MV_DRAM_INFO dram_info;
87 int ratio_2to1 = 0;
88 int tmp_ratio = 1;
89 int status;
90
91 if (debug_mode)
92 DEBUG_MAIN_S("DDR3 Training Sequence - DEBUG - 1\n");
93
94 memset(&dram_info, 0, sizeof(dram_info));
95 dram_info.num_cs = ddr3_get_cs_num_from_reg();
96 dram_info.cs_ena = ddr3_get_cs_ena_from_reg();
97 dram_info.target_frequency = target_freq;
98 dram_info.ddr_width = ddr_width;
99 dram_info.num_of_std_pups = ddr_width / PUP_SIZE;
100 dram_info.rl400_bug = 0;
101 dram_info.multi_cs_mr_support = 0;
102 #ifdef MV88F67XX
103 dram_info.rl400_bug = 1;
104 #endif
105
106 /* Ignore ECC errors - if ECC is enabled */
107 reg = reg_read(REG_SDRAM_CONFIG_ADDR);
108 if (reg & (1 << REG_SDRAM_CONFIG_ECC_OFFS)) {
109 dram_info.ecc_ena = 1;
110 reg |= (1 << REG_SDRAM_CONFIG_IERR_OFFS);
111 reg_write(REG_SDRAM_CONFIG_ADDR, reg);
112 } else {
113 dram_info.ecc_ena = 0;
114 }
115
116 reg = reg_read(REG_SDRAM_CONFIG_ADDR);
117 if (reg & (1 << REG_SDRAM_CONFIG_REGDIMM_OFFS))
118 dram_info.reg_dimm = 1;
119 else
120 dram_info.reg_dimm = 0;
121
122 dram_info.num_of_total_pups = ddr_width / PUP_SIZE + dram_info.ecc_ena;
123
124 /* Get target 2T value */
125 reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
126 dram_info.mode_2t = (reg >> REG_DUNIT_CTRL_LOW_2T_OFFS) &
127 REG_DUNIT_CTRL_LOW_2T_MASK;
128
129 /* Get target CL value */
130 #ifdef MV88F67XX
131 reg = reg_read(REG_DDR3_MR0_ADDR) >> 2;
132 #else
133 reg = reg_read(REG_DDR3_MR0_CS_ADDR) >> 2;
134 #endif
135
136 reg = (((reg >> 1) & 0xE) | (reg & 0x1)) & 0xF;
137 dram_info.cl = ddr3_valid_cl_to_cl(reg);
138
139 /* Get target CWL value */
140 #ifdef MV88F67XX
141 reg = reg_read(REG_DDR3_MR2_ADDR) >> REG_DDR3_MR2_CWL_OFFS;
142 #else
143 reg = reg_read(REG_DDR3_MR2_CS_ADDR) >> REG_DDR3_MR2_CWL_OFFS;
144 #endif
145
146 reg &= REG_DDR3_MR2_CWL_MASK;
147 dram_info.cwl = reg;
148 #if !defined(MV88F67XX)
149 /* A370 has no PBS mechanism */
150 #if defined(MV88F78X60)
151 if ((dram_info.target_frequency > DDR_400) && (ddr3_run_pbs))
152 first_loop_flag = 1;
153 #else
154 /* first_loop_flag = 1; skip mid freq at ALP/A375 */
155 if ((dram_info.target_frequency > DDR_400) && (ddr3_run_pbs) &&
156 (mv_ctrl_revision_get() >= UMC_A0))
157 first_loop_flag = 1;
158 else
159 first_loop_flag = 0;
160 #endif
161 #endif
162
163 freq = dram_info.target_frequency;
164
165 /* Set ODT to always on */
166 ddr3_odt_activate(1);
167
168 /* Init XOR */
169 mv_sys_xor_init(&dram_info);
170
171 /* Get DRAM/HCLK ratio */
172 if (reg_read(REG_DDR_IO_ADDR) & (1 << REG_DDR_IO_CLK_RATIO_OFFS))
173 ratio_2to1 = 1;
174
175 /*
176 * Xor Bypass - ECC support in AXP is currently available for 1:1
177 * modes frequency modes.
178 * Not all frequency modes support the ddr3 training sequence
179 * (Only 1200/300).
180 * Xor Bypass allows using the Xor initializations and scrubbing
181 * inside the ddr3 training sequence without running the training
182 * itself.
183 */
184 if (xor_bypass == 0) {
185 if (ddr3_run_pbs) {
186 DEBUG_MAIN_S("DDR3 Training Sequence - Run with PBS.\n");
187 } else {
188 DEBUG_MAIN_S("DDR3 Training Sequence - Run without PBS.\n");
189 }
190
191 if (dram_info.target_frequency > DFS_MARGIN) {
192 tmp_ratio = 0;
193 freq = DDR_100;
194
195 if (dram_info.reg_dimm == 1)
196 freq = DDR_300;
197
198 if (MV_OK != ddr3_dfs_high_2_low(freq, &dram_info)) {
199 /* Set low - 100Mhz DDR Frequency by HW */
200 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Dfs High2Low)\n");
201 return MV_DDR3_TRAINING_ERR_DFS_H2L;
202 }
203
204 if ((dram_info.reg_dimm == 1) &&
205 (reg_dimm_skip_wl == 0)) {
206 if (MV_OK !=
207 ddr3_write_leveling_hw_reg_dimm(freq,
208 &dram_info))
209 DEBUG_MAIN_S("DDR3 Training Sequence - Registered DIMM Low WL - SKIP\n");
210 }
211
212 if (ddr3_get_log_level() >= MV_LOG_LEVEL_1)
213 ddr3_print_freq(freq);
214
215 if (debug_mode)
216 DEBUG_MAIN_S("DDR3 Training Sequence - DEBUG - 2\n");
217 } else {
218 if (!dqs_clk_aligned) {
219 #ifdef MV88F67XX
220 /*
221 * If running training sequence without DFS,
222 * we must run Write leveling before writing
223 * the patterns
224 */
225
226 /*
227 * ODT - Multi CS system use SW WL,
228 * Single CS System use HW WL
229 */
230 if (dram_info.cs_ena > 1) {
231 if (MV_OK !=
232 ddr3_write_leveling_sw(
233 freq, tmp_ratio,
234 &dram_info)) {
235 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Write Leveling Sw)\n");
236 return MV_DDR3_TRAINING_ERR_WR_LVL_SW;
237 }
238 } else {
239 if (MV_OK !=
240 ddr3_write_leveling_hw(freq,
241 &dram_info)) {
242 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Write Leveling Hw)\n");
243 return MV_DDR3_TRAINING_ERR_WR_LVL_HW;
244 }
245 }
246 #else
247 if (MV_OK != ddr3_write_leveling_hw(
248 freq, &dram_info)) {
249 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Write Leveling Hw)\n");
250 if (ddr3_sw_wl_rl_debug) {
251 if (MV_OK !=
252 ddr3_write_leveling_sw(
253 freq, tmp_ratio,
254 &dram_info)) {
255 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Write Leveling Sw)\n");
256 return MV_DDR3_TRAINING_ERR_WR_LVL_SW;
257 }
258 } else {
259 return MV_DDR3_TRAINING_ERR_WR_LVL_HW;
260 }
261 }
262 #endif
263 }
264
265 if (debug_mode)
266 DEBUG_MAIN_S("DDR3 Training Sequence - DEBUG - 3\n");
267 }
268
269 if (MV_OK != ddr3_load_patterns(&dram_info, 0)) {
270 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Loading Patterns)\n");
271 return MV_DDR3_TRAINING_ERR_LOAD_PATTERNS;
272 }
273
274 /*
275 * TODO:
276 * The mainline U-Boot port of the bin_hdr DDR training code
277 * needs a delay of minimum 20ms here (10ms is a bit too short
278 * and the CPU hangs). The bin_hdr code doesn't have this delay.
279 * To be save here, lets add a delay of 50ms here.
280 *
281 * Tested on the Marvell DB-MV784MP-GP board
282 */
283 mdelay(50);
284
285 do {
286 freq = dram_info.target_frequency;
287 tmp_ratio = ratio_2to1;
288 DEBUG_MAIN_FULL_S("DDR3 Training Sequence - DEBUG - 4\n");
289
290 #if defined(MV88F78X60)
291 /*
292 * There is a difference on the DFS frequency at the
293 * first iteration of this loop
294 */
295 if (first_loop_flag) {
296 freq = DDR_400;
297 tmp_ratio = 0;
298 }
299 #endif
300
301 if (MV_OK != ddr3_dfs_low_2_high(freq, tmp_ratio,
302 &dram_info)) {
303 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Dfs Low2High)\n");
304 return MV_DDR3_TRAINING_ERR_DFS_H2L;
305 }
306
307 if (ddr3_get_log_level() >= MV_LOG_LEVEL_1) {
308 ddr3_print_freq(freq);
309 }
310
311 if (debug_mode)
312 DEBUG_MAIN_S("DDR3 Training Sequence - DEBUG - 5\n");
313
314 /* Write leveling */
315 if (!dqs_clk_aligned) {
316 #ifdef MV88F67XX
317 /*
318 * ODT - Multi CS system that not support Multi
319 * CS MRS commands must use SW WL
320 */
321 if (dram_info.cs_ena > 1) {
322 if (MV_OK != ddr3_write_leveling_sw(
323 freq, tmp_ratio, &dram_info)) {
324 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Write Leveling Sw)\n");
325 return MV_DDR3_TRAINING_ERR_WR_LVL_SW;
326 }
327 } else {
328 if (MV_OK != ddr3_write_leveling_hw(
329 freq, &dram_info)) {
330 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Write Leveling Hw)\n");
331 return MV_DDR3_TRAINING_ERR_WR_LVL_HW;
332 }
333 }
334 #else
335 if ((dram_info.reg_dimm == 1) &&
336 (freq == DDR_400)) {
337 if (reg_dimm_skip_wl == 0) {
338 if (MV_OK != ddr3_write_leveling_hw_reg_dimm(
339 freq, &dram_info))
340 DEBUG_MAIN_S("DDR3 Training Sequence - Registered DIMM WL - SKIP\n");
341 }
342 } else {
343 if (MV_OK != ddr3_write_leveling_hw(
344 freq, &dram_info)) {
345 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Write Leveling Hw)\n");
346 if (ddr3_sw_wl_rl_debug) {
347 if (MV_OK != ddr3_write_leveling_sw(
348 freq, tmp_ratio, &dram_info)) {
349 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Write Leveling Sw)\n");
350 return MV_DDR3_TRAINING_ERR_WR_LVL_SW;
351 }
352 } else {
353 return MV_DDR3_TRAINING_ERR_WR_LVL_HW;
354 }
355 }
356 }
357 #endif
358 if (debug_mode)
359 DEBUG_MAIN_S
360 ("DDR3 Training Sequence - DEBUG - 6\n");
361 }
362
363 /* Read Leveling */
364 /*
365 * Armada 370 - Support for HCLK @ 400MHZ - must use
366 * SW read leveling
367 */
368 if (freq == DDR_400 && dram_info.rl400_bug) {
369 status = ddr3_read_leveling_sw(freq, tmp_ratio,
370 &dram_info);
371 if (MV_OK != status) {
372 DEBUG_MAIN_S
373 ("DDR3 Training Sequence - FAILED (Read Leveling Sw)\n");
374 return status;
375 }
376 } else {
377 if (MV_OK != ddr3_read_leveling_hw(
378 freq, &dram_info)) {
379 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Read Leveling Hw)\n");
380 if (ddr3_sw_wl_rl_debug) {
381 if (MV_OK != ddr3_read_leveling_sw(
382 freq, tmp_ratio,
383 &dram_info)) {
384 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Read Leveling Sw)\n");
385 return MV_DDR3_TRAINING_ERR_WR_LVL_SW;
386 }
387 } else {
388 return MV_DDR3_TRAINING_ERR_WR_LVL_HW;
389 }
390 }
391 }
392
393 if (debug_mode)
394 DEBUG_MAIN_S("DDR3 Training Sequence - DEBUG - 7\n");
395
396 if (MV_OK != ddr3_wl_supplement(&dram_info)) {
397 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Write Leveling Hi-Freq Sup)\n");
398 return MV_DDR3_TRAINING_ERR_WR_LVL_HI_FREQ;
399 }
400
401 if (debug_mode)
402 DEBUG_MAIN_S("DDR3 Training Sequence - DEBUG - 8\n");
403 #if !defined(MV88F67XX)
404 /* A370 has no PBS mechanism */
405 #if defined(MV88F78X60) || defined(MV88F672X)
406 if (first_loop_flag == 1) {
407 first_loop_flag = 0;
408
409 status = MV_OK;
410 status = ddr3_pbs_rx(&dram_info);
411 if (MV_OK != status) {
412 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (PBS RX)\n");
413 return status;
414 }
415
416 if (debug_mode)
417 DEBUG_MAIN_S("DDR3 Training Sequence - DEBUG - 9\n");
418
419 status = ddr3_pbs_tx(&dram_info);
420 if (MV_OK != status) {
421 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (PBS TX)\n");
422 return status;
423 }
424
425 if (debug_mode)
426 DEBUG_MAIN_S("DDR3 Training Sequence - DEBUG - 10\n");
427 }
428 #endif
429 #endif
430 } while (freq != dram_info.target_frequency);
431
432 status = ddr3_dqs_centralization_rx(&dram_info);
433 if (MV_OK != status) {
434 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (DQS Centralization RX)\n");
435 return status;
436 }
437
438 if (debug_mode)
439 DEBUG_MAIN_S("DDR3 Training Sequence - DEBUG - 11\n");
440
441 status = ddr3_dqs_centralization_tx(&dram_info);
442 if (MV_OK != status) {
443 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (DQS Centralization TX)\n");
444 return status;
445 }
446
447 if (debug_mode)
448 DEBUG_MAIN_S("DDR3 Training Sequence - DEBUG - 12\n");
449 }
450
451 ddr3_set_performance_params(&dram_info);
452
453 if (dram_info.ecc_ena) {
454 /* Need to SCRUB the DRAM memory area to load U-Boot */
455 mv_sys_xor_finish();
456 dram_info.num_cs = 1;
457 dram_info.cs_ena = 1;
458 mv_sys_xor_init(&dram_info);
459 mv_xor_mem_init(0, scrub_offs, scrub_size, 0xdeadbeef,
460 0xdeadbeef);
461
462 /* Wait for previous transfer completion */
463 while (mv_xor_state_get(0) != MV_IDLE)
464 ;
465
466 if (debug_mode)
467 DEBUG_MAIN_S("DDR3 Training Sequence - DEBUG - 13\n");
468 }
469
470 /* Return XOR State */
471 mv_sys_xor_finish();
472
473 #if defined(MV88F78X60)
474 /* Save training results in memeory for resume state */
475 ddr3_save_training(&dram_info);
476 #endif
477 /* Clear ODT always on */
478 ddr3_odt_activate(0);
479
480 /* Configure Dynamic read ODT */
481 ddr3_odt_read_dynamic_config(&dram_info);
482
483 return MV_OK;
484 }
485
ddr3_set_performance_params(MV_DRAM_INFO * dram_info)486 void ddr3_set_performance_params(MV_DRAM_INFO *dram_info)
487 {
488 u32 twr2wr, trd2rd, trd2wr_wr2rd;
489 u32 tmp1, tmp2, reg;
490
491 DEBUG_MAIN_FULL_C("Max WL Phase: ", dram_info->wl_max_phase, 2);
492 DEBUG_MAIN_FULL_C("Min WL Phase: ", dram_info->wl_min_phase, 2);
493 DEBUG_MAIN_FULL_C("Max RL Phase: ", dram_info->rl_max_phase, 2);
494 DEBUG_MAIN_FULL_C("Min RL Phase: ", dram_info->rl_min_phase, 2);
495
496 if (dram_info->wl_max_phase < 2)
497 twr2wr = 0x2;
498 else
499 twr2wr = 0x3;
500
501 trd2rd = 0x1 + (dram_info->rl_max_phase + 1) / 2 +
502 (dram_info->rl_max_phase + 1) % 2;
503
504 tmp1 = (dram_info->rl_max_phase - dram_info->wl_min_phase) / 2 +
505 (((dram_info->rl_max_phase - dram_info->wl_min_phase) % 2) >
506 0 ? 1 : 0);
507 tmp2 = (dram_info->wl_max_phase - dram_info->rl_min_phase) / 2 +
508 ((dram_info->wl_max_phase - dram_info->rl_min_phase) % 2 >
509 0 ? 1 : 0);
510 trd2wr_wr2rd = (tmp1 >= tmp2) ? tmp1 : tmp2;
511
512 trd2wr_wr2rd += 2;
513 trd2rd += 2;
514 twr2wr += 2;
515
516 DEBUG_MAIN_FULL_C("WR 2 WR: ", twr2wr, 2);
517 DEBUG_MAIN_FULL_C("RD 2 RD: ", trd2rd, 2);
518 DEBUG_MAIN_FULL_C("RD 2 WR / WR 2 RD: ", trd2wr_wr2rd, 2);
519
520 reg = reg_read(REG_SDRAM_TIMING_HIGH_ADDR);
521
522 reg &= ~(REG_SDRAM_TIMING_H_W2W_MASK << REG_SDRAM_TIMING_H_W2W_OFFS);
523 reg |= ((twr2wr & REG_SDRAM_TIMING_H_W2W_MASK) <<
524 REG_SDRAM_TIMING_H_W2W_OFFS);
525
526 reg &= ~(REG_SDRAM_TIMING_H_R2R_MASK << REG_SDRAM_TIMING_H_R2R_OFFS);
527 reg &= ~(REG_SDRAM_TIMING_H_R2R_H_MASK <<
528 REG_SDRAM_TIMING_H_R2R_H_OFFS);
529 reg |= ((trd2rd & REG_SDRAM_TIMING_H_R2R_MASK) <<
530 REG_SDRAM_TIMING_H_R2R_OFFS);
531 reg |= (((trd2rd >> 2) & REG_SDRAM_TIMING_H_R2R_H_MASK) <<
532 REG_SDRAM_TIMING_H_R2R_H_OFFS);
533
534 reg &= ~(REG_SDRAM_TIMING_H_R2W_W2R_MASK <<
535 REG_SDRAM_TIMING_H_R2W_W2R_OFFS);
536 reg &= ~(REG_SDRAM_TIMING_H_R2W_W2R_H_MASK <<
537 REG_SDRAM_TIMING_H_R2W_W2R_H_OFFS);
538 reg |= ((trd2wr_wr2rd & REG_SDRAM_TIMING_H_R2W_W2R_MASK) <<
539 REG_SDRAM_TIMING_H_R2W_W2R_OFFS);
540 reg |= (((trd2wr_wr2rd >> 2) & REG_SDRAM_TIMING_H_R2W_W2R_H_MASK) <<
541 REG_SDRAM_TIMING_H_R2W_W2R_H_OFFS);
542
543 reg_write(REG_SDRAM_TIMING_HIGH_ADDR, reg);
544 }
545
546 /*
547 * Perform DDR3 PUP Indirect Write
548 */
ddr3_write_pup_reg(u32 mode,u32 cs,u32 pup,u32 phase,u32 delay)549 void ddr3_write_pup_reg(u32 mode, u32 cs, u32 pup, u32 phase, u32 delay)
550 {
551 u32 reg = 0;
552
553 if (pup == PUP_BC)
554 reg |= (1 << REG_PHY_BC_OFFS);
555 else
556 reg |= (pup << REG_PHY_PUP_OFFS);
557
558 reg |= ((0x4 * cs + mode) << REG_PHY_CS_OFFS);
559 reg |= (phase << REG_PHY_PHASE_OFFS) | delay;
560
561 if (mode == PUP_WL_MODE)
562 reg |= ((INIT_WL_DELAY + delay) << REG_PHY_DQS_REF_DLY_OFFS);
563
564 reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg); /* 0x16A0 */
565 reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
566 reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg); /* 0x16A0 */
567
568 do {
569 reg = reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR) &
570 REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
571 } while (reg); /* Wait for '0' to mark the end of the transaction */
572
573 /* If read Leveling mode - need to write to register 3 separetly */
574 if (mode == PUP_RL_MODE) {
575 reg = 0;
576
577 if (pup == PUP_BC)
578 reg |= (1 << REG_PHY_BC_OFFS);
579 else
580 reg |= (pup << REG_PHY_PUP_OFFS);
581
582 reg |= ((0x4 * cs + mode + 1) << REG_PHY_CS_OFFS);
583 reg |= (INIT_RL_DELAY);
584
585 reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg); /* 0x16A0 */
586 reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
587 reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg); /* 0x16A0 */
588
589 do {
590 reg = reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR) &
591 REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
592 } while (reg);
593 }
594 }
595
596 /*
597 * Perform DDR3 PUP Indirect Read
598 */
ddr3_read_pup_reg(u32 mode,u32 cs,u32 pup)599 u32 ddr3_read_pup_reg(u32 mode, u32 cs, u32 pup)
600 {
601 u32 reg;
602
603 reg = (pup << REG_PHY_PUP_OFFS) |
604 ((0x4 * cs + mode) << REG_PHY_CS_OFFS);
605 reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg); /* 0x16A0 */
606
607 reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_RD;
608 reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg); /* 0x16A0 */
609
610 do {
611 reg = reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR) &
612 REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
613 } while (reg); /* Wait for '0' to mark the end of the transaction */
614
615 return reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR); /* 0x16A0 */
616 }
617
618 /*
619 * Set training patterns
620 */
ddr3_load_patterns(MV_DRAM_INFO * dram_info,int resume)621 int ddr3_load_patterns(MV_DRAM_INFO *dram_info, int resume)
622 {
623 u32 reg;
624
625 /* Enable SW override - Required for the ECC Pup */
626 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
627 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
628
629 /* [0] = 1 - Enable SW override */
630 /* 0x15B8 - Training SW 2 Register */
631 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
632
633 reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);
634 reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
635
636 if (resume == 0) {
637 #if defined(MV88F78X60) || defined(MV88F672X)
638 ddr3_load_pbs_patterns(dram_info);
639 #endif
640 ddr3_load_dqs_patterns(dram_info);
641 }
642
643 /* Disable SW override - Must be in a different stage */
644 /* [0]=0 - Enable SW override */
645 reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
646 reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
647 /* 0x15B8 - Training SW 2 Register */
648 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
649
650 reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
651 (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
652 reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
653
654 /* Set Base Addr */
655 #if defined(MV88F67XX)
656 reg_write(REG_DRAM_TRAINING_PATTERN_BASE_ADDR, 0);
657 #else
658 if (resume == 0)
659 reg_write(REG_DRAM_TRAINING_PATTERN_BASE_ADDR, 0);
660 else
661 reg_write(REG_DRAM_TRAINING_PATTERN_BASE_ADDR,
662 RESUME_RL_PATTERNS_ADDR);
663 #endif
664
665 /* Set Patterns */
666 if (resume == 0) {
667 reg = (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS) |
668 (1 << REG_DRAM_TRAINING_PATTERNS_OFFS);
669 } else {
670 reg = (0x1 << REG_DRAM_TRAINING_CS_OFFS) |
671 (1 << REG_DRAM_TRAINING_PATTERNS_OFFS);
672 }
673
674 reg |= (1 << REG_DRAM_TRAINING_AUTO_OFFS);
675
676 reg_write(REG_DRAM_TRAINING_ADDR, reg);
677
678 udelay(100);
679
680 /* Check if Successful */
681 if (reg_read(REG_DRAM_TRAINING_ADDR) &
682 (1 << REG_DRAM_TRAINING_ERROR_OFFS))
683 return MV_OK;
684 else
685 return MV_FAIL;
686 }
687
688 #if !defined(MV88F67XX)
689 /*
690 * Name: ddr3_save_training(MV_DRAM_INFO *dram_info)
691 * Desc: saves the training results to memeory (RL,WL,PBS,Rx/Tx
692 * Centeralization)
693 * Args: MV_DRAM_INFO *dram_info
694 * Notes:
695 * Returns: None.
696 */
ddr3_save_training(MV_DRAM_INFO * dram_info)697 void ddr3_save_training(MV_DRAM_INFO *dram_info)
698 {
699 u32 val, pup, tmp_cs, cs, i, dq;
700 u32 crc = 0;
701 u32 regs = 0;
702 u32 *sdram_offset = (u32 *)RESUME_TRAINING_VALUES_ADDR;
703 u32 mode_config[MAX_TRAINING_MODE];
704
705 mode_config[DQS_WR_MODE] = PUP_DQS_WR;
706 mode_config[WL_MODE_] = PUP_WL_MODE;
707 mode_config[RL_MODE_] = PUP_RL_MODE;
708 mode_config[DQS_RD_MODE] = PUP_DQS_RD;
709 mode_config[PBS_TX_DM_MODE] = PUP_PBS_TX_DM;
710 mode_config[PBS_TX_MODE] = PUP_PBS_TX;
711 mode_config[PBS_RX_MODE] = PUP_PBS_RX;
712
713 /* num of training modes */
714 for (i = 0; i < MAX_TRAINING_MODE; i++) {
715 tmp_cs = dram_info->cs_ena;
716 /* num of CS */
717 for (cs = 0; cs < MAX_CS; cs++) {
718 if (tmp_cs & (1 << cs)) {
719 /* num of PUPs */
720 for (pup = 0; pup < dram_info->num_of_total_pups;
721 pup++) {
722 if (pup == dram_info->num_of_std_pups &&
723 dram_info->ecc_ena)
724 pup = ECC_PUP;
725 if (i == PBS_TX_DM_MODE) {
726 /*
727 * Change CS bitmask because
728 * PBS works only with CS0
729 */
730 tmp_cs = 0x1;
731 val = ddr3_read_pup_reg(
732 mode_config[i], CS0, pup);
733 } else if (i == PBS_TX_MODE ||
734 i == PBS_RX_MODE) {
735 /*
736 * Change CS bitmask because
737 * PBS works only with CS0
738 */
739 tmp_cs = 0x1;
740 for (dq = 0; dq <= DQ_NUM;
741 dq++) {
742 val = ddr3_read_pup_reg(
743 mode_config[i] + dq,
744 CS0,
745 pup);
746 (*sdram_offset) = val;
747 crc += *sdram_offset;
748 sdram_offset++;
749 regs++;
750 }
751 continue;
752 } else {
753 val = ddr3_read_pup_reg(
754 mode_config[i], cs, pup);
755 }
756
757 *sdram_offset = val;
758 crc += *sdram_offset;
759 sdram_offset++;
760 regs++;
761 }
762 }
763 }
764 }
765
766 *sdram_offset = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
767 crc += *sdram_offset;
768 sdram_offset++;
769 regs++;
770 *sdram_offset = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
771 crc += *sdram_offset;
772 sdram_offset++;
773 regs++;
774 sdram_offset = (u32 *)NUM_OF_REGISTER_ADDR;
775 *sdram_offset = regs;
776 DEBUG_SUSPEND_RESUME_S("Training Results CheckSum write= ");
777 DEBUG_SUSPEND_RESUME_D(crc, 8);
778 DEBUG_SUSPEND_RESUME_S("\n");
779 sdram_offset = (u32 *)CHECKSUM_RESULT_ADDR;
780 *sdram_offset = crc;
781 }
782
783 /*
784 * Name: ddr3_read_training_results()
785 * Desc: Reads the training results from memeory (RL,WL,PBS,Rx/Tx
786 * Centeralization)
787 * and writes them to the relevant registers
788 * Args: MV_DRAM_INFO *dram_info
789 * Notes:
790 * Returns: None.
791 */
ddr3_read_training_results(void)792 int ddr3_read_training_results(void)
793 {
794 u32 val, reg, idx, dqs_wr_idx = 0, crc = 0;
795 u32 *sdram_offset = (u32 *)RESUME_TRAINING_VALUES_ADDR;
796 u32 training_val[RESUME_TRAINING_VALUES_MAX] = { 0 };
797 u32 regs = *((u32 *)NUM_OF_REGISTER_ADDR);
798
799 /*
800 * Read Training results & Dunit registers from memory and write
801 * it to an array
802 */
803 for (idx = 0; idx < regs; idx++) {
804 training_val[idx] = *sdram_offset;
805 crc += *sdram_offset;
806 sdram_offset++;
807 }
808
809 sdram_offset = (u32 *)CHECKSUM_RESULT_ADDR;
810
811 if ((*sdram_offset) == crc) {
812 DEBUG_SUSPEND_RESUME_S("Training Results CheckSum read PASS= ");
813 DEBUG_SUSPEND_RESUME_D(crc, 8);
814 DEBUG_SUSPEND_RESUME_S("\n");
815 } else {
816 DEBUG_MAIN_S("Wrong Training Results CheckSum\n");
817 return MV_FAIL;
818 }
819
820 /*
821 * We iterate through all the registers except for the last 2 since
822 * they are Dunit registers (and not PHY registers)
823 */
824 for (idx = 0; idx < (regs - 2); idx++) {
825 val = training_val[idx];
826 reg = (val >> REG_PHY_CS_OFFS) & 0x3F; /*read the phy address */
827
828 /* Check if the values belongs to the DQS WR */
829 if (reg == PUP_WL_MODE) {
830 /* bit[5:0] in DQS_WR are delay */
831 val = (training_val[dqs_wr_idx++] & 0x3F);
832 /*
833 * bit[15:10] are DQS_WR delay & bit[9:0] are
834 * WL phase & delay
835 */
836 val = (val << REG_PHY_DQS_REF_DLY_OFFS) |
837 (training_val[idx] & 0x3C003FF);
838 /* Add Request pending and write operation bits */
839 val |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
840 } else if (reg == PUP_DQS_WR) {
841 /*
842 * Do nothing since DQS_WR will be done in PUP_WL_MODE
843 */
844 continue;
845 }
846
847 val |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
848 reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, val);
849 do {
850 val = (reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR)) &
851 REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
852 } while (val); /* Wait for '0' to mark the end of the transaction */
853 }
854
855 /* write last 2 Dunit configurations */
856 val = training_val[idx];
857 reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, val); /* reg 0x1538 */
858 val = training_val[idx + 1];
859 reg_write(REG_READ_DATA_READY_DELAYS_ADDR, val); /* reg 0x153c */
860
861 return MV_OK;
862 }
863
864 /*
865 * Name: ddr3_check_if_resume_mode()
866 * Desc: Reads the address (0x3000) of the Resume Magic word (0xDEADB002)
867 * Args: MV_DRAM_INFO *dram_info
868 * Notes:
869 * Returns: return (magic_word == SUSPEND_MAGIC_WORD)
870 */
ddr3_check_if_resume_mode(MV_DRAM_INFO * dram_info,u32 freq)871 int ddr3_check_if_resume_mode(MV_DRAM_INFO *dram_info, u32 freq)
872 {
873 u32 magic_word;
874 u32 *sdram_offset = (u32 *)BOOT_INFO_ADDR;
875
876 if (dram_info->reg_dimm != 1) {
877 /*
878 * Perform write levleling in order initiate the phy with
879 * low frequency
880 */
881 if (MV_OK != ddr3_write_leveling_hw(freq, dram_info)) {
882 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Write Leveling Hw)\n");
883 return MV_DDR3_TRAINING_ERR_WR_LVL_HW;
884 }
885 }
886
887 if (MV_OK != ddr3_load_patterns(dram_info, 1)) {
888 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Loading Patterns)\n");
889 return MV_DDR3_TRAINING_ERR_LOAD_PATTERNS;
890 }
891
892 /* Enable CS0 only for RL */
893 dram_info->cs_ena = 0x1;
894
895 /* Perform Read levleling in order to get stable memory */
896 if (MV_OK != ddr3_read_leveling_hw(freq, dram_info)) {
897 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Read Leveling Hw)\n");
898 return MV_DDR3_TRAINING_ERR_WR_LVL_HW;
899 }
900
901 /* Back to relevant CS */
902 dram_info->cs_ena = ddr3_get_cs_ena_from_reg();
903
904 magic_word = *sdram_offset;
905 return magic_word == SUSPEND_MAGIC_WORD;
906 }
907
908 /*
909 * Name: ddr3_training_suspend_resume()
910 * Desc: Execute the Resume state
911 * Args: MV_DRAM_INFO *dram_info
912 * Notes:
913 * Returns: return (magic_word == SUSPEND_MAGIC_WORD)
914 */
ddr3_training_suspend_resume(MV_DRAM_INFO * dram_info)915 int ddr3_training_suspend_resume(MV_DRAM_INFO *dram_info)
916 {
917 u32 freq, reg;
918 int tmp_ratio;
919
920 /* Configure DDR */
921 if (MV_OK != ddr3_read_training_results())
922 return MV_FAIL;
923
924 /* Reset read FIFO */
925 reg = reg_read(REG_DRAM_TRAINING_ADDR);
926
927 /* Start Auto Read Leveling procedure */
928 reg |= (1 << REG_DRAM_TRAINING_RL_OFFS);
929 reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
930
931 reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
932 reg |= ((1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS) +
933 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS));
934
935 /* [0] = 1 - Enable SW override, [4] = 1 - FIFO reset */
936 /* 0x15B8 - Training SW 2 Register */
937 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
938
939 udelay(2);
940
941 reg = reg_read(REG_DRAM_TRAINING_ADDR);
942 /* Clear Auto Read Leveling procedure */
943 reg &= ~(1 << REG_DRAM_TRAINING_RL_OFFS);
944 reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
945
946 /* Return to target frequency */
947 freq = dram_info->target_frequency;
948 tmp_ratio = 1;
949 if (MV_OK != ddr3_dfs_low_2_high(freq, tmp_ratio, dram_info)) {
950 DEBUG_MAIN_S("DDR3 Training Sequence - FAILED (Dfs Low2High)\n");
951 return MV_DDR3_TRAINING_ERR_DFS_H2L;
952 }
953
954 if (dram_info->ecc_ena) {
955 /* Scabbling the RL area pattern and the training area */
956 mv_sys_xor_finish();
957 dram_info->num_cs = 1;
958 dram_info->cs_ena = 1;
959 mv_sys_xor_init(dram_info);
960 mv_xor_mem_init(0, RESUME_RL_PATTERNS_ADDR,
961 RESUME_RL_PATTERNS_SIZE, 0xFFFFFFFF, 0xFFFFFFFF);
962
963 /* Wait for previous transfer completion */
964
965 while (mv_xor_state_get(0) != MV_IDLE)
966 ;
967
968 /* Return XOR State */
969 mv_sys_xor_finish();
970 }
971
972 return MV_OK;
973 }
974 #endif
975
ddr3_print_freq(u32 freq)976 void ddr3_print_freq(u32 freq)
977 {
978 u32 tmp_freq;
979
980 switch (freq) {
981 case 0:
982 tmp_freq = 100;
983 break;
984 case 1:
985 tmp_freq = 300;
986 break;
987 case 2:
988 tmp_freq = 360;
989 break;
990 case 3:
991 tmp_freq = 400;
992 break;
993 case 4:
994 tmp_freq = 444;
995 break;
996 case 5:
997 tmp_freq = 500;
998 break;
999 case 6:
1000 tmp_freq = 533;
1001 break;
1002 case 7:
1003 tmp_freq = 600;
1004 break;
1005 case 8:
1006 tmp_freq = 666;
1007 break;
1008 case 9:
1009 tmp_freq = 720;
1010 break;
1011 case 10:
1012 tmp_freq = 800;
1013 break;
1014 default:
1015 tmp_freq = 100;
1016 }
1017
1018 printf("Current frequency is: %dMHz\n", tmp_freq);
1019 }
1020
ddr3_get_min_max_read_sample_delay(u32 cs_enable,u32 reg,u32 * min,u32 * max,u32 * cs_max)1021 int ddr3_get_min_max_read_sample_delay(u32 cs_enable, u32 reg, u32 *min,
1022 u32 *max, u32 *cs_max)
1023 {
1024 u32 cs, delay;
1025
1026 *min = 0xFFFFFFFF;
1027 *max = 0x0;
1028
1029 for (cs = 0; cs < MAX_CS; cs++) {
1030 if ((cs_enable & (1 << cs)) == 0)
1031 continue;
1032
1033 delay = ((reg >> (cs * 8)) & 0x1F);
1034
1035 if (delay < *min)
1036 *min = delay;
1037
1038 if (delay > *max) {
1039 *max = delay;
1040 *cs_max = cs;
1041 }
1042 }
1043
1044 return MV_OK;
1045 }
1046
ddr3_get_min_max_rl_phase(MV_DRAM_INFO * dram_info,u32 * min,u32 * max,u32 cs)1047 int ddr3_get_min_max_rl_phase(MV_DRAM_INFO *dram_info, u32 *min, u32 *max,
1048 u32 cs)
1049 {
1050 u32 pup, reg, phase;
1051
1052 *min = 0xFFFFFFFF;
1053 *max = 0x0;
1054
1055 for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
1056 reg = ddr3_read_pup_reg(PUP_RL_MODE, cs, pup);
1057 phase = ((reg >> 8) & 0x7);
1058
1059 if (phase < *min)
1060 *min = phase;
1061
1062 if (phase > *max)
1063 *max = phase;
1064 }
1065
1066 return MV_OK;
1067 }
1068
ddr3_odt_activate(int activate)1069 int ddr3_odt_activate(int activate)
1070 {
1071 u32 reg, mask;
1072
1073 mask = (1 << REG_DUNIT_ODT_CTRL_OVRD_OFFS) |
1074 (1 << REG_DUNIT_ODT_CTRL_OVRD_VAL_OFFS);
1075 /* {0x0000149C} - DDR Dunit ODT Control Register */
1076 reg = reg_read(REG_DUNIT_ODT_CTRL_ADDR);
1077 if (activate)
1078 reg |= mask;
1079 else
1080 reg &= ~mask;
1081
1082 reg_write(REG_DUNIT_ODT_CTRL_ADDR, reg);
1083
1084 return MV_OK;
1085 }
1086
ddr3_odt_read_dynamic_config(MV_DRAM_INFO * dram_info)1087 int ddr3_odt_read_dynamic_config(MV_DRAM_INFO *dram_info)
1088 {
1089 u32 min_read_sample_delay, max_read_sample_delay, max_rl_phase;
1090 u32 min, max, cs_max;
1091 u32 cs_ena, reg;
1092
1093 reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
1094 cs_ena = ddr3_get_cs_ena_from_reg();
1095
1096 /* Get minimum and maximum of read sample delay of all CS */
1097 ddr3_get_min_max_read_sample_delay(cs_ena, reg, &min_read_sample_delay,
1098 &max_read_sample_delay, &cs_max);
1099
1100 /*
1101 * Get minimum and maximum read leveling phase which belongs to the
1102 * maximal read sample delay
1103 */
1104 ddr3_get_min_max_rl_phase(dram_info, &min, &max, cs_max);
1105 max_rl_phase = max;
1106
1107 /* DDR ODT Timing (Low) Register calculation */
1108 reg = reg_read(REG_ODT_TIME_LOW_ADDR);
1109 reg &= ~(0x1FF << REG_ODT_ON_CTL_RD_OFFS);
1110 reg |= (((min_read_sample_delay - 1) & 0xF) << REG_ODT_ON_CTL_RD_OFFS);
1111 reg |= (((max_read_sample_delay + 4 + (((max_rl_phase + 1) / 2) + 1)) &
1112 0x1F) << REG_ODT_OFF_CTL_RD_OFFS);
1113 reg_write(REG_ODT_TIME_LOW_ADDR, reg);
1114
1115 return MV_OK;
1116 }
1117