1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020 MediaTek Inc.
4 *
5 * Author: Weijie Gao <weijie.gao@mediatek.com>
6 */
7
8 #include <common.h>
9 #include <asm/addrspace.h>
10 #include <asm/cacheops.h>
11 #include <asm/global_data.h>
12 #include <linux/bitops.h>
13 #include <linux/io.h>
14 #include <mach/mc.h>
15
16 DECLARE_GLOBAL_DATA_PTR;
17
18 #define COARSE_MIN_START 6
19 #define FINE_MIN_START 15
20 #define COARSE_MAX_START 7
21 #define FINE_MAX_START 0
22
23 #define NUM_OF_CACHELINE 128
24 #define TEST_PAT_SIZE (NUM_OF_CACHELINE * CONFIG_SYS_CACHELINE_SIZE)
25
26 #define INIT_DQS_VAL ((7 << DQS1_DELAY_COARSE_TUNING_S) | \
27 (4 << DQS1_DELAY_FINE_TUNING_S) | \
28 (7 << DQS0_DELAY_COARSE_TUNING_S) | \
29 (4 << DQS0_DELAY_FINE_TUNING_S))
30
pref_op(int op,const volatile void * addr)31 static inline void pref_op(int op, const volatile void *addr)
32 {
33 __asm__ __volatile__("pref %0, 0(%1)" : : "i" (op), "r" (addr));
34 }
35
dqs_test_error(void __iomem * memc,u32 memsize,u32 dqsval,u32 bias)36 static inline bool dqs_test_error(void __iomem *memc, u32 memsize, u32 dqsval,
37 u32 bias)
38 {
39 u32 *nca, *ca;
40 u32 off;
41 int i;
42
43 for (off = 0; off < memsize - TEST_PAT_SIZE; off += (memsize >> 6)) {
44 nca = (u32 *)KSEG1ADDR(off);
45 ca = (u32 *)KSEG0ADDR(off);
46
47 writel(INIT_DQS_VAL, memc + MEMCTL_DDR_DQS_DLY_REG);
48 wmb();
49
50 for (i = 0; i < TEST_PAT_SIZE / sizeof(u32); i++)
51 ca[i] = 0x1f1f1f1f;
52
53 for (i = 0; i < TEST_PAT_SIZE / sizeof(u32); i++)
54 nca[i] = (u32)nca + i + bias;
55
56 writel(dqsval, memc + MEMCTL_DDR_DQS_DLY_REG);
57 wmb();
58
59 for (i = 0; i < TEST_PAT_SIZE; i += CONFIG_SYS_CACHELINE_SIZE)
60 mips_cache(HIT_INVALIDATE_D, (u8 *)ca + i);
61 wmb();
62
63 for (i = 0; i < TEST_PAT_SIZE; i += CONFIG_SYS_CACHELINE_SIZE)
64 pref_op(0, (u8 *)ca + i);
65
66 for (i = 0; i < TEST_PAT_SIZE / sizeof(u32); i++) {
67 if (ca[i] != (u32)nca + i + bias)
68 return true;
69 }
70 }
71
72 return false;
73 }
74
dqs_find_max(void __iomem * memc,u32 memsize,int initval,int maxval,int shift,u32 regval)75 static inline int dqs_find_max(void __iomem *memc, u32 memsize, int initval,
76 int maxval, int shift, u32 regval)
77 {
78 int fieldval;
79 u32 dqsval;
80
81 for (fieldval = initval; fieldval <= maxval; fieldval++) {
82 dqsval = regval | (fieldval << shift);
83 if (dqs_test_error(memc, memsize, dqsval, 3))
84 return max(fieldval - 1, initval);
85 }
86
87 return maxval;
88 }
89
dqs_find_min(void __iomem * memc,u32 memsize,int initval,int minval,int shift,u32 regval)90 static inline int dqs_find_min(void __iomem *memc, u32 memsize, int initval,
91 int minval, int shift, u32 regval)
92 {
93 int fieldval;
94 u32 dqsval;
95
96 for (fieldval = initval; fieldval >= minval; fieldval--) {
97 dqsval = regval | (fieldval << shift);
98 if (dqs_test_error(memc, memsize, dqsval, 1))
99 return min(fieldval + 1, initval);
100 }
101
102 return minval;
103 }
104
ddr_calibrate(void __iomem * memc,u32 memsize,u32 bw)105 void ddr_calibrate(void __iomem *memc, u32 memsize, u32 bw)
106 {
107 u32 dqs_coarse_min, dqs_coarse_max, dqs_coarse_val;
108 u32 dqs_fine_min, dqs_fine_max, dqs_fine_val;
109 u32 dqs_coarse_min_limit, dqs_fine_min_limit;
110 u32 dlls, dqs_dll, ddr_cfg2_reg;
111 u32 dqs_dly_tmp, dqs_dly, test_dqs, shift;
112 u32 rem, mask;
113 int i;
114
115 /* Disable Self-refresh */
116 clrbits_32(memc + MEMCTL_DDR_SELF_REFRESH_REG, SR_AUTO_EN);
117
118 /* Save DDR_CFG2 and modify its DQS gating window */
119 ddr_cfg2_reg = readl(memc + MEMCTL_DDR_CFG2_REG);
120 mask = DQS0_GATING_WINDOW_M;
121 if (bw == IND_SDRAM_WIDTH_16BIT)
122 mask |= DQS1_GATING_WINDOW_M;
123 clrbits_32(memc + MEMCTL_DDR_CFG2_REG, mask);
124
125 /* Get minimum available DQS value */
126 dlls = readl(memc + MEMCTL_DLL_DBG_REG);
127 dlls = (dlls & MST_DLY_SEL_M) >> MST_DLY_SEL_S;
128
129 dqs_dll = dlls >> 4;
130 if (dqs_dll <= 8)
131 dqs_coarse_min_limit = 8 - dqs_dll;
132 else
133 dqs_coarse_min_limit = 0;
134
135 dqs_dll = dlls & 0xf;
136 if (dqs_dll <= 8)
137 dqs_fine_min_limit = 8 - dqs_dll;
138 else
139 dqs_fine_min_limit = 0;
140
141 /* Initial DQS register value */
142 dqs_dly = INIT_DQS_VAL;
143
144 /* Calibrate DQS0 and/or DQS1 */
145 for (i = 0; i < bw; i++) {
146 shift = i * 8;
147 dqs_dly &= ~(0xff << shift);
148
149 /* Find maximum DQS coarse-grain */
150 dqs_dly_tmp = dqs_dly | (0xf << shift);
151 dqs_coarse_max = dqs_find_max(memc, memsize, COARSE_MAX_START,
152 0xf, 4 + shift, dqs_dly_tmp);
153
154 /* Find maximum DQS fine-grain */
155 dqs_dly_tmp = dqs_dly | (dqs_coarse_max << (4 + shift));
156 test_dqs = dqs_find_max(memc, memsize, FINE_MAX_START, 0xf,
157 shift, dqs_dly_tmp);
158
159 if (test_dqs == FINE_MAX_START) {
160 dqs_coarse_max--;
161 dqs_fine_max = 0xf;
162 } else {
163 dqs_fine_max = test_dqs - 1;
164 }
165
166 /* Find minimum DQS coarse-grain */
167 dqs_dly_tmp = dqs_dly;
168 dqs_coarse_min = dqs_find_min(memc, memsize, COARSE_MIN_START,
169 dqs_coarse_min_limit, 4 + shift,
170 dqs_dly_tmp);
171
172 /* Find minimum DQS fine-grain */
173 dqs_dly_tmp = dqs_dly | (dqs_coarse_min << (4 + shift));
174 test_dqs = dqs_find_min(memc, memsize, FINE_MIN_START,
175 dqs_fine_min_limit, shift, dqs_dly_tmp);
176
177 if (test_dqs == FINE_MIN_START + 1) {
178 dqs_coarse_min++;
179 dqs_fine_min = 0;
180 } else {
181 dqs_fine_min = test_dqs;
182 }
183
184 /* Calculate central DQS coarse/fine value */
185 dqs_coarse_val = (dqs_coarse_max + dqs_coarse_min) >> 1;
186 rem = (dqs_coarse_max + dqs_coarse_min) % 2;
187
188 dqs_fine_val = (rem * 4) + ((dqs_fine_max + dqs_fine_min) >> 1);
189 if (dqs_fine_val >= 0x10) {
190 dqs_coarse_val++;
191 dqs_fine_val -= 8;
192 }
193
194 /* Save current DQS value */
195 dqs_dly |= ((dqs_coarse_val << 4) | dqs_fine_val) << shift;
196 }
197
198 /* Set final DQS value */
199 writel(dqs_dly, memc + MEMCTL_DDR_DQS_DLY_REG);
200
201 /* Restore DDR_CFG2 */
202 writel(ddr_cfg2_reg, memc + MEMCTL_DDR_CFG2_REG);
203
204 /* Enable Self-refresh */
205 setbits_32(memc + MEMCTL_DDR_SELF_REFRESH_REG, SR_AUTO_EN);
206 }
207