1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2020 Marvell International Ltd.
4 */
5
6 #ifndef __OCTEON_DDR_H_
7 #define __OCTEON_DDR_H_
8
9 #include <env.h>
10 #include <linux/compat.h>
11 #include <linux/delay.h>
12 #include <linux/io.h>
13 #include <mach/octeon-model.h>
14 #include <mach/cvmx-lmcx-defs.h>
15 #include <mach/cvmx-regs.h>
16 #include <mach/cvmx-l2c-defs.h>
17
18 /* Some "external" (non-LMC) registers */
19 #define CVMX_IPD_CLK_COUNT 0x00014F0000000338
20 #define CVMX_FPA_CLK_COUNT 0x00012800000000F0
21
22 #define CVMX_NODE_MEM_SHIFT 40
23
24 #define DDR_INTERFACE_MAX 4
25
26 /* Private data struct */
27 struct ddr_priv {
28 void __iomem *lmc_base;
29 void __iomem *l2c_base;
30
31 bool ddr_clock_initialized[DDR_INTERFACE_MAX];
32 bool ddr_memory_preserved;
33 u32 flags;
34
35 struct ram_info info;
36 };
37
38 /* Short cut to convert a number to megabytes */
39 #define MB(X) ((u64)(X) * (u64)(1024 * 1024))
40
41 #define octeon_is_cpuid(x) (__OCTEON_IS_MODEL_COMPILE__(x, read_c0_prid()))
42
43 #define strtoull simple_strtoull
44
45 /* Access LMC registers */
lmc_rd(struct ddr_priv * priv,u64 addr)46 static inline u64 lmc_rd(struct ddr_priv *priv, u64 addr)
47 {
48 return ioread64(priv->lmc_base + addr);
49 }
50
lmc_wr(struct ddr_priv * priv,u64 addr,u64 val)51 static inline void lmc_wr(struct ddr_priv *priv, u64 addr, u64 val)
52 {
53 iowrite64(val, priv->lmc_base + addr);
54 }
55
56 /* Access L2C registers */
l2c_rd(struct ddr_priv * priv,u64 addr)57 static inline u64 l2c_rd(struct ddr_priv *priv, u64 addr)
58 {
59 return ioread64(priv->l2c_base + addr);
60 }
61
l2c_wr(struct ddr_priv * priv,u64 addr,u64 val)62 static inline void l2c_wr(struct ddr_priv *priv, u64 addr, u64 val)
63 {
64 iowrite64(val, priv->l2c_base + addr);
65 }
66
67 /* Failsafe mode */
68 #define FLAG_FAILSAFE_MODE 0x01000
69 /* Note that the DDR clock initialized flags must be contiguous */
70 /* Clock for DDR 0 initialized */
71 #define FLAG_DDR0_CLK_INITIALIZED 0x02000
72 /* Clock for DDR 1 initialized */
73 #define FLAG_DDR1_CLK_INITIALIZED 0x04000
74 /* Clock for DDR 2 initialized */
75 #define FLAG_DDR2_CLK_INITIALIZED 0x08000
76 /* Clock for DDR 3 initialized */
77 #define FLAG_DDR3_CLK_INITIALIZED 0x10000
78 /* Loaded into RAM externally */
79 #define FLAG_RAM_RESIDENT 0x20000
80 /* Verbose DDR information */
81 #define FLAG_DDR_VERBOSE 0x40000
82 /* Check env. for DDR variables */
83 #define FLAG_DDR_DEBUG 0x80000
84 #define FLAG_DDR_TRACE_INIT 0x100000
85 #define FLAG_MEMORY_PRESERVED 0x200000
86 #define FLAG_DFM_VERBOSE 0x400000
87 #define FLAG_DFM_TRACE_INIT 0x800000
88 /* DFM memory clock initialized */
89 #define FLAG_DFM_CLK_INITIALIZED 0x1000000
90 /* EEPROM clock descr. missing */
91 #define FLAG_CLOCK_DESC_MISSING 0x2000000
92 /* EEPROM board descr. missing */
93 #define FLAG_BOARD_DESC_MISSING 0x4000000
94 #define FLAG_DDR_PROMPT 0x8000000
95
96 #ifndef DDR_NO_DEBUG
ddr_verbose(struct ddr_priv * priv)97 static inline int ddr_verbose(struct ddr_priv *priv)
98 {
99 return !!(priv->flags & FLAG_DDR_VERBOSE);
100 }
101
ddr_getenv_debug(struct ddr_priv * priv,char * name)102 static inline char *ddr_getenv_debug(struct ddr_priv *priv, char *name)
103 {
104 if (priv->flags & FLAG_FAILSAFE_MODE)
105 return NULL;
106
107 if (priv->flags & FLAG_DDR_DEBUG)
108 return env_get(name);
109
110 return NULL;
111 }
112 #else
ddr_verbose(void)113 static inline int ddr_verbose(void)
114 {
115 return 0;
116 }
117 #endif
118
119 /* turn the variable name into a string */
120 #define CVMX_TMP_STR(x) CVMX_TMP_STR2(x)
121 #define CVMX_TMP_STR2(x) #x
122
123 #define CVMX_CACHE(op, address, offset) \
124 asm volatile ("cache " CVMX_TMP_STR(op) ", " \
125 CVMX_TMP_STR(offset) "(%[rbase])" \
126 : : [rbase] "d" (address))
127
128 /* unlock the state */
129 #define CVMX_CACHE_WBIL2(address, offset) \
130 CVMX_CACHE(23, address, offset)
131
132 /* complete prefetches, invalidate entire dcache */
133 #define CVMX_DCACHE_INVALIDATE \
134 { CVMX_SYNC; asm volatile ("cache 9, 0($0)" : : ); }
135
136 #define DEBUG_VALIDATE_BITMASK 0
137 #if DEBUG_VALIDATE_BITMASK
138 #define debug_bitmask_print printf
139 #else
140 #define debug_bitmask_print(...)
141 #endif
142
143 #define RLEVEL_BITMASK_TRAILING_BITS_ERROR 5
144 // FIXME? now less than TOOLONG
145 #define RLEVEL_BITMASK_BUBBLE_BITS_ERROR 11
146 #define RLEVEL_BITMASK_NARROW_ERROR 6
147 #define RLEVEL_BITMASK_BLANK_ERROR 100
148 #define RLEVEL_BITMASK_TOOLONG_ERROR 12
149 #define RLEVEL_NONSEQUENTIAL_DELAY_ERROR 50
150 #define RLEVEL_ADJACENT_DELAY_ERROR 30
151
152 /*
153 * Apply a filter to the BITMASK results returned from Octeon
154 * read-leveling to determine the most likely delay result. This
155 * computed delay may be used to qualify the delay result returned by
156 * Octeon. Accumulate an error penalty for invalid characteristics of
157 * the bitmask so that they can be used to select the most reliable
158 * results.
159 *
160 * The algorithm searches for the largest contiguous MASK within a
161 * maximum RANGE of bits beginning with the MSB.
162 *
163 * 1. a MASK with a WIDTH less than 4 will be penalized
164 * 2. Bubbles in the bitmask that occur before or after the MASK
165 * will be penalized
166 * 3. If there are no trailing bubbles then extra bits that occur
167 * beyond the maximum RANGE will be penalized.
168 *
169 * +++++++++++++++++++++++++++++++++++++++++++++++++++
170 * + +
171 * + e.g. bitmask = 27B00 +
172 * + +
173 * + 63 +--- mstart 0 +
174 * + | | | +
175 * + | +---------+ +--- fb | +
176 * + | | range | | | +
177 * + V V V V V +
178 * + +
179 * + 0 0 ... 1 0 0 1 1 1 1 0 1 1 0 0 0 0 0 0 0 0 +
180 * + +
181 * + ^ ^ ^ +
182 * + | | mask| +
183 * + lb ---+ +-----+ +
184 * + width +
185 * + +
186 * +++++++++++++++++++++++++++++++++++++++++++++++++++
187 */
188
189 struct rlevel_bitmask {
190 u64 bm;
191 u8 mstart;
192 u8 width;
193 int errs;
194 };
195
196 #define MASKRANGE_BITS 6
197 #define MASKRANGE ((1 << MASKRANGE_BITS) - 1)
198
199 /* data field addresses in the DDR2 SPD eeprom */
200 enum ddr2_spd_addrs {
201 DDR2_SPD_BYTES_PROGRAMMED = 0,
202 DDR2_SPD_TOTAL_BYTES = 1,
203 DDR2_SPD_MEM_TYPE = 2,
204 DDR2_SPD_NUM_ROW_BITS = 3,
205 DDR2_SPD_NUM_COL_BITS = 4,
206 DDR2_SPD_NUM_RANKS = 5,
207 DDR2_SPD_CYCLE_CLX = 9,
208 DDR2_SPD_CONFIG_TYPE = 11,
209 DDR2_SPD_REFRESH = 12,
210 DDR2_SPD_SDRAM_WIDTH = 13,
211 DDR2_SPD_BURST_LENGTH = 16,
212 DDR2_SPD_NUM_BANKS = 17,
213 DDR2_SPD_CAS_LATENCY = 18,
214 DDR2_SPD_DIMM_TYPE = 20,
215 DDR2_SPD_CYCLE_CLX1 = 23,
216 DDR2_SPD_CYCLE_CLX2 = 25,
217 DDR2_SPD_TRP = 27,
218 DDR2_SPD_TRRD = 28,
219 DDR2_SPD_TRCD = 29,
220 DDR2_SPD_TRAS = 30,
221 DDR2_SPD_TWR = 36,
222 DDR2_SPD_TWTR = 37,
223 DDR2_SPD_TRFC_EXT = 40,
224 DDR2_SPD_TRFC = 42,
225 DDR2_SPD_CHECKSUM = 63,
226 DDR2_SPD_MFR_ID = 64
227 };
228
229 /* data field addresses in the DDR2 SPD eeprom */
230 enum ddr3_spd_addrs {
231 DDR3_SPD_BYTES_PROGRAMMED = 0,
232 DDR3_SPD_REVISION = 1,
233 DDR3_SPD_KEY_BYTE_DEVICE_TYPE = 2,
234 DDR3_SPD_KEY_BYTE_MODULE_TYPE = 3,
235 DDR3_SPD_DENSITY_BANKS = 4,
236 DDR3_SPD_ADDRESSING_ROW_COL_BITS = 5,
237 DDR3_SPD_NOMINAL_VOLTAGE = 6,
238 DDR3_SPD_MODULE_ORGANIZATION = 7,
239 DDR3_SPD_MEMORY_BUS_WIDTH = 8,
240 DDR3_SPD_FINE_TIMEBASE_DIVIDEND_DIVISOR = 9,
241 DDR3_SPD_MEDIUM_TIMEBASE_DIVIDEND = 10,
242 DDR3_SPD_MEDIUM_TIMEBASE_DIVISOR = 11,
243 DDR3_SPD_MINIMUM_CYCLE_TIME_TCKMIN = 12,
244 DDR3_SPD_CAS_LATENCIES_LSB = 14,
245 DDR3_SPD_CAS_LATENCIES_MSB = 15,
246 DDR3_SPD_MIN_CAS_LATENCY_TAAMIN = 16,
247 DDR3_SPD_MIN_WRITE_RECOVERY_TWRMIN = 17,
248 DDR3_SPD_MIN_RAS_CAS_DELAY_TRCDMIN = 18,
249 DDR3_SPD_MIN_ROW_ACTIVE_DELAY_TRRDMIN = 19,
250 DDR3_SPD_MIN_ROW_PRECHARGE_DELAY_TRPMIN = 20,
251 DDR3_SPD_UPPER_NIBBLES_TRAS_TRC = 21,
252 DDR3_SPD_MIN_ACTIVE_PRECHARGE_LSB_TRASMIN = 22,
253 DDR3_SPD_MIN_ACTIVE_REFRESH_LSB_TRCMIN = 23,
254 DDR3_SPD_MIN_REFRESH_RECOVERY_LSB_TRFCMIN = 24,
255 DDR3_SPD_MIN_REFRESH_RECOVERY_MSB_TRFCMIN = 25,
256 DDR3_SPD_MIN_INTERNAL_WRITE_READ_CMD_TWTRMIN = 26,
257 DDR3_SPD_MIN_INTERNAL_READ_PRECHARGE_CMD_TRTPMIN = 27,
258 DDR3_SPD_UPPER_NIBBLE_TFAW = 28,
259 DDR3_SPD_MIN_FOUR_ACTIVE_WINDOW_TFAWMIN = 29,
260 DDR3_SPD_SDRAM_OPTIONAL_FEATURES = 30,
261 DDR3_SPD_SDRAM_THERMAL_REFRESH_OPTIONS = 31,
262 DDR3_SPD_MODULE_THERMAL_SENSOR = 32,
263 DDR3_SPD_SDRAM_DEVICE_TYPE = 33,
264 DDR3_SPD_MINIMUM_CYCLE_TIME_FINE_TCKMIN = 34,
265 DDR3_SPD_MIN_CAS_LATENCY_FINE_TAAMIN = 35,
266 DDR3_SPD_MIN_RAS_CAS_DELAY_FINE_TRCDMIN = 36,
267 DDR3_SPD_MIN_ROW_PRECHARGE_DELAY_FINE_TRPMIN = 37,
268 DDR3_SPD_MIN_ACTIVE_REFRESH_LSB_FINE_TRCMIN = 38,
269 DDR3_SPD_REFERENCE_RAW_CARD = 62,
270 DDR3_SPD_ADDRESS_MAPPING = 63,
271 DDR3_SPD_REGISTER_MANUFACTURER_ID_LSB = 65,
272 DDR3_SPD_REGISTER_MANUFACTURER_ID_MSB = 66,
273 DDR3_SPD_REGISTER_REVISION_NUMBER = 67,
274 DDR3_SPD_MODULE_SERIAL_NUMBER = 122,
275 DDR3_SPD_CYCLICAL_REDUNDANCY_CODE_LOWER_NIBBLE = 126,
276 DDR3_SPD_CYCLICAL_REDUNDANCY_CODE_UPPER_NIBBLE = 127,
277 DDR3_SPD_MODULE_PART_NUMBER = 128
278 };
279
280 /* data field addresses in the DDR4 SPD eeprom */
281 enum ddr4_spd_addrs {
282 DDR4_SPD_BYTES_PROGRAMMED = 0,
283 DDR4_SPD_REVISION = 1,
284 DDR4_SPD_KEY_BYTE_DEVICE_TYPE = 2,
285 DDR4_SPD_KEY_BYTE_MODULE_TYPE = 3,
286 DDR4_SPD_DENSITY_BANKS = 4,
287 DDR4_SPD_ADDRESSING_ROW_COL_BITS = 5,
288 DDR4_SPD_PACKAGE_TYPE = 6,
289 DDR4_SPD_OPTIONAL_FEATURES = 7,
290 DDR4_SPD_THERMAL_REFRESH_OPTIONS = 8,
291 DDR4_SPD_OTHER_OPTIONAL_FEATURES = 9,
292 DDR4_SPD_SECONDARY_PACKAGE_TYPE = 10,
293 DDR4_SPD_MODULE_NOMINAL_VOLTAGE = 11,
294 DDR4_SPD_MODULE_ORGANIZATION = 12,
295 DDR4_SPD_MODULE_MEMORY_BUS_WIDTH = 13,
296 DDR4_SPD_MODULE_THERMAL_SENSOR = 14,
297 DDR4_SPD_RESERVED_BYTE15 = 15,
298 DDR4_SPD_RESERVED_BYTE16 = 16,
299 DDR4_SPD_TIMEBASES = 17,
300 DDR4_SPD_MINIMUM_CYCLE_TIME_TCKAVGMIN = 18,
301 DDR4_SPD_MAXIMUM_CYCLE_TIME_TCKAVGMAX = 19,
302 DDR4_SPD_CAS_LATENCIES_BYTE0 = 20,
303 DDR4_SPD_CAS_LATENCIES_BYTE1 = 21,
304 DDR4_SPD_CAS_LATENCIES_BYTE2 = 22,
305 DDR4_SPD_CAS_LATENCIES_BYTE3 = 23,
306 DDR4_SPD_MIN_CAS_LATENCY_TAAMIN = 24,
307 DDR4_SPD_MIN_RAS_CAS_DELAY_TRCDMIN = 25,
308 DDR4_SPD_MIN_ROW_PRECHARGE_DELAY_TRPMIN = 26,
309 DDR4_SPD_UPPER_NIBBLES_TRAS_TRC = 27,
310 DDR4_SPD_MIN_ACTIVE_PRECHARGE_LSB_TRASMIN = 28,
311 DDR4_SPD_MIN_ACTIVE_REFRESH_LSB_TRCMIN = 29,
312 DDR4_SPD_MIN_REFRESH_RECOVERY_LSB_TRFC1MIN = 30,
313 DDR4_SPD_MIN_REFRESH_RECOVERY_MSB_TRFC1MIN = 31,
314 DDR4_SPD_MIN_REFRESH_RECOVERY_LSB_TRFC2MIN = 32,
315 DDR4_SPD_MIN_REFRESH_RECOVERY_MSB_TRFC2MIN = 33,
316 DDR4_SPD_MIN_REFRESH_RECOVERY_LSB_TRFC4MIN = 34,
317 DDR4_SPD_MIN_REFRESH_RECOVERY_MSB_TRFC4MIN = 35,
318 DDR4_SPD_MIN_FOUR_ACTIVE_WINDOW_MSN_TFAWMIN = 36,
319 DDR4_SPD_MIN_FOUR_ACTIVE_WINDOW_LSB_TFAWMIN = 37,
320 DDR4_SPD_MIN_ROW_ACTIVE_DELAY_SAME_TRRD_SMIN = 38,
321 DDR4_SPD_MIN_ROW_ACTIVE_DELAY_DIFF_TRRD_LMIN = 39,
322 DDR4_SPD_MIN_CAS_TO_CAS_DELAY_TCCD_LMIN = 40,
323 DDR4_SPD_MIN_CAS_TO_CAS_DELAY_FINE_TCCD_LMIN = 117,
324 DDR4_SPD_MIN_ACT_TO_ACT_DELAY_SAME_FINE_TRRD_LMIN = 118,
325 DDR4_SPD_MIN_ACT_TO_ACT_DELAY_DIFF_FINE_TRRD_SMIN = 119,
326 DDR4_SPD_MIN_ACT_TO_ACT_REFRESH_DELAY_FINE_TRCMIN = 120,
327 DDR4_SPD_MIN_ROW_PRECHARGE_DELAY_FINE_TRPMIN = 121,
328 DDR4_SPD_MIN_RAS_TO_CAS_DELAY_FINE_TRCDMIN = 122,
329 DDR4_SPD_MIN_CAS_LATENCY_FINE_TAAMIN = 123,
330 DDR4_SPD_MAX_CYCLE_TIME_FINE_TCKAVGMAX = 124,
331 DDR4_SPD_MIN_CYCLE_TIME_FINE_TCKAVGMIN = 125,
332 DDR4_SPD_CYCLICAL_REDUNDANCY_CODE_LOWER_NIBBLE = 126,
333 DDR4_SPD_CYCLICAL_REDUNDANCY_CODE_UPPER_NIBBLE = 127,
334 DDR4_SPD_REFERENCE_RAW_CARD = 130,
335 DDR4_SPD_UDIMM_ADDR_MAPPING_FROM_EDGE = 131,
336 DDR4_SPD_REGISTER_MANUFACTURER_ID_LSB = 133,
337 DDR4_SPD_REGISTER_MANUFACTURER_ID_MSB = 134,
338 DDR4_SPD_REGISTER_REVISION_NUMBER = 135,
339 DDR4_SPD_RDIMM_ADDR_MAPPING_FROM_REGISTER_TO_DRAM = 136,
340 DDR4_SPD_RDIMM_REGISTER_DRIVE_STRENGTH_CTL = 137,
341 DDR4_SPD_RDIMM_REGISTER_DRIVE_STRENGTH_CK = 138,
342 };
343
344 #define SPD_EEPROM_SIZE (DDR4_SPD_RDIMM_REGISTER_DRIVE_STRENGTH_CK + 1)
345
346 struct impedence_values {
347 unsigned char *rodt_ohms;
348 unsigned char *rtt_nom_ohms;
349 unsigned char *rtt_nom_table;
350 unsigned char *rtt_wr_ohms;
351 unsigned char *dic_ohms;
352 short *drive_strength;
353 short *dqx_strength;
354 };
355
356 #define RODT_OHMS_COUNT 8
357 #define RTT_NOM_OHMS_COUNT 8
358 #define RTT_NOM_TABLE_COUNT 8
359 #define RTT_WR_OHMS_COUNT 8
360 #define DIC_OHMS_COUNT 3
361 #define DRIVE_STRENGTH_COUNT 15
362
363 /*
364 * Structure that provides DIMM information, either in the form of an SPD
365 * TWSI address, or a pointer to an array that contains SPD data. One of
366 * the two fields must be valid.
367 */
368 struct dimm_config {
369 u16 spd_addrs[2]; /* TWSI address of SPD, 0 if not used */
370 u8 *spd_ptrs[2]; /* pointer to SPD data array, NULL if not used */
371 int spd_cached[2];
372 u8 spd_data[2][SPD_EEPROM_SIZE];
373 };
374
375 struct dimm_odt_config {
376 u8 odt_ena; /* FIX: dqx_ctl for Octeon 3 DDR4 */
377 u64 odt_mask; /* FIX: wodt_mask for Octeon 3 */
378 union cvmx_lmcx_modereg_params1 modereg_params1;
379 union cvmx_lmcx_modereg_params2 modereg_params2;
380 u8 qs_dic; /* FIX: rodt_ctl for Octeon 3 */
381 u64 rodt_ctl; /* FIX: rodt_mask for Octeon 3 */
382 u8 dic;
383 };
384
385 struct ddr_delay_config {
386 u32 ddr_board_delay;
387 u8 lmc_delay_clk;
388 u8 lmc_delay_cmd;
389 u8 lmc_delay_dq;
390 };
391
392 /*
393 * The parameters below make up the custom_lmc_config data structure.
394 * This structure is used to customize the way that the LMC DRAM
395 * Controller is configured for a particular board design.
396 *
397 * The HRM describes LMC Read Leveling which supports automatic
398 * selection of per byte-lane delays. When measuring the read delays
399 * the LMC configuration software sweeps through a range of settings
400 * for LMC0_COMP_CTL2[RODT_CTL], the Octeon II on-die-termination
401 * resistance and LMC0_MODEREG_PARAMS1[RTT_NOM_XX], the DRAM
402 * on-die-termination resistance. The minimum and maximum parameters
403 * for rtt_nom_idx and rodt_ctl listed below determine the ranges of
404 * ODT settings used for the measurements. Note that for rtt_nom an
405 * index is used into a sorted table rather than the direct csr setting
406 * in order to optimize the sweep.
407 *
408 * .min_rtt_nom_idx: 1=120ohms, 2=60ohms, 3=40ohms, 4=30ohms, 5=20ohms
409 * .max_rtt_nom_idx: 1=120ohms, 2=60ohms, 3=40ohms, 4=30ohms, 5=20ohms
410 * .min_rodt_ctl: 1=20ohms, 2=30ohms, 3=40ohms, 4=60ohms, 5=120ohms
411 * .max_rodt_ctl: 1=20ohms, 2=30ohms, 3=40ohms, 4=60ohms, 5=120ohms
412 *
413 * The settings below control the Octeon II drive strength for the CK,
414 * ADD/CMD, and DQ/DQS signals. 1=24ohms, 2=26.67ohms, 3=30ohms,
415 * 4=34.3ohms, 5=40ohms, 6=48ohms, 6=60ohms.
416 *
417 * .dqx_ctl: Drive strength control for DDR_DQX/DDR_DQS_X_P/N drivers.
418 * .ck_ctl: Drive strength control for
419 * DDR_CK_X_P/DDR_DIMMX_CSX_L/DDR_DIMMX_ODT_X drivers.
420 * .cmd_ctl: Drive strength control for CMD/A/RESET_L/CKEX drivers.
421 *
422 * The LMC controller software selects the most optimal CAS Latency
423 * that complies with the appropriate SPD values and the frequency
424 * that the DRAMS are being operated. When operating the DRAMs at
425 * frequencies substantially lower than their rated frequencies it
426 * might be necessary to limit the minimum CAS Latency the LMC
427 * controller software is allowed to select in order to make the DRAM
428 * work reliably.
429 *
430 * .min_cas_latency: Minimum allowed CAS Latency
431 *
432 * The value used for LMC0_RLEVEL_CTL[OFFSET_EN] determine how the
433 * read-leveling information that the Octeon II gathers is interpreted
434 * to determine the per-byte read delays.
435 *
436 * .offset_en: Value used for LMC0_RLEVEL_CTL[OFFSET_EN].
437 * .offset_udimm: Value used for LMC0_RLEVEL_CTL[OFFSET] for UDIMMS.
438 * .offset_rdimm: Value used for LMC0_RLEVEL_CTL[OFFSET] for RDIMMS.
439 *
440 * The LMC configuration software sweeps through a range of ODT
441 * settings while measuring the per-byte read delays. During those
442 * measurements the software makes an assessment of the quality of the
443 * measurements in order to determine which measurements provide the
444 * most accurate delays. The automatic settings provide the option to
445 * allow that same assessment to determine the most optimal RODT_CTL
446 * and/or RTT_NOM settings.
447 *
448 * The automatic approach might provide the best means to determine
449 * the settings used for initial poweron of a new design. However,
450 * the final settings should be determined by board analysis, testing,
451 * and experience.
452 *
453 * .ddr_rtt_nom_auto: 1 means automatically set RTT_NOM value.
454 * .ddr_rodt_ctl_auto: 1 means automatically set RODT_CTL value.
455 *
456 * .rlevel_compute: Enables software interpretation of per-byte read
457 * delays using the measurements collected by the
458 * Octeon II rather than completely relying on the
459 * Octeon II to determine the delays. 1=software
460 * computation is recomended since a more complete
461 * analysis is implemented in software.
462 *
463 * .rlevel_comp_offset: Set to 2 unless instructed differently by Cavium.
464 *
465 * .rlevel_average_loops: Determines the number of times the read-leveling
466 * sequence is run for each rank. The results is
467 * then averaged across the number of loops. The
468 * default setting is 1.
469 *
470 * .ddr2t_udimm:
471 * .ddr2t_rdimm: Turn on the DDR 2T mode. 2-cycle window for CMD and
472 * address. This mode helps relieve setup time pressure
473 * on the address and command bus. Please refer to
474 * Micron's tech note tn_47_01 titled DDR2-533 Memory
475 * Design Guide for Two Dimm Unbuffered Systems for
476 * physical details.
477 *
478 * .disable_sequential_delay_check: As result of the flyby topology
479 * prescribed in the JEDEC specifications the byte delays should
480 * maintain a consistent increasing or decreasing trend across
481 * the bytes on standard dimms. This setting can be used disable
482 * that check for unusual circumstances where the check is not
483 * useful.
484 *
485 * .maximum_adjacent_rlevel_delay_increment: An additional sequential
486 * delay check for the delays that result from the flyby
487 * topology. This value specifies the maximum difference between
488 * the delays of adjacent bytes. A value of 0 disables this
489 * check.
490 *
491 * .fprch2 Front Porch Enable: When set, the turn-off
492 * time for the default DDR_DQ/DQS drivers is FPRCH2 CKs earlier.
493 * 00 = 0 CKs
494 * 01 = 1 CKs
495 * 10 = 2 CKs
496 *
497 * .parity: The parity input signal PAR_IN on each dimm must be
498 * strapped high or low on the board. This bit is programmed
499 * into LMC0_DIMM_CTL[PARITY] and it must be set to match the
500 * board strapping. This signal is typically strapped low.
501 *
502 * .mode32b: Enable 32-bit datapath mode. Set to 1 if only 32 DQ pins
503 * are used. (cn61xx, cn71xx)
504 *
505 * .measured_vref: Set to 1 to measure VREF; set to 0 to compute VREF.
506 *
507 * .dram_connection: Set to 1 if discrete DRAMs; set to 0 if using DIMMs.
508 * This changes the algorithms used to compute VREF.
509 *
510 * .dll_write_offset: FIXME: Add description
511 * .dll_read_offset: FIXME: Add description
512 */
513
514 struct rlevel_table {
515 const char part[20];
516 int speed;
517 u64 rl_rank[4][4];
518 };
519
520 struct ddr3_custom_config {
521 u8 min_rtt_nom_idx;
522 u8 max_rtt_nom_idx;
523 u8 min_rodt_ctl;
524 u8 max_rodt_ctl;
525 u8 dqx_ctl;
526 u8 ck_ctl;
527 u8 cmd_ctl;
528 u8 ctl_ctl;
529 u8 min_cas_latency;
530 u8 offset_en;
531 u8 offset_udimm;
532 u8 offset_rdimm;
533 u8 rlevel_compute;
534 u8 ddr_rtt_nom_auto;
535 u8 ddr_rodt_ctl_auto;
536 u8 rlevel_comp_offset_udimm;
537 u8 rlevel_comp_offset_rdimm;
538 int8_t ptune_offset;
539 int8_t ntune_offset;
540 u8 rlevel_average_loops;
541 u8 ddr2t_udimm;
542 u8 ddr2t_rdimm;
543 u8 disable_sequential_delay_check;
544 u8 maximum_adjacent_rlevel_delay_increment;
545 u8 parity;
546 u8 fprch2;
547 u8 mode32b;
548 u8 measured_vref;
549 u8 dram_connection;
550 const int8_t *dll_write_offset;
551 const int8_t *dll_read_offset;
552 struct rlevel_table *rl_tbl;
553 };
554
555 #define DDR_CFG_T_MAX_DIMMS 5
556
557 struct ddr_conf {
558 struct dimm_config dimm_config_table[DDR_CFG_T_MAX_DIMMS];
559 struct dimm_odt_config odt_1rank_config[4];
560 struct dimm_odt_config odt_2rank_config[4];
561 struct dimm_odt_config odt_4rank_config[4];
562 struct ddr_delay_config unbuffered;
563 struct ddr_delay_config registered;
564 struct ddr3_custom_config custom_lmc_config;
565 };
566
567 /* Divide and round results to the nearest integer. */
divide_nint(u64 dividend,u64 divisor)568 static inline u64 divide_nint(u64 dividend, u64 divisor)
569 {
570 u64 quotent, remainder;
571
572 quotent = dividend / divisor;
573 remainder = dividend % divisor;
574 return (quotent + ((remainder * 2) >= divisor));
575 }
576
577 /* Divide and round results up to the next higher integer. */
divide_roundup(u64 dividend,u64 divisor)578 static inline u64 divide_roundup(u64 dividend, u64 divisor)
579 {
580 return ((dividend + divisor - 1) / divisor);
581 }
582
583 enum ddr_type {
584 DDR3_DRAM = 3,
585 DDR4_DRAM = 4,
586 };
587
588 #define rttnom_none 0 /* Rtt_Nom disabled */
589 #define rttnom_60ohm 1 /* RZQ/4 = 240/4 = 60 ohms */
590 #define rttnom_120ohm 2 /* RZQ/2 = 240/2 = 120 ohms */
591 #define rttnom_40ohm 3 /* RZQ/6 = 240/6 = 40 ohms */
592 #define rttnom_20ohm 4 /* RZQ/12 = 240/12 = 20 ohms */
593 #define rttnom_30ohm 5 /* RZQ/8 = 240/8 = 30 ohms */
594 #define rttnom_rsrv1 6 /* Reserved */
595 #define rttnom_rsrv2 7 /* Reserved */
596
597 #define rttwr_none 0 /* Dynamic ODT off */
598 #define rttwr_60ohm 1 /* RZQ/4 = 240/4 = 60 ohms */
599 #define rttwr_120ohm 2 /* RZQ/2 = 240/2 = 120 ohms */
600 #define rttwr_rsrv1 3 /* Reserved */
601
602 #define dic_40ohm 0 /* RZQ/6 = 240/6 = 40 ohms */
603 #define dic_34ohm 1 /* RZQ/7 = 240/7 = 34 ohms */
604
605 #define driver_24_ohm 1
606 #define driver_27_ohm 2
607 #define driver_30_ohm 3
608 #define driver_34_ohm 4
609 #define driver_40_ohm 5
610 #define driver_48_ohm 6
611 #define driver_60_ohm 7
612
613 #define rodt_ctl_none 0
614 #define rodt_ctl_20_ohm 1
615 #define rodt_ctl_30_ohm 2
616 #define rodt_ctl_40_ohm 3
617 #define rodt_ctl_60_ohm 4
618 #define rodt_ctl_120_ohm 5
619
620 #define ddr4_rttnom_none 0 /* Rtt_Nom disabled */
621 #define ddr4_rttnom_60ohm 1 /* RZQ/4 = 240/4 = 60 ohms */
622 #define ddr4_rttnom_120ohm 2 /* RZQ/2 = 240/2 = 120 ohms */
623 #define ddr4_rttnom_40ohm 3 /* RZQ/6 = 240/6 = 40 ohms */
624 #define ddr4_rttnom_240ohm 4 /* RZQ/1 = 240/1 = 240 ohms */
625 #define ddr4_rttnom_48ohm 5 /* RZQ/5 = 240/5 = 48 ohms */
626 #define ddr4_rttnom_80ohm 6 /* RZQ/3 = 240/3 = 80 ohms */
627 #define ddr4_rttnom_34ohm 7 /* RZQ/7 = 240/7 = 34 ohms */
628
629 #define ddr4_rttwr_none 0 /* Dynamic ODT off */
630 #define ddr4_rttwr_120ohm 1 /* RZQ/2 = 240/2 = 120 ohms */
631 #define ddr4_rttwr_240ohm 2 /* RZQ/1 = 240/1 = 240 ohms */
632 #define ddr4_rttwr_hiz 3 /* HiZ */
633 /* This setting is available for cn78xx pass 2, and cn73xx & cnf75xx pass 1 */
634 #define ddr4_rttwr_80ohm 4 /* RZQ/3 = 240/3 = 80 ohms */
635
636 #define ddr4_dic_34ohm 0 /* RZQ/7 = 240/7 = 34 ohms */
637 #define ddr4_dic_48ohm 1 /* RZQ/5 = 240/5 = 48 ohms */
638
639 #define ddr4_rttpark_none 0 /* Rtt_Park disabled */
640 #define ddr4_rttpark_60ohm 1 /* RZQ/4 = 240/4 = 60 ohms */
641 #define ddr4_rttpark_120ohm 2 /* RZQ/2 = 240/2 = 120 ohms */
642 #define ddr4_rttpark_40ohm 3 /* RZQ/6 = 240/6 = 40 ohms */
643 #define ddr4_rttpark_240ohm 4 /* RZQ/1 = 240/1 = 240 ohms */
644 #define ddr4_rttpark_48ohm 5 /* RZQ/5 = 240/5 = 48 ohms */
645 #define ddr4_rttpark_80ohm 6 /* RZQ/3 = 240/3 = 80 ohms */
646 #define ddr4_rttpark_34ohm 7 /* RZQ/7 = 240/7 = 34 ohms */
647
648 #define ddr4_driver_26_ohm 2
649 #define ddr4_driver_30_ohm 3
650 #define ddr4_driver_34_ohm 4
651 #define ddr4_driver_40_ohm 5
652 #define ddr4_driver_48_ohm 6
653
654 #define ddr4_dqx_driver_24_ohm 1
655 #define ddr4_dqx_driver_27_ohm 2
656 #define ddr4_dqx_driver_30_ohm 3
657 #define ddr4_dqx_driver_34_ohm 4
658 #define ddr4_dqx_driver_40_ohm 5
659 #define ddr4_dqx_driver_48_ohm 6
660 #define ddr4_dqx_driver_60_ohm 7
661
662 #define ddr4_rodt_ctl_none 0
663 #define ddr4_rodt_ctl_40_ohm 1
664 #define ddr4_rodt_ctl_60_ohm 2
665 #define ddr4_rodt_ctl_80_ohm 3
666 #define ddr4_rodt_ctl_120_ohm 4
667 #define ddr4_rodt_ctl_240_ohm 5
668 #define ddr4_rodt_ctl_34_ohm 6
669 #define ddr4_rodt_ctl_48_ohm 7
670
671 #define DIMM_CONFIG_TERMINATOR { {0, 0}, {NULL, NULL} }
672
673 #define SET_DDR_DLL_CTL3(field, expr) \
674 do { \
675 if (octeon_is_cpuid(OCTEON_CN66XX) || \
676 octeon_is_cpuid(OCTEON_CN63XX)) \
677 ddr_dll_ctl3.cn63xx.field = (expr); \
678 else if (octeon_is_cpuid(OCTEON_CN68XX) || \
679 octeon_is_cpuid(OCTEON_CN61XX) || \
680 octeon_is_cpuid(OCTEON_CNF71XX)) \
681 ddr_dll_ctl3.cn61xx.field = (expr); \
682 else if (octeon_is_cpuid(OCTEON_CN70XX) || \
683 octeon_is_cpuid(OCTEON_CN78XX)) \
684 ddr_dll_ctl3.cn70xx.field = (expr); \
685 else if (octeon_is_cpuid(OCTEON_CN73XX) || \
686 octeon_is_cpuid(OCTEON_CNF75XX)) \
687 ddr_dll_ctl3.cn73xx.field = (expr); \
688 else \
689 debug("%s(): " #field \
690 "not set for unknown chip\n", \
691 __func__); \
692 } while (0)
693
694 #define ENCODE_DLL90_BYTE_SEL(byte_sel) \
695 (octeon_is_cpuid(OCTEON_CN70XX) ? ((9 + 7 - (byte_sel)) % 9) : \
696 ((byte_sel) + 1))
697
698 /**
699 * If debugging is disabled the ddr_print macro is not compatible
700 * with this macro.
701 */
702 # define GET_DDR_DLL_CTL3(field) \
703 ((octeon_is_cpuid(OCTEON_CN66XX) || \
704 octeon_is_cpuid(OCTEON_CN63XX)) ? \
705 ddr_dll_ctl3.cn63xx.field : \
706 (octeon_is_cpuid(OCTEON_CN68XX) || \
707 octeon_is_cpuid(OCTEON_CN61XX) || \
708 octeon_is_cpuid(OCTEON_CNF71XX)) ? \
709 ddr_dll_ctl3.cn61xx.field : \
710 (octeon_is_cpuid(OCTEON_CN70XX) || \
711 octeon_is_cpuid(OCTEON_CN78XX)) ? \
712 ddr_dll_ctl3.cn70xx.field : \
713 (octeon_is_cpuid(OCTEON_CN73XX) || \
714 octeon_is_cpuid(OCTEON_CNF75XX)) ? \
715 ddr_dll_ctl3.cn73xx.field : 0)
716
717 extern const char *ddr3_dimm_types[];
718 extern const char *ddr4_dimm_types[];
719
720 extern const struct dimm_odt_config disable_odt_config[];
721
722 #define RLEVEL_BYTE_BITS 6
723 #define RLEVEL_BYTE_MSK ((1ULL << 6) - 1)
724
725 /* Prototypes */
726 int get_ddr_type(struct dimm_config *dimm_config, int upper_dimm);
727 int get_dimm_module_type(struct dimm_config *dimm_config, int upper_dimm,
728 int ddr_type);
729 int read_spd(struct dimm_config *dimm_config, int dimm_index, int spd_field);
730 int read_spd_init(struct dimm_config *dimm_config, int dimm_index);
731 void report_dimm(struct dimm_config *dimm_config, int upper_dimm,
732 int dimm, int if_num);
733 int validate_dimm(struct ddr_priv *priv, struct dimm_config *dimm_config,
734 int dimm_index);
735 char *printable_rank_spec(char *buffer, int num_ranks, int dram_width,
736 int spd_package);
737
738 bool ddr_memory_preserved(struct ddr_priv *priv);
739
740 int get_wl_rank(union cvmx_lmcx_wlevel_rankx *lmc_wlevel_rank, int byte);
741 int get_rl_rank(union cvmx_lmcx_rlevel_rankx *lmc_rlevel_rank, int byte);
742 void upd_wl_rank(union cvmx_lmcx_wlevel_rankx *lmc_wlevel_rank, int byte,
743 int delay);
744 void upd_rl_rank(union cvmx_lmcx_rlevel_rankx *lmc_rlevel_rank, int byte,
745 int delay);
746
747 int compute_ddr3_rlevel_delay(u8 mstart, u8 width,
748 union cvmx_lmcx_rlevel_ctl rlevel_ctl);
749
750 int encode_row_lsb_ddr3(int row_lsb);
751 int encode_pbank_lsb_ddr3(int pbank_lsb);
752
753 int initialize_ddr_clock(struct ddr_priv *priv, struct ddr_conf *ddr_conf,
754 u32 cpu_hertz, u32 ddr_hertz, u32 ddr_ref_hertz,
755 int if_num, u32 if_mask);
756
757 void process_custom_dll_offsets(struct ddr_priv *priv, int if_num,
758 const char *enable_str,
759 const int8_t *offsets, const char *byte_str,
760 int mode);
761 int nonseq_del(struct rlevel_byte_data *rlevel_byte, int start, int end,
762 int max_adj_delay_inc);
763 int roundup_ddr3_wlevel_bitmask(int bitmask);
764
765 void oct3_ddr3_seq(struct ddr_priv *priv, int rank_mask, int if_num,
766 int sequence);
767 void ddr_init_seq(struct ddr_priv *priv, int rank_mask, int if_num);
768
769 void rlevel_to_wlevel(union cvmx_lmcx_rlevel_rankx *lmc_rlevel_rank,
770 union cvmx_lmcx_wlevel_rankx *lmc_wlevel_rank, int byte);
771
772 int validate_ddr3_rlevel_bitmask(struct rlevel_bitmask *rlevel_bitmask_p,
773 int ddr_type);
774
775 void change_dll_offset_enable(struct ddr_priv *priv, int if_num, int change);
776 unsigned short load_dll_offset(struct ddr_priv *priv, int if_num,
777 int dll_offset_mode,
778 int byte_offset, int byte);
779
780 u64 lmc_ddr3_rl_dbg_read(struct ddr_priv *priv, int if_num, int idx);
781 u64 lmc_ddr3_wl_dbg_read(struct ddr_priv *priv, int if_num, int idx);
782
783 void cvmx_maybe_tune_node(struct ddr_priv *priv, u32 ddr_speed);
784 void cvmx_dbi_switchover(struct ddr_priv *priv);
785
786 int init_octeon3_ddr3_interface(struct ddr_priv *priv,
787 struct ddr_conf *ddr_conf,
788 u32 ddr_hertz, u32 cpu_hertz, u32 ddr_ref_hertz,
789 int if_num, u32 if_mask);
790
791 char *lookup_env(struct ddr_priv *priv, const char *format, ...);
792 char *lookup_env_ull(struct ddr_priv *priv, const char *format, ...);
793
794 /* Each board provides a board-specific config table via this function */
795 struct ddr_conf *octeon_ddr_conf_table_get(int *count, int *def_ddr_freq);
796
797 #endif /* __OCTEON_DDR_H_ */
798