1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved. 4 */ 5 6 #include <linux/kernel.h> 7 #include <linux/io.h> 8 #include <linux/clk.h> 9 #include <linux/delay.h> 10 #include <linux/of.h> 11 12 #include <soc/tegra/mc.h> 13 14 #include "tegra210-emc.h" 15 #include "tegra210-mc.h" 16 17 /* 18 * Enable flags for specifying verbosity. 19 */ 20 #define INFO (1 << 0) 21 #define STEPS (1 << 1) 22 #define SUB_STEPS (1 << 2) 23 #define PRELOCK (1 << 3) 24 #define PRELOCK_STEPS (1 << 4) 25 #define ACTIVE_EN (1 << 5) 26 #define PRAMP_UP (1 << 6) 27 #define PRAMP_DN (1 << 7) 28 #define EMA_WRITES (1 << 10) 29 #define EMA_UPDATES (1 << 11) 30 #define PER_TRAIN (1 << 16) 31 #define CC_PRINT (1 << 17) 32 #define CCFIFO (1 << 29) 33 #define REGS (1 << 30) 34 #define REG_LISTS (1 << 31) 35 36 #define emc_dbg(emc, flags, ...) dev_dbg(emc->dev, __VA_ARGS__) 37 38 #define DVFS_CLOCK_CHANGE_VERSION 21021 39 #define EMC_PRELOCK_VERSION 2101 40 41 enum { 42 DVFS_SEQUENCE = 1, 43 WRITE_TRAINING_SEQUENCE = 2, 44 PERIODIC_TRAINING_SEQUENCE = 3, 45 DVFS_PT1 = 10, 46 DVFS_UPDATE = 11, 47 TRAINING_PT1 = 12, 48 TRAINING_UPDATE = 13, 49 PERIODIC_TRAINING_UPDATE = 14 50 }; 51 52 /* 53 * PTFV defines - basically just indexes into the per table PTFV array. 54 */ 55 #define PTFV_DQSOSC_MOVAVG_C0D0U0_INDEX 0 56 #define PTFV_DQSOSC_MOVAVG_C0D0U1_INDEX 1 57 #define PTFV_DQSOSC_MOVAVG_C0D1U0_INDEX 2 58 #define PTFV_DQSOSC_MOVAVG_C0D1U1_INDEX 3 59 #define PTFV_DQSOSC_MOVAVG_C1D0U0_INDEX 4 60 #define PTFV_DQSOSC_MOVAVG_C1D0U1_INDEX 5 61 #define PTFV_DQSOSC_MOVAVG_C1D1U0_INDEX 6 62 #define PTFV_DQSOSC_MOVAVG_C1D1U1_INDEX 7 63 #define PTFV_DVFS_SAMPLES_INDEX 9 64 #define PTFV_MOVAVG_WEIGHT_INDEX 10 65 #define PTFV_CONFIG_CTRL_INDEX 11 66 67 #define PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA (1 << 0) 68 69 /* 70 * Do arithmetic in fixed point. 71 */ 72 #define MOVAVG_PRECISION_FACTOR 100 73 74 /* 75 * The division portion of the average operation. 76 */ 77 #define __AVERAGE_PTFV(dev) \ 78 ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \ 79 next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \ 80 next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; }) 81 82 /* 83 * Convert val to fixed point and add it to the temporary average. 84 */ 85 #define __INCREMENT_PTFV(dev, val) \ 86 ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] += \ 87 ((val) * MOVAVG_PRECISION_FACTOR); }) 88 89 /* 90 * Convert a moving average back to integral form and return the value. 91 */ 92 #define __MOVAVG_AC(timing, dev) \ 93 ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \ 94 MOVAVG_PRECISION_FACTOR) 95 96 /* Weighted update. */ 97 #define __WEIGHTED_UPDATE_PTFV(dev, nval) \ 98 do { \ 99 int w = PTFV_MOVAVG_WEIGHT_INDEX; \ 100 int dqs = PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX; \ 101 \ 102 next->ptfv_list[dqs] = \ 103 ((nval * MOVAVG_PRECISION_FACTOR) + \ 104 (next->ptfv_list[dqs] * \ 105 next->ptfv_list[w])) / \ 106 (next->ptfv_list[w] + 1); \ 107 \ 108 emc_dbg(emc, EMA_UPDATES, "%s: (s=%lu) EMA: %u\n", \ 109 __stringify(dev), nval, next->ptfv_list[dqs]); \ 110 } while (0) 111 112 /* Access a particular average. */ 113 #define __MOVAVG(timing, dev) \ 114 ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX]) 115 116 static u32 update_clock_tree_delay(struct tegra210_emc *emc, int type) 117 { 118 bool periodic_training_update = type == PERIODIC_TRAINING_UPDATE; 119 struct tegra210_emc_timing *last = emc->last; 120 struct tegra210_emc_timing *next = emc->next; 121 u32 last_timing_rate_mhz = last->rate / 1000; 122 u32 next_timing_rate_mhz = next->rate / 1000; 123 bool dvfs_update = type == DVFS_UPDATE; 124 s32 tdel = 0, tmdel = 0, adel = 0; 125 bool dvfs_pt1 = type == DVFS_PT1; 126 unsigned long cval = 0; 127 u32 temp[2][2], value; 128 unsigned int i; 129 130 /* 131 * Dev0 MSB. 132 */ 133 if (dvfs_pt1 || periodic_training_update) { 134 value = tegra210_emc_mrr_read(emc, 2, 19); 135 136 for (i = 0; i < emc->num_channels; i++) { 137 temp[i][0] = (value & 0x00ff) << 8; 138 temp[i][1] = (value & 0xff00) << 0; 139 value >>= 16; 140 } 141 142 /* 143 * Dev0 LSB. 144 */ 145 value = tegra210_emc_mrr_read(emc, 2, 18); 146 147 for (i = 0; i < emc->num_channels; i++) { 148 temp[i][0] |= (value & 0x00ff) >> 0; 149 temp[i][1] |= (value & 0xff00) >> 8; 150 value >>= 16; 151 } 152 } 153 154 if (dvfs_pt1 || periodic_training_update) { 155 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 156 cval *= 1000000; 157 cval /= last_timing_rate_mhz * 2 * temp[0][0]; 158 } 159 160 if (dvfs_pt1) 161 __INCREMENT_PTFV(C0D0U0, cval); 162 else if (dvfs_update) 163 __AVERAGE_PTFV(C0D0U0); 164 else if (periodic_training_update) 165 __WEIGHTED_UPDATE_PTFV(C0D0U0, cval); 166 167 if (dvfs_update || periodic_training_update) { 168 tdel = next->current_dram_clktree[C0D0U0] - 169 __MOVAVG_AC(next, C0D0U0); 170 tmdel = (tdel < 0) ? -1 * tdel : tdel; 171 adel = tmdel; 172 173 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 174 next->tree_margin) 175 next->current_dram_clktree[C0D0U0] = 176 __MOVAVG_AC(next, C0D0U0); 177 } 178 179 if (dvfs_pt1 || periodic_training_update) { 180 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 181 cval *= 1000000; 182 cval /= last_timing_rate_mhz * 2 * temp[0][1]; 183 } 184 185 if (dvfs_pt1) 186 __INCREMENT_PTFV(C0D0U1, cval); 187 else if (dvfs_update) 188 __AVERAGE_PTFV(C0D0U1); 189 else if (periodic_training_update) 190 __WEIGHTED_UPDATE_PTFV(C0D0U1, cval); 191 192 if (dvfs_update || periodic_training_update) { 193 tdel = next->current_dram_clktree[C0D0U1] - 194 __MOVAVG_AC(next, C0D0U1); 195 tmdel = (tdel < 0) ? -1 * tdel : tdel; 196 197 if (tmdel > adel) 198 adel = tmdel; 199 200 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 201 next->tree_margin) 202 next->current_dram_clktree[C0D0U1] = 203 __MOVAVG_AC(next, C0D0U1); 204 } 205 206 if (emc->num_channels > 1) { 207 if (dvfs_pt1 || periodic_training_update) { 208 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 209 cval *= 1000000; 210 cval /= last_timing_rate_mhz * 2 * temp[1][0]; 211 } 212 213 if (dvfs_pt1) 214 __INCREMENT_PTFV(C1D0U0, cval); 215 else if (dvfs_update) 216 __AVERAGE_PTFV(C1D0U0); 217 else if (periodic_training_update) 218 __WEIGHTED_UPDATE_PTFV(C1D0U0, cval); 219 220 if (dvfs_update || periodic_training_update) { 221 tdel = next->current_dram_clktree[C1D0U0] - 222 __MOVAVG_AC(next, C1D0U0); 223 tmdel = (tdel < 0) ? -1 * tdel : tdel; 224 225 if (tmdel > adel) 226 adel = tmdel; 227 228 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 229 next->tree_margin) 230 next->current_dram_clktree[C1D0U0] = 231 __MOVAVG_AC(next, C1D0U0); 232 } 233 234 if (dvfs_pt1 || periodic_training_update) { 235 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 236 cval *= 1000000; 237 cval /= last_timing_rate_mhz * 2 * temp[1][1]; 238 } 239 240 if (dvfs_pt1) 241 __INCREMENT_PTFV(C1D0U1, cval); 242 else if (dvfs_update) 243 __AVERAGE_PTFV(C1D0U1); 244 else if (periodic_training_update) 245 __WEIGHTED_UPDATE_PTFV(C1D0U1, cval); 246 247 if (dvfs_update || periodic_training_update) { 248 tdel = next->current_dram_clktree[C1D0U1] - 249 __MOVAVG_AC(next, C1D0U1); 250 tmdel = (tdel < 0) ? -1 * tdel : tdel; 251 252 if (tmdel > adel) 253 adel = tmdel; 254 255 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 256 next->tree_margin) 257 next->current_dram_clktree[C1D0U1] = 258 __MOVAVG_AC(next, C1D0U1); 259 } 260 } 261 262 if (emc->num_devices < 2) 263 goto done; 264 265 /* 266 * Dev1 MSB. 267 */ 268 if (dvfs_pt1 || periodic_training_update) { 269 value = tegra210_emc_mrr_read(emc, 1, 19); 270 271 for (i = 0; i < emc->num_channels; i++) { 272 temp[i][0] = (value & 0x00ff) << 8; 273 temp[i][1] = (value & 0xff00) << 0; 274 value >>= 16; 275 } 276 277 /* 278 * Dev1 LSB. 279 */ 280 value = tegra210_emc_mrr_read(emc, 1, 18); 281 282 for (i = 0; i < emc->num_channels; i++) { 283 temp[i][0] |= (value & 0x00ff) >> 0; 284 temp[i][1] |= (value & 0xff00) >> 8; 285 value >>= 16; 286 } 287 } 288 289 if (dvfs_pt1 || periodic_training_update) { 290 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 291 cval *= 1000000; 292 cval /= last_timing_rate_mhz * 2 * temp[0][0]; 293 } 294 295 if (dvfs_pt1) 296 __INCREMENT_PTFV(C0D1U0, cval); 297 else if (dvfs_update) 298 __AVERAGE_PTFV(C0D1U0); 299 else if (periodic_training_update) 300 __WEIGHTED_UPDATE_PTFV(C0D1U0, cval); 301 302 if (dvfs_update || periodic_training_update) { 303 tdel = next->current_dram_clktree[C0D1U0] - 304 __MOVAVG_AC(next, C0D1U0); 305 tmdel = (tdel < 0) ? -1 * tdel : tdel; 306 307 if (tmdel > adel) 308 adel = tmdel; 309 310 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 311 next->tree_margin) 312 next->current_dram_clktree[C0D1U0] = 313 __MOVAVG_AC(next, C0D1U0); 314 } 315 316 if (dvfs_pt1 || periodic_training_update) { 317 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 318 cval *= 1000000; 319 cval /= last_timing_rate_mhz * 2 * temp[0][1]; 320 } 321 322 if (dvfs_pt1) 323 __INCREMENT_PTFV(C0D1U1, cval); 324 else if (dvfs_update) 325 __AVERAGE_PTFV(C0D1U1); 326 else if (periodic_training_update) 327 __WEIGHTED_UPDATE_PTFV(C0D1U1, cval); 328 329 if (dvfs_update || periodic_training_update) { 330 tdel = next->current_dram_clktree[C0D1U1] - 331 __MOVAVG_AC(next, C0D1U1); 332 tmdel = (tdel < 0) ? -1 * tdel : tdel; 333 334 if (tmdel > adel) 335 adel = tmdel; 336 337 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 338 next->tree_margin) 339 next->current_dram_clktree[C0D1U1] = 340 __MOVAVG_AC(next, C0D1U1); 341 } 342 343 if (emc->num_channels > 1) { 344 if (dvfs_pt1 || periodic_training_update) { 345 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 346 cval *= 1000000; 347 cval /= last_timing_rate_mhz * 2 * temp[1][0]; 348 } 349 350 if (dvfs_pt1) 351 __INCREMENT_PTFV(C1D1U0, cval); 352 else if (dvfs_update) 353 __AVERAGE_PTFV(C1D1U0); 354 else if (periodic_training_update) 355 __WEIGHTED_UPDATE_PTFV(C1D1U0, cval); 356 357 if (dvfs_update || periodic_training_update) { 358 tdel = next->current_dram_clktree[C1D1U0] - 359 __MOVAVG_AC(next, C1D1U0); 360 tmdel = (tdel < 0) ? -1 * tdel : tdel; 361 362 if (tmdel > adel) 363 adel = tmdel; 364 365 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 366 next->tree_margin) 367 next->current_dram_clktree[C1D1U0] = 368 __MOVAVG_AC(next, C1D1U0); 369 } 370 371 if (dvfs_pt1 || periodic_training_update) { 372 cval = tegra210_emc_actual_osc_clocks(last->run_clocks); 373 cval *= 1000000; 374 cval /= last_timing_rate_mhz * 2 * temp[1][1]; 375 } 376 377 if (dvfs_pt1) 378 __INCREMENT_PTFV(C1D1U1, cval); 379 else if (dvfs_update) 380 __AVERAGE_PTFV(C1D1U1); 381 else if (periodic_training_update) 382 __WEIGHTED_UPDATE_PTFV(C1D1U1, cval); 383 384 if (dvfs_update || periodic_training_update) { 385 tdel = next->current_dram_clktree[C1D1U1] - 386 __MOVAVG_AC(next, C1D1U1); 387 tmdel = (tdel < 0) ? -1 * tdel : tdel; 388 389 if (tmdel > adel) 390 adel = tmdel; 391 392 if (tmdel * 128 * next_timing_rate_mhz / 1000000 > 393 next->tree_margin) 394 next->current_dram_clktree[C1D1U1] = 395 __MOVAVG_AC(next, C1D1U1); 396 } 397 } 398 399 done: 400 return adel; 401 } 402 403 static u32 periodic_compensation_handler(struct tegra210_emc *emc, u32 type, 404 struct tegra210_emc_timing *last, 405 struct tegra210_emc_timing *next) 406 { 407 #define __COPY_EMA(nt, lt, dev) \ 408 ({ __MOVAVG(nt, dev) = __MOVAVG(lt, dev) * \ 409 (nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; }) 410 411 u32 i, adel = 0, samples = next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; 412 u32 delay; 413 414 delay = tegra210_emc_actual_osc_clocks(last->run_clocks); 415 delay *= 1000; 416 delay = 2 + (delay / last->rate); 417 418 if (!next->periodic_training) 419 return 0; 420 421 if (type == DVFS_SEQUENCE) { 422 if (last->periodic_training && 423 (next->ptfv_list[PTFV_CONFIG_CTRL_INDEX] & 424 PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA)) { 425 /* 426 * If the previous frequency was using periodic 427 * calibration then we can reuse the previous 428 * frequencies EMA data. 429 */ 430 __COPY_EMA(next, last, C0D0U0); 431 __COPY_EMA(next, last, C0D0U1); 432 __COPY_EMA(next, last, C1D0U0); 433 __COPY_EMA(next, last, C1D0U1); 434 __COPY_EMA(next, last, C0D1U0); 435 __COPY_EMA(next, last, C0D1U1); 436 __COPY_EMA(next, last, C1D1U0); 437 __COPY_EMA(next, last, C1D1U1); 438 } else { 439 /* Reset the EMA.*/ 440 __MOVAVG(next, C0D0U0) = 0; 441 __MOVAVG(next, C0D0U1) = 0; 442 __MOVAVG(next, C1D0U0) = 0; 443 __MOVAVG(next, C1D0U1) = 0; 444 __MOVAVG(next, C0D1U0) = 0; 445 __MOVAVG(next, C0D1U1) = 0; 446 __MOVAVG(next, C1D1U0) = 0; 447 __MOVAVG(next, C1D1U1) = 0; 448 449 for (i = 0; i < samples; i++) { 450 tegra210_emc_start_periodic_compensation(emc); 451 udelay(delay); 452 453 /* 454 * Generate next sample of data. 455 */ 456 adel = update_clock_tree_delay(emc, DVFS_PT1); 457 } 458 } 459 460 /* 461 * Seems like it should be part of the 462 * 'if (last_timing->periodic_training)' conditional 463 * since is already done for the else clause. 464 */ 465 adel = update_clock_tree_delay(emc, DVFS_UPDATE); 466 } 467 468 if (type == PERIODIC_TRAINING_SEQUENCE) { 469 tegra210_emc_start_periodic_compensation(emc); 470 udelay(delay); 471 472 adel = update_clock_tree_delay(emc, PERIODIC_TRAINING_UPDATE); 473 } 474 475 return adel; 476 } 477 478 static u32 tegra210_emc_r21021_periodic_compensation(struct tegra210_emc *emc) 479 { 480 u32 emc_cfg, emc_cfg_o, emc_cfg_update, del, value; 481 static const u32 list[] = { 482 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0, 483 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1, 484 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2, 485 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3, 486 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0, 487 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1, 488 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2, 489 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3, 490 EMC_DATA_BRLSHFT_0, 491 EMC_DATA_BRLSHFT_1 492 }; 493 struct tegra210_emc_timing *last = emc->last; 494 unsigned int items = ARRAY_SIZE(list), i; 495 unsigned long delay; 496 497 if (last->periodic_training) { 498 emc_dbg(emc, PER_TRAIN, "Periodic training starting\n"); 499 500 value = emc_readl(emc, EMC_DBG); 501 emc_cfg_o = emc_readl(emc, EMC_CFG); 502 emc_cfg = emc_cfg_o & ~(EMC_CFG_DYN_SELF_REF | 503 EMC_CFG_DRAM_ACPD | 504 EMC_CFG_DRAM_CLKSTOP_PD); 505 506 507 /* 508 * 1. Power optimizations should be off. 509 */ 510 emc_writel(emc, emc_cfg, EMC_CFG); 511 512 /* Does emc_timing_update() for above changes. */ 513 tegra210_emc_dll_disable(emc); 514 515 for (i = 0; i < emc->num_channels; i++) 516 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, 517 EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, 518 0); 519 520 for (i = 0; i < emc->num_channels; i++) 521 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, 522 EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, 523 0); 524 525 emc_cfg_update = value = emc_readl(emc, EMC_CFG_UPDATE); 526 value &= ~EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_MASK; 527 value |= (2 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT); 528 emc_writel(emc, value, EMC_CFG_UPDATE); 529 530 /* 531 * 2. osc kick off - this assumes training and dvfs have set 532 * correct MR23. 533 */ 534 tegra210_emc_start_periodic_compensation(emc); 535 536 /* 537 * 3. Let dram capture its clock tree delays. 538 */ 539 delay = tegra210_emc_actual_osc_clocks(last->run_clocks); 540 delay *= 1000; 541 delay /= last->rate + 1; 542 udelay(delay); 543 544 /* 545 * 4. Check delta wrt previous values (save value if margin 546 * exceeds what is set in table). 547 */ 548 del = periodic_compensation_handler(emc, 549 PERIODIC_TRAINING_SEQUENCE, 550 last, last); 551 552 /* 553 * 5. Apply compensation w.r.t. trained values (if clock tree 554 * has drifted more than the set margin). 555 */ 556 if (last->tree_margin < ((del * 128 * (last->rate / 1000)) / 1000000)) { 557 for (i = 0; i < items; i++) { 558 value = tegra210_emc_compensate(last, list[i]); 559 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", 560 list[i], value); 561 emc_writel(emc, value, list[i]); 562 } 563 } 564 565 emc_writel(emc, emc_cfg_o, EMC_CFG); 566 567 /* 568 * 6. Timing update actally applies the new trimmers. 569 */ 570 tegra210_emc_timing_update(emc); 571 572 /* 6.1. Restore the UPDATE_DLL_IN_UPDATE field. */ 573 emc_writel(emc, emc_cfg_update, EMC_CFG_UPDATE); 574 575 /* 6.2. Restore the DLL. */ 576 tegra210_emc_dll_enable(emc); 577 } 578 579 return 0; 580 } 581 582 /* 583 * Do the clock change sequence. 584 */ 585 static void tegra210_emc_r21021_set_clock(struct tegra210_emc *emc, u32 clksrc) 586 { 587 /* state variables */ 588 static bool fsp_for_next_freq; 589 /* constant configuration parameters */ 590 const bool save_restore_clkstop_pd = true; 591 const u32 zqcal_before_cc_cutoff = 2400; 592 const bool cya_allow_ref_cc = false; 593 const bool cya_issue_pc_ref = false; 594 const bool opt_cc_short_zcal = true; 595 const bool ref_b4_sref_en = false; 596 const u32 tZQCAL_lpddr4 = 1000000; 597 const bool opt_short_zcal = true; 598 const bool opt_do_sw_qrst = true; 599 const u32 opt_dvfs_mode = MAN_SR; 600 /* 601 * This is the timing table for the source frequency. It does _not_ 602 * necessarily correspond to the actual timing values in the EMC at the 603 * moment. If the boot BCT differs from the table then this can happen. 604 * However, we need it for accessing the dram_timings (which are not 605 * really registers) array for the current frequency. 606 */ 607 struct tegra210_emc_timing *fake, *last = emc->last, *next = emc->next; 608 u32 tRTM, RP_war, R2P_war, TRPab_war, deltaTWATM, W2P_war, tRPST; 609 u32 mr13_flip_fspwr, mr13_flip_fspop, ramp_up_wait, ramp_down_wait; 610 u32 zq_wait_long, zq_latch_dvfs_wait_time, tZQCAL_lpddr4_fc_adj; 611 u32 emc_auto_cal_config, auto_cal_en, emc_cfg, emc_sel_dpd_ctrl; 612 u32 tFC_lpddr4 = 1000 * next->dram_timings[T_FC_LPDDR4]; 613 u32 bg_reg_mode_change, enable_bglp_reg, enable_bg_reg; 614 bool opt_zcal_en_cc = false, is_lpddr3 = false; 615 bool compensate_trimmer_applicable = false; 616 u32 emc_dbg, emc_cfg_pipe_clk, emc_pin; 617 u32 src_clk_period, dst_clk_period; /* in picoseconds */ 618 bool shared_zq_resistor = false; 619 u32 value, dram_type; 620 u32 opt_dll_mode = 0; 621 unsigned long delay; 622 unsigned int i; 623 624 emc_dbg(emc, INFO, "Running clock change.\n"); 625 626 /* XXX fake == last */ 627 fake = tegra210_emc_find_timing(emc, last->rate * 1000UL); 628 fsp_for_next_freq = !fsp_for_next_freq; 629 630 value = emc_readl(emc, EMC_FBIO_CFG5) & EMC_FBIO_CFG5_DRAM_TYPE_MASK; 631 dram_type = value >> EMC_FBIO_CFG5_DRAM_TYPE_SHIFT; 632 633 if (last->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] & BIT(31)) 634 shared_zq_resistor = true; 635 636 if ((next->burst_regs[EMC_ZCAL_INTERVAL_INDEX] != 0 && 637 last->burst_regs[EMC_ZCAL_INTERVAL_INDEX] == 0) || 638 dram_type == DRAM_TYPE_LPDDR4) 639 opt_zcal_en_cc = true; 640 641 if (dram_type == DRAM_TYPE_DDR3) 642 opt_dll_mode = tegra210_emc_get_dll_state(next); 643 644 if ((next->burst_regs[EMC_FBIO_CFG5_INDEX] & BIT(25)) && 645 (dram_type == DRAM_TYPE_LPDDR2)) 646 is_lpddr3 = true; 647 648 emc_readl(emc, EMC_CFG); 649 emc_readl(emc, EMC_AUTO_CAL_CONFIG); 650 651 src_clk_period = 1000000000 / last->rate; 652 dst_clk_period = 1000000000 / next->rate; 653 654 if (dst_clk_period <= zqcal_before_cc_cutoff) 655 tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4 - tFC_lpddr4; 656 else 657 tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4; 658 659 tZQCAL_lpddr4_fc_adj /= dst_clk_period; 660 661 emc_dbg = emc_readl(emc, EMC_DBG); 662 emc_pin = emc_readl(emc, EMC_PIN); 663 emc_cfg_pipe_clk = emc_readl(emc, EMC_CFG_PIPE_CLK); 664 665 emc_cfg = next->burst_regs[EMC_CFG_INDEX]; 666 emc_cfg &= ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD | 667 EMC_CFG_DRAM_CLKSTOP_SR | EMC_CFG_DRAM_CLKSTOP_PD); 668 emc_sel_dpd_ctrl = next->emc_sel_dpd_ctrl; 669 emc_sel_dpd_ctrl &= ~(EMC_SEL_DPD_CTRL_CLK_SEL_DPD_EN | 670 EMC_SEL_DPD_CTRL_CA_SEL_DPD_EN | 671 EMC_SEL_DPD_CTRL_RESET_SEL_DPD_EN | 672 EMC_SEL_DPD_CTRL_ODT_SEL_DPD_EN | 673 EMC_SEL_DPD_CTRL_DATA_SEL_DPD_EN); 674 675 emc_dbg(emc, INFO, "Clock change version: %d\n", 676 DVFS_CLOCK_CHANGE_VERSION); 677 emc_dbg(emc, INFO, "DRAM type = %d\n", dram_type); 678 emc_dbg(emc, INFO, "DRAM dev #: %u\n", emc->num_devices); 679 emc_dbg(emc, INFO, "Next EMC clksrc: 0x%08x\n", clksrc); 680 emc_dbg(emc, INFO, "DLL clksrc: 0x%08x\n", next->dll_clk_src); 681 emc_dbg(emc, INFO, "last rate: %u, next rate %u\n", last->rate, 682 next->rate); 683 emc_dbg(emc, INFO, "last period: %u, next period: %u\n", 684 src_clk_period, dst_clk_period); 685 emc_dbg(emc, INFO, " shared_zq_resistor: %d\n", !!shared_zq_resistor); 686 emc_dbg(emc, INFO, " num_channels: %u\n", emc->num_channels); 687 emc_dbg(emc, INFO, " opt_dll_mode: %d\n", opt_dll_mode); 688 689 /* 690 * Step 1: 691 * Pre DVFS SW sequence. 692 */ 693 emc_dbg(emc, STEPS, "Step 1\n"); 694 emc_dbg(emc, STEPS, "Step 1.1: Disable DLL temporarily.\n"); 695 696 value = emc_readl(emc, EMC_CFG_DIG_DLL); 697 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; 698 emc_writel(emc, value, EMC_CFG_DIG_DLL); 699 700 tegra210_emc_timing_update(emc); 701 702 for (i = 0; i < emc->num_channels; i++) 703 tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL, 704 EMC_CFG_DIG_DLL_CFG_DLL_EN, 0); 705 706 emc_dbg(emc, STEPS, "Step 1.2: Disable AUTOCAL temporarily.\n"); 707 708 emc_auto_cal_config = next->emc_auto_cal_config; 709 auto_cal_en = emc_auto_cal_config & EMC_AUTO_CAL_CONFIG_AUTO_CAL_ENABLE; 710 emc_auto_cal_config &= ~EMC_AUTO_CAL_CONFIG_AUTO_CAL_START; 711 emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_MEASURE_STALL; 712 emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_UPDATE_STALL; 713 emc_auto_cal_config |= auto_cal_en; 714 emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); 715 emc_readl(emc, EMC_AUTO_CAL_CONFIG); /* Flush write. */ 716 717 emc_dbg(emc, STEPS, "Step 1.3: Disable other power features.\n"); 718 719 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 720 emc_writel(emc, emc_cfg, EMC_CFG); 721 emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); 722 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 723 724 if (next->periodic_training) { 725 tegra210_emc_reset_dram_clktree_values(next); 726 727 for (i = 0; i < emc->num_channels; i++) 728 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, 729 EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, 730 0); 731 732 for (i = 0; i < emc->num_channels; i++) 733 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, 734 EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, 735 0); 736 737 tegra210_emc_start_periodic_compensation(emc); 738 739 delay = 1000 * tegra210_emc_actual_osc_clocks(last->run_clocks); 740 udelay((delay / last->rate) + 2); 741 742 value = periodic_compensation_handler(emc, DVFS_SEQUENCE, fake, 743 next); 744 value = (value * 128 * next->rate / 1000) / 1000000; 745 746 if (next->periodic_training && value > next->tree_margin) 747 compensate_trimmer_applicable = true; 748 } 749 750 emc_writel(emc, EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS); 751 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 752 emc_writel(emc, emc_cfg, EMC_CFG); 753 emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); 754 emc_writel(emc, emc_cfg_pipe_clk | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON, 755 EMC_CFG_PIPE_CLK); 756 emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp & 757 ~EMC_FDPD_CTRL_CMD_NO_RAMP_CMD_DPD_NO_RAMP_ENABLE, 758 EMC_FDPD_CTRL_CMD_NO_RAMP); 759 760 bg_reg_mode_change = 761 ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 762 EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) ^ 763 (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 764 EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD)) || 765 ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 766 EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) ^ 767 (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 768 EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD)); 769 enable_bglp_reg = 770 (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 771 EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) == 0; 772 enable_bg_reg = 773 (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 774 EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) == 0; 775 776 if (bg_reg_mode_change) { 777 if (enable_bg_reg) 778 emc_writel(emc, last->burst_regs 779 [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 780 ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, 781 EMC_PMACRO_BG_BIAS_CTRL_0); 782 783 if (enable_bglp_reg) 784 emc_writel(emc, last->burst_regs 785 [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 786 ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, 787 EMC_PMACRO_BG_BIAS_CTRL_0); 788 } 789 790 /* Check if we need to turn on VREF generator. */ 791 if ((((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & 792 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 0) && 793 ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & 794 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 1)) || 795 (((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & 796 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) == 0) && 797 ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & 798 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) != 0))) { 799 u32 pad_tx_ctrl = 800 next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; 801 u32 last_pad_tx_ctrl = 802 last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; 803 u32 next_dq_e_ivref, next_dqs_e_ivref; 804 805 next_dqs_e_ivref = pad_tx_ctrl & 806 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF; 807 next_dq_e_ivref = pad_tx_ctrl & 808 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF; 809 value = (last_pad_tx_ctrl & 810 ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF & 811 ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) | 812 next_dq_e_ivref | next_dqs_e_ivref; 813 emc_writel(emc, value, EMC_PMACRO_DATA_PAD_TX_CTRL); 814 udelay(1); 815 } else if (bg_reg_mode_change) { 816 udelay(1); 817 } 818 819 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 820 821 /* 822 * Step 2: 823 * Prelock the DLL. 824 */ 825 emc_dbg(emc, STEPS, "Step 2\n"); 826 827 if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & 828 EMC_CFG_DIG_DLL_CFG_DLL_EN) { 829 emc_dbg(emc, INFO, "Prelock enabled for target frequency.\n"); 830 value = tegra210_emc_dll_prelock(emc, clksrc); 831 emc_dbg(emc, INFO, "DLL out: 0x%03x\n", value); 832 } else { 833 emc_dbg(emc, INFO, "Disabling DLL for target frequency.\n"); 834 tegra210_emc_dll_disable(emc); 835 } 836 837 /* 838 * Step 3: 839 * Prepare autocal for the clock change. 840 */ 841 emc_dbg(emc, STEPS, "Step 3\n"); 842 843 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 844 emc_writel(emc, next->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2); 845 emc_writel(emc, next->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3); 846 emc_writel(emc, next->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4); 847 emc_writel(emc, next->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5); 848 emc_writel(emc, next->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6); 849 emc_writel(emc, next->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7); 850 emc_writel(emc, next->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8); 851 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 852 853 emc_auto_cal_config |= (EMC_AUTO_CAL_CONFIG_AUTO_CAL_COMPUTE_START | 854 auto_cal_en); 855 emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); 856 857 /* 858 * Step 4: 859 * Update EMC_CFG. (??) 860 */ 861 emc_dbg(emc, STEPS, "Step 4\n"); 862 863 if (src_clk_period > 50000 && dram_type == DRAM_TYPE_LPDDR4) 864 ccfifo_writel(emc, 1, EMC_SELF_REF, 0); 865 else 866 emc_writel(emc, next->emc_cfg_2, EMC_CFG_2); 867 868 /* 869 * Step 5: 870 * Prepare reference variables for ZQCAL regs. 871 */ 872 emc_dbg(emc, STEPS, "Step 5\n"); 873 874 if (dram_type == DRAM_TYPE_LPDDR4) 875 zq_wait_long = max((u32)1, div_o3(1000000, dst_clk_period)); 876 else if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3) 877 zq_wait_long = max(next->min_mrs_wait, 878 div_o3(360000, dst_clk_period)) + 4; 879 else if (dram_type == DRAM_TYPE_DDR3) 880 zq_wait_long = max((u32)256, 881 div_o3(320000, dst_clk_period) + 2); 882 else 883 zq_wait_long = 0; 884 885 /* 886 * Step 6: 887 * Training code - removed. 888 */ 889 emc_dbg(emc, STEPS, "Step 6\n"); 890 891 /* 892 * Step 7: 893 * Program FSP reference registers and send MRWs to new FSPWR. 894 */ 895 emc_dbg(emc, STEPS, "Step 7\n"); 896 emc_dbg(emc, SUB_STEPS, "Step 7.1: Bug 200024907 - Patch RP R2P"); 897 898 /* WAR 200024907 */ 899 if (dram_type == DRAM_TYPE_LPDDR4) { 900 u32 nRTP = 16; 901 902 if (src_clk_period >= 1000000 / 1866) /* 535.91 ps */ 903 nRTP = 14; 904 905 if (src_clk_period >= 1000000 / 1600) /* 625.00 ps */ 906 nRTP = 12; 907 908 if (src_clk_period >= 1000000 / 1333) /* 750.19 ps */ 909 nRTP = 10; 910 911 if (src_clk_period >= 1000000 / 1066) /* 938.09 ps */ 912 nRTP = 8; 913 914 deltaTWATM = max_t(u32, div_o3(7500, src_clk_period), 8); 915 916 /* 917 * Originally there was a + .5 in the tRPST calculation. 918 * However since we can't do FP in the kernel and the tRTM 919 * computation was in a floating point ceiling function, adding 920 * one to tRTP should be ok. There is no other source of non 921 * integer values, so the result was always going to be 922 * something for the form: f_ceil(N + .5) = N + 1; 923 */ 924 tRPST = (last->emc_mrw & 0x80) >> 7; 925 tRTM = fake->dram_timings[RL] + div_o3(3600, src_clk_period) + 926 max_t(u32, div_o3(7500, src_clk_period), 8) + tRPST + 927 1 + nRTP; 928 929 emc_dbg(emc, INFO, "tRTM = %u, EMC_RP = %u\n", tRTM, 930 next->burst_regs[EMC_RP_INDEX]); 931 932 if (last->burst_regs[EMC_RP_INDEX] < tRTM) { 933 if (tRTM > (last->burst_regs[EMC_R2P_INDEX] + 934 last->burst_regs[EMC_RP_INDEX])) { 935 R2P_war = tRTM - last->burst_regs[EMC_RP_INDEX]; 936 RP_war = last->burst_regs[EMC_RP_INDEX]; 937 TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; 938 939 if (R2P_war > 63) { 940 RP_war = R2P_war + 941 last->burst_regs[EMC_RP_INDEX] - 63; 942 943 if (TRPab_war < RP_war) 944 TRPab_war = RP_war; 945 946 R2P_war = 63; 947 } 948 } else { 949 R2P_war = last->burst_regs[EMC_R2P_INDEX]; 950 RP_war = last->burst_regs[EMC_RP_INDEX]; 951 TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; 952 } 953 954 if (RP_war < deltaTWATM) { 955 W2P_war = last->burst_regs[EMC_W2P_INDEX] 956 + deltaTWATM - RP_war; 957 if (W2P_war > 63) { 958 RP_war = RP_war + W2P_war - 63; 959 if (TRPab_war < RP_war) 960 TRPab_war = RP_war; 961 W2P_war = 63; 962 } 963 } else { 964 W2P_war = last->burst_regs[ 965 EMC_W2P_INDEX]; 966 } 967 968 if ((last->burst_regs[EMC_W2P_INDEX] ^ W2P_war) || 969 (last->burst_regs[EMC_R2P_INDEX] ^ R2P_war) || 970 (last->burst_regs[EMC_RP_INDEX] ^ RP_war) || 971 (last->burst_regs[EMC_TRPAB_INDEX] ^ TRPab_war)) { 972 emc_writel(emc, RP_war, EMC_RP); 973 emc_writel(emc, R2P_war, EMC_R2P); 974 emc_writel(emc, W2P_war, EMC_W2P); 975 emc_writel(emc, TRPab_war, EMC_TRPAB); 976 } 977 978 tegra210_emc_timing_update(emc); 979 } else { 980 emc_dbg(emc, INFO, "Skipped WAR\n"); 981 } 982 } 983 984 if (!fsp_for_next_freq) { 985 mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x80; 986 mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0x00; 987 } else { 988 mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x40; 989 mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0xc0; 990 } 991 992 if (dram_type == DRAM_TYPE_LPDDR4) { 993 emc_writel(emc, mr13_flip_fspwr, EMC_MRW3); 994 emc_writel(emc, next->emc_mrw, EMC_MRW); 995 emc_writel(emc, next->emc_mrw2, EMC_MRW2); 996 } 997 998 /* 999 * Step 8: 1000 * Program the shadow registers. 1001 */ 1002 emc_dbg(emc, STEPS, "Step 8\n"); 1003 emc_dbg(emc, SUB_STEPS, "Writing burst_regs\n"); 1004 1005 for (i = 0; i < next->num_burst; i++) { 1006 const u16 *offsets = emc->offsets->burst; 1007 u16 offset; 1008 1009 if (!offsets[i]) 1010 continue; 1011 1012 value = next->burst_regs[i]; 1013 offset = offsets[i]; 1014 1015 if (dram_type != DRAM_TYPE_LPDDR4 && 1016 (offset == EMC_MRW6 || offset == EMC_MRW7 || 1017 offset == EMC_MRW8 || offset == EMC_MRW9 || 1018 offset == EMC_MRW10 || offset == EMC_MRW11 || 1019 offset == EMC_MRW12 || offset == EMC_MRW13 || 1020 offset == EMC_MRW14 || offset == EMC_MRW15 || 1021 offset == EMC_TRAINING_CTRL)) 1022 continue; 1023 1024 /* Pain... And suffering. */ 1025 if (offset == EMC_CFG) { 1026 value &= ~EMC_CFG_DRAM_ACPD; 1027 value &= ~EMC_CFG_DYN_SELF_REF; 1028 1029 if (dram_type == DRAM_TYPE_LPDDR4) { 1030 value &= ~EMC_CFG_DRAM_CLKSTOP_SR; 1031 value &= ~EMC_CFG_DRAM_CLKSTOP_PD; 1032 } 1033 } else if (offset == EMC_MRS_WAIT_CNT && 1034 dram_type == DRAM_TYPE_LPDDR2 && 1035 opt_zcal_en_cc && !opt_cc_short_zcal && 1036 opt_short_zcal) { 1037 value = (value & ~(EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK << 1038 EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)) | 1039 ((zq_wait_long & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK) << 1040 EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT); 1041 } else if (offset == EMC_ZCAL_WAIT_CNT && 1042 dram_type == DRAM_TYPE_DDR3 && opt_zcal_en_cc && 1043 !opt_cc_short_zcal && opt_short_zcal) { 1044 value = (value & ~(EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK << 1045 EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT)) | 1046 ((zq_wait_long & EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) << 1047 EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT); 1048 } else if (offset == EMC_ZCAL_INTERVAL && opt_zcal_en_cc) { 1049 value = 0; /* EMC_ZCAL_INTERVAL reset value. */ 1050 } else if (offset == EMC_PMACRO_AUTOCAL_CFG_COMMON) { 1051 value |= EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS; 1052 } else if (offset == EMC_PMACRO_DATA_PAD_TX_CTRL) { 1053 value &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | 1054 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC | 1055 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | 1056 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC); 1057 } else if (offset == EMC_PMACRO_CMD_PAD_TX_CTRL) { 1058 value |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON; 1059 value &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | 1060 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC | 1061 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | 1062 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC); 1063 } else if (offset == EMC_PMACRO_BRICK_CTRL_RFU1) { 1064 value &= 0xf800f800; 1065 } else if (offset == EMC_PMACRO_COMMON_PAD_TX_CTRL) { 1066 value &= 0xfffffff0; 1067 } 1068 1069 emc_writel(emc, value, offset); 1070 } 1071 1072 /* SW addition: do EMC refresh adjustment here. */ 1073 tegra210_emc_adjust_timing(emc, next); 1074 1075 if (dram_type == DRAM_TYPE_LPDDR4) { 1076 value = (23 << EMC_MRW_MRW_MA_SHIFT) | 1077 (next->run_clocks & EMC_MRW_MRW_OP_MASK); 1078 emc_writel(emc, value, EMC_MRW); 1079 } 1080 1081 /* Per channel burst registers. */ 1082 emc_dbg(emc, SUB_STEPS, "Writing burst_regs_per_ch\n"); 1083 1084 for (i = 0; i < next->num_burst_per_ch; i++) { 1085 const struct tegra210_emc_per_channel_regs *burst = 1086 emc->offsets->burst_per_channel; 1087 1088 if (!burst[i].offset) 1089 continue; 1090 1091 if (dram_type != DRAM_TYPE_LPDDR4 && 1092 (burst[i].offset == EMC_MRW6 || 1093 burst[i].offset == EMC_MRW7 || 1094 burst[i].offset == EMC_MRW8 || 1095 burst[i].offset == EMC_MRW9 || 1096 burst[i].offset == EMC_MRW10 || 1097 burst[i].offset == EMC_MRW11 || 1098 burst[i].offset == EMC_MRW12 || 1099 burst[i].offset == EMC_MRW13 || 1100 burst[i].offset == EMC_MRW14 || 1101 burst[i].offset == EMC_MRW15)) 1102 continue; 1103 1104 /* Filter out second channel if not in DUAL_CHANNEL mode. */ 1105 if (emc->num_channels < 2 && burst[i].bank >= 1) 1106 continue; 1107 1108 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1109 next->burst_reg_per_ch[i], burst[i].offset); 1110 emc_channel_writel(emc, burst[i].bank, 1111 next->burst_reg_per_ch[i], 1112 burst[i].offset); 1113 } 1114 1115 /* Vref regs. */ 1116 emc_dbg(emc, SUB_STEPS, "Writing vref_regs\n"); 1117 1118 for (i = 0; i < next->vref_num; i++) { 1119 const struct tegra210_emc_per_channel_regs *vref = 1120 emc->offsets->vref_per_channel; 1121 1122 if (!vref[i].offset) 1123 continue; 1124 1125 if (emc->num_channels < 2 && vref[i].bank >= 1) 1126 continue; 1127 1128 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1129 next->vref_perch_regs[i], vref[i].offset); 1130 emc_channel_writel(emc, vref[i].bank, next->vref_perch_regs[i], 1131 vref[i].offset); 1132 } 1133 1134 /* Trimmers. */ 1135 emc_dbg(emc, SUB_STEPS, "Writing trim_regs\n"); 1136 1137 for (i = 0; i < next->num_trim; i++) { 1138 const u16 *offsets = emc->offsets->trim; 1139 1140 if (!offsets[i]) 1141 continue; 1142 1143 if (compensate_trimmer_applicable && 1144 (offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 || 1145 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 || 1146 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 || 1147 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 || 1148 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 || 1149 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 || 1150 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 || 1151 offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 || 1152 offsets[i] == EMC_DATA_BRLSHFT_0 || 1153 offsets[i] == EMC_DATA_BRLSHFT_1)) { 1154 value = tegra210_emc_compensate(next, offsets[i]); 1155 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1156 value, offsets[i]); 1157 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", 1158 (u32)(u64)offsets[i], value); 1159 emc_writel(emc, value, offsets[i]); 1160 } else { 1161 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1162 next->trim_regs[i], offsets[i]); 1163 emc_writel(emc, next->trim_regs[i], offsets[i]); 1164 } 1165 } 1166 1167 /* Per channel trimmers. */ 1168 emc_dbg(emc, SUB_STEPS, "Writing trim_regs_per_ch\n"); 1169 1170 for (i = 0; i < next->num_trim_per_ch; i++) { 1171 const struct tegra210_emc_per_channel_regs *trim = 1172 &emc->offsets->trim_per_channel[0]; 1173 unsigned int offset; 1174 1175 if (!trim[i].offset) 1176 continue; 1177 1178 if (emc->num_channels < 2 && trim[i].bank >= 1) 1179 continue; 1180 1181 offset = trim[i].offset; 1182 1183 if (compensate_trimmer_applicable && 1184 (offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 || 1185 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 || 1186 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 || 1187 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 || 1188 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 || 1189 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 || 1190 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 || 1191 offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 || 1192 offset == EMC_DATA_BRLSHFT_0 || 1193 offset == EMC_DATA_BRLSHFT_1)) { 1194 value = tegra210_emc_compensate(next, offset); 1195 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1196 value, offset); 1197 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", offset, 1198 value); 1199 emc_channel_writel(emc, trim[i].bank, value, offset); 1200 } else { 1201 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1202 next->trim_perch_regs[i], offset); 1203 emc_channel_writel(emc, trim[i].bank, 1204 next->trim_perch_regs[i], offset); 1205 } 1206 } 1207 1208 emc_dbg(emc, SUB_STEPS, "Writing burst_mc_regs\n"); 1209 1210 for (i = 0; i < next->num_mc_regs; i++) { 1211 const u16 *offsets = emc->offsets->burst_mc; 1212 u32 *values = next->burst_mc_regs; 1213 1214 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1215 values[i], offsets[i]); 1216 mc_writel(emc->mc, values[i], offsets[i]); 1217 } 1218 1219 /* Registers to be programmed on the faster clock. */ 1220 if (next->rate < last->rate) { 1221 const u16 *la = emc->offsets->la_scale; 1222 1223 emc_dbg(emc, SUB_STEPS, "Writing la_scale_regs\n"); 1224 1225 for (i = 0; i < next->num_up_down; i++) { 1226 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, 1227 next->la_scale_regs[i], la[i]); 1228 mc_writel(emc->mc, next->la_scale_regs[i], la[i]); 1229 } 1230 } 1231 1232 /* Flush all the burst register writes. */ 1233 mc_readl(emc->mc, MC_EMEM_ADR_CFG); 1234 1235 /* 1236 * Step 9: 1237 * LPDDR4 section A. 1238 */ 1239 emc_dbg(emc, STEPS, "Step 9\n"); 1240 1241 value = next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX]; 1242 value &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK; 1243 1244 if (dram_type == DRAM_TYPE_LPDDR4) { 1245 emc_writel(emc, 0, EMC_ZCAL_INTERVAL); 1246 emc_writel(emc, value, EMC_ZCAL_WAIT_CNT); 1247 1248 value = emc_dbg | (EMC_DBG_WRITE_MUX_ACTIVE | 1249 EMC_DBG_WRITE_ACTIVE_ONLY); 1250 1251 emc_writel(emc, value, EMC_DBG); 1252 emc_writel(emc, 0, EMC_ZCAL_INTERVAL); 1253 emc_writel(emc, emc_dbg, EMC_DBG); 1254 } 1255 1256 /* 1257 * Step 10: 1258 * LPDDR4 and DDR3 common section. 1259 */ 1260 emc_dbg(emc, STEPS, "Step 10\n"); 1261 1262 if (opt_dvfs_mode == MAN_SR || dram_type == DRAM_TYPE_LPDDR4) { 1263 if (dram_type == DRAM_TYPE_LPDDR4) 1264 ccfifo_writel(emc, 0x101, EMC_SELF_REF, 0); 1265 else 1266 ccfifo_writel(emc, 0x1, EMC_SELF_REF, 0); 1267 1268 if (dram_type == DRAM_TYPE_LPDDR4 && 1269 dst_clk_period <= zqcal_before_cc_cutoff) { 1270 ccfifo_writel(emc, mr13_flip_fspwr ^ 0x40, EMC_MRW3, 0); 1271 ccfifo_writel(emc, (next->burst_regs[EMC_MRW6_INDEX] & 1272 0xFFFF3F3F) | 1273 (last->burst_regs[EMC_MRW6_INDEX] & 1274 0x0000C0C0), EMC_MRW6, 0); 1275 ccfifo_writel(emc, (next->burst_regs[EMC_MRW14_INDEX] & 1276 0xFFFF0707) | 1277 (last->burst_regs[EMC_MRW14_INDEX] & 1278 0x00003838), EMC_MRW14, 0); 1279 1280 if (emc->num_devices > 1) { 1281 ccfifo_writel(emc, 1282 (next->burst_regs[EMC_MRW7_INDEX] & 1283 0xFFFF3F3F) | 1284 (last->burst_regs[EMC_MRW7_INDEX] & 1285 0x0000C0C0), EMC_MRW7, 0); 1286 ccfifo_writel(emc, 1287 (next->burst_regs[EMC_MRW15_INDEX] & 1288 0xFFFF0707) | 1289 (last->burst_regs[EMC_MRW15_INDEX] & 1290 0x00003838), EMC_MRW15, 0); 1291 } 1292 1293 if (opt_zcal_en_cc) { 1294 if (emc->num_devices < 2) 1295 ccfifo_writel(emc, 1296 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT 1297 | EMC_ZQ_CAL_ZQ_CAL_CMD, 1298 EMC_ZQ_CAL, 0); 1299 else if (shared_zq_resistor) 1300 ccfifo_writel(emc, 1301 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT 1302 | EMC_ZQ_CAL_ZQ_CAL_CMD, 1303 EMC_ZQ_CAL, 0); 1304 else 1305 ccfifo_writel(emc, 1306 EMC_ZQ_CAL_ZQ_CAL_CMD, 1307 EMC_ZQ_CAL, 0); 1308 } 1309 } 1310 } 1311 1312 if (dram_type == DRAM_TYPE_LPDDR4) { 1313 value = (1000 * fake->dram_timings[T_RP]) / src_clk_period; 1314 ccfifo_writel(emc, mr13_flip_fspop | 0x8, EMC_MRW3, value); 1315 ccfifo_writel(emc, 0, 0, tFC_lpddr4 / src_clk_period); 1316 } 1317 1318 if (dram_type == DRAM_TYPE_LPDDR4 || opt_dvfs_mode != MAN_SR) { 1319 delay = 30; 1320 1321 if (cya_allow_ref_cc) { 1322 delay += (1000 * fake->dram_timings[T_RP]) / 1323 src_clk_period; 1324 delay += 4000 * fake->dram_timings[T_RFC]; 1325 } 1326 1327 ccfifo_writel(emc, emc_pin & ~(EMC_PIN_PIN_CKE_PER_DEV | 1328 EMC_PIN_PIN_CKEB | 1329 EMC_PIN_PIN_CKE), 1330 EMC_PIN, delay); 1331 } 1332 1333 /* calculate reference delay multiplier */ 1334 value = 1; 1335 1336 if (ref_b4_sref_en) 1337 value++; 1338 1339 if (cya_allow_ref_cc) 1340 value++; 1341 1342 if (cya_issue_pc_ref) 1343 value++; 1344 1345 if (dram_type != DRAM_TYPE_LPDDR4) { 1346 delay = ((1000 * fake->dram_timings[T_RP] / src_clk_period) + 1347 (1000 * fake->dram_timings[T_RFC] / src_clk_period)); 1348 delay = value * delay + 20; 1349 } else { 1350 delay = 0; 1351 } 1352 1353 /* 1354 * Step 11: 1355 * Ramp down. 1356 */ 1357 emc_dbg(emc, STEPS, "Step 11\n"); 1358 1359 ccfifo_writel(emc, 0x0, EMC_CFG_SYNC, delay); 1360 1361 value = emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE | EMC_DBG_WRITE_ACTIVE_ONLY; 1362 ccfifo_writel(emc, value, EMC_DBG, 0); 1363 1364 ramp_down_wait = tegra210_emc_dvfs_power_ramp_down(emc, src_clk_period, 1365 0); 1366 1367 /* 1368 * Step 12: 1369 * And finally - trigger the clock change. 1370 */ 1371 emc_dbg(emc, STEPS, "Step 12\n"); 1372 1373 ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 0); 1374 value &= ~EMC_DBG_WRITE_ACTIVE_ONLY; 1375 ccfifo_writel(emc, value, EMC_DBG, 0); 1376 1377 /* 1378 * Step 13: 1379 * Ramp up. 1380 */ 1381 emc_dbg(emc, STEPS, "Step 13\n"); 1382 1383 ramp_up_wait = tegra210_emc_dvfs_power_ramp_up(emc, dst_clk_period, 0); 1384 ccfifo_writel(emc, emc_dbg, EMC_DBG, 0); 1385 1386 /* 1387 * Step 14: 1388 * Bringup CKE pins. 1389 */ 1390 emc_dbg(emc, STEPS, "Step 14\n"); 1391 1392 if (dram_type == DRAM_TYPE_LPDDR4) { 1393 value = emc_pin | EMC_PIN_PIN_CKE; 1394 1395 if (emc->num_devices <= 1) 1396 value &= ~(EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV); 1397 else 1398 value |= EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV; 1399 1400 ccfifo_writel(emc, value, EMC_PIN, 0); 1401 } 1402 1403 /* 1404 * Step 15: (two step 15s ??) 1405 * Calculate zqlatch wait time; has dependency on ramping times. 1406 */ 1407 emc_dbg(emc, STEPS, "Step 15\n"); 1408 1409 if (dst_clk_period <= zqcal_before_cc_cutoff) { 1410 s32 t = (s32)(ramp_up_wait + ramp_down_wait) / 1411 (s32)dst_clk_period; 1412 zq_latch_dvfs_wait_time = (s32)tZQCAL_lpddr4_fc_adj - t; 1413 } else { 1414 zq_latch_dvfs_wait_time = tZQCAL_lpddr4_fc_adj - 1415 div_o3(1000 * next->dram_timings[T_PDEX], 1416 dst_clk_period); 1417 } 1418 1419 emc_dbg(emc, INFO, "tZQCAL_lpddr4_fc_adj = %u\n", tZQCAL_lpddr4_fc_adj); 1420 emc_dbg(emc, INFO, "dst_clk_period = %u\n", 1421 dst_clk_period); 1422 emc_dbg(emc, INFO, "next->dram_timings[T_PDEX] = %u\n", 1423 next->dram_timings[T_PDEX]); 1424 emc_dbg(emc, INFO, "zq_latch_dvfs_wait_time = %d\n", 1425 max_t(s32, 0, zq_latch_dvfs_wait_time)); 1426 1427 if (dram_type == DRAM_TYPE_LPDDR4 && opt_zcal_en_cc) { 1428 delay = div_o3(1000 * next->dram_timings[T_PDEX], 1429 dst_clk_period); 1430 1431 if (emc->num_devices < 2) { 1432 if (dst_clk_period > zqcal_before_cc_cutoff) 1433 ccfifo_writel(emc, 1434 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1435 EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, 1436 delay); 1437 1438 value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; 1439 ccfifo_writel(emc, value, EMC_MRW3, delay); 1440 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); 1441 ccfifo_writel(emc, 0, EMC_REF, 0); 1442 ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1443 EMC_ZQ_CAL_ZQ_LATCH_CMD, 1444 EMC_ZQ_CAL, 1445 max_t(s32, 0, zq_latch_dvfs_wait_time)); 1446 } else if (shared_zq_resistor) { 1447 if (dst_clk_period > zqcal_before_cc_cutoff) 1448 ccfifo_writel(emc, 1449 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1450 EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, 1451 delay); 1452 1453 ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1454 EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, 1455 max_t(s32, 0, zq_latch_dvfs_wait_time) + 1456 delay); 1457 ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1458 EMC_ZQ_CAL_ZQ_LATCH_CMD, 1459 EMC_ZQ_CAL, 0); 1460 1461 value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; 1462 ccfifo_writel(emc, value, EMC_MRW3, 0); 1463 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); 1464 ccfifo_writel(emc, 0, EMC_REF, 0); 1465 1466 ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1467 EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, 1468 tZQCAL_lpddr4 / dst_clk_period); 1469 } else { 1470 if (dst_clk_period > zqcal_before_cc_cutoff) 1471 ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_CAL_CMD, 1472 EMC_ZQ_CAL, delay); 1473 1474 value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; 1475 ccfifo_writel(emc, value, EMC_MRW3, delay); 1476 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); 1477 ccfifo_writel(emc, 0, EMC_REF, 0); 1478 1479 ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, 1480 max_t(s32, 0, zq_latch_dvfs_wait_time)); 1481 } 1482 } 1483 1484 /* WAR: delay for zqlatch */ 1485 ccfifo_writel(emc, 0, 0, 10); 1486 1487 /* 1488 * Step 16: 1489 * LPDDR4 Conditional Training Kickoff. Removed. 1490 */ 1491 1492 /* 1493 * Step 17: 1494 * MANSR exit self refresh. 1495 */ 1496 emc_dbg(emc, STEPS, "Step 17\n"); 1497 1498 if (opt_dvfs_mode == MAN_SR && dram_type != DRAM_TYPE_LPDDR4) 1499 ccfifo_writel(emc, 0, EMC_SELF_REF, 0); 1500 1501 /* 1502 * Step 18: 1503 * Send MRWs to LPDDR3/DDR3. 1504 */ 1505 emc_dbg(emc, STEPS, "Step 18\n"); 1506 1507 if (dram_type == DRAM_TYPE_LPDDR2) { 1508 ccfifo_writel(emc, next->emc_mrw2, EMC_MRW2, 0); 1509 ccfifo_writel(emc, next->emc_mrw, EMC_MRW, 0); 1510 if (is_lpddr3) 1511 ccfifo_writel(emc, next->emc_mrw4, EMC_MRW4, 0); 1512 } else if (dram_type == DRAM_TYPE_DDR3) { 1513 if (opt_dll_mode) 1514 ccfifo_writel(emc, next->emc_emrs & 1515 ~EMC_EMRS_USE_EMRS_LONG_CNT, EMC_EMRS, 0); 1516 ccfifo_writel(emc, next->emc_emrs2 & 1517 ~EMC_EMRS2_USE_EMRS2_LONG_CNT, EMC_EMRS2, 0); 1518 ccfifo_writel(emc, next->emc_mrs | 1519 EMC_EMRS_USE_EMRS_LONG_CNT, EMC_MRS, 0); 1520 } 1521 1522 /* 1523 * Step 19: 1524 * ZQCAL for LPDDR3/DDR3 1525 */ 1526 emc_dbg(emc, STEPS, "Step 19\n"); 1527 1528 if (opt_zcal_en_cc) { 1529 if (dram_type == DRAM_TYPE_LPDDR2) { 1530 value = opt_cc_short_zcal ? 90000 : 360000; 1531 value = div_o3(value, dst_clk_period); 1532 value = value << 1533 EMC_MRS_WAIT_CNT2_MRS_EXT2_WAIT_CNT_SHIFT | 1534 value << 1535 EMC_MRS_WAIT_CNT2_MRS_EXT1_WAIT_CNT_SHIFT; 1536 ccfifo_writel(emc, value, EMC_MRS_WAIT_CNT2, 0); 1537 1538 value = opt_cc_short_zcal ? 0x56 : 0xab; 1539 ccfifo_writel(emc, 2 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | 1540 EMC_MRW_USE_MRW_EXT_CNT | 1541 10 << EMC_MRW_MRW_MA_SHIFT | 1542 value << EMC_MRW_MRW_OP_SHIFT, 1543 EMC_MRW, 0); 1544 1545 if (emc->num_devices > 1) { 1546 value = 1 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | 1547 EMC_MRW_USE_MRW_EXT_CNT | 1548 10 << EMC_MRW_MRW_MA_SHIFT | 1549 value << EMC_MRW_MRW_OP_SHIFT; 1550 ccfifo_writel(emc, value, EMC_MRW, 0); 1551 } 1552 } else if (dram_type == DRAM_TYPE_DDR3) { 1553 value = opt_cc_short_zcal ? 0 : EMC_ZQ_CAL_LONG; 1554 1555 ccfifo_writel(emc, value | 1556 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1557 EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, 1558 0); 1559 1560 if (emc->num_devices > 1) { 1561 value = value | 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | 1562 EMC_ZQ_CAL_ZQ_CAL_CMD; 1563 ccfifo_writel(emc, value, EMC_ZQ_CAL, 0); 1564 } 1565 } 1566 } 1567 1568 if (bg_reg_mode_change) { 1569 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 1570 1571 if (ramp_up_wait <= 1250000) 1572 delay = (1250000 - ramp_up_wait) / dst_clk_period; 1573 else 1574 delay = 0; 1575 1576 ccfifo_writel(emc, 1577 next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX], 1578 EMC_PMACRO_BG_BIAS_CTRL_0, delay); 1579 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 1580 } 1581 1582 /* 1583 * Step 20: 1584 * Issue ref and optional QRST. 1585 */ 1586 emc_dbg(emc, STEPS, "Step 20\n"); 1587 1588 if (dram_type != DRAM_TYPE_LPDDR4) 1589 ccfifo_writel(emc, 0, EMC_REF, 0); 1590 1591 if (opt_do_sw_qrst) { 1592 ccfifo_writel(emc, 1, EMC_ISSUE_QRST, 0); 1593 ccfifo_writel(emc, 0, EMC_ISSUE_QRST, 2); 1594 } 1595 1596 /* 1597 * Step 21: 1598 * Restore ZCAL and ZCAL interval. 1599 */ 1600 emc_dbg(emc, STEPS, "Step 21\n"); 1601 1602 if (save_restore_clkstop_pd || opt_zcal_en_cc) { 1603 ccfifo_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, 1604 EMC_DBG, 0); 1605 if (opt_zcal_en_cc && dram_type != DRAM_TYPE_LPDDR4) 1606 ccfifo_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], 1607 EMC_ZCAL_INTERVAL, 0); 1608 1609 if (save_restore_clkstop_pd) 1610 ccfifo_writel(emc, next->burst_regs[EMC_CFG_INDEX] & 1611 ~EMC_CFG_DYN_SELF_REF, 1612 EMC_CFG, 0); 1613 ccfifo_writel(emc, emc_dbg, EMC_DBG, 0); 1614 } 1615 1616 /* 1617 * Step 22: 1618 * Restore EMC_CFG_PIPE_CLK. 1619 */ 1620 emc_dbg(emc, STEPS, "Step 22\n"); 1621 1622 ccfifo_writel(emc, emc_cfg_pipe_clk, EMC_CFG_PIPE_CLK, 0); 1623 1624 if (bg_reg_mode_change) { 1625 if (enable_bg_reg) 1626 emc_writel(emc, 1627 next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 1628 ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, 1629 EMC_PMACRO_BG_BIAS_CTRL_0); 1630 else 1631 emc_writel(emc, 1632 next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & 1633 ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, 1634 EMC_PMACRO_BG_BIAS_CTRL_0); 1635 } 1636 1637 /* 1638 * Step 23: 1639 */ 1640 emc_dbg(emc, STEPS, "Step 23\n"); 1641 1642 value = emc_readl(emc, EMC_CFG_DIG_DLL); 1643 value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC; 1644 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK; 1645 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK; 1646 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; 1647 value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | 1648 (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); 1649 emc_writel(emc, value, EMC_CFG_DIG_DLL); 1650 1651 tegra210_emc_do_clock_change(emc, clksrc); 1652 1653 /* 1654 * Step 24: 1655 * Save training results. Removed. 1656 */ 1657 1658 /* 1659 * Step 25: 1660 * Program MC updown registers. 1661 */ 1662 emc_dbg(emc, STEPS, "Step 25\n"); 1663 1664 if (next->rate > last->rate) { 1665 for (i = 0; i < next->num_up_down; i++) 1666 mc_writel(emc->mc, next->la_scale_regs[i], 1667 emc->offsets->la_scale[i]); 1668 1669 tegra210_emc_timing_update(emc); 1670 } 1671 1672 /* 1673 * Step 26: 1674 * Restore ZCAL registers. 1675 */ 1676 emc_dbg(emc, STEPS, "Step 26\n"); 1677 1678 if (dram_type == DRAM_TYPE_LPDDR4) { 1679 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 1680 emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], 1681 EMC_ZCAL_WAIT_CNT); 1682 emc_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], 1683 EMC_ZCAL_INTERVAL); 1684 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 1685 } 1686 1687 if (dram_type != DRAM_TYPE_LPDDR4 && opt_zcal_en_cc && 1688 !opt_short_zcal && opt_cc_short_zcal) { 1689 udelay(2); 1690 1691 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 1692 if (dram_type == DRAM_TYPE_LPDDR2) 1693 emc_writel(emc, next->burst_regs[EMC_MRS_WAIT_CNT_INDEX], 1694 EMC_MRS_WAIT_CNT); 1695 else if (dram_type == DRAM_TYPE_DDR3) 1696 emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], 1697 EMC_ZCAL_WAIT_CNT); 1698 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 1699 } 1700 1701 /* 1702 * Step 27: 1703 * Restore EMC_CFG, FDPD registers. 1704 */ 1705 emc_dbg(emc, STEPS, "Step 27\n"); 1706 1707 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 1708 emc_writel(emc, next->burst_regs[EMC_CFG_INDEX], EMC_CFG); 1709 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 1710 emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp, 1711 EMC_FDPD_CTRL_CMD_NO_RAMP); 1712 emc_writel(emc, next->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); 1713 1714 /* 1715 * Step 28: 1716 * Training recover. Removed. 1717 */ 1718 emc_dbg(emc, STEPS, "Step 28\n"); 1719 1720 tegra210_emc_set_shadow_bypass(emc, ACTIVE); 1721 emc_writel(emc, 1722 next->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX], 1723 EMC_PMACRO_AUTOCAL_CFG_COMMON); 1724 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); 1725 1726 /* 1727 * Step 29: 1728 * Power fix WAR. 1729 */ 1730 emc_dbg(emc, STEPS, "Step 29\n"); 1731 1732 emc_writel(emc, EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 | 1733 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE1 | 1734 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE2 | 1735 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE3 | 1736 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE4 | 1737 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE5 | 1738 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE6 | 1739 EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE7, 1740 EMC_PMACRO_CFG_PM_GLOBAL_0); 1741 emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR, 1742 EMC_PMACRO_TRAINING_CTRL_0); 1743 emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR, 1744 EMC_PMACRO_TRAINING_CTRL_1); 1745 emc_writel(emc, 0, EMC_PMACRO_CFG_PM_GLOBAL_0); 1746 1747 /* 1748 * Step 30: 1749 * Re-enable autocal. 1750 */ 1751 emc_dbg(emc, STEPS, "Step 30: Re-enable DLL and AUTOCAL\n"); 1752 1753 if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) { 1754 value = emc_readl(emc, EMC_CFG_DIG_DLL); 1755 value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC; 1756 value |= EMC_CFG_DIG_DLL_CFG_DLL_EN; 1757 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK; 1758 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK; 1759 value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | 1760 (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); 1761 emc_writel(emc, value, EMC_CFG_DIG_DLL); 1762 tegra210_emc_timing_update(emc); 1763 } 1764 1765 emc_writel(emc, next->emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); 1766 1767 /* Done! Yay. */ 1768 } 1769 1770 const struct tegra210_emc_sequence tegra210_emc_r21021 = { 1771 .revision = 0x7, 1772 .set_clock = tegra210_emc_r21021_set_clock, 1773 .periodic_compensation = tegra210_emc_r21021_periodic_compensation, 1774 }; 1775