1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Tegra20 External Memory Controller driver 4 * 5 * Author: Dmitry Osipenko <digetx@gmail.com> 6 */ 7 8 #include <linux/clk.h> 9 #include <linux/clk/tegra.h> 10 #include <linux/completion.h> 11 #include <linux/err.h> 12 #include <linux/interrupt.h> 13 #include <linux/io.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/of.h> 17 #include <linux/platform_device.h> 18 #include <linux/sort.h> 19 #include <linux/types.h> 20 21 #include <soc/tegra/fuse.h> 22 23 #define EMC_INTSTATUS 0x000 24 #define EMC_INTMASK 0x004 25 #define EMC_DBG 0x008 26 #define EMC_TIMING_CONTROL 0x028 27 #define EMC_RC 0x02c 28 #define EMC_RFC 0x030 29 #define EMC_RAS 0x034 30 #define EMC_RP 0x038 31 #define EMC_R2W 0x03c 32 #define EMC_W2R 0x040 33 #define EMC_R2P 0x044 34 #define EMC_W2P 0x048 35 #define EMC_RD_RCD 0x04c 36 #define EMC_WR_RCD 0x050 37 #define EMC_RRD 0x054 38 #define EMC_REXT 0x058 39 #define EMC_WDV 0x05c 40 #define EMC_QUSE 0x060 41 #define EMC_QRST 0x064 42 #define EMC_QSAFE 0x068 43 #define EMC_RDV 0x06c 44 #define EMC_REFRESH 0x070 45 #define EMC_BURST_REFRESH_NUM 0x074 46 #define EMC_PDEX2WR 0x078 47 #define EMC_PDEX2RD 0x07c 48 #define EMC_PCHG2PDEN 0x080 49 #define EMC_ACT2PDEN 0x084 50 #define EMC_AR2PDEN 0x088 51 #define EMC_RW2PDEN 0x08c 52 #define EMC_TXSR 0x090 53 #define EMC_TCKE 0x094 54 #define EMC_TFAW 0x098 55 #define EMC_TRPAB 0x09c 56 #define EMC_TCLKSTABLE 0x0a0 57 #define EMC_TCLKSTOP 0x0a4 58 #define EMC_TREFBW 0x0a8 59 #define EMC_QUSE_EXTRA 0x0ac 60 #define EMC_ODT_WRITE 0x0b0 61 #define EMC_ODT_READ 0x0b4 62 #define EMC_FBIO_CFG5 0x104 63 #define EMC_FBIO_CFG6 0x114 64 #define EMC_AUTO_CAL_INTERVAL 0x2a8 65 #define EMC_CFG_2 0x2b8 66 #define EMC_CFG_DIG_DLL 0x2bc 67 #define EMC_DLL_XFORM_DQS 0x2c0 68 #define EMC_DLL_XFORM_QUSE 0x2c4 69 #define EMC_ZCAL_REF_CNT 0x2e0 70 #define EMC_ZCAL_WAIT_CNT 0x2e4 71 #define EMC_CFG_CLKTRIM_0 0x2d0 72 #define EMC_CFG_CLKTRIM_1 0x2d4 73 #define EMC_CFG_CLKTRIM_2 0x2d8 74 75 #define EMC_CLKCHANGE_REQ_ENABLE BIT(0) 76 #define EMC_CLKCHANGE_PD_ENABLE BIT(1) 77 #define EMC_CLKCHANGE_SR_ENABLE BIT(2) 78 79 #define EMC_TIMING_UPDATE BIT(0) 80 81 #define EMC_REFRESH_OVERFLOW_INT BIT(3) 82 #define EMC_CLKCHANGE_COMPLETE_INT BIT(4) 83 84 #define EMC_DBG_READ_MUX_ASSEMBLY BIT(0) 85 #define EMC_DBG_WRITE_MUX_ACTIVE BIT(1) 86 #define EMC_DBG_FORCE_UPDATE BIT(2) 87 #define EMC_DBG_READ_DQM_CTRL BIT(9) 88 #define EMC_DBG_CFG_PRIORITY BIT(24) 89 90 static const u16 emc_timing_registers[] = { 91 EMC_RC, 92 EMC_RFC, 93 EMC_RAS, 94 EMC_RP, 95 EMC_R2W, 96 EMC_W2R, 97 EMC_R2P, 98 EMC_W2P, 99 EMC_RD_RCD, 100 EMC_WR_RCD, 101 EMC_RRD, 102 EMC_REXT, 103 EMC_WDV, 104 EMC_QUSE, 105 EMC_QRST, 106 EMC_QSAFE, 107 EMC_RDV, 108 EMC_REFRESH, 109 EMC_BURST_REFRESH_NUM, 110 EMC_PDEX2WR, 111 EMC_PDEX2RD, 112 EMC_PCHG2PDEN, 113 EMC_ACT2PDEN, 114 EMC_AR2PDEN, 115 EMC_RW2PDEN, 116 EMC_TXSR, 117 EMC_TCKE, 118 EMC_TFAW, 119 EMC_TRPAB, 120 EMC_TCLKSTABLE, 121 EMC_TCLKSTOP, 122 EMC_TREFBW, 123 EMC_QUSE_EXTRA, 124 EMC_FBIO_CFG6, 125 EMC_ODT_WRITE, 126 EMC_ODT_READ, 127 EMC_FBIO_CFG5, 128 EMC_CFG_DIG_DLL, 129 EMC_DLL_XFORM_DQS, 130 EMC_DLL_XFORM_QUSE, 131 EMC_ZCAL_REF_CNT, 132 EMC_ZCAL_WAIT_CNT, 133 EMC_AUTO_CAL_INTERVAL, 134 EMC_CFG_CLKTRIM_0, 135 EMC_CFG_CLKTRIM_1, 136 EMC_CFG_CLKTRIM_2, 137 }; 138 139 struct emc_timing { 140 unsigned long rate; 141 u32 data[ARRAY_SIZE(emc_timing_registers)]; 142 }; 143 144 struct tegra_emc { 145 struct device *dev; 146 struct completion clk_handshake_complete; 147 struct notifier_block clk_nb; 148 struct clk *clk; 149 void __iomem *regs; 150 151 struct emc_timing *timings; 152 unsigned int num_timings; 153 }; 154 155 static irqreturn_t tegra_emc_isr(int irq, void *data) 156 { 157 struct tegra_emc *emc = data; 158 u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT; 159 u32 status; 160 161 status = readl_relaxed(emc->regs + EMC_INTSTATUS) & intmask; 162 if (!status) 163 return IRQ_NONE; 164 165 /* notify about EMC-CAR handshake completion */ 166 if (status & EMC_CLKCHANGE_COMPLETE_INT) 167 complete(&emc->clk_handshake_complete); 168 169 /* notify about HW problem */ 170 if (status & EMC_REFRESH_OVERFLOW_INT) 171 dev_err_ratelimited(emc->dev, 172 "refresh request overflow timeout\n"); 173 174 /* clear interrupts */ 175 writel_relaxed(status, emc->regs + EMC_INTSTATUS); 176 177 return IRQ_HANDLED; 178 } 179 180 static struct emc_timing *tegra_emc_find_timing(struct tegra_emc *emc, 181 unsigned long rate) 182 { 183 struct emc_timing *timing = NULL; 184 unsigned int i; 185 186 for (i = 0; i < emc->num_timings; i++) { 187 if (emc->timings[i].rate >= rate) { 188 timing = &emc->timings[i]; 189 break; 190 } 191 } 192 193 if (!timing) { 194 dev_err(emc->dev, "no timing for rate %lu\n", rate); 195 return NULL; 196 } 197 198 return timing; 199 } 200 201 static int emc_prepare_timing_change(struct tegra_emc *emc, unsigned long rate) 202 { 203 struct emc_timing *timing = tegra_emc_find_timing(emc, rate); 204 unsigned int i; 205 206 if (!timing) 207 return -EINVAL; 208 209 dev_dbg(emc->dev, "%s: using timing rate %lu for requested rate %lu\n", 210 __func__, timing->rate, rate); 211 212 /* program shadow registers */ 213 for (i = 0; i < ARRAY_SIZE(timing->data); i++) 214 writel_relaxed(timing->data[i], 215 emc->regs + emc_timing_registers[i]); 216 217 /* wait until programming has settled */ 218 readl_relaxed(emc->regs + emc_timing_registers[i - 1]); 219 220 reinit_completion(&emc->clk_handshake_complete); 221 222 return 0; 223 } 224 225 static int emc_complete_timing_change(struct tegra_emc *emc, bool flush) 226 { 227 unsigned long timeout; 228 229 dev_dbg(emc->dev, "%s: flush %d\n", __func__, flush); 230 231 if (flush) { 232 /* manually initiate memory timing update */ 233 writel_relaxed(EMC_TIMING_UPDATE, 234 emc->regs + EMC_TIMING_CONTROL); 235 return 0; 236 } 237 238 timeout = wait_for_completion_timeout(&emc->clk_handshake_complete, 239 msecs_to_jiffies(100)); 240 if (timeout == 0) { 241 dev_err(emc->dev, "EMC-CAR handshake failed\n"); 242 return -EIO; 243 } 244 245 return 0; 246 } 247 248 static int tegra_emc_clk_change_notify(struct notifier_block *nb, 249 unsigned long msg, void *data) 250 { 251 struct tegra_emc *emc = container_of(nb, struct tegra_emc, clk_nb); 252 struct clk_notifier_data *cnd = data; 253 int err; 254 255 switch (msg) { 256 case PRE_RATE_CHANGE: 257 err = emc_prepare_timing_change(emc, cnd->new_rate); 258 break; 259 260 case ABORT_RATE_CHANGE: 261 err = emc_prepare_timing_change(emc, cnd->old_rate); 262 if (err) 263 break; 264 265 err = emc_complete_timing_change(emc, true); 266 break; 267 268 case POST_RATE_CHANGE: 269 err = emc_complete_timing_change(emc, false); 270 break; 271 272 default: 273 return NOTIFY_DONE; 274 } 275 276 return notifier_from_errno(err); 277 } 278 279 static int load_one_timing_from_dt(struct tegra_emc *emc, 280 struct emc_timing *timing, 281 struct device_node *node) 282 { 283 u32 rate; 284 int err; 285 286 if (!of_device_is_compatible(node, "nvidia,tegra20-emc-table")) { 287 dev_err(emc->dev, "incompatible DT node: %pOF\n", node); 288 return -EINVAL; 289 } 290 291 err = of_property_read_u32(node, "clock-frequency", &rate); 292 if (err) { 293 dev_err(emc->dev, "timing %pOF: failed to read rate: %d\n", 294 node, err); 295 return err; 296 } 297 298 err = of_property_read_u32_array(node, "nvidia,emc-registers", 299 timing->data, 300 ARRAY_SIZE(emc_timing_registers)); 301 if (err) { 302 dev_err(emc->dev, 303 "timing %pOF: failed to read emc timing data: %d\n", 304 node, err); 305 return err; 306 } 307 308 /* 309 * The EMC clock rate is twice the bus rate, and the bus rate is 310 * measured in kHz. 311 */ 312 timing->rate = rate * 2 * 1000; 313 314 dev_dbg(emc->dev, "%s: %pOF: EMC rate %lu\n", 315 __func__, node, timing->rate); 316 317 return 0; 318 } 319 320 static int cmp_timings(const void *_a, const void *_b) 321 { 322 const struct emc_timing *a = _a; 323 const struct emc_timing *b = _b; 324 325 if (a->rate < b->rate) 326 return -1; 327 328 if (a->rate > b->rate) 329 return 1; 330 331 return 0; 332 } 333 334 static int tegra_emc_load_timings_from_dt(struct tegra_emc *emc, 335 struct device_node *node) 336 { 337 struct device_node *child; 338 struct emc_timing *timing; 339 int child_count; 340 int err; 341 342 child_count = of_get_child_count(node); 343 if (!child_count) { 344 dev_err(emc->dev, "no memory timings in DT node: %pOF\n", node); 345 return -EINVAL; 346 } 347 348 emc->timings = devm_kcalloc(emc->dev, child_count, sizeof(*timing), 349 GFP_KERNEL); 350 if (!emc->timings) 351 return -ENOMEM; 352 353 emc->num_timings = child_count; 354 timing = emc->timings; 355 356 for_each_child_of_node(node, child) { 357 err = load_one_timing_from_dt(emc, timing++, child); 358 if (err) { 359 of_node_put(child); 360 return err; 361 } 362 } 363 364 sort(emc->timings, emc->num_timings, sizeof(*timing), cmp_timings, 365 NULL); 366 367 dev_info(emc->dev, 368 "got %u timings for RAM code %u (min %luMHz max %luMHz)\n", 369 emc->num_timings, 370 tegra_read_ram_code(), 371 emc->timings[0].rate / 1000000, 372 emc->timings[emc->num_timings - 1].rate / 1000000); 373 374 return 0; 375 } 376 377 static struct device_node * 378 tegra_emc_find_node_by_ram_code(struct device *dev) 379 { 380 struct device_node *np; 381 u32 value, ram_code; 382 int err; 383 384 if (!of_property_read_bool(dev->of_node, "nvidia,use-ram-code")) 385 return of_node_get(dev->of_node); 386 387 ram_code = tegra_read_ram_code(); 388 389 for (np = of_find_node_by_name(dev->of_node, "emc-tables"); np; 390 np = of_find_node_by_name(np, "emc-tables")) { 391 err = of_property_read_u32(np, "nvidia,ram-code", &value); 392 if (err || value != ram_code) { 393 of_node_put(np); 394 continue; 395 } 396 397 return np; 398 } 399 400 dev_err(dev, "no memory timings for RAM code %u found in device tree\n", 401 ram_code); 402 403 return NULL; 404 } 405 406 static int emc_setup_hw(struct tegra_emc *emc) 407 { 408 u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT; 409 u32 emc_cfg, emc_dbg; 410 411 emc_cfg = readl_relaxed(emc->regs + EMC_CFG_2); 412 413 /* 414 * Depending on a memory type, DRAM should enter either self-refresh 415 * or power-down state on EMC clock change. 416 */ 417 if (!(emc_cfg & EMC_CLKCHANGE_PD_ENABLE) && 418 !(emc_cfg & EMC_CLKCHANGE_SR_ENABLE)) { 419 dev_err(emc->dev, 420 "bootloader didn't specify DRAM auto-suspend mode\n"); 421 return -EINVAL; 422 } 423 424 /* enable EMC and CAR to handshake on PLL divider/source changes */ 425 emc_cfg |= EMC_CLKCHANGE_REQ_ENABLE; 426 writel_relaxed(emc_cfg, emc->regs + EMC_CFG_2); 427 428 /* initialize interrupt */ 429 writel_relaxed(intmask, emc->regs + EMC_INTMASK); 430 writel_relaxed(intmask, emc->regs + EMC_INTSTATUS); 431 432 /* ensure that unwanted debug features are disabled */ 433 emc_dbg = readl_relaxed(emc->regs + EMC_DBG); 434 emc_dbg |= EMC_DBG_CFG_PRIORITY; 435 emc_dbg &= ~EMC_DBG_READ_MUX_ASSEMBLY; 436 emc_dbg &= ~EMC_DBG_WRITE_MUX_ACTIVE; 437 emc_dbg &= ~EMC_DBG_FORCE_UPDATE; 438 writel_relaxed(emc_dbg, emc->regs + EMC_DBG); 439 440 return 0; 441 } 442 443 static long emc_round_rate(unsigned long rate, 444 unsigned long min_rate, 445 unsigned long max_rate, 446 void *arg) 447 { 448 struct emc_timing *timing = NULL; 449 struct tegra_emc *emc = arg; 450 unsigned int i; 451 452 min_rate = min(min_rate, emc->timings[emc->num_timings - 1].rate); 453 454 for (i = 0; i < emc->num_timings; i++) { 455 if (emc->timings[i].rate < rate && i != emc->num_timings - 1) 456 continue; 457 458 if (emc->timings[i].rate > max_rate) { 459 i = max(i, 1u) - 1; 460 461 if (emc->timings[i].rate < min_rate) 462 break; 463 } 464 465 if (emc->timings[i].rate < min_rate) 466 continue; 467 468 timing = &emc->timings[i]; 469 break; 470 } 471 472 if (!timing) { 473 dev_err(emc->dev, "no timing for rate %lu min %lu max %lu\n", 474 rate, min_rate, max_rate); 475 return -EINVAL; 476 } 477 478 return timing->rate; 479 } 480 481 static int tegra_emc_probe(struct platform_device *pdev) 482 { 483 struct device_node *np; 484 struct tegra_emc *emc; 485 struct resource *res; 486 int irq, err; 487 488 /* driver has nothing to do in a case of memory timing absence */ 489 if (of_get_child_count(pdev->dev.of_node) == 0) { 490 dev_info(&pdev->dev, 491 "EMC device tree node doesn't have memory timings\n"); 492 return 0; 493 } 494 495 irq = platform_get_irq(pdev, 0); 496 if (irq < 0) { 497 dev_err(&pdev->dev, "interrupt not specified\n"); 498 dev_err(&pdev->dev, "please update your device tree\n"); 499 return irq; 500 } 501 502 np = tegra_emc_find_node_by_ram_code(&pdev->dev); 503 if (!np) 504 return -EINVAL; 505 506 emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL); 507 if (!emc) { 508 of_node_put(np); 509 return -ENOMEM; 510 } 511 512 init_completion(&emc->clk_handshake_complete); 513 emc->clk_nb.notifier_call = tegra_emc_clk_change_notify; 514 emc->dev = &pdev->dev; 515 516 err = tegra_emc_load_timings_from_dt(emc, np); 517 of_node_put(np); 518 if (err) 519 return err; 520 521 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 522 emc->regs = devm_ioremap_resource(&pdev->dev, res); 523 if (IS_ERR(emc->regs)) 524 return PTR_ERR(emc->regs); 525 526 err = emc_setup_hw(emc); 527 if (err) 528 return err; 529 530 err = devm_request_irq(&pdev->dev, irq, tegra_emc_isr, 0, 531 dev_name(&pdev->dev), emc); 532 if (err) { 533 dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", irq, err); 534 return err; 535 } 536 537 tegra20_clk_set_emc_round_callback(emc_round_rate, emc); 538 539 emc->clk = devm_clk_get(&pdev->dev, "emc"); 540 if (IS_ERR(emc->clk)) { 541 err = PTR_ERR(emc->clk); 542 dev_err(&pdev->dev, "failed to get emc clock: %d\n", err); 543 goto unset_cb; 544 } 545 546 err = clk_notifier_register(emc->clk, &emc->clk_nb); 547 if (err) { 548 dev_err(&pdev->dev, "failed to register clk notifier: %d\n", 549 err); 550 goto unset_cb; 551 } 552 553 return 0; 554 555 unset_cb: 556 tegra20_clk_set_emc_round_callback(NULL, NULL); 557 558 return err; 559 } 560 561 static const struct of_device_id tegra_emc_of_match[] = { 562 { .compatible = "nvidia,tegra20-emc", }, 563 {}, 564 }; 565 566 static struct platform_driver tegra_emc_driver = { 567 .probe = tegra_emc_probe, 568 .driver = { 569 .name = "tegra20-emc", 570 .of_match_table = tegra_emc_of_match, 571 .suppress_bind_attrs = true, 572 }, 573 }; 574 575 static int __init tegra_emc_init(void) 576 { 577 return platform_driver_register(&tegra_emc_driver); 578 } 579 subsys_initcall(tegra_emc_init); 580