1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Tegra20 External Memory Controller driver 4 * 5 * Author: Dmitry Osipenko <digetx@gmail.com> 6 */ 7 8 #include <linux/clk.h> 9 #include <linux/completion.h> 10 #include <linux/err.h> 11 #include <linux/interrupt.h> 12 #include <linux/iopoll.h> 13 #include <linux/kernel.h> 14 #include <linux/module.h> 15 #include <linux/of.h> 16 #include <linux/platform_device.h> 17 #include <linux/sort.h> 18 #include <linux/types.h> 19 20 #include <soc/tegra/fuse.h> 21 22 #define EMC_INTSTATUS 0x000 23 #define EMC_INTMASK 0x004 24 #define EMC_TIMING_CONTROL 0x028 25 #define EMC_RC 0x02c 26 #define EMC_RFC 0x030 27 #define EMC_RAS 0x034 28 #define EMC_RP 0x038 29 #define EMC_R2W 0x03c 30 #define EMC_W2R 0x040 31 #define EMC_R2P 0x044 32 #define EMC_W2P 0x048 33 #define EMC_RD_RCD 0x04c 34 #define EMC_WR_RCD 0x050 35 #define EMC_RRD 0x054 36 #define EMC_REXT 0x058 37 #define EMC_WDV 0x05c 38 #define EMC_QUSE 0x060 39 #define EMC_QRST 0x064 40 #define EMC_QSAFE 0x068 41 #define EMC_RDV 0x06c 42 #define EMC_REFRESH 0x070 43 #define EMC_BURST_REFRESH_NUM 0x074 44 #define EMC_PDEX2WR 0x078 45 #define EMC_PDEX2RD 0x07c 46 #define EMC_PCHG2PDEN 0x080 47 #define EMC_ACT2PDEN 0x084 48 #define EMC_AR2PDEN 0x088 49 #define EMC_RW2PDEN 0x08c 50 #define EMC_TXSR 0x090 51 #define EMC_TCKE 0x094 52 #define EMC_TFAW 0x098 53 #define EMC_TRPAB 0x09c 54 #define EMC_TCLKSTABLE 0x0a0 55 #define EMC_TCLKSTOP 0x0a4 56 #define EMC_TREFBW 0x0a8 57 #define EMC_QUSE_EXTRA 0x0ac 58 #define EMC_ODT_WRITE 0x0b0 59 #define EMC_ODT_READ 0x0b4 60 #define EMC_FBIO_CFG5 0x104 61 #define EMC_FBIO_CFG6 0x114 62 #define EMC_AUTO_CAL_INTERVAL 0x2a8 63 #define EMC_CFG_2 0x2b8 64 #define EMC_CFG_DIG_DLL 0x2bc 65 #define EMC_DLL_XFORM_DQS 0x2c0 66 #define EMC_DLL_XFORM_QUSE 0x2c4 67 #define EMC_ZCAL_REF_CNT 0x2e0 68 #define EMC_ZCAL_WAIT_CNT 0x2e4 69 #define EMC_CFG_CLKTRIM_0 0x2d0 70 #define EMC_CFG_CLKTRIM_1 0x2d4 71 #define EMC_CFG_CLKTRIM_2 0x2d8 72 73 #define EMC_CLKCHANGE_REQ_ENABLE BIT(0) 74 #define EMC_CLKCHANGE_PD_ENABLE BIT(1) 75 #define EMC_CLKCHANGE_SR_ENABLE BIT(2) 76 77 #define EMC_TIMING_UPDATE BIT(0) 78 79 #define EMC_REFRESH_OVERFLOW_INT BIT(3) 80 #define EMC_CLKCHANGE_COMPLETE_INT BIT(4) 81 82 static const u16 emc_timing_registers[] = { 83 EMC_RC, 84 EMC_RFC, 85 EMC_RAS, 86 EMC_RP, 87 EMC_R2W, 88 EMC_W2R, 89 EMC_R2P, 90 EMC_W2P, 91 EMC_RD_RCD, 92 EMC_WR_RCD, 93 EMC_RRD, 94 EMC_REXT, 95 EMC_WDV, 96 EMC_QUSE, 97 EMC_QRST, 98 EMC_QSAFE, 99 EMC_RDV, 100 EMC_REFRESH, 101 EMC_BURST_REFRESH_NUM, 102 EMC_PDEX2WR, 103 EMC_PDEX2RD, 104 EMC_PCHG2PDEN, 105 EMC_ACT2PDEN, 106 EMC_AR2PDEN, 107 EMC_RW2PDEN, 108 EMC_TXSR, 109 EMC_TCKE, 110 EMC_TFAW, 111 EMC_TRPAB, 112 EMC_TCLKSTABLE, 113 EMC_TCLKSTOP, 114 EMC_TREFBW, 115 EMC_QUSE_EXTRA, 116 EMC_FBIO_CFG6, 117 EMC_ODT_WRITE, 118 EMC_ODT_READ, 119 EMC_FBIO_CFG5, 120 EMC_CFG_DIG_DLL, 121 EMC_DLL_XFORM_DQS, 122 EMC_DLL_XFORM_QUSE, 123 EMC_ZCAL_REF_CNT, 124 EMC_ZCAL_WAIT_CNT, 125 EMC_AUTO_CAL_INTERVAL, 126 EMC_CFG_CLKTRIM_0, 127 EMC_CFG_CLKTRIM_1, 128 EMC_CFG_CLKTRIM_2, 129 }; 130 131 struct emc_timing { 132 unsigned long rate; 133 u32 data[ARRAY_SIZE(emc_timing_registers)]; 134 }; 135 136 struct tegra_emc { 137 struct device *dev; 138 struct completion clk_handshake_complete; 139 struct notifier_block clk_nb; 140 struct clk *backup_clk; 141 struct clk *emc_mux; 142 struct clk *pll_m; 143 struct clk *clk; 144 void __iomem *regs; 145 146 struct emc_timing *timings; 147 unsigned int num_timings; 148 }; 149 150 static irqreturn_t tegra_emc_isr(int irq, void *data) 151 { 152 struct tegra_emc *emc = data; 153 u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT; 154 u32 status; 155 156 status = readl_relaxed(emc->regs + EMC_INTSTATUS) & intmask; 157 if (!status) 158 return IRQ_NONE; 159 160 /* notify about EMC-CAR handshake completion */ 161 if (status & EMC_CLKCHANGE_COMPLETE_INT) 162 complete(&emc->clk_handshake_complete); 163 164 /* notify about HW problem */ 165 if (status & EMC_REFRESH_OVERFLOW_INT) 166 dev_err_ratelimited(emc->dev, 167 "refresh request overflow timeout\n"); 168 169 /* clear interrupts */ 170 writel_relaxed(status, emc->regs + EMC_INTSTATUS); 171 172 return IRQ_HANDLED; 173 } 174 175 static struct emc_timing *tegra_emc_find_timing(struct tegra_emc *emc, 176 unsigned long rate) 177 { 178 struct emc_timing *timing = NULL; 179 unsigned int i; 180 181 for (i = 0; i < emc->num_timings; i++) { 182 if (emc->timings[i].rate >= rate) { 183 timing = &emc->timings[i]; 184 break; 185 } 186 } 187 188 if (!timing) { 189 dev_err(emc->dev, "no timing for rate %lu\n", rate); 190 return NULL; 191 } 192 193 return timing; 194 } 195 196 static int emc_prepare_timing_change(struct tegra_emc *emc, unsigned long rate) 197 { 198 struct emc_timing *timing = tegra_emc_find_timing(emc, rate); 199 unsigned int i; 200 201 if (!timing) 202 return -EINVAL; 203 204 dev_dbg(emc->dev, "%s: using timing rate %lu for requested rate %lu\n", 205 __func__, timing->rate, rate); 206 207 /* program shadow registers */ 208 for (i = 0; i < ARRAY_SIZE(timing->data); i++) 209 writel_relaxed(timing->data[i], 210 emc->regs + emc_timing_registers[i]); 211 212 /* wait until programming has settled */ 213 readl_relaxed(emc->regs + emc_timing_registers[i - 1]); 214 215 reinit_completion(&emc->clk_handshake_complete); 216 217 return 0; 218 } 219 220 static int emc_complete_timing_change(struct tegra_emc *emc, bool flush) 221 { 222 long timeout; 223 224 dev_dbg(emc->dev, "%s: flush %d\n", __func__, flush); 225 226 if (flush) { 227 /* manually initiate memory timing update */ 228 writel_relaxed(EMC_TIMING_UPDATE, 229 emc->regs + EMC_TIMING_CONTROL); 230 return 0; 231 } 232 233 timeout = wait_for_completion_timeout(&emc->clk_handshake_complete, 234 usecs_to_jiffies(100)); 235 if (timeout == 0) { 236 dev_err(emc->dev, "EMC-CAR handshake failed\n"); 237 return -EIO; 238 } else if (timeout < 0) { 239 dev_err(emc->dev, "failed to wait for EMC-CAR handshake: %ld\n", 240 timeout); 241 return timeout; 242 } 243 244 return 0; 245 } 246 247 static int tegra_emc_clk_change_notify(struct notifier_block *nb, 248 unsigned long msg, void *data) 249 { 250 struct tegra_emc *emc = container_of(nb, struct tegra_emc, clk_nb); 251 struct clk_notifier_data *cnd = data; 252 int err; 253 254 switch (msg) { 255 case PRE_RATE_CHANGE: 256 err = emc_prepare_timing_change(emc, cnd->new_rate); 257 break; 258 259 case ABORT_RATE_CHANGE: 260 err = emc_prepare_timing_change(emc, cnd->old_rate); 261 if (err) 262 break; 263 264 err = emc_complete_timing_change(emc, true); 265 break; 266 267 case POST_RATE_CHANGE: 268 err = emc_complete_timing_change(emc, false); 269 break; 270 271 default: 272 return NOTIFY_DONE; 273 } 274 275 return notifier_from_errno(err); 276 } 277 278 static int load_one_timing_from_dt(struct tegra_emc *emc, 279 struct emc_timing *timing, 280 struct device_node *node) 281 { 282 u32 rate; 283 int err; 284 285 if (!of_device_is_compatible(node, "nvidia,tegra20-emc-table")) { 286 dev_err(emc->dev, "incompatible DT node: %pOF\n", node); 287 return -EINVAL; 288 } 289 290 err = of_property_read_u32(node, "clock-frequency", &rate); 291 if (err) { 292 dev_err(emc->dev, "timing %pOF: failed to read rate: %d\n", 293 node, err); 294 return err; 295 } 296 297 err = of_property_read_u32_array(node, "nvidia,emc-registers", 298 timing->data, 299 ARRAY_SIZE(emc_timing_registers)); 300 if (err) { 301 dev_err(emc->dev, 302 "timing %pOF: failed to read emc timing data: %d\n", 303 node, err); 304 return err; 305 } 306 307 /* 308 * The EMC clock rate is twice the bus rate, and the bus rate is 309 * measured in kHz. 310 */ 311 timing->rate = rate * 2 * 1000; 312 313 dev_dbg(emc->dev, "%s: %pOF: EMC rate %lu\n", 314 __func__, node, timing->rate); 315 316 return 0; 317 } 318 319 static int cmp_timings(const void *_a, const void *_b) 320 { 321 const struct emc_timing *a = _a; 322 const struct emc_timing *b = _b; 323 324 if (a->rate < b->rate) 325 return -1; 326 327 if (a->rate > b->rate) 328 return 1; 329 330 return 0; 331 } 332 333 static int tegra_emc_load_timings_from_dt(struct tegra_emc *emc, 334 struct device_node *node) 335 { 336 struct device_node *child; 337 struct emc_timing *timing; 338 int child_count; 339 int err; 340 341 child_count = of_get_child_count(node); 342 if (!child_count) { 343 dev_err(emc->dev, "no memory timings in DT node: %pOF\n", node); 344 return -EINVAL; 345 } 346 347 emc->timings = devm_kcalloc(emc->dev, child_count, sizeof(*timing), 348 GFP_KERNEL); 349 if (!emc->timings) 350 return -ENOMEM; 351 352 emc->num_timings = child_count; 353 timing = emc->timings; 354 355 for_each_child_of_node(node, child) { 356 err = load_one_timing_from_dt(emc, timing++, child); 357 if (err) { 358 of_node_put(child); 359 return err; 360 } 361 } 362 363 sort(emc->timings, emc->num_timings, sizeof(*timing), cmp_timings, 364 NULL); 365 366 return 0; 367 } 368 369 static struct device_node * 370 tegra_emc_find_node_by_ram_code(struct device *dev) 371 { 372 struct device_node *np; 373 u32 value, ram_code; 374 int err; 375 376 if (!of_property_read_bool(dev->of_node, "nvidia,use-ram-code")) 377 return of_node_get(dev->of_node); 378 379 ram_code = tegra_read_ram_code(); 380 381 for (np = of_find_node_by_name(dev->of_node, "emc-tables"); np; 382 np = of_find_node_by_name(np, "emc-tables")) { 383 err = of_property_read_u32(np, "nvidia,ram-code", &value); 384 if (err || value != ram_code) { 385 of_node_put(np); 386 continue; 387 } 388 389 return np; 390 } 391 392 dev_err(dev, "no memory timings for RAM code %u found in device tree\n", 393 ram_code); 394 395 return NULL; 396 } 397 398 static int emc_setup_hw(struct tegra_emc *emc) 399 { 400 u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT; 401 u32 emc_cfg; 402 403 emc_cfg = readl_relaxed(emc->regs + EMC_CFG_2); 404 405 /* 406 * Depending on a memory type, DRAM should enter either self-refresh 407 * or power-down state on EMC clock change. 408 */ 409 if (!(emc_cfg & EMC_CLKCHANGE_PD_ENABLE) && 410 !(emc_cfg & EMC_CLKCHANGE_SR_ENABLE)) { 411 dev_err(emc->dev, 412 "bootloader didn't specify DRAM auto-suspend mode\n"); 413 return -EINVAL; 414 } 415 416 /* enable EMC and CAR to handshake on PLL divider/source changes */ 417 emc_cfg |= EMC_CLKCHANGE_REQ_ENABLE; 418 writel_relaxed(emc_cfg, emc->regs + EMC_CFG_2); 419 420 /* initialize interrupt */ 421 writel_relaxed(intmask, emc->regs + EMC_INTMASK); 422 writel_relaxed(intmask, emc->regs + EMC_INTSTATUS); 423 424 return 0; 425 } 426 427 static int emc_init(struct tegra_emc *emc, unsigned long rate) 428 { 429 int err; 430 431 err = clk_set_parent(emc->emc_mux, emc->backup_clk); 432 if (err) { 433 dev_err(emc->dev, 434 "failed to reparent to backup source: %d\n", err); 435 return err; 436 } 437 438 err = clk_set_rate(emc->pll_m, rate); 439 if (err) { 440 dev_err(emc->dev, 441 "failed to change pll_m rate: %d\n", err); 442 return err; 443 } 444 445 err = clk_set_parent(emc->emc_mux, emc->pll_m); 446 if (err) { 447 dev_err(emc->dev, 448 "failed to reparent to pll_m: %d\n", err); 449 return err; 450 } 451 452 err = clk_set_rate(emc->clk, rate); 453 if (err) { 454 dev_err(emc->dev, 455 "failed to change emc rate: %d\n", err); 456 return err; 457 } 458 459 return 0; 460 } 461 462 static int tegra_emc_probe(struct platform_device *pdev) 463 { 464 struct device_node *np; 465 struct tegra_emc *emc; 466 struct resource *res; 467 int irq, err; 468 469 /* driver has nothing to do in a case of memory timing absence */ 470 if (of_get_child_count(pdev->dev.of_node) == 0) { 471 dev_info(&pdev->dev, 472 "EMC device tree node doesn't have memory timings\n"); 473 return 0; 474 } 475 476 irq = platform_get_irq(pdev, 0); 477 if (irq < 0) { 478 dev_err(&pdev->dev, "interrupt not specified\n"); 479 dev_err(&pdev->dev, "please update your device tree\n"); 480 return irq; 481 } 482 483 np = tegra_emc_find_node_by_ram_code(&pdev->dev); 484 if (!np) 485 return -EINVAL; 486 487 emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL); 488 if (!emc) { 489 of_node_put(np); 490 return -ENOMEM; 491 } 492 493 init_completion(&emc->clk_handshake_complete); 494 emc->clk_nb.notifier_call = tegra_emc_clk_change_notify; 495 emc->dev = &pdev->dev; 496 497 err = tegra_emc_load_timings_from_dt(emc, np); 498 of_node_put(np); 499 if (err) 500 return err; 501 502 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 503 emc->regs = devm_ioremap_resource(&pdev->dev, res); 504 if (IS_ERR(emc->regs)) 505 return PTR_ERR(emc->regs); 506 507 err = emc_setup_hw(emc); 508 if (err) 509 return err; 510 511 err = devm_request_irq(&pdev->dev, irq, tegra_emc_isr, 0, 512 dev_name(&pdev->dev), emc); 513 if (err) { 514 dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", irq, err); 515 return err; 516 } 517 518 emc->clk = devm_clk_get(&pdev->dev, "emc"); 519 if (IS_ERR(emc->clk)) { 520 err = PTR_ERR(emc->clk); 521 dev_err(&pdev->dev, "failed to get emc clock: %d\n", err); 522 return err; 523 } 524 525 emc->pll_m = clk_get_sys(NULL, "pll_m"); 526 if (IS_ERR(emc->pll_m)) { 527 err = PTR_ERR(emc->pll_m); 528 dev_err(&pdev->dev, "failed to get pll_m clock: %d\n", err); 529 return err; 530 } 531 532 emc->backup_clk = clk_get_sys(NULL, "pll_p"); 533 if (IS_ERR(emc->backup_clk)) { 534 err = PTR_ERR(emc->backup_clk); 535 dev_err(&pdev->dev, "failed to get pll_p clock: %d\n", err); 536 goto put_pll_m; 537 } 538 539 emc->emc_mux = clk_get_parent(emc->clk); 540 if (IS_ERR(emc->emc_mux)) { 541 err = PTR_ERR(emc->emc_mux); 542 dev_err(&pdev->dev, "failed to get emc_mux clock: %d\n", err); 543 goto put_backup; 544 } 545 546 err = clk_notifier_register(emc->clk, &emc->clk_nb); 547 if (err) { 548 dev_err(&pdev->dev, "failed to register clk notifier: %d\n", 549 err); 550 goto put_backup; 551 } 552 553 /* set DRAM clock rate to maximum */ 554 err = emc_init(emc, emc->timings[emc->num_timings - 1].rate); 555 if (err) { 556 dev_err(&pdev->dev, "failed to initialize EMC clock rate: %d\n", 557 err); 558 goto unreg_notifier; 559 } 560 561 return 0; 562 563 unreg_notifier: 564 clk_notifier_unregister(emc->clk, &emc->clk_nb); 565 put_backup: 566 clk_put(emc->backup_clk); 567 put_pll_m: 568 clk_put(emc->pll_m); 569 570 return err; 571 } 572 573 static const struct of_device_id tegra_emc_of_match[] = { 574 { .compatible = "nvidia,tegra20-emc", }, 575 {}, 576 }; 577 578 static struct platform_driver tegra_emc_driver = { 579 .probe = tegra_emc_probe, 580 .driver = { 581 .name = "tegra20-emc", 582 .of_match_table = tegra_emc_of_match, 583 .suppress_bind_attrs = true, 584 }, 585 }; 586 587 static int __init tegra_emc_init(void) 588 { 589 return platform_driver_register(&tegra_emc_driver); 590 } 591 subsys_initcall(tegra_emc_init); 592