1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2011 The FreeBSD Foundation 5 * Copyright (c) 2013 Ruslan Bukin <br@bsdpad.com> 6 * All rights reserved. 7 * 8 * Based on mpcore_timer.c developed by Ben Gray <ben.r.gray@gmail.com> 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. The name of the company nor the name of the author may be used to 19 * endorse or promote products derived from this software without specific 20 * prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /** 36 * Cortex-A7, Cortex-A15, ARMv8 and later Generic Timer 37 */ 38 39 #include "opt_acpi.h" 40 #include "opt_platform.h" 41 42 #include <sys/cdefs.h> 43 __FBSDID("$FreeBSD$"); 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/bus.h> 48 #include <sys/kernel.h> 49 #include <sys/module.h> 50 #include <sys/malloc.h> 51 #include <sys/rman.h> 52 #include <sys/timeet.h> 53 #include <sys/timetc.h> 54 #include <sys/smp.h> 55 #include <sys/vdso.h> 56 #include <sys/watchdog.h> 57 58 #include <machine/bus.h> 59 #include <machine/cpu.h> 60 #include <machine/intr.h> 61 #include <machine/machdep.h> 62 #include <machine/md_var.h> 63 64 #if defined(__aarch64__) 65 #include <machine/undefined.h> 66 #endif 67 68 #ifdef FDT 69 #include <dev/ofw/openfirm.h> 70 #include <dev/ofw/ofw_bus.h> 71 #include <dev/ofw/ofw_bus_subr.h> 72 #endif 73 74 #ifdef DEV_ACPI 75 #include <contrib/dev/acpica/include/acpi.h> 76 #include <dev/acpica/acpivar.h> 77 #endif 78 79 #define GT_PHYS_SECURE 0 80 #define GT_PHYS_NONSECURE 1 81 #define GT_VIRT 2 82 #define GT_HYP 3 83 #define GT_IRQ_COUNT 4 84 85 #define GT_CTRL_ENABLE (1 << 0) 86 #define GT_CTRL_INT_MASK (1 << 1) 87 #define GT_CTRL_INT_STAT (1 << 2) 88 #define GT_REG_CTRL 0 89 #define GT_REG_TVAL 1 90 91 #define GT_CNTKCTL_PL0PTEN (1 << 9) /* PL0 Physical timer reg access */ 92 #define GT_CNTKCTL_PL0VTEN (1 << 8) /* PL0 Virtual timer reg access */ 93 #define GT_CNTKCTL_EVNTI (0xf << 4) /* Virtual counter event bits */ 94 #define GT_CNTKCTL_EVNTDIR (1 << 3) /* Virtual counter event transition */ 95 #define GT_CNTKCTL_EVNTEN (1 << 2) /* Enables virtual counter events */ 96 #define GT_CNTKCTL_PL0VCTEN (1 << 1) /* PL0 CNTVCT and CNTFRQ access */ 97 #define GT_CNTKCTL_PL0PCTEN (1 << 0) /* PL0 CNTPCT and CNTFRQ access */ 98 99 struct arm_tmr_softc { 100 struct resource *res[GT_IRQ_COUNT]; 101 void *ihl[GT_IRQ_COUNT]; 102 uint64_t (*get_cntxct)(bool); 103 uint32_t clkfreq; 104 struct eventtimer et; 105 bool physical; 106 }; 107 108 static struct arm_tmr_softc *arm_tmr_sc = NULL; 109 110 static struct resource_spec timer_spec[] = { 111 { SYS_RES_IRQ, GT_PHYS_SECURE, RF_ACTIVE }, 112 { SYS_RES_IRQ, GT_PHYS_NONSECURE, RF_ACTIVE }, 113 { SYS_RES_IRQ, GT_VIRT, RF_ACTIVE | RF_OPTIONAL }, 114 { SYS_RES_IRQ, GT_HYP, RF_ACTIVE | RF_OPTIONAL }, 115 { -1, 0 } 116 }; 117 118 static uint32_t arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th, 119 struct timecounter *tc); 120 static void arm_tmr_do_delay(int usec, void *); 121 122 static timecounter_get_t arm_tmr_get_timecount; 123 124 static struct timecounter arm_tmr_timecount = { 125 .tc_name = "ARM MPCore Timecounter", 126 .tc_get_timecount = arm_tmr_get_timecount, 127 .tc_poll_pps = NULL, 128 .tc_counter_mask = ~0u, 129 .tc_frequency = 0, 130 .tc_quality = 1000, 131 .tc_fill_vdso_timehands = arm_tmr_fill_vdso_timehands, 132 }; 133 134 #ifdef __arm__ 135 #define get_el0(x) cp15_## x ##_get() 136 #define get_el1(x) cp15_## x ##_get() 137 #define set_el0(x, val) cp15_## x ##_set(val) 138 #define set_el1(x, val) cp15_## x ##_set(val) 139 #define HAS_PHYS true 140 #else /* __aarch64__ */ 141 #define get_el0(x) READ_SPECIALREG(x ##_el0) 142 #define get_el1(x) READ_SPECIALREG(x ##_el1) 143 #define set_el0(x, val) WRITE_SPECIALREG(x ##_el0, val) 144 #define set_el1(x, val) WRITE_SPECIALREG(x ##_el1, val) 145 #define HAS_PHYS has_hyp() 146 #endif 147 148 static int 149 get_freq(void) 150 { 151 return (get_el0(cntfrq)); 152 } 153 154 static uint64_t 155 get_cntxct_a64_unstable(bool physical) 156 { 157 uint64_t val 158 ; 159 isb(); 160 if (physical) { 161 do { 162 val = get_el0(cntpct); 163 } 164 while (((val + 1) & 0x7FF) <= 1); 165 } 166 else { 167 do { 168 val = get_el0(cntvct); 169 } 170 while (((val + 1) & 0x7FF) <= 1); 171 } 172 173 return (val); 174 } 175 176 static uint64_t 177 get_cntxct(bool physical) 178 { 179 uint64_t val; 180 181 isb(); 182 if (physical) 183 val = get_el0(cntpct); 184 else 185 val = get_el0(cntvct); 186 187 return (val); 188 } 189 190 static int 191 set_ctrl(uint32_t val, bool physical) 192 { 193 194 if (physical) 195 set_el0(cntp_ctl, val); 196 else 197 set_el0(cntv_ctl, val); 198 isb(); 199 200 return (0); 201 } 202 203 static int 204 set_tval(uint32_t val, bool physical) 205 { 206 207 if (physical) 208 set_el0(cntp_tval, val); 209 else 210 set_el0(cntv_tval, val); 211 isb(); 212 213 return (0); 214 } 215 216 static int 217 get_ctrl(bool physical) 218 { 219 uint32_t val; 220 221 if (physical) 222 val = get_el0(cntp_ctl); 223 else 224 val = get_el0(cntv_ctl); 225 226 return (val); 227 } 228 229 static void 230 setup_user_access(void *arg __unused) 231 { 232 uint32_t cntkctl; 233 234 cntkctl = get_el1(cntkctl); 235 cntkctl &= ~(GT_CNTKCTL_PL0PTEN | GT_CNTKCTL_PL0VTEN | 236 GT_CNTKCTL_EVNTEN); 237 if (arm_tmr_sc->physical) { 238 cntkctl |= GT_CNTKCTL_PL0PCTEN; 239 cntkctl &= ~GT_CNTKCTL_PL0VCTEN; 240 } else { 241 cntkctl |= GT_CNTKCTL_PL0VCTEN; 242 cntkctl &= ~GT_CNTKCTL_PL0PCTEN; 243 } 244 set_el1(cntkctl, cntkctl); 245 isb(); 246 } 247 248 #ifdef __aarch64__ 249 static int 250 cntpct_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame, 251 uint32_t esr) 252 { 253 uint64_t val; 254 int reg; 255 256 if ((insn & MRS_MASK) != MRS_VALUE) 257 return (0); 258 259 if (MRS_SPECIAL(insn) != MRS_SPECIAL(CNTPCT_EL0)) 260 return (0); 261 262 reg = MRS_REGISTER(insn); 263 val = READ_SPECIALREG(cntvct_el0); 264 if (reg < nitems(frame->tf_x)) { 265 frame->tf_x[reg] = val; 266 } else if (reg == 30) { 267 frame->tf_lr = val; 268 } 269 270 /* 271 * We will handle this instruction, move to the next so we 272 * don't trap here again. 273 */ 274 frame->tf_elr += INSN_SIZE; 275 276 return (1); 277 } 278 #endif 279 280 static void 281 tmr_setup_user_access(void *arg __unused) 282 { 283 #ifdef __aarch64__ 284 int emulate; 285 #endif 286 287 if (arm_tmr_sc != NULL) { 288 smp_rendezvous(NULL, setup_user_access, NULL, NULL); 289 #ifdef __aarch64__ 290 if (TUNABLE_INT_FETCH("hw.emulate_phys_counter", &emulate) && 291 emulate != 0) { 292 install_undef_handler(true, cntpct_handler); 293 } 294 #endif 295 } 296 } 297 SYSINIT(tmr_ua, SI_SUB_SMP, SI_ORDER_ANY, tmr_setup_user_access, NULL); 298 299 static unsigned 300 arm_tmr_get_timecount(struct timecounter *tc) 301 { 302 303 return (arm_tmr_sc->get_cntxct(arm_tmr_sc->physical)); 304 } 305 306 static int 307 arm_tmr_start(struct eventtimer *et, sbintime_t first, 308 sbintime_t period __unused) 309 { 310 struct arm_tmr_softc *sc; 311 int counts, ctrl; 312 313 sc = (struct arm_tmr_softc *)et->et_priv; 314 315 if (first != 0) { 316 counts = ((uint32_t)et->et_frequency * first) >> 32; 317 ctrl = get_ctrl(sc->physical); 318 ctrl &= ~GT_CTRL_INT_MASK; 319 ctrl |= GT_CTRL_ENABLE; 320 set_tval(counts, sc->physical); 321 set_ctrl(ctrl, sc->physical); 322 return (0); 323 } 324 325 return (EINVAL); 326 327 } 328 329 static void 330 arm_tmr_disable(bool physical) 331 { 332 int ctrl; 333 334 ctrl = get_ctrl(physical); 335 ctrl &= ~GT_CTRL_ENABLE; 336 set_ctrl(ctrl, physical); 337 } 338 339 static int 340 arm_tmr_stop(struct eventtimer *et) 341 { 342 struct arm_tmr_softc *sc; 343 344 sc = (struct arm_tmr_softc *)et->et_priv; 345 arm_tmr_disable(sc->physical); 346 347 return (0); 348 } 349 350 static int 351 arm_tmr_intr(void *arg) 352 { 353 struct arm_tmr_softc *sc; 354 int ctrl; 355 356 sc = (struct arm_tmr_softc *)arg; 357 ctrl = get_ctrl(sc->physical); 358 if (ctrl & GT_CTRL_INT_STAT) { 359 ctrl |= GT_CTRL_INT_MASK; 360 set_ctrl(ctrl, sc->physical); 361 } 362 363 if (sc->et.et_active) 364 sc->et.et_event_cb(&sc->et, sc->et.et_arg); 365 366 return (FILTER_HANDLED); 367 } 368 369 #ifdef FDT 370 static int 371 arm_tmr_fdt_probe(device_t dev) 372 { 373 374 if (!ofw_bus_status_okay(dev)) 375 return (ENXIO); 376 377 if (ofw_bus_is_compatible(dev, "arm,armv8-timer")) { 378 device_set_desc(dev, "ARMv8 Generic Timer"); 379 return (BUS_PROBE_DEFAULT); 380 } else if (ofw_bus_is_compatible(dev, "arm,armv7-timer")) { 381 device_set_desc(dev, "ARMv7 Generic Timer"); 382 return (BUS_PROBE_DEFAULT); 383 } 384 385 return (ENXIO); 386 } 387 #endif 388 389 #ifdef DEV_ACPI 390 static void 391 arm_tmr_acpi_add_irq(device_t parent, device_t dev, int rid, u_int irq) 392 { 393 394 BUS_SET_RESOURCE(parent, dev, SYS_RES_IRQ, rid, irq, 1); 395 } 396 397 static void 398 arm_tmr_acpi_identify(driver_t *driver, device_t parent) 399 { 400 ACPI_TABLE_GTDT *gtdt; 401 vm_paddr_t physaddr; 402 device_t dev; 403 404 physaddr = acpi_find_table(ACPI_SIG_GTDT); 405 if (physaddr == 0) 406 return; 407 408 gtdt = acpi_map_table(physaddr, ACPI_SIG_GTDT); 409 if (gtdt == NULL) { 410 device_printf(parent, "gic: Unable to map the GTDT\n"); 411 return; 412 } 413 414 dev = BUS_ADD_CHILD(parent, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE, 415 "generic_timer", -1); 416 if (dev == NULL) { 417 device_printf(parent, "add gic child failed\n"); 418 goto out; 419 } 420 421 arm_tmr_acpi_add_irq(parent, dev, GT_PHYS_SECURE, 422 gtdt->SecureEl1Interrupt); 423 arm_tmr_acpi_add_irq(parent, dev, GT_PHYS_NONSECURE, 424 gtdt->NonSecureEl1Interrupt); 425 arm_tmr_acpi_add_irq(parent, dev, GT_VIRT, 426 gtdt->VirtualTimerInterrupt); 427 428 out: 429 acpi_unmap_table(gtdt); 430 } 431 432 static int 433 arm_tmr_acpi_probe(device_t dev) 434 { 435 436 device_set_desc(dev, "ARM Generic Timer"); 437 return (BUS_PROBE_NOWILDCARD); 438 } 439 #endif 440 441 static int 442 arm_tmr_attach(device_t dev) 443 { 444 struct arm_tmr_softc *sc; 445 #ifdef FDT 446 phandle_t node; 447 pcell_t clock; 448 #endif 449 int error; 450 int i, first_timer, last_timer; 451 452 sc = device_get_softc(dev); 453 if (arm_tmr_sc) 454 return (ENXIO); 455 456 sc->get_cntxct = &get_cntxct; 457 #ifdef FDT 458 /* Get the base clock frequency */ 459 node = ofw_bus_get_node(dev); 460 if (node > 0) { 461 error = OF_getencprop(node, "clock-frequency", &clock, 462 sizeof(clock)); 463 if (error > 0) 464 sc->clkfreq = clock; 465 466 if (OF_hasprop(node, "allwinner,sun50i-a64-unstable-timer")) { 467 sc->get_cntxct = &get_cntxct_a64_unstable; 468 if (bootverbose) 469 device_printf(dev, 470 "Enabling allwinner unstable timer workaround\n"); 471 } 472 } 473 #endif 474 475 if (sc->clkfreq == 0) { 476 /* Try to get clock frequency from timer */ 477 sc->clkfreq = get_freq(); 478 } 479 480 if (sc->clkfreq == 0) { 481 device_printf(dev, "No clock frequency specified\n"); 482 return (ENXIO); 483 } 484 485 if (bus_alloc_resources(dev, timer_spec, sc->res)) { 486 device_printf(dev, "could not allocate resources\n"); 487 return (ENXIO); 488 } 489 490 #ifdef __aarch64__ 491 /* Use the virtual timer if we have one. */ 492 if (sc->res[GT_VIRT] != NULL) { 493 sc->physical = false; 494 first_timer = GT_VIRT; 495 last_timer = GT_VIRT; 496 } else 497 #endif 498 /* Otherwise set up the secure and non-secure physical timers. */ 499 { 500 sc->physical = true; 501 first_timer = GT_PHYS_SECURE; 502 last_timer = GT_PHYS_NONSECURE; 503 } 504 505 arm_tmr_sc = sc; 506 507 /* Setup secure, non-secure and virtual IRQs handler */ 508 for (i = first_timer; i <= last_timer; i++) { 509 /* If we do not have the interrupt, skip it. */ 510 if (sc->res[i] == NULL) 511 continue; 512 error = bus_setup_intr(dev, sc->res[i], INTR_TYPE_CLK, 513 arm_tmr_intr, NULL, sc, &sc->ihl[i]); 514 if (error) { 515 device_printf(dev, "Unable to alloc int resource.\n"); 516 return (ENXIO); 517 } 518 } 519 520 /* Disable the virtual timer until we are ready */ 521 if (sc->res[GT_VIRT] != NULL) 522 arm_tmr_disable(false); 523 /* And the physical */ 524 if ((sc->res[GT_PHYS_SECURE] != NULL || 525 sc->res[GT_PHYS_NONSECURE] != NULL) && HAS_PHYS) 526 arm_tmr_disable(true); 527 528 arm_tmr_timecount.tc_frequency = sc->clkfreq; 529 tc_init(&arm_tmr_timecount); 530 531 sc->et.et_name = "ARM MPCore Eventtimer"; 532 sc->et.et_flags = ET_FLAGS_ONESHOT | ET_FLAGS_PERCPU; 533 sc->et.et_quality = 1000; 534 535 sc->et.et_frequency = sc->clkfreq; 536 sc->et.et_min_period = (0x00000010LLU << 32) / sc->et.et_frequency; 537 sc->et.et_max_period = (0xfffffffeLLU << 32) / sc->et.et_frequency; 538 sc->et.et_start = arm_tmr_start; 539 sc->et.et_stop = arm_tmr_stop; 540 sc->et.et_priv = sc; 541 et_register(&sc->et); 542 543 #if defined(__arm__) 544 arm_set_delay(arm_tmr_do_delay, sc); 545 #endif 546 547 return (0); 548 } 549 550 #ifdef FDT 551 static device_method_t arm_tmr_fdt_methods[] = { 552 DEVMETHOD(device_probe, arm_tmr_fdt_probe), 553 DEVMETHOD(device_attach, arm_tmr_attach), 554 { 0, 0 } 555 }; 556 557 static DEFINE_CLASS_0(generic_timer, arm_tmr_fdt_driver, arm_tmr_fdt_methods, 558 sizeof(struct arm_tmr_softc)); 559 560 EARLY_DRIVER_MODULE(timer, simplebus, arm_tmr_fdt_driver, 0, 0, 561 BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); 562 EARLY_DRIVER_MODULE(timer, ofwbus, arm_tmr_fdt_driver, 0, 0, 563 BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); 564 #endif 565 566 #ifdef DEV_ACPI 567 static device_method_t arm_tmr_acpi_methods[] = { 568 DEVMETHOD(device_identify, arm_tmr_acpi_identify), 569 DEVMETHOD(device_probe, arm_tmr_acpi_probe), 570 DEVMETHOD(device_attach, arm_tmr_attach), 571 { 0, 0 } 572 }; 573 574 static DEFINE_CLASS_0(generic_timer, arm_tmr_acpi_driver, arm_tmr_acpi_methods, 575 sizeof(struct arm_tmr_softc)); 576 577 EARLY_DRIVER_MODULE(timer, acpi, arm_tmr_acpi_driver, 0, 0, 578 BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); 579 #endif 580 581 static void 582 arm_tmr_do_delay(int usec, void *arg) 583 { 584 struct arm_tmr_softc *sc = arg; 585 int32_t counts, counts_per_usec; 586 uint32_t first, last; 587 588 /* Get the number of times to count */ 589 counts_per_usec = ((arm_tmr_timecount.tc_frequency / 1000000) + 1); 590 591 /* 592 * Clamp the timeout at a maximum value (about 32 seconds with 593 * a 66MHz clock). *Nobody* should be delay()ing for anywhere 594 * near that length of time and if they are, they should be hung 595 * out to dry. 596 */ 597 if (usec >= (0x80000000U / counts_per_usec)) 598 counts = (0x80000000U / counts_per_usec) - 1; 599 else 600 counts = usec * counts_per_usec; 601 602 first = sc->get_cntxct(sc->physical); 603 604 while (counts > 0) { 605 last = sc->get_cntxct(sc->physical); 606 counts -= (int32_t)(last - first); 607 first = last; 608 } 609 } 610 611 #if defined(__aarch64__) 612 void 613 DELAY(int usec) 614 { 615 int32_t counts; 616 617 TSENTER(); 618 /* 619 * Check the timers are setup, if not just 620 * use a for loop for the meantime 621 */ 622 if (arm_tmr_sc == NULL) { 623 for (; usec > 0; usec--) 624 for (counts = 200; counts > 0; counts--) 625 /* 626 * Prevent the compiler from optimizing 627 * out the loop 628 */ 629 cpufunc_nullop(); 630 } else 631 arm_tmr_do_delay(usec, arm_tmr_sc); 632 TSEXIT(); 633 } 634 #endif 635 636 static uint32_t 637 arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th, 638 struct timecounter *tc) 639 { 640 641 vdso_th->th_algo = VDSO_TH_ALGO_ARM_GENTIM; 642 vdso_th->th_physical = arm_tmr_sc->physical; 643 bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); 644 return (1); 645 } 646