1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2011 The FreeBSD Foundation 5 * Copyright (c) 2013 Ruslan Bukin <br@bsdpad.com> 6 * All rights reserved. 7 * 8 * Based on mpcore_timer.c developed by Ben Gray <ben.r.gray@gmail.com> 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. The name of the company nor the name of the author may be used to 19 * endorse or promote products derived from this software without specific 20 * prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /** 36 * Cortex-A7, Cortex-A15, ARMv8 and later Generic Timer 37 */ 38 39 #include "opt_acpi.h" 40 #include "opt_platform.h" 41 42 #include <sys/cdefs.h> 43 __FBSDID("$FreeBSD$"); 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/bus.h> 48 #include <sys/kernel.h> 49 #include <sys/module.h> 50 #include <sys/malloc.h> 51 #include <sys/rman.h> 52 #include <sys/timeet.h> 53 #include <sys/timetc.h> 54 #include <sys/smp.h> 55 #include <sys/vdso.h> 56 #include <sys/watchdog.h> 57 #include <machine/bus.h> 58 #include <machine/cpu.h> 59 #include <machine/intr.h> 60 #include <machine/md_var.h> 61 62 #if defined(__arm__) 63 #include <machine/machdep.h> /* For arm_set_delay */ 64 #endif 65 66 #ifdef FDT 67 #include <dev/ofw/openfirm.h> 68 #include <dev/ofw/ofw_bus.h> 69 #include <dev/ofw/ofw_bus_subr.h> 70 #endif 71 72 #ifdef DEV_ACPI 73 #include <contrib/dev/acpica/include/acpi.h> 74 #include <dev/acpica/acpivar.h> 75 #include "acpi_bus_if.h" 76 #endif 77 78 #define GT_CTRL_ENABLE (1 << 0) 79 #define GT_CTRL_INT_MASK (1 << 1) 80 #define GT_CTRL_INT_STAT (1 << 2) 81 #define GT_REG_CTRL 0 82 #define GT_REG_TVAL 1 83 84 #define GT_CNTKCTL_PL0PTEN (1 << 9) /* PL0 Physical timer reg access */ 85 #define GT_CNTKCTL_PL0VTEN (1 << 8) /* PL0 Virtual timer reg access */ 86 #define GT_CNTKCTL_EVNTI (0xf << 4) /* Virtual counter event bits */ 87 #define GT_CNTKCTL_EVNTDIR (1 << 3) /* Virtual counter event transition */ 88 #define GT_CNTKCTL_EVNTEN (1 << 2) /* Enables virtual counter events */ 89 #define GT_CNTKCTL_PL0VCTEN (1 << 1) /* PL0 CNTVCT and CNTFRQ access */ 90 #define GT_CNTKCTL_PL0PCTEN (1 << 0) /* PL0 CNTPCT and CNTFRQ access */ 91 92 struct arm_tmr_softc { 93 struct resource *res[4]; 94 void *ihl[4]; 95 uint32_t clkfreq; 96 struct eventtimer et; 97 bool physical; 98 }; 99 100 static struct arm_tmr_softc *arm_tmr_sc = NULL; 101 102 static struct resource_spec timer_spec[] = { 103 { SYS_RES_IRQ, 0, RF_ACTIVE }, /* Secure */ 104 { SYS_RES_IRQ, 1, RF_ACTIVE }, /* Non-secure */ 105 { SYS_RES_IRQ, 2, RF_ACTIVE | RF_OPTIONAL }, /* Virt */ 106 { SYS_RES_IRQ, 3, RF_ACTIVE | RF_OPTIONAL }, /* Hyp */ 107 { -1, 0 } 108 }; 109 110 static uint32_t arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th, 111 struct timecounter *tc); 112 static void arm_tmr_do_delay(int usec, void *); 113 114 static timecounter_get_t arm_tmr_get_timecount; 115 116 static struct timecounter arm_tmr_timecount = { 117 .tc_name = "ARM MPCore Timecounter", 118 .tc_get_timecount = arm_tmr_get_timecount, 119 .tc_poll_pps = NULL, 120 .tc_counter_mask = ~0u, 121 .tc_frequency = 0, 122 .tc_quality = 1000, 123 .tc_fill_vdso_timehands = arm_tmr_fill_vdso_timehands, 124 }; 125 126 #ifdef __arm__ 127 #define get_el0(x) cp15_## x ##_get() 128 #define get_el1(x) cp15_## x ##_get() 129 #define set_el0(x, val) cp15_## x ##_set(val) 130 #define set_el1(x, val) cp15_## x ##_set(val) 131 #else /* __aarch64__ */ 132 #define get_el0(x) READ_SPECIALREG(x ##_el0) 133 #define get_el1(x) READ_SPECIALREG(x ##_el1) 134 #define set_el0(x, val) WRITE_SPECIALREG(x ##_el0, val) 135 #define set_el1(x, val) WRITE_SPECIALREG(x ##_el1, val) 136 #endif 137 138 static int 139 get_freq(void) 140 { 141 return (get_el0(cntfrq)); 142 } 143 144 static uint64_t 145 get_cntxct(bool physical) 146 { 147 uint64_t val; 148 149 isb(); 150 if (physical) 151 val = get_el0(cntpct); 152 else 153 val = get_el0(cntvct); 154 155 return (val); 156 } 157 158 static int 159 set_ctrl(uint32_t val, bool physical) 160 { 161 162 if (physical) 163 set_el0(cntp_ctl, val); 164 else 165 set_el0(cntv_ctl, val); 166 isb(); 167 168 return (0); 169 } 170 171 static int 172 set_tval(uint32_t val, bool physical) 173 { 174 175 if (physical) 176 set_el0(cntp_tval, val); 177 else 178 set_el0(cntv_tval, val); 179 isb(); 180 181 return (0); 182 } 183 184 static int 185 get_ctrl(bool physical) 186 { 187 uint32_t val; 188 189 if (physical) 190 val = get_el0(cntp_ctl); 191 else 192 val = get_el0(cntv_ctl); 193 194 return (val); 195 } 196 197 static void 198 setup_user_access(void *arg __unused) 199 { 200 uint32_t cntkctl; 201 202 cntkctl = get_el1(cntkctl); 203 cntkctl &= ~(GT_CNTKCTL_PL0PTEN | GT_CNTKCTL_PL0VTEN | 204 GT_CNTKCTL_EVNTEN); 205 if (arm_tmr_sc->physical) { 206 cntkctl |= GT_CNTKCTL_PL0PCTEN; 207 cntkctl &= ~GT_CNTKCTL_PL0VCTEN; 208 } else { 209 cntkctl |= GT_CNTKCTL_PL0VCTEN; 210 cntkctl &= ~GT_CNTKCTL_PL0PCTEN; 211 } 212 set_el1(cntkctl, cntkctl); 213 isb(); 214 } 215 216 static void 217 tmr_setup_user_access(void *arg __unused) 218 { 219 220 if (arm_tmr_sc != NULL) 221 smp_rendezvous(NULL, setup_user_access, NULL, NULL); 222 } 223 SYSINIT(tmr_ua, SI_SUB_SMP, SI_ORDER_SECOND, tmr_setup_user_access, NULL); 224 225 static unsigned 226 arm_tmr_get_timecount(struct timecounter *tc) 227 { 228 229 return (get_cntxct(arm_tmr_sc->physical)); 230 } 231 232 static int 233 arm_tmr_start(struct eventtimer *et, sbintime_t first, 234 sbintime_t period __unused) 235 { 236 struct arm_tmr_softc *sc; 237 int counts, ctrl; 238 239 sc = (struct arm_tmr_softc *)et->et_priv; 240 241 if (first != 0) { 242 counts = ((uint32_t)et->et_frequency * first) >> 32; 243 ctrl = get_ctrl(sc->physical); 244 ctrl &= ~GT_CTRL_INT_MASK; 245 ctrl |= GT_CTRL_ENABLE; 246 set_tval(counts, sc->physical); 247 set_ctrl(ctrl, sc->physical); 248 return (0); 249 } 250 251 return (EINVAL); 252 253 } 254 255 static void 256 arm_tmr_disable(bool physical) 257 { 258 int ctrl; 259 260 ctrl = get_ctrl(physical); 261 ctrl &= ~GT_CTRL_ENABLE; 262 set_ctrl(ctrl, physical); 263 } 264 265 static int 266 arm_tmr_stop(struct eventtimer *et) 267 { 268 struct arm_tmr_softc *sc; 269 270 sc = (struct arm_tmr_softc *)et->et_priv; 271 arm_tmr_disable(sc->physical); 272 273 return (0); 274 } 275 276 static int 277 arm_tmr_intr(void *arg) 278 { 279 struct arm_tmr_softc *sc; 280 int ctrl; 281 282 sc = (struct arm_tmr_softc *)arg; 283 ctrl = get_ctrl(sc->physical); 284 if (ctrl & GT_CTRL_INT_STAT) { 285 ctrl |= GT_CTRL_INT_MASK; 286 set_ctrl(ctrl, sc->physical); 287 } 288 289 if (sc->et.et_active) 290 sc->et.et_event_cb(&sc->et, sc->et.et_arg); 291 292 return (FILTER_HANDLED); 293 } 294 295 #ifdef FDT 296 static int 297 arm_tmr_fdt_probe(device_t dev) 298 { 299 300 if (!ofw_bus_status_okay(dev)) 301 return (ENXIO); 302 303 if (ofw_bus_is_compatible(dev, "arm,armv7-timer")) { 304 device_set_desc(dev, "ARMv7 Generic Timer"); 305 return (BUS_PROBE_DEFAULT); 306 } else if (ofw_bus_is_compatible(dev, "arm,armv8-timer")) { 307 device_set_desc(dev, "ARMv8 Generic Timer"); 308 return (BUS_PROBE_DEFAULT); 309 } 310 311 return (ENXIO); 312 } 313 #endif 314 315 #ifdef DEV_ACPI 316 static void 317 arm_tmr_acpi_add_irq(device_t parent, device_t dev, int rid, u_int irq) 318 { 319 320 irq = ACPI_BUS_MAP_INTR(parent, dev, irq, 321 INTR_TRIGGER_LEVEL, INTR_POLARITY_HIGH); 322 BUS_SET_RESOURCE(parent, dev, SYS_RES_IRQ, rid, irq, 1); 323 } 324 325 static void 326 arm_tmr_acpi_identify(driver_t *driver, device_t parent) 327 { 328 ACPI_TABLE_GTDT *gtdt; 329 vm_paddr_t physaddr; 330 device_t dev; 331 332 physaddr = acpi_find_table(ACPI_SIG_GTDT); 333 if (physaddr == 0) 334 return; 335 336 gtdt = acpi_map_table(physaddr, ACPI_SIG_GTDT); 337 if (gtdt == NULL) { 338 device_printf(parent, "gic: Unable to map the GTDT\n"); 339 return; 340 } 341 342 dev = BUS_ADD_CHILD(parent, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE, 343 "generic_timer", -1); 344 if (dev == NULL) { 345 device_printf(parent, "add gic child failed\n"); 346 goto out; 347 } 348 349 arm_tmr_acpi_add_irq(parent, dev, 0, gtdt->SecureEl1Interrupt); 350 arm_tmr_acpi_add_irq(parent, dev, 1, gtdt->NonSecureEl1Interrupt); 351 arm_tmr_acpi_add_irq(parent, dev, 2, gtdt->VirtualTimerInterrupt); 352 353 out: 354 acpi_unmap_table(gtdt); 355 } 356 357 static int 358 arm_tmr_acpi_probe(device_t dev) 359 { 360 361 device_set_desc(dev, "ARM Generic Timer"); 362 return (BUS_PROBE_NOWILDCARD); 363 } 364 #endif 365 366 367 static int 368 arm_tmr_attach(device_t dev) 369 { 370 struct arm_tmr_softc *sc; 371 #ifdef FDT 372 phandle_t node; 373 pcell_t clock; 374 #endif 375 int error; 376 int i; 377 378 sc = device_get_softc(dev); 379 if (arm_tmr_sc) 380 return (ENXIO); 381 382 #ifdef FDT 383 /* Get the base clock frequency */ 384 node = ofw_bus_get_node(dev); 385 if (node > 0) { 386 error = OF_getencprop(node, "clock-frequency", &clock, 387 sizeof(clock)); 388 if (error > 0) 389 sc->clkfreq = clock; 390 } 391 #endif 392 393 if (sc->clkfreq == 0) { 394 /* Try to get clock frequency from timer */ 395 sc->clkfreq = get_freq(); 396 } 397 398 if (sc->clkfreq == 0) { 399 device_printf(dev, "No clock frequency specified\n"); 400 return (ENXIO); 401 } 402 403 if (bus_alloc_resources(dev, timer_spec, sc->res)) { 404 device_printf(dev, "could not allocate resources\n"); 405 return (ENXIO); 406 } 407 408 #ifdef __arm__ 409 sc->physical = true; 410 #else /* __aarch64__ */ 411 /* If we do not have a virtual timer use the physical. */ 412 sc->physical = (sc->res[2] == NULL) ? true : false; 413 #endif 414 415 arm_tmr_sc = sc; 416 417 /* Setup secure, non-secure and virtual IRQs handler */ 418 for (i = 0; i < 3; i++) { 419 /* If we do not have the interrupt, skip it. */ 420 if (sc->res[i] == NULL) 421 continue; 422 error = bus_setup_intr(dev, sc->res[i], INTR_TYPE_CLK, 423 arm_tmr_intr, NULL, sc, &sc->ihl[i]); 424 if (error) { 425 device_printf(dev, "Unable to alloc int resource.\n"); 426 return (ENXIO); 427 } 428 } 429 430 /* Disable the virtual timer until we are ready */ 431 if (sc->res[2] != NULL) 432 arm_tmr_disable(false); 433 /* And the physical */ 434 if (sc->physical) 435 arm_tmr_disable(true); 436 437 arm_tmr_timecount.tc_frequency = sc->clkfreq; 438 tc_init(&arm_tmr_timecount); 439 440 sc->et.et_name = "ARM MPCore Eventtimer"; 441 sc->et.et_flags = ET_FLAGS_ONESHOT | ET_FLAGS_PERCPU; 442 sc->et.et_quality = 1000; 443 444 sc->et.et_frequency = sc->clkfreq; 445 sc->et.et_min_period = (0x00000010LLU << 32) / sc->et.et_frequency; 446 sc->et.et_max_period = (0xfffffffeLLU << 32) / sc->et.et_frequency; 447 sc->et.et_start = arm_tmr_start; 448 sc->et.et_stop = arm_tmr_stop; 449 sc->et.et_priv = sc; 450 et_register(&sc->et); 451 452 #if defined(__arm__) 453 arm_set_delay(arm_tmr_do_delay, sc); 454 #endif 455 456 return (0); 457 } 458 459 #ifdef FDT 460 static device_method_t arm_tmr_fdt_methods[] = { 461 DEVMETHOD(device_probe, arm_tmr_fdt_probe), 462 DEVMETHOD(device_attach, arm_tmr_attach), 463 { 0, 0 } 464 }; 465 466 static driver_t arm_tmr_fdt_driver = { 467 "generic_timer", 468 arm_tmr_fdt_methods, 469 sizeof(struct arm_tmr_softc), 470 }; 471 472 static devclass_t arm_tmr_fdt_devclass; 473 474 EARLY_DRIVER_MODULE(timer, simplebus, arm_tmr_fdt_driver, arm_tmr_fdt_devclass, 475 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); 476 EARLY_DRIVER_MODULE(timer, ofwbus, arm_tmr_fdt_driver, arm_tmr_fdt_devclass, 477 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); 478 #endif 479 480 #ifdef DEV_ACPI 481 static device_method_t arm_tmr_acpi_methods[] = { 482 DEVMETHOD(device_identify, arm_tmr_acpi_identify), 483 DEVMETHOD(device_probe, arm_tmr_acpi_probe), 484 DEVMETHOD(device_attach, arm_tmr_attach), 485 { 0, 0 } 486 }; 487 488 static driver_t arm_tmr_acpi_driver = { 489 "generic_timer", 490 arm_tmr_acpi_methods, 491 sizeof(struct arm_tmr_softc), 492 }; 493 494 static devclass_t arm_tmr_acpi_devclass; 495 496 EARLY_DRIVER_MODULE(timer, acpi, arm_tmr_acpi_driver, arm_tmr_acpi_devclass, 497 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); 498 #endif 499 500 static void 501 arm_tmr_do_delay(int usec, void *arg) 502 { 503 struct arm_tmr_softc *sc = arg; 504 int32_t counts, counts_per_usec; 505 uint32_t first, last; 506 507 /* Get the number of times to count */ 508 counts_per_usec = ((arm_tmr_timecount.tc_frequency / 1000000) + 1); 509 510 /* 511 * Clamp the timeout at a maximum value (about 32 seconds with 512 * a 66MHz clock). *Nobody* should be delay()ing for anywhere 513 * near that length of time and if they are, they should be hung 514 * out to dry. 515 */ 516 if (usec >= (0x80000000U / counts_per_usec)) 517 counts = (0x80000000U / counts_per_usec) - 1; 518 else 519 counts = usec * counts_per_usec; 520 521 first = get_cntxct(sc->physical); 522 523 while (counts > 0) { 524 last = get_cntxct(sc->physical); 525 counts -= (int32_t)(last - first); 526 first = last; 527 } 528 } 529 530 #if defined(__aarch64__) 531 void 532 DELAY(int usec) 533 { 534 int32_t counts; 535 536 TSENTER(); 537 /* 538 * Check the timers are setup, if not just 539 * use a for loop for the meantime 540 */ 541 if (arm_tmr_sc == NULL) { 542 for (; usec > 0; usec--) 543 for (counts = 200; counts > 0; counts--) 544 /* 545 * Prevent the compiler from optimizing 546 * out the loop 547 */ 548 cpufunc_nullop(); 549 } else 550 arm_tmr_do_delay(usec, arm_tmr_sc); 551 TSEXIT(); 552 } 553 #endif 554 555 static uint32_t 556 arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th, 557 struct timecounter *tc) 558 { 559 560 vdso_th->th_algo = VDSO_TH_ALGO_ARM_GENTIM; 561 vdso_th->th_physical = arm_tmr_sc->physical; 562 bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); 563 return (1); 564 } 565