1 /* $OpenBSD: agtimer.c,v 1.19 2021/10/24 17:52:28 mpi Exp $ */ 2 /* 3 * Copyright (c) 2011 Dale Rahn <drahn@openbsd.org> 4 * Copyright (c) 2013 Patrick Wildt <patrick@blueri.se> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/param.h> 20 #include <sys/systm.h> 21 #include <sys/queue.h> 22 #include <sys/malloc.h> 23 #include <sys/device.h> 24 #include <sys/kernel.h> 25 #include <sys/timetc.h> 26 #include <sys/evcount.h> 27 28 #include <machine/intr.h> 29 #include <machine/bus.h> 30 #include <machine/fdt.h> 31 32 #include <dev/ofw/fdt.h> 33 #include <dev/ofw/openfirm.h> 34 35 /* registers */ 36 #define GTIMER_CNTV_CTL_ENABLE (1 << 0) 37 #define GTIMER_CNTV_CTL_IMASK (1 << 1) 38 #define GTIMER_CNTV_CTL_ISTATUS (1 << 2) 39 40 #define TIMER_FREQUENCY 24 * 1000 * 1000 /* ARM core clock */ 41 int32_t agtimer_frequency = TIMER_FREQUENCY; 42 43 u_int agtimer_get_timecount(struct timecounter *); 44 45 static struct timecounter agtimer_timecounter = { 46 .tc_get_timecount = agtimer_get_timecount, 47 .tc_poll_pps = NULL, 48 .tc_counter_mask = 0xffffffff, 49 .tc_frequency = 0, 50 .tc_name = "agtimer", 51 .tc_quality = 0, 52 .tc_priv = NULL, 53 .tc_user = TC_AGTIMER, 54 }; 55 56 struct agtimer_pcpu_softc { 57 uint64_t pc_nexttickevent; 58 uint64_t pc_nextstatevent; 59 u_int32_t pc_ticks_err_sum; 60 }; 61 62 struct agtimer_softc { 63 struct device sc_dev; 64 int sc_node; 65 66 struct agtimer_pcpu_softc sc_pstat[MAXCPUS]; 67 68 u_int32_t sc_ticks_err_cnt; 69 u_int32_t sc_ticks_per_second; 70 u_int32_t sc_ticks_per_intr; 71 u_int32_t sc_statvar; 72 u_int32_t sc_statmin; 73 74 #ifdef AMPTIMER_DEBUG 75 struct evcount sc_clk_count; 76 struct evcount sc_stat_count; 77 #endif 78 void *sc_ih; 79 }; 80 81 int agtimer_match(struct device *, void *, void *); 82 void agtimer_attach(struct device *, struct device *, void *); 83 uint64_t agtimer_readcnt64(void); 84 int agtimer_intr(void *); 85 void agtimer_cpu_initclocks(void); 86 void agtimer_delay(u_int); 87 void agtimer_setstatclockrate(int stathz); 88 void agtimer_set_clockrate(int32_t new_frequency); 89 void agtimer_startclock(void); 90 91 const struct cfattach agtimer_ca = { 92 sizeof (struct agtimer_softc), agtimer_match, agtimer_attach 93 }; 94 95 struct cfdriver agtimer_cd = { 96 NULL, "agtimer", DV_DULL 97 }; 98 99 uint64_t 100 agtimer_readcnt64(void) 101 { 102 uint64_t val0, val1; 103 104 /* 105 * Work around Cortex-A73 errata 858921, where there is a 106 * one-cycle window where the read might return the old value 107 * for the low 32 bits and the new value for the high 32 bits 108 * upon roll-over of the low 32 bits. 109 */ 110 __asm volatile("isb" ::: "memory"); 111 __asm volatile("mrs %x0, CNTVCT_EL0" : "=r" (val0)); 112 __asm volatile("mrs %x0, CNTVCT_EL0" : "=r" (val1)); 113 return ((val0 ^ val1) & 0x100000000ULL) ? val0 : val1; 114 } 115 116 static inline uint64_t 117 agtimer_get_freq(void) 118 { 119 uint64_t val; 120 121 __asm volatile("mrs %x0, CNTFRQ_EL0" : "=r" (val)); 122 123 return (val); 124 } 125 126 static inline int 127 agtimer_get_ctrl(void) 128 { 129 uint32_t val; 130 131 __asm volatile("mrs %x0, CNTV_CTL_EL0" : "=r" (val)); 132 133 return (val); 134 } 135 136 static inline int 137 agtimer_set_ctrl(uint32_t val) 138 { 139 __asm volatile("msr CNTV_CTL_EL0, %x0" :: "r" (val)); 140 __asm volatile("isb" ::: "memory"); 141 142 return (0); 143 } 144 145 static inline int 146 agtimer_set_tval(uint32_t val) 147 { 148 __asm volatile("msr CNTV_TVAL_EL0, %x0" :: "r" (val)); 149 __asm volatile("isb" ::: "memory"); 150 151 return (0); 152 } 153 154 int 155 agtimer_match(struct device *parent, void *cfdata, void *aux) 156 { 157 struct fdt_attach_args *faa = (struct fdt_attach_args *)aux; 158 159 return (OF_is_compatible(faa->fa_node, "arm,armv7-timer") || 160 OF_is_compatible(faa->fa_node, "arm,armv8-timer")); 161 } 162 163 void 164 agtimer_attach(struct device *parent, struct device *self, void *aux) 165 { 166 struct agtimer_softc *sc = (struct agtimer_softc *)self; 167 struct fdt_attach_args *faa = aux; 168 169 sc->sc_node = faa->fa_node; 170 171 if (agtimer_get_freq() != 0) 172 agtimer_frequency = agtimer_get_freq(); 173 agtimer_frequency = 174 OF_getpropint(sc->sc_node, "clock-frequency", agtimer_frequency); 175 sc->sc_ticks_per_second = agtimer_frequency; 176 177 printf(": %d kHz\n", sc->sc_ticks_per_second / 1000); 178 179 #ifdef AMPTIMER_DEBUG 180 evcount_attach(&sc->sc_clk_count, "clock", NULL); 181 evcount_attach(&sc->sc_stat_count, "stat", NULL); 182 #endif 183 184 /* 185 * private timer and interrupts not enabled until 186 * timer configures 187 */ 188 189 arm_clock_register(agtimer_cpu_initclocks, agtimer_delay, 190 agtimer_setstatclockrate, agtimer_startclock); 191 192 agtimer_timecounter.tc_frequency = sc->sc_ticks_per_second; 193 agtimer_timecounter.tc_priv = sc; 194 195 tc_init(&agtimer_timecounter); 196 } 197 198 u_int 199 agtimer_get_timecount(struct timecounter *tc) 200 { 201 uint64_t val; 202 203 /* 204 * No need to work around Cortex-A73 errata 858921 since we 205 * only look at the low 32 bits here. 206 */ 207 __asm volatile("isb" ::: "memory"); 208 __asm volatile("mrs %x0, CNTVCT_EL0" : "=r" (val)); 209 return (val & 0xffffffff); 210 } 211 212 int 213 agtimer_intr(void *frame) 214 { 215 struct agtimer_softc *sc = agtimer_cd.cd_devs[0]; 216 struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())]; 217 uint64_t now; 218 uint64_t nextevent; 219 uint32_t r; 220 #if defined(USE_GTIMER_CMP) 221 int skip = 1; 222 #else 223 int64_t delay; 224 #endif 225 int rc = 0; 226 227 /* 228 * DSR - I know that the tick timer is 64 bits, but the following 229 * code deals with rollover, so there is no point in dealing 230 * with the 64 bit math, just let the 32 bit rollover 231 * do the right thing 232 */ 233 234 now = agtimer_readcnt64(); 235 236 while (pc->pc_nexttickevent <= now) { 237 pc->pc_nexttickevent += sc->sc_ticks_per_intr; 238 pc->pc_ticks_err_sum += sc->sc_ticks_err_cnt; 239 240 /* looping a few times is faster than divide */ 241 while (pc->pc_ticks_err_sum > hz) { 242 pc->pc_nexttickevent += 1; 243 pc->pc_ticks_err_sum -= hz; 244 } 245 246 #ifdef AMPTIMER_DEBUG 247 sc->sc_clk_count.ec_count++; 248 #endif 249 rc = 1; 250 hardclock(frame); 251 } 252 while (pc->pc_nextstatevent <= now) { 253 do { 254 r = random() & (sc->sc_statvar -1); 255 } while (r == 0); /* random == 0 not allowed */ 256 pc->pc_nextstatevent += sc->sc_statmin + r; 257 258 /* XXX - correct nextstatevent? */ 259 #ifdef AMPTIMER_DEBUG 260 sc->sc_stat_count.ec_count++; 261 #endif 262 rc = 1; 263 statclock(frame); 264 } 265 266 if (pc->pc_nexttickevent < pc->pc_nextstatevent) 267 nextevent = pc->pc_nexttickevent; 268 else 269 nextevent = pc->pc_nextstatevent; 270 271 delay = nextevent - now; 272 if (delay < 0) 273 delay = 1; 274 275 agtimer_set_tval(delay); 276 277 return (rc); 278 } 279 280 void 281 agtimer_set_clockrate(int32_t new_frequency) 282 { 283 struct agtimer_softc *sc = agtimer_cd.cd_devs[0]; 284 285 agtimer_frequency = new_frequency; 286 287 if (sc == NULL) 288 return; 289 290 sc->sc_ticks_per_second = agtimer_frequency; 291 agtimer_timecounter.tc_frequency = sc->sc_ticks_per_second; 292 printf("agtimer0: adjusting clock: new tick rate %d kHz\n", 293 sc->sc_ticks_per_second / 1000); 294 } 295 296 void 297 agtimer_cpu_initclocks(void) 298 { 299 struct agtimer_softc *sc = agtimer_cd.cd_devs[0]; 300 struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())]; 301 uint32_t reg; 302 uint64_t next; 303 uint64_t kctl; 304 305 stathz = hz; 306 profhz = hz * 10; 307 308 if (sc->sc_ticks_per_second != agtimer_frequency) { 309 agtimer_set_clockrate(agtimer_frequency); 310 } 311 312 agtimer_setstatclockrate(stathz); 313 314 sc->sc_ticks_per_intr = sc->sc_ticks_per_second / hz; 315 sc->sc_ticks_err_cnt = sc->sc_ticks_per_second % hz; 316 pc->pc_ticks_err_sum = 0; 317 318 /* configure virtual timer interrupt */ 319 sc->sc_ih = arm_intr_establish_fdt_idx(sc->sc_node, 2, 320 IPL_CLOCK|IPL_MPSAFE, agtimer_intr, NULL, "tick"); 321 322 next = agtimer_readcnt64() + sc->sc_ticks_per_intr; 323 pc->pc_nexttickevent = pc->pc_nextstatevent = next; 324 325 reg = agtimer_get_ctrl(); 326 reg &= ~GTIMER_CNTV_CTL_IMASK; 327 reg |= GTIMER_CNTV_CTL_ENABLE; 328 agtimer_set_tval(sc->sc_ticks_per_second); 329 agtimer_set_ctrl(reg); 330 331 /* enable userland access to virtual counter */ 332 kctl = READ_SPECIALREG(CNTKCTL_EL1); 333 WRITE_SPECIALREG(CNTKCTL_EL1, kctl | CNTKCTL_EL0VCTEN); 334 } 335 336 void 337 agtimer_delay(u_int usecs) 338 { 339 uint64_t clock, oclock, delta, delaycnt; 340 uint64_t csec, usec; 341 volatile int j; 342 343 if (usecs > (0x80000000 / agtimer_frequency)) { 344 csec = usecs / 10000; 345 usec = usecs % 10000; 346 347 delaycnt = (agtimer_frequency / 100) * csec + 348 (agtimer_frequency / 100) * usec / 10000; 349 } else { 350 delaycnt = agtimer_frequency * usecs / 1000000; 351 } 352 if (delaycnt <= 1) 353 for (j = 100; j > 0; j--) 354 ; 355 356 oclock = agtimer_readcnt64(); 357 while (1) { 358 for (j = 100; j > 0; j--) 359 ; 360 clock = agtimer_readcnt64(); 361 delta = clock - oclock; 362 if (delta > delaycnt) 363 break; 364 } 365 } 366 367 void 368 agtimer_setstatclockrate(int newhz) 369 { 370 struct agtimer_softc *sc = agtimer_cd.cd_devs[0]; 371 int minint, statint; 372 int s; 373 374 s = splclock(); 375 376 statint = sc->sc_ticks_per_second / newhz; 377 /* calculate largest 2^n which is smaller that just over half statint */ 378 sc->sc_statvar = 0x40000000; /* really big power of two */ 379 minint = statint / 2 + 100; 380 while (sc->sc_statvar > minint) 381 sc->sc_statvar >>= 1; 382 383 sc->sc_statmin = statint - (sc->sc_statvar >> 1); 384 385 splx(s); 386 387 /* 388 * XXX this allows the next stat timer to occur then it switches 389 * to the new frequency. Rather than switching instantly. 390 */ 391 } 392 393 void 394 agtimer_startclock(void) 395 { 396 struct agtimer_softc *sc = agtimer_cd.cd_devs[0]; 397 struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())]; 398 uint64_t nextevent; 399 uint64_t kctl; 400 uint32_t reg; 401 402 nextevent = agtimer_readcnt64() + sc->sc_ticks_per_intr; 403 pc->pc_nexttickevent = pc->pc_nextstatevent = nextevent; 404 405 arm_intr_route(sc->sc_ih, 1, curcpu()); 406 407 reg = agtimer_get_ctrl(); 408 reg &= ~GTIMER_CNTV_CTL_IMASK; 409 reg |= GTIMER_CNTV_CTL_ENABLE; 410 agtimer_set_tval(sc->sc_ticks_per_second); 411 agtimer_set_ctrl(reg); 412 413 /* enable userland access to virtual counter */ 414 kctl = READ_SPECIALREG(CNTKCTL_EL1); 415 WRITE_SPECIALREG(CNTKCTL_EL1, kctl | CNTKCTL_EL0VCTEN); 416 } 417 418 void 419 agtimer_init(void) 420 { 421 uint64_t cntfrq = 0; 422 423 /* XXX: Check for Generic Timer support. */ 424 cntfrq = agtimer_get_freq(); 425 426 if (cntfrq != 0) { 427 agtimer_frequency = cntfrq; 428 arm_clock_register(NULL, agtimer_delay, NULL, NULL); 429 } 430 } 431