1 /* 2 * Copyright (c) 2005 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 /* 35 * Generic cputimer - access to a reliable, free-running counter. 36 */ 37 38 #include <sys/param.h> 39 #include <sys/kernel.h> 40 #include <sys/systm.h> 41 #include <sys/thread.h> 42 #include <sys/globaldata.h> 43 #include <sys/serialize.h> 44 #include <sys/systimer.h> 45 #include <sys/sysctl.h> 46 47 extern void pcpu_timer_process(void); 48 extern void pcpu_timer_process_frame(struct intrframe *); 49 50 static uint64_t dummy_cpucounter_count(void); 51 52 static sysclock_t dummy_cputimer_count(void); 53 54 static struct cputimer dummy_cputimer = { 55 .next = SLIST_ENTRY_INITIALIZER, 56 .name = "dummy", 57 .pri = CPUTIMER_PRI_DUMMY, 58 .type = CPUTIMER_DUMMY, 59 .count = dummy_cputimer_count, 60 .fromhz = cputimer_default_fromhz, 61 .fromus = cputimer_default_fromus, 62 .construct = cputimer_default_construct, 63 .destruct = cputimer_default_destruct, 64 .freq = 1000000, 65 .freq64_usec = (1000000LL << 32) / 1000000, 66 .freq64_nsec = (1000000000LL << 32) / 1000000 67 }; 68 69 static struct cpucounter dummy_cpucounter = { 70 .freq = 1000000ULL, 71 .count = dummy_cpucounter_count, 72 .flags = CPUCOUNTER_FLAG_MPSYNC, 73 .prio = CPUCOUNTER_PRIO_DUMMY, 74 .type = CPUCOUNTER_DUMMY 75 }; 76 77 struct cputimer *sys_cputimer = &dummy_cputimer; 78 SLIST_HEAD(, cputimer) cputimerhead = SLIST_HEAD_INITIALIZER(&cputimerhead); 79 80 static SLIST_HEAD(, cpucounter) cpucounterhead = 81 SLIST_HEAD_INITIALIZER(cpucounterhead); 82 83 static int cputimer_intr_ps_reqs; 84 static struct lwkt_serialize cputimer_intr_ps_slize = 85 LWKT_SERIALIZE_INITIALIZER; 86 87 /* 88 * Generic cputimer API 89 */ 90 void 91 cputimer_select(struct cputimer *timer, int pri) 92 { 93 sysclock_t oldclock; 94 95 /* 96 * Calculate helper fields 97 */ 98 cputimer_set_frequency(timer, timer->freq); 99 100 /* 101 * Install a new cputimer if its priority allows it. If timer is 102 * passed as NULL we deinstall the current timer and revert to our 103 * dummy. 104 */ 105 if (pri == 0) 106 pri = timer->pri; 107 if (timer == NULL || pri >= sys_cputimer->pri) { 108 oldclock = sys_cputimer->count(); 109 sys_cputimer->destruct(sys_cputimer); 110 sys_cputimer = &dummy_cputimer; 111 if (timer) { 112 sys_cputimer = timer; 113 timer->construct(timer, oldclock); 114 cputimer_intr_config(timer); 115 systimer_changed(); 116 } 117 } 118 } 119 120 /* 121 * Register a timer. If the timer has already been registered, do nothing. 122 */ 123 void 124 cputimer_register(struct cputimer *timer) 125 { 126 struct cputimer *scan; 127 128 /* 129 * Initialize dummy_cputimer if the slist is empty, it does not get 130 * registered the normal way. 131 */ 132 if (SLIST_EMPTY(&cputimerhead)) 133 SLIST_FIRST(&cputimerhead) = &dummy_cputimer; 134 SLIST_FOREACH(scan, &cputimerhead, next) { 135 if (scan == timer) 136 return; 137 } 138 SLIST_INSERT_HEAD(&cputimerhead, timer, next); 139 } 140 141 /* 142 * Deregister a timer. If the timer has already been deregistered, do nothing. 143 */ 144 void 145 cputimer_deregister(struct cputimer *timer) 146 { 147 struct cputimer *scan; 148 struct cputimer *best; 149 150 /* 151 * Locate and remove the timer. If the timer is our currently active 152 * timer, revert to the dummy timer. 153 */ 154 SLIST_FOREACH(scan, &cputimerhead, next) { 155 if (timer == scan) { 156 if (timer == sys_cputimer) 157 cputimer_select(&dummy_cputimer, 0x7FFFFFFF); 158 SLIST_REMOVE(&cputimerhead, timer, cputimer, next); 159 break; 160 } 161 } 162 163 /* 164 * If sys_cputimer reverted to the dummy, select the best one 165 */ 166 if (sys_cputimer == &dummy_cputimer) { 167 best = NULL; 168 SLIST_FOREACH(scan, &cputimerhead, next) { 169 if (best == NULL || scan->pri > best->pri) 170 best = scan; 171 } 172 if (best) 173 cputimer_select(best, 0x7FFFFFFF); 174 } 175 } 176 177 /* 178 * Calculate usec / tick and nsec / tick, scaled by (1 << 32). 179 * 180 * so e.g. a 3 mhz timer would be 3 usec / tick x (1 << 32), 181 * or 3000 nsec / tick x (1 << 32) 182 */ 183 void 184 cputimer_set_frequency(struct cputimer *timer, sysclock_t freq) 185 { 186 timer->freq = freq; 187 timer->freq64_usec = (1000000LL << 32) / freq; 188 timer->freq64_nsec = (1000000000LL << 32) / freq; 189 if (timer == sys_cputimer) 190 cputimer_intr_config(timer); 191 } 192 193 sysclock_t 194 cputimer_default_fromhz(int64_t freq) 195 { 196 return(sys_cputimer->freq / freq + 1); 197 } 198 199 sysclock_t 200 cputimer_default_fromus(int64_t us) 201 { 202 return muldivu64(us, sys_cputimer->freq, 1000000); 203 } 204 205 /* 206 * Dummy counter implementation 207 */ 208 static 209 sysclock_t 210 dummy_cputimer_count(void) 211 { 212 return(++dummy_cputimer.base); 213 } 214 215 void 216 cputimer_default_construct(struct cputimer *cputimer, sysclock_t oldclock) 217 { 218 cputimer->base = oldclock; 219 } 220 221 void 222 cputimer_default_destruct(struct cputimer *cputimer) 223 { 224 } 225 226 /************************************************************************ 227 * SYSCTL SUPPORT * 228 ************************************************************************ 229 * 230 * Note: the ability to change the systimer is not currently enabled 231 * because it will mess up systimer calculations. You have to live 232 * with what is configured at boot. 233 */ 234 static int 235 sysctl_cputimer_reglist(SYSCTL_HANDLER_ARGS) 236 { 237 struct cputimer *scan; 238 int error = 0; 239 int loop = 0; 240 241 /* 242 * Build a list of available timers 243 */ 244 SLIST_FOREACH(scan, &cputimerhead, next) { 245 if (error == 0 && loop) 246 error = SYSCTL_OUT(req, " ", 1); 247 if (error == 0) 248 error = SYSCTL_OUT(req, scan->name, strlen(scan->name)); 249 ++loop; 250 } 251 return (error); 252 } 253 254 static int 255 sysctl_cputimer_name(SYSCTL_HANDLER_ARGS) 256 { 257 int error; 258 259 error = SYSCTL_OUT(req, sys_cputimer->name, strlen(sys_cputimer->name)); 260 return (error); 261 } 262 263 static int 264 sysctl_cputimer_clock(SYSCTL_HANDLER_ARGS) 265 { 266 sysclock_t clock; 267 int error; 268 269 clock = sys_cputimer->count(); 270 error = SYSCTL_OUT(req, &clock, sizeof(clock)); 271 return (error); 272 } 273 274 static int 275 sysctl_cputimer_freq(SYSCTL_HANDLER_ARGS) 276 { 277 int error; 278 279 error = SYSCTL_OUT(req, &sys_cputimer->freq, sizeof(sys_cputimer->freq)); 280 return (error); 281 } 282 283 SYSCTL_DECL(_kern_cputimer); 284 SYSCTL_NODE(_kern, OID_AUTO, cputimer, CTLFLAG_RW, NULL, "cputimer"); 285 286 SYSCTL_PROC(_kern_cputimer, OID_AUTO, select, CTLTYPE_STRING|CTLFLAG_RD, 287 NULL, 0, sysctl_cputimer_reglist, "A", ""); 288 SYSCTL_PROC(_kern_cputimer, OID_AUTO, name, CTLTYPE_STRING|CTLFLAG_RD, 289 NULL, 0, sysctl_cputimer_name, "A", ""); 290 SYSCTL_PROC(_kern_cputimer, OID_AUTO, clock, CTLTYPE_ULONG|CTLFLAG_RD, 291 NULL, 0, sysctl_cputimer_clock, "LU", ""); 292 SYSCTL_PROC(_kern_cputimer, OID_AUTO, freq, CTLTYPE_LONG|CTLFLAG_RD, 293 NULL, 0, sysctl_cputimer_freq, "L", ""); 294 295 static struct cputimer_intr *sys_cputimer_intr; 296 static uint32_t cputimer_intr_caps; 297 SLIST_HEAD(, cputimer_intr) cputimer_intr_head = 298 SLIST_HEAD_INITIALIZER(&cputimer_intr_head); 299 300 void 301 cputimer_intr_register(struct cputimer_intr *cti) 302 { 303 struct cputimer_intr *scan; 304 305 SLIST_FOREACH(scan, &cputimer_intr_head, next) { 306 if (scan == cti) 307 return; 308 } 309 cti->config(cti, sys_cputimer); 310 SLIST_INSERT_HEAD(&cputimer_intr_head, cti, next); 311 } 312 313 void 314 cputimer_intr_deregister(struct cputimer_intr *cti) 315 { 316 KKASSERT(cti != sys_cputimer_intr); 317 SLIST_REMOVE(&cputimer_intr_head, cti, cputimer_intr, next); 318 } 319 320 int 321 cputimer_intr_select(struct cputimer_intr *cti, int prio) 322 { 323 KKASSERT(cti != NULL); 324 325 if (prio == 0) 326 prio = cti->prio; 327 328 if (sys_cputimer_intr == NULL) { 329 KKASSERT(cputimer_intr_caps == 0); 330 sys_cputimer_intr = cti; 331 return 0; 332 } 333 334 if ((cti->caps & cputimer_intr_caps) == cputimer_intr_caps) { 335 if (prio > sys_cputimer_intr->prio) { 336 sys_cputimer_intr = cti; 337 return 0; 338 } else { 339 return EBUSY; 340 } 341 } else { 342 return EOPNOTSUPP; 343 } 344 } 345 346 void 347 cputimer_intr_default_enable(struct cputimer_intr *cti __unused) 348 { 349 } 350 351 void 352 cputimer_intr_default_restart(struct cputimer_intr *cti) 353 { 354 cti->reload(cti, 0); 355 } 356 357 void 358 cputimer_intr_default_config(struct cputimer_intr *cti __unused, 359 const struct cputimer *timer __unused) 360 { 361 } 362 363 void 364 cputimer_intr_default_pmfixup(struct cputimer_intr *cti __unused) 365 { 366 } 367 368 void 369 cputimer_intr_default_initclock(struct cputimer_intr *cti __unused, 370 boolean_t selected __unused) 371 { 372 } 373 374 void 375 cputimer_intr_enable(void) 376 { 377 struct cputimer_intr *cti; 378 379 SLIST_FOREACH(cti, &cputimer_intr_head, next) 380 cti->enable(cti); 381 } 382 383 void 384 cputimer_intr_config(const struct cputimer *timer) 385 { 386 struct cputimer_intr *cti; 387 388 SLIST_FOREACH(cti, &cputimer_intr_head, next) 389 cti->config(cti, timer); 390 } 391 392 void 393 cputimer_intr_pmfixup(void) 394 { 395 struct cputimer_intr *cti; 396 397 SLIST_FOREACH(cti, &cputimer_intr_head, next) 398 cti->pmfixup(cti); 399 } 400 401 void 402 cputimer_intr_reload(sysclock_t reload) 403 { 404 struct cputimer_intr *cti = sys_cputimer_intr; 405 406 cti->reload(cti, reload); 407 } 408 409 void 410 cputimer_intr_restart(void) 411 { 412 struct cputimer_intr *cti = sys_cputimer_intr; 413 414 cti->restart(cti); 415 } 416 417 int 418 cputimer_intr_select_caps(uint32_t caps) 419 { 420 struct cputimer_intr *cti, *maybe; 421 int error; 422 423 maybe = NULL; 424 SLIST_FOREACH(cti, &cputimer_intr_head, next) { 425 if ((cti->caps & caps) == caps) { 426 if (maybe == NULL) 427 maybe = cti; 428 else if (cti->prio > maybe->prio) 429 maybe = cti; 430 } 431 } 432 if (maybe == NULL) 433 return ENOENT; 434 435 if (sys_cputimer_intr == maybe) 436 return 0; 437 438 cputimer_intr_caps = caps; 439 error = cputimer_intr_select(maybe, CPUTIMER_INTR_PRIO_MAX); 440 KKASSERT(!error); 441 442 return ERESTART; 443 } 444 445 static void 446 cputimer_intr_initclocks(void) 447 { 448 struct cputimer_intr *cti, *ncti; 449 450 /* 451 * An interrupt cputimer may deregister itself, 452 * so use SLIST_FOREACH_MUTABLE here. 453 */ 454 SLIST_FOREACH_MUTABLE(cti, &cputimer_intr_head, next, ncti) { 455 boolean_t selected = FALSE; 456 457 if (cti == sys_cputimer_intr) 458 selected = TRUE; 459 cti->initclock(cti, selected); 460 } 461 } 462 /* NOTE: Must be SECOND to allow platform initialization to go first */ 463 SYSINIT(cputimer_intr, SI_BOOT2_CLOCKREG, SI_ORDER_SECOND, 464 cputimer_intr_initclocks, NULL); 465 466 static int 467 sysctl_cputimer_intr_reglist(SYSCTL_HANDLER_ARGS) 468 { 469 struct cputimer_intr *scan; 470 int error = 0; 471 int loop = 0; 472 473 /* 474 * Build a list of available interrupt cputimers 475 */ 476 SLIST_FOREACH(scan, &cputimer_intr_head, next) { 477 if (error == 0 && loop) 478 error = SYSCTL_OUT(req, " ", 1); 479 if (error == 0) 480 error = SYSCTL_OUT(req, scan->name, strlen(scan->name)); 481 ++loop; 482 } 483 return (error); 484 } 485 486 static int 487 sysctl_cputimer_intr_freq(SYSCTL_HANDLER_ARGS) 488 { 489 int error; 490 491 error = SYSCTL_OUT(req, &sys_cputimer_intr->freq, 492 sizeof(sys_cputimer_intr->freq)); 493 return (error); 494 } 495 496 static int 497 sysctl_cputimer_intr_select(SYSCTL_HANDLER_ARGS) 498 { 499 struct cputimer_intr *cti; 500 char name[32]; 501 int error; 502 503 ksnprintf(name, sizeof(name), "%s", sys_cputimer_intr->name); 504 error = sysctl_handle_string(oidp, name, sizeof(name), req); 505 if (error != 0 || req->newptr == NULL) 506 return error; 507 508 SLIST_FOREACH(cti, &cputimer_intr_head, next) { 509 if (strcmp(cti->name, name) == 0) 510 break; 511 } 512 if (cti == NULL) 513 return ENOENT; 514 if (cti == sys_cputimer_intr) 515 return 0; 516 517 error = cputimer_intr_select(cti, CPUTIMER_INTR_PRIO_MAX); 518 if (!error) 519 cputimer_intr_restart(); 520 return error; 521 } 522 523 SYSCTL_NODE(_kern_cputimer, OID_AUTO, intr, CTLFLAG_RW, NULL, 524 "interrupt cputimer"); 525 526 SYSCTL_PROC(_kern_cputimer_intr, OID_AUTO, reglist, CTLTYPE_STRING|CTLFLAG_RD, 527 NULL, 0, sysctl_cputimer_intr_reglist, "A", ""); 528 SYSCTL_PROC(_kern_cputimer_intr, OID_AUTO, freq, CTLTYPE_LONG|CTLFLAG_RD, 529 NULL, 0, sysctl_cputimer_intr_freq, "L", ""); 530 SYSCTL_PROC(_kern_cputimer_intr, OID_AUTO, select, CTLTYPE_STRING|CTLFLAG_RW, 531 NULL, 0, sysctl_cputimer_intr_select, "A", ""); 532 533 int 534 cputimer_intr_powersave_addreq(void) 535 { 536 int error = 0; 537 538 lwkt_serialize_enter(&cputimer_intr_ps_slize); 539 540 ++cputimer_intr_ps_reqs; 541 if (cputimer_intr_ps_reqs == 1) { 542 /* 543 * Upon the first power saving request, switch to an one shot 544 * timer, which would not stop in the any power saving state. 545 */ 546 error = cputimer_intr_select_caps(CPUTIMER_INTR_CAP_PS); 547 if (error == ERESTART) { 548 error = 0; 549 if (bootverbose) 550 kprintf("cputimer: first power save request, restart\n"); 551 cputimer_intr_restart(); 552 } else if (error) { 553 kprintf("no suitable intr cputimer found\n"); 554 --cputimer_intr_ps_reqs; 555 } else if (bootverbose) { 556 kprintf("cputimer: first power save request\n"); 557 } 558 } 559 560 lwkt_serialize_exit(&cputimer_intr_ps_slize); 561 562 return error; 563 } 564 565 void 566 cputimer_intr_powersave_remreq(void) 567 { 568 lwkt_serialize_enter(&cputimer_intr_ps_slize); 569 570 KASSERT(cputimer_intr_ps_reqs > 0, 571 ("invalid # of powersave reqs %d", cputimer_intr_ps_reqs)); 572 --cputimer_intr_ps_reqs; 573 if (cputimer_intr_ps_reqs == 0) { 574 int error; 575 576 /* No one needs power saving, use a better one shot timer. */ 577 error = cputimer_intr_select_caps(CPUTIMER_INTR_CAP_NONE); 578 KKASSERT(!error || error == ERESTART); 579 if (error == ERESTART) { 580 if (bootverbose) 581 kprintf("cputimer: no powser save request, restart\n"); 582 cputimer_intr_restart(); 583 } else if (bootverbose) { 584 kprintf("cputimer: no power save request\n"); 585 } 586 } 587 588 lwkt_serialize_exit(&cputimer_intr_ps_slize); 589 } 590 591 static __inline void 592 cputimer_intr_pcpuhand(void) 593 { 594 struct cputimer_intr *cti = sys_cputimer_intr; 595 596 if (cti->pcpuhand != NULL) 597 cti->pcpuhand(cti); 598 } 599 600 static void 601 pcpu_timer_process_oncpu(struct globaldata *gd, struct intrframe *frame) 602 { 603 sysclock_t count; 604 605 cputimer_intr_pcpuhand(); 606 607 gd->gd_timer_running = 0; 608 609 count = sys_cputimer->count(); 610 if (TAILQ_FIRST(&gd->gd_systimerq) != NULL) 611 systimer_intr(&count, 0, frame); 612 } 613 614 void 615 pcpu_timer_process(void) 616 { 617 pcpu_timer_process_oncpu(mycpu, NULL); 618 } 619 620 void 621 pcpu_timer_process_frame(struct intrframe *frame) 622 { 623 pcpu_timer_process_oncpu(mycpu, frame); 624 } 625 626 static uint64_t 627 dummy_cpucounter_count(void) 628 { 629 struct timeval tv; 630 631 microuptime(&tv); 632 return ((tv.tv_sec * 1000000ULL) + tv.tv_usec); 633 } 634 635 const struct cpucounter * 636 cpucounter_find_pcpu(void) 637 { 638 const struct cpucounter *cc, *ret; 639 640 ret = &dummy_cpucounter; 641 SLIST_FOREACH(cc, &cpucounterhead, link) { 642 if (cc->prio > ret->prio) 643 ret = cc; 644 } 645 return (ret); 646 } 647 648 const struct cpucounter * 649 cpucounter_find(void) 650 { 651 const struct cpucounter *cc, *ret; 652 653 ret = &dummy_cpucounter; 654 SLIST_FOREACH(cc, &cpucounterhead, link) { 655 if ((cc->flags & CPUCOUNTER_FLAG_MPSYNC) && 656 cc->prio > ret->prio) 657 ret = cc; 658 } 659 KASSERT(ret->flags & CPUCOUNTER_FLAG_MPSYNC, 660 ("cpucounter %u is not MPsync", ret->type)); 661 return (ret); 662 } 663 664 void 665 cpucounter_register(struct cpucounter *cc) 666 { 667 668 SLIST_INSERT_HEAD(&cpucounterhead, cc, link); 669 } 670