1 /* 2 * Copyright (c) 2005 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 /* 35 * Generic cputimer - access to a reliable, free-running counter. 36 */ 37 38 #include <sys/param.h> 39 #include <sys/kernel.h> 40 #include <sys/systm.h> 41 #include <sys/thread.h> 42 #include <sys/globaldata.h> 43 #include <sys/serialize.h> 44 #include <sys/systimer.h> 45 #include <sys/sysctl.h> 46 47 extern void pcpu_timer_process(void); 48 extern void pcpu_timer_process_frame(struct intrframe *); 49 50 static uint64_t dummy_cpucounter_count(void); 51 52 static sysclock_t dummy_cputimer_count(void); 53 54 static struct cputimer dummy_cputimer = { 55 .next = SLIST_ENTRY_INITIALIZER, 56 .name = "dummy", 57 .pri = CPUTIMER_PRI_DUMMY, 58 .type = CPUTIMER_DUMMY, 59 .count = dummy_cputimer_count, 60 .fromhz = cputimer_default_fromhz, 61 .fromus = cputimer_default_fromus, 62 .construct = cputimer_default_construct, 63 .destruct = cputimer_default_destruct, 64 .freq = 1000000, 65 .freq64_usec = (1000000LL << 32) / 1000000, 66 .freq64_nsec = (1000000000LL << 32) / 1000000 67 }; 68 69 static struct cpucounter dummy_cpucounter = { 70 .freq = 1000000ULL, 71 .count = dummy_cpucounter_count, 72 .flags = CPUCOUNTER_FLAG_MPSYNC, 73 .prio = CPUCOUNTER_PRIO_DUMMY, 74 .type = CPUCOUNTER_DUMMY 75 }; 76 77 struct cputimer *sys_cputimer = &dummy_cputimer; 78 SLIST_HEAD(, cputimer) cputimerhead = SLIST_HEAD_INITIALIZER(&cputimerhead); 79 80 static SLIST_HEAD(, cpucounter) cpucounterhead = 81 SLIST_HEAD_INITIALIZER(cpucounterhead); 82 83 static int cputimer_intr_ps_reqs; 84 static struct lwkt_serialize cputimer_intr_ps_slize = 85 LWKT_SERIALIZE_INITIALIZER; 86 87 /* 88 * Generic cputimer API 89 */ 90 void 91 cputimer_select(struct cputimer *timer, int pri) 92 { 93 sysclock_t oldclock; 94 95 /* 96 * Calculate helper fields 97 */ 98 cputimer_set_frequency(timer, timer->freq); 99 100 /* 101 * Install a new cputimer if its priority allows it. If timer is 102 * passed as NULL we deinstall the current timer and revert to our 103 * dummy. 104 */ 105 if (pri == 0) 106 pri = timer->pri; 107 if (timer == NULL || pri >= sys_cputimer->pri) { 108 oldclock = sys_cputimer->count(); 109 sys_cputimer->destruct(sys_cputimer); 110 sys_cputimer = &dummy_cputimer; 111 if (timer) { 112 sys_cputimer = timer; 113 timer->construct(timer, oldclock); 114 cputimer_intr_config(timer); 115 } 116 } 117 } 118 119 /* 120 * Register a timer. If the timer has already been registered, do nothing. 121 */ 122 void 123 cputimer_register(struct cputimer *timer) 124 { 125 struct cputimer *scan; 126 127 /* 128 * Initialize dummy_cputimer if the slist is empty, it does not get 129 * registered the normal way. 130 */ 131 if (SLIST_EMPTY(&cputimerhead)) 132 SLIST_FIRST(&cputimerhead) = &dummy_cputimer; 133 SLIST_FOREACH(scan, &cputimerhead, next) { 134 if (scan == timer) 135 return; 136 } 137 SLIST_INSERT_HEAD(&cputimerhead, timer, next); 138 } 139 140 /* 141 * Deregister a timer. If the timer has already been deregistered, do nothing. 142 */ 143 void 144 cputimer_deregister(struct cputimer *timer) 145 { 146 struct cputimer *scan; 147 struct cputimer *best; 148 149 /* 150 * Locate and remove the timer. If the timer is our currently active 151 * timer, revert to the dummy timer. 152 */ 153 SLIST_FOREACH(scan, &cputimerhead, next) { 154 if (timer == scan) { 155 if (timer == sys_cputimer) 156 cputimer_select(&dummy_cputimer, 0x7FFFFFFF); 157 SLIST_REMOVE(&cputimerhead, timer, cputimer, next); 158 break; 159 } 160 } 161 162 /* 163 * If sys_cputimer reverted to the dummy, select the best one 164 */ 165 if (sys_cputimer == &dummy_cputimer) { 166 best = NULL; 167 SLIST_FOREACH(scan, &cputimerhead, next) { 168 if (best == NULL || scan->pri > best->pri) 169 best = scan; 170 } 171 if (best) 172 cputimer_select(best, 0x7FFFFFFF); 173 } 174 } 175 176 /* 177 * Calculate usec / tick and nsec / tick, scaled by (1 << 32). 178 * 179 * so e.g. a 3 mhz timer would be 3 usec / tick x (1 << 32), 180 * or 3000 nsec / tick x (1 << 32) 181 */ 182 void 183 cputimer_set_frequency(struct cputimer *timer, sysclock_t freq) 184 { 185 timer->freq = freq; 186 timer->freq64_usec = (1000000LL << 32) / freq; 187 timer->freq64_nsec = (1000000000LL << 32) / freq; 188 if (timer == sys_cputimer) 189 cputimer_intr_config(timer); 190 } 191 192 sysclock_t 193 cputimer_default_fromhz(int freq) 194 { 195 return(sys_cputimer->freq / freq + 1); 196 } 197 198 sysclock_t 199 cputimer_default_fromus(int us) 200 { 201 return((int64_t)sys_cputimer->freq * us / 1000000); 202 } 203 204 /* 205 * Dummy counter implementation 206 */ 207 static 208 sysclock_t 209 dummy_cputimer_count(void) 210 { 211 return(++dummy_cputimer.base); 212 } 213 214 void 215 cputimer_default_construct(struct cputimer *cputimer, sysclock_t oldclock) 216 { 217 cputimer->base = oldclock; 218 } 219 220 void 221 cputimer_default_destruct(struct cputimer *cputimer) 222 { 223 } 224 225 /************************************************************************ 226 * SYSCTL SUPPORT * 227 ************************************************************************ 228 * 229 * Note: the ability to change the systimer is not currently enabled 230 * because it will mess up systimer calculations. You have to live 231 * with what is configured at boot. 232 */ 233 static int 234 sysctl_cputimer_reglist(SYSCTL_HANDLER_ARGS) 235 { 236 struct cputimer *scan; 237 int error = 0; 238 int loop = 0; 239 240 /* 241 * Build a list of available timers 242 */ 243 SLIST_FOREACH(scan, &cputimerhead, next) { 244 if (error == 0 && loop) 245 error = SYSCTL_OUT(req, " ", 1); 246 if (error == 0) 247 error = SYSCTL_OUT(req, scan->name, strlen(scan->name)); 248 ++loop; 249 } 250 return (error); 251 } 252 253 static int 254 sysctl_cputimer_name(SYSCTL_HANDLER_ARGS) 255 { 256 int error; 257 258 error = SYSCTL_OUT(req, sys_cputimer->name, strlen(sys_cputimer->name)); 259 return (error); 260 } 261 262 static int 263 sysctl_cputimer_clock(SYSCTL_HANDLER_ARGS) 264 { 265 sysclock_t clock; 266 int error; 267 268 clock = sys_cputimer->count(); 269 error = SYSCTL_OUT(req, &clock, sizeof(clock)); 270 return (error); 271 } 272 273 static int 274 sysctl_cputimer_freq(SYSCTL_HANDLER_ARGS) 275 { 276 int error; 277 278 error = SYSCTL_OUT(req, &sys_cputimer->freq, sizeof(sys_cputimer->freq)); 279 return (error); 280 } 281 282 SYSCTL_DECL(_kern_cputimer); 283 SYSCTL_NODE(_kern, OID_AUTO, cputimer, CTLFLAG_RW, NULL, "cputimer"); 284 285 SYSCTL_PROC(_kern_cputimer, OID_AUTO, select, CTLTYPE_STRING|CTLFLAG_RD, 286 NULL, 0, sysctl_cputimer_reglist, "A", ""); 287 SYSCTL_PROC(_kern_cputimer, OID_AUTO, name, CTLTYPE_STRING|CTLFLAG_RD, 288 NULL, 0, sysctl_cputimer_name, "A", ""); 289 SYSCTL_PROC(_kern_cputimer, OID_AUTO, clock, CTLTYPE_UINT|CTLFLAG_RD, 290 NULL, 0, sysctl_cputimer_clock, "IU", ""); 291 SYSCTL_PROC(_kern_cputimer, OID_AUTO, freq, CTLTYPE_INT|CTLFLAG_RD, 292 NULL, 0, sysctl_cputimer_freq, "I", ""); 293 294 static struct cputimer_intr *sys_cputimer_intr; 295 static uint32_t cputimer_intr_caps; 296 SLIST_HEAD(, cputimer_intr) cputimer_intr_head = 297 SLIST_HEAD_INITIALIZER(&cputimer_intr_head); 298 299 void 300 cputimer_intr_register(struct cputimer_intr *cti) 301 { 302 struct cputimer_intr *scan; 303 304 SLIST_FOREACH(scan, &cputimer_intr_head, next) { 305 if (scan == cti) 306 return; 307 } 308 cti->config(cti, sys_cputimer); 309 SLIST_INSERT_HEAD(&cputimer_intr_head, cti, next); 310 } 311 312 void 313 cputimer_intr_deregister(struct cputimer_intr *cti) 314 { 315 KKASSERT(cti != sys_cputimer_intr); 316 SLIST_REMOVE(&cputimer_intr_head, cti, cputimer_intr, next); 317 } 318 319 int 320 cputimer_intr_select(struct cputimer_intr *cti, int prio) 321 { 322 KKASSERT(cti != NULL); 323 324 if (prio == 0) 325 prio = cti->prio; 326 327 if (sys_cputimer_intr == NULL) { 328 KKASSERT(cputimer_intr_caps == 0); 329 sys_cputimer_intr = cti; 330 return 0; 331 } 332 333 if ((cti->caps & cputimer_intr_caps) == cputimer_intr_caps) { 334 if (prio > sys_cputimer_intr->prio) { 335 sys_cputimer_intr = cti; 336 return 0; 337 } else { 338 return EBUSY; 339 } 340 } else { 341 return EOPNOTSUPP; 342 } 343 } 344 345 void 346 cputimer_intr_default_enable(struct cputimer_intr *cti __unused) 347 { 348 } 349 350 void 351 cputimer_intr_default_restart(struct cputimer_intr *cti) 352 { 353 cti->reload(cti, 0); 354 } 355 356 void 357 cputimer_intr_default_config(struct cputimer_intr *cti __unused, 358 const struct cputimer *timer __unused) 359 { 360 } 361 362 void 363 cputimer_intr_default_pmfixup(struct cputimer_intr *cti __unused) 364 { 365 } 366 367 void 368 cputimer_intr_default_initclock(struct cputimer_intr *cti __unused, 369 boolean_t selected __unused) 370 { 371 } 372 373 void 374 cputimer_intr_enable(void) 375 { 376 struct cputimer_intr *cti; 377 378 SLIST_FOREACH(cti, &cputimer_intr_head, next) 379 cti->enable(cti); 380 } 381 382 void 383 cputimer_intr_config(const struct cputimer *timer) 384 { 385 struct cputimer_intr *cti; 386 387 SLIST_FOREACH(cti, &cputimer_intr_head, next) 388 cti->config(cti, timer); 389 } 390 391 void 392 cputimer_intr_pmfixup(void) 393 { 394 struct cputimer_intr *cti; 395 396 SLIST_FOREACH(cti, &cputimer_intr_head, next) 397 cti->pmfixup(cti); 398 } 399 400 void 401 cputimer_intr_reload(sysclock_t reload) 402 { 403 struct cputimer_intr *cti = sys_cputimer_intr; 404 405 cti->reload(cti, reload); 406 } 407 408 void 409 cputimer_intr_restart(void) 410 { 411 struct cputimer_intr *cti = sys_cputimer_intr; 412 413 cti->restart(cti); 414 } 415 416 int 417 cputimer_intr_select_caps(uint32_t caps) 418 { 419 struct cputimer_intr *cti, *maybe; 420 int error; 421 422 maybe = NULL; 423 SLIST_FOREACH(cti, &cputimer_intr_head, next) { 424 if ((cti->caps & caps) == caps) { 425 if (maybe == NULL) 426 maybe = cti; 427 else if (cti->prio > maybe->prio) 428 maybe = cti; 429 } 430 } 431 if (maybe == NULL) 432 return ENOENT; 433 434 if (sys_cputimer_intr == maybe) 435 return 0; 436 437 cputimer_intr_caps = caps; 438 error = cputimer_intr_select(maybe, CPUTIMER_INTR_PRIO_MAX); 439 KKASSERT(!error); 440 441 return ERESTART; 442 } 443 444 static void 445 cputimer_intr_initclocks(void) 446 { 447 struct cputimer_intr *cti, *ncti; 448 449 /* 450 * An interrupt cputimer may deregister itself, 451 * so use SLIST_FOREACH_MUTABLE here. 452 */ 453 SLIST_FOREACH_MUTABLE(cti, &cputimer_intr_head, next, ncti) { 454 boolean_t selected = FALSE; 455 456 if (cti == sys_cputimer_intr) 457 selected = TRUE; 458 cti->initclock(cti, selected); 459 } 460 } 461 /* NOTE: Must be SECOND to allow platform initialization to go first */ 462 SYSINIT(cputimer_intr, SI_BOOT2_CLOCKREG, SI_ORDER_SECOND, 463 cputimer_intr_initclocks, NULL); 464 465 static int 466 sysctl_cputimer_intr_reglist(SYSCTL_HANDLER_ARGS) 467 { 468 struct cputimer_intr *scan; 469 int error = 0; 470 int loop = 0; 471 472 /* 473 * Build a list of available interrupt cputimers 474 */ 475 SLIST_FOREACH(scan, &cputimer_intr_head, next) { 476 if (error == 0 && loop) 477 error = SYSCTL_OUT(req, " ", 1); 478 if (error == 0) 479 error = SYSCTL_OUT(req, scan->name, strlen(scan->name)); 480 ++loop; 481 } 482 return (error); 483 } 484 485 static int 486 sysctl_cputimer_intr_freq(SYSCTL_HANDLER_ARGS) 487 { 488 int error; 489 490 error = SYSCTL_OUT(req, &sys_cputimer_intr->freq, 491 sizeof(sys_cputimer_intr->freq)); 492 return (error); 493 } 494 495 static int 496 sysctl_cputimer_intr_select(SYSCTL_HANDLER_ARGS) 497 { 498 struct cputimer_intr *cti; 499 char name[32]; 500 int error; 501 502 ksnprintf(name, sizeof(name), "%s", sys_cputimer_intr->name); 503 error = sysctl_handle_string(oidp, name, sizeof(name), req); 504 if (error != 0 || req->newptr == NULL) 505 return error; 506 507 SLIST_FOREACH(cti, &cputimer_intr_head, next) { 508 if (strcmp(cti->name, name) == 0) 509 break; 510 } 511 if (cti == NULL) 512 return ENOENT; 513 if (cti == sys_cputimer_intr) 514 return 0; 515 516 error = cputimer_intr_select(cti, CPUTIMER_INTR_PRIO_MAX); 517 if (!error) 518 cputimer_intr_restart(); 519 return error; 520 } 521 522 SYSCTL_NODE(_kern_cputimer, OID_AUTO, intr, CTLFLAG_RW, NULL, 523 "interrupt cputimer"); 524 525 SYSCTL_PROC(_kern_cputimer_intr, OID_AUTO, reglist, CTLTYPE_STRING|CTLFLAG_RD, 526 NULL, 0, sysctl_cputimer_intr_reglist, "A", ""); 527 SYSCTL_PROC(_kern_cputimer_intr, OID_AUTO, freq, CTLTYPE_INT|CTLFLAG_RD, 528 NULL, 0, sysctl_cputimer_intr_freq, "I", ""); 529 SYSCTL_PROC(_kern_cputimer_intr, OID_AUTO, select, CTLTYPE_STRING|CTLFLAG_RW, 530 NULL, 0, sysctl_cputimer_intr_select, "A", ""); 531 532 int 533 cputimer_intr_powersave_addreq(void) 534 { 535 int error = 0; 536 537 lwkt_serialize_enter(&cputimer_intr_ps_slize); 538 539 ++cputimer_intr_ps_reqs; 540 if (cputimer_intr_ps_reqs == 1) { 541 /* 542 * Upon the first power saving request, switch to an one shot 543 * timer, which would not stop in the any power saving state. 544 */ 545 error = cputimer_intr_select_caps(CPUTIMER_INTR_CAP_PS); 546 if (error == ERESTART) { 547 error = 0; 548 if (bootverbose) 549 kprintf("cputimer: first power save request, restart\n"); 550 cputimer_intr_restart(); 551 } else if (error) { 552 kprintf("no suitable intr cputimer found\n"); 553 --cputimer_intr_ps_reqs; 554 } else if (bootverbose) { 555 kprintf("cputimer: first power save request\n"); 556 } 557 } 558 559 lwkt_serialize_exit(&cputimer_intr_ps_slize); 560 561 return error; 562 } 563 564 void 565 cputimer_intr_powersave_remreq(void) 566 { 567 lwkt_serialize_enter(&cputimer_intr_ps_slize); 568 569 KASSERT(cputimer_intr_ps_reqs > 0, 570 ("invalid # of powersave reqs %d", cputimer_intr_ps_reqs)); 571 --cputimer_intr_ps_reqs; 572 if (cputimer_intr_ps_reqs == 0) { 573 int error; 574 575 /* No one needs power saving, use a better one shot timer. */ 576 error = cputimer_intr_select_caps(CPUTIMER_INTR_CAP_NONE); 577 KKASSERT(!error || error == ERESTART); 578 if (error == ERESTART) { 579 if (bootverbose) 580 kprintf("cputimer: no powser save request, restart\n"); 581 cputimer_intr_restart(); 582 } else if (bootverbose) { 583 kprintf("cputimer: no power save request\n"); 584 } 585 } 586 587 lwkt_serialize_exit(&cputimer_intr_ps_slize); 588 } 589 590 static __inline void 591 cputimer_intr_pcpuhand(void) 592 { 593 struct cputimer_intr *cti = sys_cputimer_intr; 594 595 if (cti->pcpuhand != NULL) 596 cti->pcpuhand(cti); 597 } 598 599 static void 600 pcpu_timer_process_oncpu(struct globaldata *gd, struct intrframe *frame) 601 { 602 sysclock_t count; 603 604 cputimer_intr_pcpuhand(); 605 606 gd->gd_timer_running = 0; 607 608 count = sys_cputimer->count(); 609 if (TAILQ_FIRST(&gd->gd_systimerq) != NULL) 610 systimer_intr(&count, 0, frame); 611 } 612 613 void 614 pcpu_timer_process(void) 615 { 616 pcpu_timer_process_oncpu(mycpu, NULL); 617 } 618 619 void 620 pcpu_timer_process_frame(struct intrframe *frame) 621 { 622 pcpu_timer_process_oncpu(mycpu, frame); 623 } 624 625 static uint64_t 626 dummy_cpucounter_count(void) 627 { 628 struct timeval tv; 629 630 microuptime(&tv); 631 return ((tv.tv_sec * 1000000ULL) + tv.tv_usec); 632 } 633 634 const struct cpucounter * 635 cpucounter_find_pcpu(void) 636 { 637 const struct cpucounter *cc, *ret; 638 639 ret = &dummy_cpucounter; 640 SLIST_FOREACH(cc, &cpucounterhead, link) { 641 if (cc->prio > ret->prio) 642 ret = cc; 643 } 644 return (ret); 645 } 646 647 const struct cpucounter * 648 cpucounter_find(void) 649 { 650 const struct cpucounter *cc, *ret; 651 652 ret = &dummy_cpucounter; 653 SLIST_FOREACH(cc, &cpucounterhead, link) { 654 if ((cc->flags & CPUCOUNTER_FLAG_MPSYNC) && 655 cc->prio > ret->prio) 656 ret = cc; 657 } 658 KASSERT(ret->flags & CPUCOUNTER_FLAG_MPSYNC, 659 ("cpucounter %u is not MPsync", ret->type)); 660 return (ret); 661 } 662 663 void 664 cpucounter_register(struct cpucounter *cc) 665 { 666 667 SLIST_INSERT_HEAD(&cpucounterhead, cc, link); 668 } 669