1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1982, 1986, 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * (c) UNIX System Laboratories, Inc. 7 * All or some portions of this file are derived from material licensed 8 * to the University of California by American Telephone and Telegraph 9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 10 * the permission of UNIX System Laboratories, Inc. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95 37 */ 38 39 #include <sys/cdefs.h> 40 __FBSDID("$FreeBSD$"); 41 42 #include "opt_ktrace.h" 43 #include "opt_sched.h" 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/condvar.h> 48 #include <sys/kdb.h> 49 #include <sys/kernel.h> 50 #include <sys/ktr.h> 51 #include <sys/lock.h> 52 #include <sys/mutex.h> 53 #include <sys/proc.h> 54 #include <sys/resourcevar.h> 55 #include <sys/sched.h> 56 #include <sys/sdt.h> 57 #include <sys/signalvar.h> 58 #include <sys/sleepqueue.h> 59 #include <sys/smp.h> 60 #include <sys/sx.h> 61 #include <sys/sysctl.h> 62 #include <sys/sysproto.h> 63 #include <sys/vmmeter.h> 64 #ifdef KTRACE 65 #include <sys/uio.h> 66 #include <sys/ktrace.h> 67 #endif 68 69 #include <machine/cpu.h> 70 71 static void synch_setup(void *dummy); 72 SYSINIT(synch_setup, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, synch_setup, 73 NULL); 74 75 int hogticks; 76 static uint8_t pause_wchan[MAXCPU]; 77 78 static struct callout loadav_callout; 79 80 struct loadavg averunnable = 81 { {0, 0, 0}, FSCALE }; /* load average, of runnable procs */ 82 /* 83 * Constants for averages over 1, 5, and 15 minutes 84 * when sampling at 5 second intervals. 85 */ 86 static fixpt_t cexp[3] = { 87 0.9200444146293232 * FSCALE, /* exp(-1/12) */ 88 0.9834714538216174 * FSCALE, /* exp(-1/60) */ 89 0.9944598480048967 * FSCALE, /* exp(-1/180) */ 90 }; 91 92 /* kernel uses `FSCALE', userland (SHOULD) use kern.fscale */ 93 SYSCTL_INT(_kern, OID_AUTO, fscale, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, FSCALE, ""); 94 95 static void loadav(void *arg); 96 97 SDT_PROVIDER_DECLARE(sched); 98 SDT_PROBE_DEFINE(sched, , , preempt); 99 100 static void 101 sleepinit(void *unused) 102 { 103 104 hogticks = (hz / 10) * 2; /* Default only. */ 105 init_sleepqueues(); 106 } 107 108 /* 109 * vmem tries to lock the sleepq mutexes when free'ing kva, so make sure 110 * it is available. 111 */ 112 SYSINIT(sleepinit, SI_SUB_KMEM, SI_ORDER_ANY, sleepinit, NULL); 113 114 /* 115 * General sleep call. Suspends the current thread until a wakeup is 116 * performed on the specified identifier. The thread will then be made 117 * runnable with the specified priority. Sleeps at most sbt units of time 118 * (0 means no timeout). If pri includes the PCATCH flag, let signals 119 * interrupt the sleep, otherwise ignore them while sleeping. Returns 0 if 120 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a 121 * signal becomes pending, ERESTART is returned if the current system 122 * call should be restarted if possible, and EINTR is returned if the system 123 * call should be interrupted by the signal (return EINTR). 124 * 125 * The lock argument is unlocked before the caller is suspended, and 126 * re-locked before _sleep() returns. If priority includes the PDROP 127 * flag the lock is not re-locked before returning. 128 */ 129 int 130 _sleep(void *ident, struct lock_object *lock, int priority, 131 const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags) 132 { 133 struct thread *td; 134 struct lock_class *class; 135 uintptr_t lock_state; 136 int catch, pri, rval, sleepq_flags; 137 WITNESS_SAVE_DECL(lock_witness); 138 139 td = curthread; 140 #ifdef KTRACE 141 if (KTRPOINT(td, KTR_CSW)) 142 ktrcsw(1, 0, wmesg); 143 #endif 144 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 145 "Sleeping on \"%s\"", wmesg); 146 KASSERT(sbt != 0 || mtx_owned(&Giant) || lock != NULL, 147 ("sleeping without a lock")); 148 KASSERT(ident != NULL, ("_sleep: NULL ident")); 149 KASSERT(TD_IS_RUNNING(td), ("_sleep: curthread not running")); 150 KASSERT(td->td_epochnest == 0, ("sleeping in an epoch section")); 151 if (priority & PDROP) 152 KASSERT(lock != NULL && lock != &Giant.lock_object, 153 ("PDROP requires a non-Giant lock")); 154 if (lock != NULL) 155 class = LOCK_CLASS(lock); 156 else 157 class = NULL; 158 159 if (SCHEDULER_STOPPED_TD(td)) { 160 if (lock != NULL && priority & PDROP) 161 class->lc_unlock(lock); 162 return (0); 163 } 164 catch = priority & PCATCH; 165 pri = priority & PRIMASK; 166 167 KASSERT(!TD_ON_SLEEPQ(td), ("recursive sleep")); 168 169 if ((uint8_t *)ident >= &pause_wchan[0] && 170 (uint8_t *)ident <= &pause_wchan[MAXCPU - 1]) 171 sleepq_flags = SLEEPQ_PAUSE; 172 else 173 sleepq_flags = SLEEPQ_SLEEP; 174 if (catch) 175 sleepq_flags |= SLEEPQ_INTERRUPTIBLE; 176 177 sleepq_lock(ident); 178 CTR5(KTR_PROC, "sleep: thread %ld (pid %ld, %s) on %s (%p)", 179 td->td_tid, td->td_proc->p_pid, td->td_name, wmesg, ident); 180 181 if (lock == &Giant.lock_object) 182 mtx_assert(&Giant, MA_OWNED); 183 DROP_GIANT(); 184 if (lock != NULL && lock != &Giant.lock_object && 185 !(class->lc_flags & LC_SLEEPABLE)) { 186 WITNESS_SAVE(lock, lock_witness); 187 lock_state = class->lc_unlock(lock); 188 } else 189 /* GCC needs to follow the Yellow Brick Road */ 190 lock_state = -1; 191 192 /* 193 * We put ourselves on the sleep queue and start our timeout 194 * before calling thread_suspend_check, as we could stop there, 195 * and a wakeup or a SIGCONT (or both) could occur while we were 196 * stopped without resuming us. Thus, we must be ready for sleep 197 * when cursig() is called. If the wakeup happens while we're 198 * stopped, then td will no longer be on a sleep queue upon 199 * return from cursig(). 200 */ 201 sleepq_add(ident, lock, wmesg, sleepq_flags, 0); 202 if (sbt != 0) 203 sleepq_set_timeout_sbt(ident, sbt, pr, flags); 204 if (lock != NULL && class->lc_flags & LC_SLEEPABLE) { 205 sleepq_release(ident); 206 WITNESS_SAVE(lock, lock_witness); 207 lock_state = class->lc_unlock(lock); 208 sleepq_lock(ident); 209 } 210 if (sbt != 0 && catch) 211 rval = sleepq_timedwait_sig(ident, pri); 212 else if (sbt != 0) 213 rval = sleepq_timedwait(ident, pri); 214 else if (catch) 215 rval = sleepq_wait_sig(ident, pri); 216 else { 217 sleepq_wait(ident, pri); 218 rval = 0; 219 } 220 #ifdef KTRACE 221 if (KTRPOINT(td, KTR_CSW)) 222 ktrcsw(0, 0, wmesg); 223 #endif 224 PICKUP_GIANT(); 225 if (lock != NULL && lock != &Giant.lock_object && !(priority & PDROP)) { 226 class->lc_lock(lock, lock_state); 227 WITNESS_RESTORE(lock, lock_witness); 228 } 229 return (rval); 230 } 231 232 int 233 msleep_spin_sbt(void *ident, struct mtx *mtx, const char *wmesg, 234 sbintime_t sbt, sbintime_t pr, int flags) 235 { 236 struct thread *td; 237 int rval; 238 WITNESS_SAVE_DECL(mtx); 239 240 td = curthread; 241 KASSERT(mtx != NULL, ("sleeping without a mutex")); 242 KASSERT(ident != NULL, ("msleep_spin_sbt: NULL ident")); 243 KASSERT(TD_IS_RUNNING(td), ("msleep_spin_sbt: curthread not running")); 244 245 if (SCHEDULER_STOPPED_TD(td)) 246 return (0); 247 248 sleepq_lock(ident); 249 CTR5(KTR_PROC, "msleep_spin: thread %ld (pid %ld, %s) on %s (%p)", 250 td->td_tid, td->td_proc->p_pid, td->td_name, wmesg, ident); 251 252 DROP_GIANT(); 253 mtx_assert(mtx, MA_OWNED | MA_NOTRECURSED); 254 WITNESS_SAVE(&mtx->lock_object, mtx); 255 mtx_unlock_spin(mtx); 256 257 /* 258 * We put ourselves on the sleep queue and start our timeout. 259 */ 260 sleepq_add(ident, &mtx->lock_object, wmesg, SLEEPQ_SLEEP, 0); 261 if (sbt != 0) 262 sleepq_set_timeout_sbt(ident, sbt, pr, flags); 263 264 /* 265 * Can't call ktrace with any spin locks held so it can lock the 266 * ktrace_mtx lock, and WITNESS_WARN considers it an error to hold 267 * any spin lock. Thus, we have to drop the sleepq spin lock while 268 * we handle those requests. This is safe since we have placed our 269 * thread on the sleep queue already. 270 */ 271 #ifdef KTRACE 272 if (KTRPOINT(td, KTR_CSW)) { 273 sleepq_release(ident); 274 ktrcsw(1, 0, wmesg); 275 sleepq_lock(ident); 276 } 277 #endif 278 #ifdef WITNESS 279 sleepq_release(ident); 280 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "Sleeping on \"%s\"", 281 wmesg); 282 sleepq_lock(ident); 283 #endif 284 if (sbt != 0) 285 rval = sleepq_timedwait(ident, 0); 286 else { 287 sleepq_wait(ident, 0); 288 rval = 0; 289 } 290 #ifdef KTRACE 291 if (KTRPOINT(td, KTR_CSW)) 292 ktrcsw(0, 0, wmesg); 293 #endif 294 PICKUP_GIANT(); 295 mtx_lock_spin(mtx); 296 WITNESS_RESTORE(&mtx->lock_object, mtx); 297 return (rval); 298 } 299 300 /* 301 * pause_sbt() delays the calling thread by the given signed binary 302 * time. During cold bootup, pause_sbt() uses the DELAY() function 303 * instead of the _sleep() function to do the waiting. The "sbt" 304 * argument must be greater than or equal to zero. A "sbt" value of 305 * zero is equivalent to a "sbt" value of one tick. 306 */ 307 int 308 pause_sbt(const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags) 309 { 310 KASSERT(sbt >= 0, ("pause_sbt: timeout must be >= 0")); 311 312 /* silently convert invalid timeouts */ 313 if (sbt == 0) 314 sbt = tick_sbt; 315 316 if ((cold && curthread == &thread0) || kdb_active || 317 SCHEDULER_STOPPED()) { 318 /* 319 * We delay one second at a time to avoid overflowing the 320 * system specific DELAY() function(s): 321 */ 322 while (sbt >= SBT_1S) { 323 DELAY(1000000); 324 sbt -= SBT_1S; 325 } 326 /* Do the delay remainder, if any */ 327 sbt = howmany(sbt, SBT_1US); 328 if (sbt > 0) 329 DELAY(sbt); 330 return (EWOULDBLOCK); 331 } 332 return (_sleep(&pause_wchan[curcpu], NULL, 333 (flags & C_CATCH) ? PCATCH : 0, wmesg, sbt, pr, flags)); 334 } 335 336 /* 337 * Make all threads sleeping on the specified identifier runnable. 338 */ 339 void 340 wakeup(void *ident) 341 { 342 int wakeup_swapper; 343 344 sleepq_lock(ident); 345 wakeup_swapper = sleepq_broadcast(ident, SLEEPQ_SLEEP, 0, 0); 346 sleepq_release(ident); 347 if (wakeup_swapper) { 348 KASSERT(ident != &proc0, 349 ("wakeup and wakeup_swapper and proc0")); 350 kick_proc0(); 351 } 352 } 353 354 /* 355 * Make a thread sleeping on the specified identifier runnable. 356 * May wake more than one thread if a target thread is currently 357 * swapped out. 358 */ 359 void 360 wakeup_one(void *ident) 361 { 362 int wakeup_swapper; 363 364 sleepq_lock(ident); 365 wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP, 0, 0); 366 sleepq_release(ident); 367 if (wakeup_swapper) 368 kick_proc0(); 369 } 370 371 static void 372 kdb_switch(void) 373 { 374 thread_unlock(curthread); 375 kdb_backtrace(); 376 kdb_reenter(); 377 panic("%s: did not reenter debugger", __func__); 378 } 379 380 /* 381 * The machine independent parts of context switching. 382 */ 383 void 384 mi_switch(int flags, struct thread *newtd) 385 { 386 uint64_t runtime, new_switchtime; 387 struct thread *td; 388 389 td = curthread; /* XXX */ 390 THREAD_LOCK_ASSERT(td, MA_OWNED | MA_NOTRECURSED); 391 KASSERT(!TD_ON_RUNQ(td), ("mi_switch: called by old code")); 392 #ifdef INVARIANTS 393 if (!TD_ON_LOCK(td) && !TD_IS_RUNNING(td)) 394 mtx_assert(&Giant, MA_NOTOWNED); 395 #endif 396 KASSERT(td->td_critnest == 1 || panicstr, 397 ("mi_switch: switch in a critical section")); 398 KASSERT((flags & (SW_INVOL | SW_VOL)) != 0, 399 ("mi_switch: switch must be voluntary or involuntary")); 400 KASSERT(newtd != curthread, ("mi_switch: preempting back to ourself")); 401 402 /* 403 * Don't perform context switches from the debugger. 404 */ 405 if (kdb_active) 406 kdb_switch(); 407 if (SCHEDULER_STOPPED_TD(td)) 408 return; 409 if (flags & SW_VOL) { 410 td->td_ru.ru_nvcsw++; 411 td->td_swvoltick = ticks; 412 } else { 413 td->td_ru.ru_nivcsw++; 414 td->td_swinvoltick = ticks; 415 } 416 #ifdef SCHED_STATS 417 SCHED_STAT_INC(sched_switch_stats[flags & SW_TYPE_MASK]); 418 #endif 419 /* 420 * Compute the amount of time during which the current 421 * thread was running, and add that to its total so far. 422 */ 423 new_switchtime = cpu_ticks(); 424 runtime = new_switchtime - PCPU_GET(switchtime); 425 td->td_runtime += runtime; 426 td->td_incruntime += runtime; 427 PCPU_SET(switchtime, new_switchtime); 428 td->td_generation++; /* bump preempt-detect counter */ 429 VM_CNT_INC(v_swtch); 430 PCPU_SET(switchticks, ticks); 431 CTR4(KTR_PROC, "mi_switch: old thread %ld (td_sched %p, pid %ld, %s)", 432 td->td_tid, td_get_sched(td), td->td_proc->p_pid, td->td_name); 433 #ifdef KDTRACE_HOOKS 434 if (__predict_false(sdt_probes_enabled) && 435 ((flags & SW_PREEMPT) != 0 || ((flags & SW_INVOL) != 0 && 436 (flags & SW_TYPE_MASK) == SWT_NEEDRESCHED))) 437 SDT_PROBE0(sched, , , preempt); 438 #endif 439 sched_switch(td, newtd, flags); 440 CTR4(KTR_PROC, "mi_switch: new thread %ld (td_sched %p, pid %ld, %s)", 441 td->td_tid, td_get_sched(td), td->td_proc->p_pid, td->td_name); 442 443 /* 444 * If the last thread was exiting, finish cleaning it up. 445 */ 446 if ((td = PCPU_GET(deadthread))) { 447 PCPU_SET(deadthread, NULL); 448 thread_stash(td); 449 } 450 } 451 452 /* 453 * Change thread state to be runnable, placing it on the run queue if 454 * it is in memory. If it is swapped out, return true so our caller 455 * will know to awaken the swapper. 456 */ 457 int 458 setrunnable(struct thread *td) 459 { 460 461 THREAD_LOCK_ASSERT(td, MA_OWNED); 462 KASSERT(td->td_proc->p_state != PRS_ZOMBIE, 463 ("setrunnable: pid %d is a zombie", td->td_proc->p_pid)); 464 switch (td->td_state) { 465 case TDS_RUNNING: 466 case TDS_RUNQ: 467 return (0); 468 case TDS_INHIBITED: 469 /* 470 * If we are only inhibited because we are swapped out 471 * then arange to swap in this process. Otherwise just return. 472 */ 473 if (td->td_inhibitors != TDI_SWAPPED) 474 return (0); 475 /* FALLTHROUGH */ 476 case TDS_CAN_RUN: 477 break; 478 default: 479 printf("state is 0x%x", td->td_state); 480 panic("setrunnable(2)"); 481 } 482 if ((td->td_flags & TDF_INMEM) == 0) { 483 if ((td->td_flags & TDF_SWAPINREQ) == 0) { 484 td->td_flags |= TDF_SWAPINREQ; 485 return (1); 486 } 487 } else 488 sched_wakeup(td); 489 return (0); 490 } 491 492 /* 493 * Compute a tenex style load average of a quantity on 494 * 1, 5 and 15 minute intervals. 495 */ 496 static void 497 loadav(void *arg) 498 { 499 int i, nrun; 500 struct loadavg *avg; 501 502 nrun = sched_load(); 503 avg = &averunnable; 504 505 for (i = 0; i < 3; i++) 506 avg->ldavg[i] = (cexp[i] * avg->ldavg[i] + 507 nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT; 508 509 /* 510 * Schedule the next update to occur after 5 seconds, but add a 511 * random variation to avoid synchronisation with processes that 512 * run at regular intervals. 513 */ 514 callout_reset_sbt(&loadav_callout, 515 SBT_1US * (4000000 + (int)(random() % 2000001)), SBT_1US, 516 loadav, NULL, C_DIRECT_EXEC | C_PREL(32)); 517 } 518 519 /* ARGSUSED */ 520 static void 521 synch_setup(void *dummy) 522 { 523 callout_init(&loadav_callout, 1); 524 525 /* Kick off timeout driven events by calling first time. */ 526 loadav(NULL); 527 } 528 529 int 530 should_yield(void) 531 { 532 533 return ((u_int)ticks - (u_int)curthread->td_swvoltick >= hogticks); 534 } 535 536 void 537 maybe_yield(void) 538 { 539 540 if (should_yield()) 541 kern_yield(PRI_USER); 542 } 543 544 void 545 kern_yield(int prio) 546 { 547 struct thread *td; 548 549 td = curthread; 550 DROP_GIANT(); 551 thread_lock(td); 552 if (prio == PRI_USER) 553 prio = td->td_user_pri; 554 if (prio >= 0) 555 sched_prio(td, prio); 556 mi_switch(SW_VOL | SWT_RELINQUISH, NULL); 557 thread_unlock(td); 558 PICKUP_GIANT(); 559 } 560 561 /* 562 * General purpose yield system call. 563 */ 564 int 565 sys_yield(struct thread *td, struct yield_args *uap) 566 { 567 568 thread_lock(td); 569 if (PRI_BASE(td->td_pri_class) == PRI_TIMESHARE) 570 sched_prio(td, PRI_MAX_TIMESHARE); 571 mi_switch(SW_VOL | SWT_RELINQUISH, NULL); 572 thread_unlock(td); 573 td->td_retval[0] = 0; 574 return (0); 575 } 576