1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 #include "opt_ktrace.h" 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/limits.h> 35 #include <sys/lock.h> 36 #include <sys/mutex.h> 37 #include <sys/proc.h> 38 #include <sys/kernel.h> 39 #include <sys/ktr.h> 40 #include <sys/ktrace.h> 41 #include <sys/condvar.h> 42 #include <sys/sched.h> 43 #include <sys/signalvar.h> 44 #include <sys/sleepqueue.h> 45 #include <sys/resourcevar.h> 46 #ifdef KTRACE 47 #include <sys/uio.h> 48 #include <sys/user.h> 49 #endif 50 51 /* 52 * A bound below which cv_waiters is valid. Once cv_waiters reaches this bound, 53 * cv_signal must manually check the wait queue for threads. 54 */ 55 #define CV_WAITERS_BOUND INT_MAX 56 57 #define CV_WAITERS_INC(cvp) do { \ 58 if ((cvp)->cv_waiters < CV_WAITERS_BOUND) \ 59 (cvp)->cv_waiters++; \ 60 } while (0) 61 62 /* 63 * Common sanity checks for cv_wait* functions. 64 */ 65 #define CV_ASSERT(cvp, lock, td) do { \ 66 KASSERT((td) != NULL, ("%s: td NULL", __func__)); \ 67 KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__)); \ 68 KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__)); \ 69 KASSERT((lock) != NULL, ("%s: lock NULL", __func__)); \ 70 } while (0) 71 72 /* 73 * Initialize a condition variable. Must be called before use. 74 */ 75 void 76 cv_init(struct cv *cvp, const char *desc) 77 { 78 79 cvp->cv_description = desc; 80 cvp->cv_waiters = 0; 81 } 82 83 /* 84 * Destroy a condition variable. The condition variable must be re-initialized 85 * in order to be re-used. 86 */ 87 void 88 cv_destroy(struct cv *cvp) 89 { 90 #ifdef INVARIANTS 91 struct sleepqueue *sq; 92 93 sleepq_lock(cvp); 94 sq = sleepq_lookup(cvp); 95 sleepq_release(cvp); 96 KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__)); 97 #endif 98 } 99 100 /* 101 * Wait on a condition variable. The current thread is placed on the condition 102 * variable's wait queue and suspended. A cv_signal or cv_broadcast on the same 103 * condition variable will resume the thread. The mutex is released before 104 * sleeping and will be held on return. It is recommended that the mutex be 105 * held when cv_signal or cv_broadcast are called. 106 */ 107 void 108 _cv_wait(struct cv *cvp, struct lock_object *lock) 109 { 110 WITNESS_SAVE_DECL(lock_witness); 111 #ifdef KTRACE 112 char wmesg[WMESGLEN + 1]; 113 #endif 114 struct lock_class *class; 115 struct thread *td __ktrace_used; 116 uintptr_t lock_state; 117 118 td = curthread; 119 CV_ASSERT(cvp, lock, td); 120 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 121 "Waiting on \"%s\"", cvp->cv_description); 122 123 if (SCHEDULER_STOPPED()) 124 return; 125 126 #ifdef KTRACE 127 if (KTRPOINT(td, KTR_CSW)) { 128 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg)); 129 ktrcsw(1, 0, wmesg); 130 } else { 131 wmesg[0] = '\0'; 132 } 133 #endif 134 135 class = LOCK_CLASS(lock); 136 lock_state = 0; 137 sleepq_lock(cvp); 138 139 CV_WAITERS_INC(cvp); 140 if (lock == &Giant.lock_object) 141 mtx_assert(&Giant, MA_OWNED); 142 DROP_GIANT(); 143 144 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 145 if (lock != &Giant.lock_object) { 146 if (class->lc_flags & LC_SLEEPABLE) 147 sleepq_release(cvp); 148 WITNESS_SAVE(lock, lock_witness); 149 lock_state = class->lc_unlock(lock); 150 if (class->lc_flags & LC_SLEEPABLE) 151 sleepq_lock(cvp); 152 } 153 sleepq_wait(cvp, 0); 154 155 #ifdef KTRACE 156 if (KTRPOINT(td, KTR_CSW)) 157 ktrcsw(0, 0, wmesg); 158 #endif 159 PICKUP_GIANT(); 160 if (lock != &Giant.lock_object) { 161 class->lc_lock(lock, lock_state); 162 WITNESS_RESTORE(lock, lock_witness); 163 } 164 } 165 166 /* 167 * Wait on a condition variable. This function differs from cv_wait by 168 * not acquiring the mutex after condition variable was signaled. 169 */ 170 void 171 _cv_wait_unlock(struct cv *cvp, struct lock_object *lock) 172 { 173 #ifdef KTRACE 174 char wmesg[WMESGLEN + 1]; 175 #endif 176 struct lock_class *class; 177 struct thread *td __ktrace_used; 178 179 td = curthread; 180 CV_ASSERT(cvp, lock, td); 181 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 182 "Waiting on \"%s\"", cvp->cv_description); 183 KASSERT(lock != &Giant.lock_object, 184 ("cv_wait_unlock cannot be used with Giant")); 185 class = LOCK_CLASS(lock); 186 187 if (SCHEDULER_STOPPED()) { 188 class->lc_unlock(lock); 189 return; 190 } 191 192 #ifdef KTRACE 193 if (KTRPOINT(td, KTR_CSW)) { 194 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg)); 195 ktrcsw(1, 0, wmesg); 196 } else { 197 wmesg[0] = '\0'; 198 } 199 #endif 200 201 sleepq_lock(cvp); 202 203 CV_WAITERS_INC(cvp); 204 DROP_GIANT(); 205 206 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 207 if (class->lc_flags & LC_SLEEPABLE) 208 sleepq_release(cvp); 209 class->lc_unlock(lock); 210 if (class->lc_flags & LC_SLEEPABLE) 211 sleepq_lock(cvp); 212 sleepq_wait(cvp, 0); 213 214 #ifdef KTRACE 215 if (KTRPOINT(td, KTR_CSW)) 216 ktrcsw(0, 0, wmesg); 217 #endif 218 PICKUP_GIANT(); 219 } 220 221 /* 222 * Wait on a condition variable, allowing interruption by signals. Return 0 if 223 * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if 224 * a signal was caught. If ERESTART is returned the system call should be 225 * restarted if possible. 226 */ 227 int 228 _cv_wait_sig(struct cv *cvp, struct lock_object *lock) 229 { 230 WITNESS_SAVE_DECL(lock_witness); 231 #ifdef KTRACE 232 char wmesg[WMESGLEN + 1]; 233 #endif 234 struct lock_class *class; 235 struct thread *td __ktrace_used; 236 uintptr_t lock_state; 237 int rval; 238 239 td = curthread; 240 CV_ASSERT(cvp, lock, td); 241 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 242 "Waiting on \"%s\"", cvp->cv_description); 243 244 if (SCHEDULER_STOPPED()) 245 return (0); 246 247 #ifdef KTRACE 248 if (KTRPOINT(td, KTR_CSW)) { 249 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg)); 250 ktrcsw(1, 0, wmesg); 251 } else { 252 wmesg[0] = '\0'; 253 } 254 #endif 255 256 class = LOCK_CLASS(lock); 257 lock_state = 0; 258 sleepq_lock(cvp); 259 260 CV_WAITERS_INC(cvp); 261 if (lock == &Giant.lock_object) 262 mtx_assert(&Giant, MA_OWNED); 263 DROP_GIANT(); 264 265 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 266 SLEEPQ_INTERRUPTIBLE, 0); 267 if (lock != &Giant.lock_object) { 268 if (class->lc_flags & LC_SLEEPABLE) 269 sleepq_release(cvp); 270 WITNESS_SAVE(lock, lock_witness); 271 lock_state = class->lc_unlock(lock); 272 if (class->lc_flags & LC_SLEEPABLE) 273 sleepq_lock(cvp); 274 } 275 rval = sleepq_wait_sig(cvp, 0); 276 277 #ifdef KTRACE 278 if (KTRPOINT(td, KTR_CSW)) 279 ktrcsw(0, 0, wmesg); 280 #endif 281 PICKUP_GIANT(); 282 if (lock != &Giant.lock_object) { 283 class->lc_lock(lock, lock_state); 284 WITNESS_RESTORE(lock, lock_witness); 285 } 286 287 return (rval); 288 } 289 290 /* 291 * Wait on a condition variable for (at most) the value specified in sbt 292 * argument. Returns 0 if the process was resumed by cv_signal or cv_broadcast, 293 * EWOULDBLOCK if the timeout expires. 294 */ 295 int 296 _cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt, 297 sbintime_t pr, int flags) 298 { 299 WITNESS_SAVE_DECL(lock_witness); 300 #ifdef KTRACE 301 char wmesg[WMESGLEN + 1]; 302 #endif 303 struct lock_class *class; 304 struct thread *td __ktrace_used; 305 int lock_state, rval; 306 307 td = curthread; 308 CV_ASSERT(cvp, lock, td); 309 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 310 "Waiting on \"%s\"", cvp->cv_description); 311 312 if (SCHEDULER_STOPPED()) 313 return (0); 314 315 #ifdef KTRACE 316 if (KTRPOINT(td, KTR_CSW)) { 317 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg)); 318 ktrcsw(1, 0, wmesg); 319 } else { 320 wmesg[0] = '\0'; 321 } 322 #endif 323 324 class = LOCK_CLASS(lock); 325 lock_state = 0; 326 sleepq_lock(cvp); 327 328 CV_WAITERS_INC(cvp); 329 if (lock == &Giant.lock_object) 330 mtx_assert(&Giant, MA_OWNED); 331 DROP_GIANT(); 332 333 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 334 sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 335 if (lock != &Giant.lock_object) { 336 if (class->lc_flags & LC_SLEEPABLE) 337 sleepq_release(cvp); 338 WITNESS_SAVE(lock, lock_witness); 339 lock_state = class->lc_unlock(lock); 340 if (class->lc_flags & LC_SLEEPABLE) 341 sleepq_lock(cvp); 342 } 343 rval = sleepq_timedwait(cvp, 0); 344 345 #ifdef KTRACE 346 if (KTRPOINT(td, KTR_CSW)) 347 ktrcsw(0, 0, wmesg); 348 #endif 349 PICKUP_GIANT(); 350 if (lock != &Giant.lock_object) { 351 class->lc_lock(lock, lock_state); 352 WITNESS_RESTORE(lock, lock_witness); 353 } 354 355 return (rval); 356 } 357 358 /* 359 * Wait on a condition variable for (at most) the value specified in sbt 360 * argument, allowing interruption by signals. 361 * Returns 0 if the thread was resumed by cv_signal or cv_broadcast, 362 * EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if a signal 363 * was caught. 364 */ 365 int 366 _cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock, 367 sbintime_t sbt, sbintime_t pr, int flags) 368 { 369 WITNESS_SAVE_DECL(lock_witness); 370 #ifdef KTRACE 371 char wmesg[WMESGLEN + 1]; 372 #endif 373 struct lock_class *class; 374 struct thread *td __ktrace_used; 375 int lock_state, rval; 376 377 td = curthread; 378 CV_ASSERT(cvp, lock, td); 379 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 380 "Waiting on \"%s\"", cvp->cv_description); 381 382 if (SCHEDULER_STOPPED()) 383 return (0); 384 385 #ifdef KTRACE 386 if (KTRPOINT(td, KTR_CSW)) { 387 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg)); 388 ktrcsw(1, 0, wmesg); 389 } else { 390 wmesg[0] = '\0'; 391 } 392 #endif 393 394 class = LOCK_CLASS(lock); 395 lock_state = 0; 396 sleepq_lock(cvp); 397 398 CV_WAITERS_INC(cvp); 399 if (lock == &Giant.lock_object) 400 mtx_assert(&Giant, MA_OWNED); 401 DROP_GIANT(); 402 403 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 404 SLEEPQ_INTERRUPTIBLE, 0); 405 sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 406 if (lock != &Giant.lock_object) { 407 if (class->lc_flags & LC_SLEEPABLE) 408 sleepq_release(cvp); 409 WITNESS_SAVE(lock, lock_witness); 410 lock_state = class->lc_unlock(lock); 411 if (class->lc_flags & LC_SLEEPABLE) 412 sleepq_lock(cvp); 413 } 414 rval = sleepq_timedwait_sig(cvp, 0); 415 416 #ifdef KTRACE 417 if (KTRPOINT(td, KTR_CSW)) 418 ktrcsw(0, 0, wmesg); 419 #endif 420 PICKUP_GIANT(); 421 if (lock != &Giant.lock_object) { 422 class->lc_lock(lock, lock_state); 423 WITNESS_RESTORE(lock, lock_witness); 424 } 425 426 return (rval); 427 } 428 429 /* 430 * Signal a condition variable, wakes up one waiting thread. Will also wakeup 431 * the swapper if the process is not in memory, so that it can bring the 432 * sleeping process in. Note that this may also result in additional threads 433 * being made runnable. Should be called with the same mutex as was passed to 434 * cv_wait held. 435 */ 436 void 437 cv_signal(struct cv *cvp) 438 { 439 440 if (cvp->cv_waiters == 0) 441 return; 442 sleepq_lock(cvp); 443 if (cvp->cv_waiters == 0) { 444 sleepq_release(cvp); 445 return; 446 } 447 if (cvp->cv_waiters == CV_WAITERS_BOUND && sleepq_lookup(cvp) == NULL) { 448 cvp->cv_waiters = 0; 449 sleepq_release(cvp); 450 } else { 451 if (cvp->cv_waiters < CV_WAITERS_BOUND) 452 cvp->cv_waiters--; 453 if (sleepq_signal(cvp, SLEEPQ_CONDVAR | SLEEPQ_DROP, 0, 0)) 454 kick_proc0(); 455 } 456 } 457 458 /* 459 * Broadcast a signal to a condition variable. Wakes up all waiting threads. 460 * Should be called with the same mutex as was passed to cv_wait held. 461 */ 462 void 463 cv_broadcastpri(struct cv *cvp, int pri) 464 { 465 int wakeup_swapper; 466 467 if (cvp->cv_waiters == 0) 468 return; 469 /* 470 * XXX sleepq_broadcast pri argument changed from -1 meaning 471 * no pri to 0 meaning no pri. 472 */ 473 wakeup_swapper = 0; 474 if (pri == -1) 475 pri = 0; 476 sleepq_lock(cvp); 477 if (cvp->cv_waiters > 0) { 478 cvp->cv_waiters = 0; 479 wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0); 480 } 481 sleepq_release(cvp); 482 if (wakeup_swapper) 483 kick_proc0(); 484 } 485