1 /*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * Machine independent bits of reader/writer lock implementation. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_ddb.h" 35 #include "opt_hwpmc_hooks.h" 36 #include "opt_no_adaptive_rwlocks.h" 37 38 #include <sys/param.h> 39 #include <sys/kdb.h> 40 #include <sys/ktr.h> 41 #include <sys/kernel.h> 42 #include <sys/lock.h> 43 #include <sys/mutex.h> 44 #include <sys/proc.h> 45 #include <sys/rwlock.h> 46 #include <sys/sched.h> 47 #include <sys/smp.h> 48 #include <sys/sysctl.h> 49 #include <sys/systm.h> 50 #include <sys/turnstile.h> 51 52 #include <machine/cpu.h> 53 54 #if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS) 55 #define ADAPTIVE_RWLOCKS 56 #endif 57 58 #ifdef HWPMC_HOOKS 59 #include <sys/pmckern.h> 60 PMC_SOFT_DECLARE( , , lock, failed); 61 #endif 62 63 /* 64 * Return the rwlock address when the lock cookie address is provided. 65 * This functionality assumes that struct rwlock* have a member named rw_lock. 66 */ 67 #define rwlock2rw(c) (__containerof(c, struct rwlock, rw_lock)) 68 69 #ifdef DDB 70 #include <ddb/ddb.h> 71 72 static void db_show_rwlock(const struct lock_object *lock); 73 #endif 74 static void assert_rw(const struct lock_object *lock, int what); 75 static void lock_rw(struct lock_object *lock, uintptr_t how); 76 #ifdef KDTRACE_HOOKS 77 static int owner_rw(const struct lock_object *lock, struct thread **owner); 78 #endif 79 static uintptr_t unlock_rw(struct lock_object *lock); 80 81 struct lock_class lock_class_rw = { 82 .lc_name = "rw", 83 .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE | LC_UPGRADABLE, 84 .lc_assert = assert_rw, 85 #ifdef DDB 86 .lc_ddb_show = db_show_rwlock, 87 #endif 88 .lc_lock = lock_rw, 89 .lc_unlock = unlock_rw, 90 #ifdef KDTRACE_HOOKS 91 .lc_owner = owner_rw, 92 #endif 93 }; 94 95 #ifdef ADAPTIVE_RWLOCKS 96 static int __read_frequently rowner_retries = 10; 97 static int __read_frequently rowner_loops = 10000; 98 static SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL, 99 "rwlock debugging"); 100 SYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, ""); 101 SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, ""); 102 103 static struct lock_delay_config __read_frequently rw_delay; 104 105 SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_base, CTLFLAG_RW, &rw_delay.base, 106 0, ""); 107 SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_max, CTLFLAG_RW, &rw_delay.max, 108 0, ""); 109 110 LOCK_DELAY_SYSINIT_DEFAULT(rw_delay); 111 #endif 112 113 /* 114 * Return a pointer to the owning thread if the lock is write-locked or 115 * NULL if the lock is unlocked or read-locked. 116 */ 117 118 #define lv_rw_wowner(v) \ 119 ((v) & RW_LOCK_READ ? NULL : \ 120 (struct thread *)RW_OWNER((v))) 121 122 #define rw_wowner(rw) lv_rw_wowner(RW_READ_VALUE(rw)) 123 124 /* 125 * Returns if a write owner is recursed. Write ownership is not assured 126 * here and should be previously checked. 127 */ 128 #define rw_recursed(rw) ((rw)->rw_recurse != 0) 129 130 /* 131 * Return true if curthread helds the lock. 132 */ 133 #define rw_wlocked(rw) (rw_wowner((rw)) == curthread) 134 135 /* 136 * Return a pointer to the owning thread for this lock who should receive 137 * any priority lent by threads that block on this lock. Currently this 138 * is identical to rw_wowner(). 139 */ 140 #define rw_owner(rw) rw_wowner(rw) 141 142 #ifndef INVARIANTS 143 #define __rw_assert(c, what, file, line) 144 #endif 145 146 void 147 assert_rw(const struct lock_object *lock, int what) 148 { 149 150 rw_assert((const struct rwlock *)lock, what); 151 } 152 153 void 154 lock_rw(struct lock_object *lock, uintptr_t how) 155 { 156 struct rwlock *rw; 157 158 rw = (struct rwlock *)lock; 159 if (how) 160 rw_rlock(rw); 161 else 162 rw_wlock(rw); 163 } 164 165 uintptr_t 166 unlock_rw(struct lock_object *lock) 167 { 168 struct rwlock *rw; 169 170 rw = (struct rwlock *)lock; 171 rw_assert(rw, RA_LOCKED | LA_NOTRECURSED); 172 if (rw->rw_lock & RW_LOCK_READ) { 173 rw_runlock(rw); 174 return (1); 175 } else { 176 rw_wunlock(rw); 177 return (0); 178 } 179 } 180 181 #ifdef KDTRACE_HOOKS 182 int 183 owner_rw(const struct lock_object *lock, struct thread **owner) 184 { 185 const struct rwlock *rw = (const struct rwlock *)lock; 186 uintptr_t x = rw->rw_lock; 187 188 *owner = rw_wowner(rw); 189 return ((x & RW_LOCK_READ) != 0 ? (RW_READERS(x) != 0) : 190 (*owner != NULL)); 191 } 192 #endif 193 194 void 195 _rw_init_flags(volatile uintptr_t *c, const char *name, int opts) 196 { 197 struct rwlock *rw; 198 int flags; 199 200 rw = rwlock2rw(c); 201 202 MPASS((opts & ~(RW_DUPOK | RW_NOPROFILE | RW_NOWITNESS | RW_QUIET | 203 RW_RECURSE | RW_NEW)) == 0); 204 ASSERT_ATOMIC_LOAD_PTR(rw->rw_lock, 205 ("%s: rw_lock not aligned for %s: %p", __func__, name, 206 &rw->rw_lock)); 207 208 flags = LO_UPGRADABLE; 209 if (opts & RW_DUPOK) 210 flags |= LO_DUPOK; 211 if (opts & RW_NOPROFILE) 212 flags |= LO_NOPROFILE; 213 if (!(opts & RW_NOWITNESS)) 214 flags |= LO_WITNESS; 215 if (opts & RW_RECURSE) 216 flags |= LO_RECURSABLE; 217 if (opts & RW_QUIET) 218 flags |= LO_QUIET; 219 if (opts & RW_NEW) 220 flags |= LO_NEW; 221 222 lock_init(&rw->lock_object, &lock_class_rw, name, NULL, flags); 223 rw->rw_lock = RW_UNLOCKED; 224 rw->rw_recurse = 0; 225 } 226 227 void 228 _rw_destroy(volatile uintptr_t *c) 229 { 230 struct rwlock *rw; 231 232 rw = rwlock2rw(c); 233 234 KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock %p not unlocked", rw)); 235 KASSERT(rw->rw_recurse == 0, ("rw lock %p still recursed", rw)); 236 rw->rw_lock = RW_DESTROYED; 237 lock_destroy(&rw->lock_object); 238 } 239 240 void 241 rw_sysinit(void *arg) 242 { 243 struct rw_args *args; 244 245 args = arg; 246 rw_init_flags((struct rwlock *)args->ra_rw, args->ra_desc, 247 args->ra_flags); 248 } 249 250 int 251 _rw_wowned(const volatile uintptr_t *c) 252 { 253 254 return (rw_wowner(rwlock2rw(c)) == curthread); 255 } 256 257 void 258 _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line) 259 { 260 struct rwlock *rw; 261 uintptr_t tid, v; 262 263 rw = rwlock2rw(c); 264 265 KASSERT(kdb_active != 0 || SCHEDULER_STOPPED() || 266 !TD_IS_IDLETHREAD(curthread), 267 ("rw_wlock() by idle thread %p on rwlock %s @ %s:%d", 268 curthread, rw->lock_object.lo_name, file, line)); 269 KASSERT(rw->rw_lock != RW_DESTROYED, 270 ("rw_wlock() of destroyed rwlock @ %s:%d", file, line)); 271 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 272 line, NULL); 273 tid = (uintptr_t)curthread; 274 v = RW_UNLOCKED; 275 if (!_rw_write_lock_fetch(rw, &v, tid)) 276 _rw_wlock_hard(rw, v, tid, file, line); 277 else 278 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, 279 0, 0, file, line, LOCKSTAT_WRITER); 280 281 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); 282 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 283 TD_LOCKS_INC(curthread); 284 } 285 286 int 287 __rw_try_wlock(volatile uintptr_t *c, const char *file, int line) 288 { 289 struct rwlock *rw; 290 struct thread *td; 291 uintptr_t tid, v; 292 int rval; 293 bool recursed; 294 295 td = curthread; 296 tid = (uintptr_t)td; 297 if (SCHEDULER_STOPPED_TD(td)) 298 return (1); 299 300 rw = rwlock2rw(c); 301 302 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(td), 303 ("rw_try_wlock() by idle thread %p on rwlock %s @ %s:%d", 304 curthread, rw->lock_object.lo_name, file, line)); 305 KASSERT(rw->rw_lock != RW_DESTROYED, 306 ("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line)); 307 308 rval = 1; 309 recursed = false; 310 v = RW_UNLOCKED; 311 for (;;) { 312 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid)) 313 break; 314 if (v == RW_UNLOCKED) 315 continue; 316 if (v == tid && (rw->lock_object.lo_flags & LO_RECURSABLE)) { 317 rw->rw_recurse++; 318 atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); 319 break; 320 } 321 rval = 0; 322 break; 323 } 324 325 LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line); 326 if (rval) { 327 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 328 file, line); 329 if (!recursed) 330 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, 331 rw, 0, 0, file, line, LOCKSTAT_WRITER); 332 TD_LOCKS_INC(curthread); 333 } 334 return (rval); 335 } 336 337 void 338 _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line) 339 { 340 struct rwlock *rw; 341 342 rw = rwlock2rw(c); 343 344 KASSERT(rw->rw_lock != RW_DESTROYED, 345 ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line)); 346 __rw_assert(c, RA_WLOCKED, file, line); 347 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 348 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file, 349 line); 350 351 #ifdef LOCK_PROFILING 352 _rw_wunlock_hard(rw, (uintptr_t)curthread, file, line); 353 #else 354 __rw_wunlock(rw, curthread, file, line); 355 #endif 356 357 TD_LOCKS_DEC(curthread); 358 } 359 360 /* 361 * Determines whether a new reader can acquire a lock. Succeeds if the 362 * reader already owns a read lock and the lock is locked for read to 363 * prevent deadlock from reader recursion. Also succeeds if the lock 364 * is unlocked and has no writer waiters or spinners. Failing otherwise 365 * prioritizes writers before readers. 366 */ 367 #define RW_CAN_READ(td, _rw) \ 368 (((_rw) & (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER)) ==\ 369 RW_LOCK_READ || ((td)->td_rw_rlocks && (_rw) & RW_LOCK_READ)) 370 371 static bool __always_inline 372 __rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp, 373 const char *file, int line) 374 { 375 376 /* 377 * Handle the easy case. If no other thread has a write 378 * lock, then try to bump up the count of read locks. Note 379 * that we have to preserve the current state of the 380 * RW_LOCK_WRITE_WAITERS flag. If we fail to acquire a 381 * read lock, then rw_lock must have changed, so restart 382 * the loop. Note that this handles the case of a 383 * completely unlocked rwlock since such a lock is encoded 384 * as a read lock with no waiters. 385 */ 386 while (RW_CAN_READ(td, *vp)) { 387 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, vp, 388 *vp + RW_ONE_READER)) { 389 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 390 CTR4(KTR_LOCK, 391 "%s: %p succeed %p -> %p", __func__, 392 rw, (void *)*vp, 393 (void *)(*vp + RW_ONE_READER)); 394 td->td_rw_rlocks++; 395 return (true); 396 } 397 } 398 return (false); 399 } 400 401 static void __noinline 402 __rw_rlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v, 403 const char *file, int line) 404 { 405 struct rwlock *rw; 406 struct turnstile *ts; 407 #ifdef ADAPTIVE_RWLOCKS 408 volatile struct thread *owner; 409 int spintries = 0; 410 int i, n; 411 #endif 412 #ifdef LOCK_PROFILING 413 uint64_t waittime = 0; 414 int contested = 0; 415 #endif 416 #if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS) 417 struct lock_delay_arg lda; 418 #endif 419 #ifdef KDTRACE_HOOKS 420 u_int sleep_cnt = 0; 421 int64_t sleep_time = 0; 422 int64_t all_time = 0; 423 #endif 424 #if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 425 uintptr_t state; 426 int doing_lockprof; 427 #endif 428 429 if (SCHEDULER_STOPPED()) 430 return; 431 432 #if defined(ADAPTIVE_RWLOCKS) 433 lock_delay_arg_init(&lda, &rw_delay); 434 #elif defined(KDTRACE_HOOKS) 435 lock_delay_arg_init(&lda, NULL); 436 #endif 437 rw = rwlock2rw(c); 438 439 #ifdef HWPMC_HOOKS 440 PMC_SOFT_CALL( , , lock, failed); 441 #endif 442 lock_profile_obtain_lock_failed(&rw->lock_object, 443 &contested, &waittime); 444 445 #ifdef LOCK_PROFILING 446 doing_lockprof = 1; 447 state = v; 448 #elif defined(KDTRACE_HOOKS) 449 doing_lockprof = lockstat_enabled; 450 if (__predict_false(doing_lockprof)) { 451 all_time -= lockstat_nsecs(&rw->lock_object); 452 state = v; 453 } 454 #endif 455 456 for (;;) { 457 if (__rw_rlock_try(rw, td, &v, file, line)) 458 break; 459 #ifdef KDTRACE_HOOKS 460 lda.spin_cnt++; 461 #endif 462 463 #ifdef ADAPTIVE_RWLOCKS 464 /* 465 * If the owner is running on another CPU, spin until 466 * the owner stops running or the state of the lock 467 * changes. 468 */ 469 if ((v & RW_LOCK_READ) == 0) { 470 owner = (struct thread *)RW_OWNER(v); 471 if (TD_IS_RUNNING(owner)) { 472 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 473 CTR3(KTR_LOCK, 474 "%s: spinning on %p held by %p", 475 __func__, rw, owner); 476 KTR_STATE1(KTR_SCHED, "thread", 477 sched_tdname(curthread), "spinning", 478 "lockname:\"%s\"", rw->lock_object.lo_name); 479 do { 480 lock_delay(&lda); 481 v = RW_READ_VALUE(rw); 482 owner = lv_rw_wowner(v); 483 } while (owner != NULL && TD_IS_RUNNING(owner)); 484 KTR_STATE0(KTR_SCHED, "thread", 485 sched_tdname(curthread), "running"); 486 continue; 487 } 488 } else if (spintries < rowner_retries) { 489 spintries++; 490 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), 491 "spinning", "lockname:\"%s\"", 492 rw->lock_object.lo_name); 493 for (i = 0; i < rowner_loops; i += n) { 494 n = RW_READERS(v); 495 lock_delay_spin(n); 496 v = RW_READ_VALUE(rw); 497 if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(td, v)) 498 break; 499 } 500 #ifdef KDTRACE_HOOKS 501 lda.spin_cnt += rowner_loops - i; 502 #endif 503 KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), 504 "running"); 505 if (i != rowner_loops) 506 continue; 507 } 508 #endif 509 510 /* 511 * Okay, now it's the hard case. Some other thread already 512 * has a write lock or there are write waiters present, 513 * acquire the turnstile lock so we can begin the process 514 * of blocking. 515 */ 516 ts = turnstile_trywait(&rw->lock_object); 517 518 /* 519 * The lock might have been released while we spun, so 520 * recheck its state and restart the loop if needed. 521 */ 522 v = RW_READ_VALUE(rw); 523 if (RW_CAN_READ(td, v)) { 524 turnstile_cancel(ts); 525 continue; 526 } 527 528 #ifdef ADAPTIVE_RWLOCKS 529 /* 530 * The current lock owner might have started executing 531 * on another CPU (or the lock could have changed 532 * owners) while we were waiting on the turnstile 533 * chain lock. If so, drop the turnstile lock and try 534 * again. 535 */ 536 if ((v & RW_LOCK_READ) == 0) { 537 owner = (struct thread *)RW_OWNER(v); 538 if (TD_IS_RUNNING(owner)) { 539 turnstile_cancel(ts); 540 continue; 541 } 542 } 543 #endif 544 545 /* 546 * The lock is held in write mode or it already has waiters. 547 */ 548 MPASS(!RW_CAN_READ(td, v)); 549 550 /* 551 * If the RW_LOCK_READ_WAITERS flag is already set, then 552 * we can go ahead and block. If it is not set then try 553 * to set it. If we fail to set it drop the turnstile 554 * lock and restart the loop. 555 */ 556 if (!(v & RW_LOCK_READ_WAITERS)) { 557 if (!atomic_cmpset_ptr(&rw->rw_lock, v, 558 v | RW_LOCK_READ_WAITERS)) { 559 turnstile_cancel(ts); 560 v = RW_READ_VALUE(rw); 561 continue; 562 } 563 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 564 CTR2(KTR_LOCK, "%s: %p set read waiters flag", 565 __func__, rw); 566 } 567 568 /* 569 * We were unable to acquire the lock and the read waiters 570 * flag is set, so we must block on the turnstile. 571 */ 572 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 573 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 574 rw); 575 #ifdef KDTRACE_HOOKS 576 sleep_time -= lockstat_nsecs(&rw->lock_object); 577 #endif 578 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE); 579 #ifdef KDTRACE_HOOKS 580 sleep_time += lockstat_nsecs(&rw->lock_object); 581 sleep_cnt++; 582 #endif 583 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 584 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 585 __func__, rw); 586 v = RW_READ_VALUE(rw); 587 } 588 #if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 589 if (__predict_true(!doing_lockprof)) 590 return; 591 #endif 592 #ifdef KDTRACE_HOOKS 593 all_time += lockstat_nsecs(&rw->lock_object); 594 if (sleep_time) 595 LOCKSTAT_RECORD4(rw__block, rw, sleep_time, 596 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 597 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 598 599 /* Record only the loops spinning and not sleeping. */ 600 if (lda.spin_cnt > sleep_cnt) 601 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time, 602 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 603 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 604 #endif 605 /* 606 * TODO: acquire "owner of record" here. Here be turnstile dragons 607 * however. turnstiles don't like owners changing between calls to 608 * turnstile_wait() currently. 609 */ 610 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested, 611 waittime, file, line, LOCKSTAT_READER); 612 } 613 614 void 615 __rw_rlock(volatile uintptr_t *c, const char *file, int line) 616 { 617 struct rwlock *rw; 618 struct thread *td; 619 uintptr_t v; 620 621 td = curthread; 622 rw = rwlock2rw(c); 623 624 KASSERT(kdb_active != 0 || SCHEDULER_STOPPED_TD(td) || 625 !TD_IS_IDLETHREAD(td), 626 ("rw_rlock() by idle thread %p on rwlock %s @ %s:%d", 627 td, rw->lock_object.lo_name, file, line)); 628 KASSERT(rw->rw_lock != RW_DESTROYED, 629 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); 630 KASSERT(rw_wowner(rw) != td, 631 ("rw_rlock: wlock already held for %s @ %s:%d", 632 rw->lock_object.lo_name, file, line)); 633 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); 634 635 v = RW_READ_VALUE(rw); 636 if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__acquire) || 637 !__rw_rlock_try(rw, td, &v, file, line))) 638 __rw_rlock_hard(c, td, v, file, line); 639 640 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); 641 WITNESS_LOCK(&rw->lock_object, 0, file, line); 642 TD_LOCKS_INC(curthread); 643 } 644 645 int 646 __rw_try_rlock(volatile uintptr_t *c, const char *file, int line) 647 { 648 struct rwlock *rw; 649 uintptr_t x; 650 651 if (SCHEDULER_STOPPED()) 652 return (1); 653 654 rw = rwlock2rw(c); 655 656 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 657 ("rw_try_rlock() by idle thread %p on rwlock %s @ %s:%d", 658 curthread, rw->lock_object.lo_name, file, line)); 659 660 x = rw->rw_lock; 661 for (;;) { 662 KASSERT(rw->rw_lock != RW_DESTROYED, 663 ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line)); 664 if (!(x & RW_LOCK_READ)) 665 break; 666 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &x, x + RW_ONE_READER)) { 667 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file, 668 line); 669 WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line); 670 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, 671 rw, 0, 0, file, line, LOCKSTAT_READER); 672 TD_LOCKS_INC(curthread); 673 curthread->td_rw_rlocks++; 674 return (1); 675 } 676 } 677 678 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line); 679 return (0); 680 } 681 682 static bool __always_inline 683 __rw_runlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp) 684 { 685 686 for (;;) { 687 /* 688 * See if there is more than one read lock held. If so, 689 * just drop one and return. 690 */ 691 if (RW_READERS(*vp) > 1) { 692 if (atomic_fcmpset_rel_ptr(&rw->rw_lock, vp, 693 *vp - RW_ONE_READER)) { 694 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 695 CTR4(KTR_LOCK, 696 "%s: %p succeeded %p -> %p", 697 __func__, rw, (void *)*vp, 698 (void *)(*vp - RW_ONE_READER)); 699 td->td_rw_rlocks--; 700 return (true); 701 } 702 continue; 703 } 704 /* 705 * If there aren't any waiters for a write lock, then try 706 * to drop it quickly. 707 */ 708 if (!(*vp & RW_LOCK_WAITERS)) { 709 MPASS((*vp & ~RW_LOCK_WRITE_SPINNER) == 710 RW_READERS_LOCK(1)); 711 if (atomic_fcmpset_rel_ptr(&rw->rw_lock, vp, 712 RW_UNLOCKED)) { 713 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 714 CTR2(KTR_LOCK, "%s: %p last succeeded", 715 __func__, rw); 716 td->td_rw_rlocks--; 717 return (true); 718 } 719 continue; 720 } 721 break; 722 } 723 return (false); 724 } 725 726 static void __noinline 727 __rw_runlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v, 728 const char *file, int line) 729 { 730 struct rwlock *rw; 731 struct turnstile *ts; 732 uintptr_t x, queue; 733 734 if (SCHEDULER_STOPPED()) 735 return; 736 737 rw = rwlock2rw(c); 738 739 for (;;) { 740 if (__rw_runlock_try(rw, td, &v)) 741 break; 742 743 /* 744 * Ok, we know we have waiters and we think we are the 745 * last reader, so grab the turnstile lock. 746 */ 747 turnstile_chain_lock(&rw->lock_object); 748 v = rw->rw_lock & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER); 749 MPASS(v & RW_LOCK_WAITERS); 750 751 /* 752 * Try to drop our lock leaving the lock in a unlocked 753 * state. 754 * 755 * If you wanted to do explicit lock handoff you'd have to 756 * do it here. You'd also want to use turnstile_signal() 757 * and you'd have to handle the race where a higher 758 * priority thread blocks on the write lock before the 759 * thread you wakeup actually runs and have the new thread 760 * "steal" the lock. For now it's a lot simpler to just 761 * wakeup all of the waiters. 762 * 763 * As above, if we fail, then another thread might have 764 * acquired a read lock, so drop the turnstile lock and 765 * restart. 766 */ 767 x = RW_UNLOCKED; 768 if (v & RW_LOCK_WRITE_WAITERS) { 769 queue = TS_EXCLUSIVE_QUEUE; 770 x |= (v & RW_LOCK_READ_WAITERS); 771 } else 772 queue = TS_SHARED_QUEUE; 773 if (!atomic_cmpset_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v, 774 x)) { 775 turnstile_chain_unlock(&rw->lock_object); 776 v = RW_READ_VALUE(rw); 777 continue; 778 } 779 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 780 CTR2(KTR_LOCK, "%s: %p last succeeded with waiters", 781 __func__, rw); 782 783 /* 784 * Ok. The lock is released and all that's left is to 785 * wake up the waiters. Note that the lock might not be 786 * free anymore, but in that case the writers will just 787 * block again if they run before the new lock holder(s) 788 * release the lock. 789 */ 790 ts = turnstile_lookup(&rw->lock_object); 791 MPASS(ts != NULL); 792 turnstile_broadcast(ts, queue); 793 turnstile_unpend(ts, TS_SHARED_LOCK); 794 turnstile_chain_unlock(&rw->lock_object); 795 td->td_rw_rlocks--; 796 break; 797 } 798 LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_READER); 799 } 800 801 void 802 _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line) 803 { 804 struct rwlock *rw; 805 struct thread *td; 806 uintptr_t v; 807 808 rw = rwlock2rw(c); 809 810 KASSERT(rw->rw_lock != RW_DESTROYED, 811 ("rw_runlock() of destroyed rwlock @ %s:%d", file, line)); 812 __rw_assert(c, RA_RLOCKED, file, line); 813 WITNESS_UNLOCK(&rw->lock_object, 0, file, line); 814 LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line); 815 816 td = curthread; 817 v = RW_READ_VALUE(rw); 818 819 if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__release) || 820 !__rw_runlock_try(rw, td, &v))) 821 __rw_runlock_hard(c, td, v, file, line); 822 823 TD_LOCKS_DEC(curthread); 824 } 825 826 /* 827 * This function is called when we are unable to obtain a write lock on the 828 * first try. This means that at least one other thread holds either a 829 * read or write lock. 830 */ 831 void 832 __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid, 833 const char *file, int line) 834 { 835 struct rwlock *rw; 836 struct turnstile *ts; 837 #ifdef ADAPTIVE_RWLOCKS 838 volatile struct thread *owner; 839 int spintries = 0; 840 int i, n; 841 #endif 842 uintptr_t x; 843 #ifdef LOCK_PROFILING 844 uint64_t waittime = 0; 845 int contested = 0; 846 #endif 847 #if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS) 848 struct lock_delay_arg lda; 849 #endif 850 #ifdef KDTRACE_HOOKS 851 u_int sleep_cnt = 0; 852 int64_t sleep_time = 0; 853 int64_t all_time = 0; 854 #endif 855 #if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 856 uintptr_t state; 857 int doing_lockprof; 858 #endif 859 860 if (SCHEDULER_STOPPED()) 861 return; 862 863 #if defined(ADAPTIVE_RWLOCKS) 864 lock_delay_arg_init(&lda, &rw_delay); 865 #elif defined(KDTRACE_HOOKS) 866 lock_delay_arg_init(&lda, NULL); 867 #endif 868 rw = rwlock2rw(c); 869 if (__predict_false(v == RW_UNLOCKED)) 870 v = RW_READ_VALUE(rw); 871 872 if (__predict_false(lv_rw_wowner(v) == (struct thread *)tid)) { 873 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE, 874 ("%s: recursing but non-recursive rw %s @ %s:%d\n", 875 __func__, rw->lock_object.lo_name, file, line)); 876 rw->rw_recurse++; 877 atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); 878 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 879 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); 880 return; 881 } 882 883 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 884 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 885 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); 886 887 #ifdef HWPMC_HOOKS 888 PMC_SOFT_CALL( , , lock, failed); 889 #endif 890 lock_profile_obtain_lock_failed(&rw->lock_object, 891 &contested, &waittime); 892 893 #ifdef LOCK_PROFILING 894 doing_lockprof = 1; 895 state = v; 896 #elif defined(KDTRACE_HOOKS) 897 doing_lockprof = lockstat_enabled; 898 if (__predict_false(doing_lockprof)) { 899 all_time -= lockstat_nsecs(&rw->lock_object); 900 state = v; 901 } 902 #endif 903 904 for (;;) { 905 if (v == RW_UNLOCKED) { 906 if (_rw_write_lock_fetch(rw, &v, tid)) 907 break; 908 continue; 909 } 910 #ifdef KDTRACE_HOOKS 911 lda.spin_cnt++; 912 #endif 913 914 #ifdef ADAPTIVE_RWLOCKS 915 /* 916 * If the lock is write locked and the owner is 917 * running on another CPU, spin until the owner stops 918 * running or the state of the lock changes. 919 */ 920 owner = lv_rw_wowner(v); 921 if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) { 922 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 923 CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 924 __func__, rw, owner); 925 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), 926 "spinning", "lockname:\"%s\"", 927 rw->lock_object.lo_name); 928 do { 929 lock_delay(&lda); 930 v = RW_READ_VALUE(rw); 931 owner = lv_rw_wowner(v); 932 } while (owner != NULL && TD_IS_RUNNING(owner)); 933 KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), 934 "running"); 935 continue; 936 } 937 if ((v & RW_LOCK_READ) && RW_READERS(v) && 938 spintries < rowner_retries) { 939 if (!(v & RW_LOCK_WRITE_SPINNER)) { 940 if (!atomic_fcmpset_ptr(&rw->rw_lock, &v, 941 v | RW_LOCK_WRITE_SPINNER)) { 942 continue; 943 } 944 } 945 spintries++; 946 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), 947 "spinning", "lockname:\"%s\"", 948 rw->lock_object.lo_name); 949 for (i = 0; i < rowner_loops; i += n) { 950 n = RW_READERS(v); 951 lock_delay_spin(n); 952 v = RW_READ_VALUE(rw); 953 if ((v & RW_LOCK_WRITE_SPINNER) == 0) 954 break; 955 } 956 KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), 957 "running"); 958 #ifdef KDTRACE_HOOKS 959 lda.spin_cnt += rowner_loops - i; 960 #endif 961 if (i != rowner_loops) 962 continue; 963 } 964 #endif 965 ts = turnstile_trywait(&rw->lock_object); 966 v = RW_READ_VALUE(rw); 967 968 #ifdef ADAPTIVE_RWLOCKS 969 /* 970 * The current lock owner might have started executing 971 * on another CPU (or the lock could have changed 972 * owners) while we were waiting on the turnstile 973 * chain lock. If so, drop the turnstile lock and try 974 * again. 975 */ 976 if (!(v & RW_LOCK_READ)) { 977 owner = (struct thread *)RW_OWNER(v); 978 if (TD_IS_RUNNING(owner)) { 979 turnstile_cancel(ts); 980 continue; 981 } 982 } 983 #endif 984 /* 985 * Check for the waiters flags about this rwlock. 986 * If the lock was released, without maintain any pending 987 * waiters queue, simply try to acquire it. 988 * If a pending waiters queue is present, claim the lock 989 * ownership and maintain the pending queue. 990 */ 991 x = v & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER); 992 if ((v & ~x) == RW_UNLOCKED) { 993 x &= ~RW_LOCK_WRITE_SPINNER; 994 if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, tid | x)) { 995 if (x) 996 turnstile_claim(ts); 997 else 998 turnstile_cancel(ts); 999 break; 1000 } 1001 turnstile_cancel(ts); 1002 v = RW_READ_VALUE(rw); 1003 continue; 1004 } 1005 /* 1006 * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to 1007 * set it. If we fail to set it, then loop back and try 1008 * again. 1009 */ 1010 if (!(v & RW_LOCK_WRITE_WAITERS)) { 1011 if (!atomic_cmpset_ptr(&rw->rw_lock, v, 1012 v | RW_LOCK_WRITE_WAITERS)) { 1013 turnstile_cancel(ts); 1014 v = RW_READ_VALUE(rw); 1015 continue; 1016 } 1017 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 1018 CTR2(KTR_LOCK, "%s: %p set write waiters flag", 1019 __func__, rw); 1020 } 1021 /* 1022 * We were unable to acquire the lock and the write waiters 1023 * flag is set, so we must block on the turnstile. 1024 */ 1025 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 1026 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 1027 rw); 1028 #ifdef KDTRACE_HOOKS 1029 sleep_time -= lockstat_nsecs(&rw->lock_object); 1030 #endif 1031 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE); 1032 #ifdef KDTRACE_HOOKS 1033 sleep_time += lockstat_nsecs(&rw->lock_object); 1034 sleep_cnt++; 1035 #endif 1036 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 1037 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 1038 __func__, rw); 1039 #ifdef ADAPTIVE_RWLOCKS 1040 spintries = 0; 1041 #endif 1042 v = RW_READ_VALUE(rw); 1043 } 1044 #if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 1045 if (__predict_true(!doing_lockprof)) 1046 return; 1047 #endif 1048 #ifdef KDTRACE_HOOKS 1049 all_time += lockstat_nsecs(&rw->lock_object); 1050 if (sleep_time) 1051 LOCKSTAT_RECORD4(rw__block, rw, sleep_time, 1052 LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0, 1053 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 1054 1055 /* Record only the loops spinning and not sleeping. */ 1056 if (lda.spin_cnt > sleep_cnt) 1057 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time, 1058 LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0, 1059 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 1060 #endif 1061 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested, 1062 waittime, file, line, LOCKSTAT_WRITER); 1063 } 1064 1065 /* 1066 * This function is called if lockstat is active or the first try at releasing 1067 * a write lock failed. The latter means that the lock is recursed or one of 1068 * the 2 waiter bits must be set indicating that at least one thread is waiting 1069 * on this lock. 1070 */ 1071 void 1072 __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file, 1073 int line) 1074 { 1075 struct rwlock *rw; 1076 struct turnstile *ts; 1077 uintptr_t v, setv; 1078 int queue; 1079 1080 if (SCHEDULER_STOPPED()) 1081 return; 1082 1083 rw = rwlock2rw(c); 1084 v = RW_READ_VALUE(rw); 1085 if (v & RW_LOCK_WRITER_RECURSED) { 1086 if (--(rw->rw_recurse) == 0) 1087 atomic_clear_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); 1088 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 1089 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw); 1090 return; 1091 } 1092 1093 LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_WRITER); 1094 if (v == tid && _rw_write_unlock(rw, tid)) 1095 return; 1096 1097 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), 1098 ("%s: neither of the waiter flags are set", __func__)); 1099 1100 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 1101 CTR2(KTR_LOCK, "%s: %p contested", __func__, rw); 1102 1103 turnstile_chain_lock(&rw->lock_object); 1104 1105 /* 1106 * Use the same algo as sx locks for now. Prefer waking up shared 1107 * waiters if we have any over writers. This is probably not ideal. 1108 * 1109 * 'v' is the value we are going to write back to rw_lock. If we 1110 * have waiters on both queues, we need to preserve the state of 1111 * the waiter flag for the queue we don't wake up. For now this is 1112 * hardcoded for the algorithm mentioned above. 1113 * 1114 * In the case of both readers and writers waiting we wakeup the 1115 * readers but leave the RW_LOCK_WRITE_WAITERS flag set. If a 1116 * new writer comes in before a reader it will claim the lock up 1117 * above. There is probably a potential priority inversion in 1118 * there that could be worked around either by waking both queues 1119 * of waiters or doing some complicated lock handoff gymnastics. 1120 */ 1121 setv = RW_UNLOCKED; 1122 v = RW_READ_VALUE(rw); 1123 queue = TS_SHARED_QUEUE; 1124 if (v & RW_LOCK_WRITE_WAITERS) { 1125 queue = TS_EXCLUSIVE_QUEUE; 1126 setv |= (v & RW_LOCK_READ_WAITERS); 1127 } 1128 atomic_store_rel_ptr(&rw->rw_lock, setv); 1129 1130 /* Wake up all waiters for the specific queue. */ 1131 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 1132 CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw, 1133 queue == TS_SHARED_QUEUE ? "read" : "write"); 1134 1135 ts = turnstile_lookup(&rw->lock_object); 1136 MPASS(ts != NULL); 1137 turnstile_broadcast(ts, queue); 1138 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 1139 turnstile_chain_unlock(&rw->lock_object); 1140 } 1141 1142 /* 1143 * Attempt to do a non-blocking upgrade from a read lock to a write 1144 * lock. This will only succeed if this thread holds a single read 1145 * lock. Returns true if the upgrade succeeded and false otherwise. 1146 */ 1147 int 1148 __rw_try_upgrade(volatile uintptr_t *c, const char *file, int line) 1149 { 1150 struct rwlock *rw; 1151 uintptr_t v, x, tid; 1152 struct turnstile *ts; 1153 int success; 1154 1155 if (SCHEDULER_STOPPED()) 1156 return (1); 1157 1158 rw = rwlock2rw(c); 1159 1160 KASSERT(rw->rw_lock != RW_DESTROYED, 1161 ("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line)); 1162 __rw_assert(c, RA_RLOCKED, file, line); 1163 1164 /* 1165 * Attempt to switch from one reader to a writer. If there 1166 * are any write waiters, then we will have to lock the 1167 * turnstile first to prevent races with another writer 1168 * calling turnstile_wait() before we have claimed this 1169 * turnstile. So, do the simple case of no waiters first. 1170 */ 1171 tid = (uintptr_t)curthread; 1172 success = 0; 1173 for (;;) { 1174 v = rw->rw_lock; 1175 if (RW_READERS(v) > 1) 1176 break; 1177 if (!(v & RW_LOCK_WAITERS)) { 1178 success = atomic_cmpset_acq_ptr(&rw->rw_lock, v, tid); 1179 if (!success) 1180 continue; 1181 break; 1182 } 1183 1184 /* 1185 * Ok, we think we have waiters, so lock the turnstile. 1186 */ 1187 ts = turnstile_trywait(&rw->lock_object); 1188 v = rw->rw_lock; 1189 if (RW_READERS(v) > 1) { 1190 turnstile_cancel(ts); 1191 break; 1192 } 1193 /* 1194 * Try to switch from one reader to a writer again. This time 1195 * we honor the current state of the waiters flags. 1196 * If we obtain the lock with the flags set, then claim 1197 * ownership of the turnstile. 1198 */ 1199 x = rw->rw_lock & RW_LOCK_WAITERS; 1200 success = atomic_cmpset_ptr(&rw->rw_lock, v, tid | x); 1201 if (success) { 1202 if (x) 1203 turnstile_claim(ts); 1204 else 1205 turnstile_cancel(ts); 1206 break; 1207 } 1208 turnstile_cancel(ts); 1209 } 1210 LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line); 1211 if (success) { 1212 curthread->td_rw_rlocks--; 1213 WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 1214 file, line); 1215 LOCKSTAT_RECORD0(rw__upgrade, rw); 1216 } 1217 return (success); 1218 } 1219 1220 /* 1221 * Downgrade a write lock into a single read lock. 1222 */ 1223 void 1224 __rw_downgrade(volatile uintptr_t *c, const char *file, int line) 1225 { 1226 struct rwlock *rw; 1227 struct turnstile *ts; 1228 uintptr_t tid, v; 1229 int rwait, wwait; 1230 1231 if (SCHEDULER_STOPPED()) 1232 return; 1233 1234 rw = rwlock2rw(c); 1235 1236 KASSERT(rw->rw_lock != RW_DESTROYED, 1237 ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line)); 1238 __rw_assert(c, RA_WLOCKED | RA_NOTRECURSED, file, line); 1239 #ifndef INVARIANTS 1240 if (rw_recursed(rw)) 1241 panic("downgrade of a recursed lock"); 1242 #endif 1243 1244 WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line); 1245 1246 /* 1247 * Convert from a writer to a single reader. First we handle 1248 * the easy case with no waiters. If there are any waiters, we 1249 * lock the turnstile and "disown" the lock. 1250 */ 1251 tid = (uintptr_t)curthread; 1252 if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1))) 1253 goto out; 1254 1255 /* 1256 * Ok, we think we have waiters, so lock the turnstile so we can 1257 * read the waiter flags without any races. 1258 */ 1259 turnstile_chain_lock(&rw->lock_object); 1260 v = rw->rw_lock & RW_LOCK_WAITERS; 1261 rwait = v & RW_LOCK_READ_WAITERS; 1262 wwait = v & RW_LOCK_WRITE_WAITERS; 1263 MPASS(rwait | wwait); 1264 1265 /* 1266 * Downgrade from a write lock while preserving waiters flag 1267 * and give up ownership of the turnstile. 1268 */ 1269 ts = turnstile_lookup(&rw->lock_object); 1270 MPASS(ts != NULL); 1271 if (!wwait) 1272 v &= ~RW_LOCK_READ_WAITERS; 1273 atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v); 1274 /* 1275 * Wake other readers if there are no writers pending. Otherwise they 1276 * won't be able to acquire the lock anyway. 1277 */ 1278 if (rwait && !wwait) { 1279 turnstile_broadcast(ts, TS_SHARED_QUEUE); 1280 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 1281 } else 1282 turnstile_disown(ts); 1283 turnstile_chain_unlock(&rw->lock_object); 1284 out: 1285 curthread->td_rw_rlocks++; 1286 LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line); 1287 LOCKSTAT_RECORD0(rw__downgrade, rw); 1288 } 1289 1290 #ifdef INVARIANT_SUPPORT 1291 #ifndef INVARIANTS 1292 #undef __rw_assert 1293 #endif 1294 1295 /* 1296 * In the non-WITNESS case, rw_assert() can only detect that at least 1297 * *some* thread owns an rlock, but it cannot guarantee that *this* 1298 * thread owns an rlock. 1299 */ 1300 void 1301 __rw_assert(const volatile uintptr_t *c, int what, const char *file, int line) 1302 { 1303 const struct rwlock *rw; 1304 1305 if (panicstr != NULL) 1306 return; 1307 1308 rw = rwlock2rw(c); 1309 1310 switch (what) { 1311 case RA_LOCKED: 1312 case RA_LOCKED | RA_RECURSED: 1313 case RA_LOCKED | RA_NOTRECURSED: 1314 case RA_RLOCKED: 1315 case RA_RLOCKED | RA_RECURSED: 1316 case RA_RLOCKED | RA_NOTRECURSED: 1317 #ifdef WITNESS 1318 witness_assert(&rw->lock_object, what, file, line); 1319 #else 1320 /* 1321 * If some other thread has a write lock or we have one 1322 * and are asserting a read lock, fail. Also, if no one 1323 * has a lock at all, fail. 1324 */ 1325 if (rw->rw_lock == RW_UNLOCKED || 1326 (!(rw->rw_lock & RW_LOCK_READ) && (what & RA_RLOCKED || 1327 rw_wowner(rw) != curthread))) 1328 panic("Lock %s not %slocked @ %s:%d\n", 1329 rw->lock_object.lo_name, (what & RA_RLOCKED) ? 1330 "read " : "", file, line); 1331 1332 if (!(rw->rw_lock & RW_LOCK_READ) && !(what & RA_RLOCKED)) { 1333 if (rw_recursed(rw)) { 1334 if (what & RA_NOTRECURSED) 1335 panic("Lock %s recursed @ %s:%d\n", 1336 rw->lock_object.lo_name, file, 1337 line); 1338 } else if (what & RA_RECURSED) 1339 panic("Lock %s not recursed @ %s:%d\n", 1340 rw->lock_object.lo_name, file, line); 1341 } 1342 #endif 1343 break; 1344 case RA_WLOCKED: 1345 case RA_WLOCKED | RA_RECURSED: 1346 case RA_WLOCKED | RA_NOTRECURSED: 1347 if (rw_wowner(rw) != curthread) 1348 panic("Lock %s not exclusively locked @ %s:%d\n", 1349 rw->lock_object.lo_name, file, line); 1350 if (rw_recursed(rw)) { 1351 if (what & RA_NOTRECURSED) 1352 panic("Lock %s recursed @ %s:%d\n", 1353 rw->lock_object.lo_name, file, line); 1354 } else if (what & RA_RECURSED) 1355 panic("Lock %s not recursed @ %s:%d\n", 1356 rw->lock_object.lo_name, file, line); 1357 break; 1358 case RA_UNLOCKED: 1359 #ifdef WITNESS 1360 witness_assert(&rw->lock_object, what, file, line); 1361 #else 1362 /* 1363 * If we hold a write lock fail. We can't reliably check 1364 * to see if we hold a read lock or not. 1365 */ 1366 if (rw_wowner(rw) == curthread) 1367 panic("Lock %s exclusively locked @ %s:%d\n", 1368 rw->lock_object.lo_name, file, line); 1369 #endif 1370 break; 1371 default: 1372 panic("Unknown rw lock assertion: %d @ %s:%d", what, file, 1373 line); 1374 } 1375 } 1376 #endif /* INVARIANT_SUPPORT */ 1377 1378 #ifdef DDB 1379 void 1380 db_show_rwlock(const struct lock_object *lock) 1381 { 1382 const struct rwlock *rw; 1383 struct thread *td; 1384 1385 rw = (const struct rwlock *)lock; 1386 1387 db_printf(" state: "); 1388 if (rw->rw_lock == RW_UNLOCKED) 1389 db_printf("UNLOCKED\n"); 1390 else if (rw->rw_lock == RW_DESTROYED) { 1391 db_printf("DESTROYED\n"); 1392 return; 1393 } else if (rw->rw_lock & RW_LOCK_READ) 1394 db_printf("RLOCK: %ju locks\n", 1395 (uintmax_t)(RW_READERS(rw->rw_lock))); 1396 else { 1397 td = rw_wowner(rw); 1398 db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 1399 td->td_tid, td->td_proc->p_pid, td->td_name); 1400 if (rw_recursed(rw)) 1401 db_printf(" recursed: %u\n", rw->rw_recurse); 1402 } 1403 db_printf(" waiters: "); 1404 switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) { 1405 case RW_LOCK_READ_WAITERS: 1406 db_printf("readers\n"); 1407 break; 1408 case RW_LOCK_WRITE_WAITERS: 1409 db_printf("writers\n"); 1410 break; 1411 case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS: 1412 db_printf("readers and writers\n"); 1413 break; 1414 default: 1415 db_printf("none\n"); 1416 break; 1417 } 1418 } 1419 1420 #endif 1421