1 /* 2 * Copyright (c) 1995 3 * The Regents of the University of California. All rights reserved. 4 * Copyright (C) 1997 5 * John S. Dyson. All rights reserved. 6 * Copyright (C) 2013-2014 7 * Matthew Dillon, All rights reserved. 8 * 9 * This code contains ideas from software contributed to Berkeley by 10 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 11 * System project at Carnegie-Mellon University. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. Neither the name of the University nor the names of its contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "opt_lint.h" 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/kernel.h> 43 #include <sys/proc.h> 44 #include <sys/lock.h> 45 #include <sys/sysctl.h> 46 #include <sys/spinlock.h> 47 #include <sys/thread2.h> 48 #include <sys/spinlock2.h> 49 50 extern struct lock sysctllock; 51 52 static void undo_upreq(struct lock *lkp); 53 54 #ifdef DEBUG_CANCEL_LOCKS 55 56 static int sysctl_cancel_lock(SYSCTL_HANDLER_ARGS); 57 static int sysctl_cancel_test(SYSCTL_HANDLER_ARGS); 58 59 static struct lock cancel_lk; 60 LOCK_SYSINIT(cancellk, &cancel_lk, "cancel", 0); 61 SYSCTL_PROC(_kern, OID_AUTO, cancel_lock, CTLTYPE_INT|CTLFLAG_RW, 0, 0, 62 sysctl_cancel_lock, "I", "test cancelable locks"); 63 SYSCTL_PROC(_kern, OID_AUTO, cancel_test, CTLTYPE_INT|CTLFLAG_RW, 0, 0, 64 sysctl_cancel_test, "I", "test cancelable locks"); 65 66 #endif 67 68 /* 69 * Locking primitives implementation. 70 * Locks provide shared/exclusive sychronization. 71 */ 72 73 #ifdef DEBUG_LOCKS 74 #define COUNT(td, x) (td)->td_locks += (x) 75 #else 76 #define COUNT(td, x) 77 #endif 78 79 #define LOCK_WAIT_TIME 100 80 #define LOCK_SAMPLE_WAIT 7 81 82 /* 83 * Set, change, or release a lock. 84 */ 85 int 86 #ifndef DEBUG_LOCKS 87 lockmgr(struct lock *lkp, u_int flags) 88 #else 89 debuglockmgr(struct lock *lkp, u_int flags, 90 const char *name, const char *file, int line) 91 #endif 92 { 93 thread_t td; 94 thread_t otd; 95 int error; 96 int extflags; 97 int count; 98 int pflags; 99 int wflags; 100 int timo; 101 #ifdef DEBUG_LOCKS 102 int i; 103 #endif 104 105 error = 0; 106 107 if (mycpu->gd_intr_nesting_level && 108 (flags & LK_NOWAIT) == 0 && 109 (flags & LK_TYPE_MASK) != LK_RELEASE && 110 panic_cpu_gd != mycpu 111 ) { 112 113 #ifndef DEBUG_LOCKS 114 panic("lockmgr %s from %p: called from interrupt, ipi, " 115 "or hard code section", 116 lkp->lk_wmesg, ((int **)&lkp)[-1]); 117 #else 118 panic("lockmgr %s from %s:%d: called from interrupt, ipi, " 119 "or hard code section", 120 lkp->lk_wmesg, file, line); 121 #endif 122 } 123 124 #ifdef DEBUG_LOCKS 125 if (mycpu->gd_spinlocks && ((flags & LK_NOWAIT) == 0)) { 126 panic("lockmgr %s from %s:%d: called with %d spinlocks held", 127 lkp->lk_wmesg, file, line, mycpu->gd_spinlocks); 128 } 129 #endif 130 131 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 132 td = curthread; 133 134 again: 135 count = lkp->lk_count; 136 cpu_ccfence(); 137 138 switch (flags & LK_TYPE_MASK) { 139 case LK_SHARED: 140 /* 141 * Shared lock critical path case 142 */ 143 if ((count & (LKC_EXREQ|LKC_UPREQ|LKC_EXCL)) == 0) { 144 if (atomic_cmpset_int(&lkp->lk_count, 145 count, count + 1)) { 146 COUNT(td, 1); 147 break; 148 } 149 goto again; 150 } 151 152 /* 153 * If the caller already holds the lock exclusively then 154 * we silently obtain another count on the exclusive lock. 155 * 156 * WARNING! The old FreeBSD behavior was to downgrade, 157 * but this creates a problem when recursions 158 * return to the caller and the caller expects 159 * its original exclusive lock to remain exclusively 160 * locked. 161 */ 162 if (lkp->lk_lockholder == td) { 163 KKASSERT(count & LKC_EXCL); 164 if ((extflags & LK_CANRECURSE) == 0) { 165 if (extflags & LK_NOWAIT) { 166 error = EBUSY; 167 break; 168 } 169 panic("lockmgr: locking against myself"); 170 } 171 atomic_add_int(&lkp->lk_count, 1); 172 COUNT(td, 1); 173 break; 174 } 175 176 /* 177 * Slow path 178 */ 179 pflags = (extflags & LK_PCATCH) ? PCATCH : 0; 180 timo = (extflags & LK_TIMELOCK) ? lkp->lk_timo : 0; 181 wflags = (td->td_flags & TDF_DEADLKTREAT) ? 182 LKC_EXCL : (LKC_EXCL|LKC_EXREQ|LKC_UPREQ); 183 184 /* 185 * Block while the lock is held exclusively or, conditionally, 186 * if other threads are tring to obtain an exclusive lock or 187 * upgrade to one. 188 */ 189 if (count & wflags) { 190 if (extflags & LK_CANCELABLE) { 191 if (count & LKC_CANCEL) { 192 error = ENOLCK; 193 break; 194 } 195 } 196 if (extflags & LK_NOWAIT) { 197 error = EBUSY; 198 break; 199 } 200 tsleep_interlock(lkp, pflags); 201 if (!atomic_cmpset_int(&lkp->lk_count, count, 202 count | LKC_SHREQ)) { 203 goto again; 204 } 205 206 mycpu->gd_cnt.v_lock_name[0] = 'S'; 207 strncpy(mycpu->gd_cnt.v_lock_name + 1, 208 lkp->lk_wmesg, 209 sizeof(mycpu->gd_cnt.v_lock_name) - 2); 210 ++mycpu->gd_cnt.v_lock_colls; 211 212 error = tsleep(lkp, pflags | PINTERLOCKED, 213 lkp->lk_wmesg, timo); 214 if (error) 215 break; 216 if (extflags & LK_SLEEPFAIL) { 217 error = ENOLCK; 218 break; 219 } 220 goto again; 221 } 222 223 /* 224 * Otherwise we can bump the count 225 */ 226 if (atomic_cmpset_int(&lkp->lk_count, count, count + 1)) { 227 COUNT(td, 1); 228 break; 229 } 230 goto again; 231 232 case LK_EXCLUSIVE: 233 /* 234 * Exclusive lock critical path. 235 */ 236 if (count == 0) { 237 if (atomic_cmpset_int(&lkp->lk_count, count, 238 LKC_EXCL | (count + 1))) { 239 lkp->lk_lockholder = td; 240 COUNT(td, 1); 241 break; 242 } 243 goto again; 244 } 245 246 /* 247 * Recursive lock if we already hold it exclusively. 248 */ 249 if (lkp->lk_lockholder == td) { 250 KKASSERT(count & LKC_EXCL); 251 if ((extflags & LK_CANRECURSE) == 0) { 252 if (extflags & LK_NOWAIT) { 253 error = EBUSY; 254 break; 255 } 256 panic("lockmgr: locking against myself"); 257 } 258 atomic_add_int(&lkp->lk_count, 1); 259 COUNT(td, 1); 260 break; 261 } 262 263 /* 264 * We will block, handle LK_NOWAIT 265 */ 266 if (extflags & LK_NOWAIT) { 267 error = EBUSY; 268 break; 269 } 270 if (extflags & LK_CANCELABLE) { 271 if (count & LKC_CANCEL) { 272 error = ENOLCK; 273 break; 274 } 275 } 276 277 /* 278 * Wait until we can obtain the exclusive lock. EXREQ is 279 * automatically cleared when all current holders release 280 * so if we abort the operation we can safely leave it set. 281 * There might be other exclusive requesters. 282 */ 283 pflags = (extflags & LK_PCATCH) ? PCATCH : 0; 284 timo = (extflags & LK_TIMELOCK) ? lkp->lk_timo : 0; 285 286 tsleep_interlock(lkp, pflags); 287 if (!atomic_cmpset_int(&lkp->lk_count, count, 288 count | LKC_EXREQ)) { 289 goto again; 290 } 291 292 mycpu->gd_cnt.v_lock_name[0] = 'X'; 293 strncpy(mycpu->gd_cnt.v_lock_name + 1, 294 lkp->lk_wmesg, 295 sizeof(mycpu->gd_cnt.v_lock_name) - 2); 296 ++mycpu->gd_cnt.v_lock_colls; 297 298 error = tsleep(lkp, pflags | PINTERLOCKED, 299 lkp->lk_wmesg, timo); 300 if (error) 301 break; 302 if (extflags & LK_SLEEPFAIL) { 303 error = ENOLCK; 304 break; 305 } 306 goto again; 307 308 case LK_DOWNGRADE: 309 /* 310 * Downgrade an exclusive lock into a shared lock. All 311 * counts on a recursive exclusive lock become shared. 312 * 313 * This function always succeeds. 314 */ 315 if (lkp->lk_lockholder != td || 316 (count & (LKC_EXCL|LKC_MASK)) != (LKC_EXCL|1)) { 317 panic("lockmgr: not holding exclusive lock"); 318 } 319 320 #ifdef DEBUG_LOCKS 321 for (i = 0; i < LOCKMGR_DEBUG_ARRAY_SIZE; i++) { 322 if (td->td_lockmgr_stack[i] == lkp && 323 td->td_lockmgr_stack_id[i] > 0 324 ) { 325 td->td_lockmgr_stack_id[i]--; 326 break; 327 } 328 } 329 #endif 330 /* 331 * NOTE! Must NULL-out lockholder before releasing LKC_EXCL. 332 */ 333 otd = lkp->lk_lockholder; 334 lkp->lk_lockholder = NULL; 335 if (atomic_cmpset_int(&lkp->lk_count, count, 336 count & ~(LKC_EXCL|LKC_SHREQ))) { 337 if (count & LKC_SHREQ) 338 wakeup(lkp); 339 break; 340 } 341 lkp->lk_lockholder = otd; 342 goto again; 343 344 case LK_EXCLUPGRADE: 345 /* 346 * Upgrade from a single shared lock to an exclusive lock. 347 * 348 * If another process is ahead of us to get an upgrade, 349 * then we want to fail rather than have an intervening 350 * exclusive access. The shared lock is released on 351 * failure. 352 */ 353 if (count & LKC_UPREQ) { 354 flags = LK_RELEASE; 355 error = EBUSY; 356 goto again; 357 } 358 /* fall through into normal upgrade */ 359 360 case LK_UPGRADE: 361 /* 362 * Upgrade a shared lock to an exclusive one. This can cause 363 * the lock to be temporarily released and stolen by other 364 * threads. LK_SLEEPFAIL or LK_NOWAIT may be used to detect 365 * this case, or use LK_EXCLUPGRADE. 366 * 367 * If the lock is already exclusively owned by us, this 368 * operation is a NOP. 369 * 370 * If we return an error (even NOWAIT), the current lock will 371 * be released. 372 * 373 * Start with the critical path. 374 */ 375 if ((count & (LKC_UPREQ|LKC_EXCL|LKC_MASK)) == 1) { 376 if (atomic_cmpset_int(&lkp->lk_count, count, 377 count | LKC_EXCL)) { 378 lkp->lk_lockholder = td; 379 break; 380 } 381 goto again; 382 } 383 384 /* 385 * If we already hold the lock exclusively this operation 386 * succeeds and is a NOP. 387 */ 388 if (count & LKC_EXCL) { 389 if (lkp->lk_lockholder == td) 390 break; 391 panic("lockmgr: upgrade unowned lock"); 392 } 393 if ((count & LKC_MASK) == 0) 394 panic("lockmgr: upgrade unowned lock"); 395 396 /* 397 * We cannot upgrade without blocking at this point. 398 */ 399 if (extflags & LK_NOWAIT) { 400 flags = LK_RELEASE; 401 error = EBUSY; 402 goto again; 403 } 404 if (extflags & LK_CANCELABLE) { 405 if (count & LKC_CANCEL) { 406 error = ENOLCK; 407 break; 408 } 409 } 410 411 /* 412 * Release the shared lock and request the upgrade. 413 */ 414 pflags = (extflags & LK_PCATCH) ? PCATCH : 0; 415 timo = (extflags & LK_TIMELOCK) ? lkp->lk_timo : 0; 416 tsleep_interlock(lkp, pflags); 417 wflags = (count & LKC_UPREQ) ? LKC_EXREQ : LKC_UPREQ; 418 419 /* 420 * If someone else owns UPREQ and this transition would 421 * allow it to be granted, we have to grant it. Otherwise 422 * we release the shared lock. 423 */ 424 if ((count & (LKC_UPREQ|LKC_MASK)) == (LKC_UPREQ | 1)) { 425 wflags |= LKC_EXCL | LKC_UPGRANT; 426 wflags |= count; 427 wflags &= ~LKC_UPREQ; 428 } else { 429 wflags |= (count - 1); 430 } 431 432 if (atomic_cmpset_int(&lkp->lk_count, count, wflags)) { 433 COUNT(td, -1); 434 435 /* 436 * Must wakeup the thread granted the upgrade. 437 */ 438 if ((count & (LKC_UPREQ|LKC_MASK)) == (LKC_UPREQ | 1)) 439 wakeup(lkp); 440 441 mycpu->gd_cnt.v_lock_name[0] = 'U'; 442 strncpy(mycpu->gd_cnt.v_lock_name + 1, 443 lkp->lk_wmesg, 444 sizeof(mycpu->gd_cnt.v_lock_name) - 2); 445 ++mycpu->gd_cnt.v_lock_colls; 446 447 error = tsleep(lkp, pflags | PINTERLOCKED, 448 lkp->lk_wmesg, timo); 449 if (error) 450 break; 451 if (extflags & LK_SLEEPFAIL) { 452 error = ENOLCK; 453 break; 454 } 455 456 /* 457 * Refactor to either LK_EXCLUSIVE or LK_WAITUPGRADE, 458 * depending on whether we were able to acquire the 459 * LKC_UPREQ bit. 460 */ 461 if (count & LKC_UPREQ) 462 flags = LK_EXCLUSIVE; /* someone else */ 463 else 464 flags = LK_WAITUPGRADE; /* we own the bit */ 465 } 466 goto again; 467 468 case LK_WAITUPGRADE: 469 /* 470 * We own the LKC_UPREQ bit, wait until we are granted the 471 * exclusive lock (LKC_UPGRANT is set). 472 * 473 * IF THE OPERATION FAILS (tsleep error tsleep+LK_SLEEPFAIL), 474 * we have to undo the upgrade request and clean up any lock 475 * that might have been granted via a race. 476 */ 477 if (count & LKC_UPGRANT) { 478 if (atomic_cmpset_int(&lkp->lk_count, count, 479 count & ~LKC_UPGRANT)) { 480 lkp->lk_lockholder = td; 481 KKASSERT(count & LKC_EXCL); 482 break; 483 } 484 /* retry */ 485 } else if ((count & LKC_CANCEL) && (extflags & LK_CANCELABLE)) { 486 undo_upreq(lkp); 487 error = ENOLCK; 488 break; 489 } else { 490 pflags = (extflags & LK_PCATCH) ? PCATCH : 0; 491 timo = (extflags & LK_TIMELOCK) ? lkp->lk_timo : 0; 492 tsleep_interlock(lkp, pflags); 493 if (atomic_cmpset_int(&lkp->lk_count, count, count)) { 494 495 mycpu->gd_cnt.v_lock_name[0] = 'U'; 496 strncpy(mycpu->gd_cnt.v_lock_name + 1, 497 lkp->lk_wmesg, 498 sizeof(mycpu->gd_cnt.v_lock_name) - 2); 499 ++mycpu->gd_cnt.v_lock_colls; 500 501 error = tsleep(lkp, pflags | PINTERLOCKED, 502 lkp->lk_wmesg, timo); 503 if (error) { 504 undo_upreq(lkp); 505 break; 506 } 507 if (extflags & LK_SLEEPFAIL) { 508 error = ENOLCK; 509 undo_upreq(lkp); 510 break; 511 } 512 } 513 /* retry */ 514 } 515 goto again; 516 517 case LK_RELEASE: 518 /* 519 * Release the currently held lock. If releasing the current 520 * lock as part of an error return, error will ALREADY be 521 * non-zero. 522 * 523 * When releasing the last lock we automatically transition 524 * LKC_UPREQ to LKC_EXCL|1. 525 * 526 * WARNING! We cannot detect when there are multiple exclusive 527 * requests pending. We clear EXREQ unconditionally 528 * on the 1->0 transition so it is possible for 529 * shared requests to race the next exclusive 530 * request. 531 * 532 * Always succeeds. 533 */ 534 if ((count & LKC_MASK) == 0) 535 panic("lockmgr: LK_RELEASE: no lock held"); 536 537 if (count & LKC_EXCL) { 538 if (lkp->lk_lockholder != LK_KERNTHREAD && 539 lkp->lk_lockholder != td) { 540 panic("lockmgr: pid %d, not exlusive " 541 "lock holder thr %p/%p unlocking", 542 (td->td_proc ? td->td_proc->p_pid : -1), 543 td, lkp->lk_lockholder); 544 } 545 if ((count & (LKC_UPREQ|LKC_MASK)) == 1) { 546 /* 547 * Last exclusive count is being released 548 */ 549 otd = lkp->lk_lockholder; 550 lkp->lk_lockholder = NULL; 551 if (!atomic_cmpset_int(&lkp->lk_count, count, 552 (count - 1) & 553 ~(LKC_EXCL | LKC_EXREQ | 554 LKC_SHREQ| LKC_CANCEL))) { 555 lkp->lk_lockholder = otd; 556 goto again; 557 } 558 if (count & (LKC_EXREQ|LKC_SHREQ)) 559 wakeup(lkp); 560 /* success */ 561 } else if ((count & (LKC_UPREQ|LKC_MASK)) == 562 (LKC_UPREQ | 1)) { 563 /* 564 * Last exclusive count is being released but 565 * an upgrade request is present, automatically 566 * grant an exclusive state to the owner of 567 * the upgrade request. 568 */ 569 otd = lkp->lk_lockholder; 570 lkp->lk_lockholder = NULL; 571 if (!atomic_cmpset_int(&lkp->lk_count, count, 572 (count & ~LKC_UPREQ) | 573 LKC_UPGRANT)) { 574 lkp->lk_lockholder = otd; 575 } 576 wakeup(lkp); 577 /* success */ 578 } else { 579 otd = lkp->lk_lockholder; 580 if (!atomic_cmpset_int(&lkp->lk_count, count, 581 count - 1)) { 582 goto again; 583 } 584 /* success */ 585 } 586 /* success */ 587 if (otd != LK_KERNTHREAD) 588 COUNT(td, -1); 589 } else { 590 if ((count & (LKC_UPREQ|LKC_MASK)) == 1) { 591 /* 592 * Last shared count is being released. 593 */ 594 if (!atomic_cmpset_int(&lkp->lk_count, count, 595 (count - 1) & 596 ~(LKC_EXREQ | LKC_SHREQ | 597 LKC_CANCEL))) { 598 goto again; 599 } 600 if (count & (LKC_EXREQ|LKC_SHREQ)) 601 wakeup(lkp); 602 /* success */ 603 } else if ((count & (LKC_UPREQ|LKC_MASK)) == 604 (LKC_UPREQ | 1)) { 605 /* 606 * Last shared count is being released but 607 * an upgrade request is present, automatically 608 * grant an exclusive state to the owner of 609 * the upgrade request. Masked count 610 * remains 1. 611 */ 612 if (!atomic_cmpset_int(&lkp->lk_count, count, 613 (count & ~(LKC_UPREQ | 614 LKC_CANCEL)) | 615 LKC_EXCL | LKC_UPGRANT)) { 616 goto again; 617 } 618 wakeup(lkp); 619 } else { 620 if (!atomic_cmpset_int(&lkp->lk_count, count, 621 count - 1)) { 622 goto again; 623 } 624 } 625 /* success */ 626 COUNT(td, -1); 627 } 628 break; 629 630 case LK_CANCEL_BEG: 631 /* 632 * Start canceling blocked requestors or later requestors. 633 * requestors must use CANCELABLE. Don't waste time issuing 634 * a wakeup if nobody is pending. 635 */ 636 KKASSERT((count & LKC_CANCEL) == 0); /* disallowed case */ 637 KKASSERT((count & LKC_MASK) != 0); /* issue w/lock held */ 638 if (!atomic_cmpset_int(&lkp->lk_count, 639 count, count | LKC_CANCEL)) { 640 goto again; 641 } 642 if (count & (LKC_EXREQ|LKC_SHREQ|LKC_UPREQ)) { 643 wakeup(lkp); 644 } 645 break; 646 647 case LK_CANCEL_END: 648 atomic_clear_int(&lkp->lk_count, LKC_CANCEL); 649 break; 650 651 default: 652 panic("lockmgr: unknown locktype request %d", 653 flags & LK_TYPE_MASK); 654 /* NOTREACHED */ 655 } 656 return (error); 657 } 658 659 /* 660 * Undo an upgrade request 661 */ 662 static 663 void 664 undo_upreq(struct lock *lkp) 665 { 666 int count; 667 668 for (;;) { 669 count = lkp->lk_count; 670 cpu_ccfence(); 671 if (count & LKC_UPGRANT) { 672 /* 673 * UPREQ was shifted to UPGRANT. We own UPGRANT now, 674 * another thread might own UPREQ. Clear UPGRANT 675 * and release the granted lock. 676 */ 677 if (atomic_cmpset_int(&lkp->lk_count, count, 678 count & ~LKC_UPGRANT)) { 679 lockmgr(lkp, LK_RELEASE); 680 break; 681 } 682 } else if (count & LKC_EXCL) { 683 /* 684 * Clear the UPREQ we still own. Nobody to wakeup 685 * here because there is an existing exclusive 686 * holder. 687 */ 688 KKASSERT(count & LKC_UPREQ); 689 KKASSERT((count & LKC_MASK) > 0); 690 if (atomic_cmpset_int(&lkp->lk_count, count, 691 count & ~LKC_UPREQ)) { 692 wakeup(lkp); 693 break; 694 } 695 } else if (count & LKC_EXREQ) { 696 /* 697 * Clear the UPREQ we still own. We cannot wakeup any 698 * shared waiters because there is an exclusive 699 * request pending. 700 */ 701 KKASSERT(count & LKC_UPREQ); 702 KKASSERT((count & LKC_MASK) > 0); 703 if (atomic_cmpset_int(&lkp->lk_count, count, 704 count & ~LKC_UPREQ)) { 705 break; 706 } 707 } else { 708 /* 709 * Clear the UPREQ we still own. Wakeup any shared 710 * waiters. 711 */ 712 KKASSERT(count & LKC_UPREQ); 713 KKASSERT((count & LKC_MASK) > 0); 714 if (atomic_cmpset_int(&lkp->lk_count, count, 715 count & 716 ~(LKC_UPREQ | LKC_SHREQ))) { 717 if (count & LKC_SHREQ) 718 wakeup(lkp); 719 break; 720 } 721 } 722 /* retry */ 723 } 724 } 725 726 void 727 lockmgr_kernproc(struct lock *lp) 728 { 729 struct thread *td __debugvar = curthread; 730 731 if (lp->lk_lockholder != LK_KERNTHREAD) { 732 KASSERT(lp->lk_lockholder == td, 733 ("lockmgr_kernproc: lock not owned by curthread %p", td)); 734 lp->lk_lockholder = LK_KERNTHREAD; 735 COUNT(td, -1); 736 } 737 } 738 739 /* 740 * Initialize a lock; required before use. 741 */ 742 void 743 lockinit(struct lock *lkp, const char *wmesg, int timo, int flags) 744 { 745 lkp->lk_flags = (flags & LK_EXTFLG_MASK); 746 lkp->lk_count = 0; 747 lkp->lk_wmesg = wmesg; 748 lkp->lk_timo = timo; 749 lkp->lk_lockholder = LK_NOTHREAD; 750 } 751 752 /* 753 * Reinitialize a lock that is being reused for a different purpose, but 754 * which may have pending (blocked) threads sitting on it. The caller 755 * must already hold the interlock. 756 */ 757 void 758 lockreinit(struct lock *lkp, const char *wmesg, int timo, int flags) 759 { 760 lkp->lk_wmesg = wmesg; 761 lkp->lk_timo = timo; 762 } 763 764 /* 765 * De-initialize a lock. The structure must no longer be used by anyone. 766 */ 767 void 768 lockuninit(struct lock *lkp) 769 { 770 KKASSERT((lkp->lk_count & (LKC_EXREQ|LKC_SHREQ|LKC_UPREQ)) == 0); 771 } 772 773 /* 774 * Determine the status of a lock. 775 */ 776 int 777 lockstatus(struct lock *lkp, struct thread *td) 778 { 779 int lock_type = 0; 780 int count; 781 782 count = lkp->lk_count; 783 cpu_ccfence(); 784 785 if (count & LKC_EXCL) { 786 if (td == NULL || lkp->lk_lockholder == td) 787 lock_type = LK_EXCLUSIVE; 788 else 789 lock_type = LK_EXCLOTHER; 790 } else if (count & LKC_MASK) { 791 lock_type = LK_SHARED; 792 } 793 return (lock_type); 794 } 795 796 /* 797 * Return non-zero if the caller owns the lock shared or exclusive. 798 * We can only guess re: shared locks. 799 */ 800 int 801 lockowned(struct lock *lkp) 802 { 803 thread_t td = curthread; 804 int count; 805 806 count = lkp->lk_count; 807 cpu_ccfence(); 808 809 if (count & LKC_EXCL) 810 return(lkp->lk_lockholder == td); 811 else 812 return((count & LKC_MASK) != 0); 813 } 814 815 /* 816 * Determine the number of holders of a lock. 817 * 818 * The non-blocking version can usually be used for assertions. 819 */ 820 int 821 lockcount(struct lock *lkp) 822 { 823 return(lkp->lk_count & LKC_MASK); 824 } 825 826 int 827 lockcountnb(struct lock *lkp) 828 { 829 return(lkp->lk_count & LKC_MASK); 830 } 831 832 /* 833 * Print out information about state of a lock. Used by VOP_PRINT 834 * routines to display status about contained locks. 835 */ 836 void 837 lockmgr_printinfo(struct lock *lkp) 838 { 839 struct thread *td = lkp->lk_lockholder; 840 struct proc *p; 841 int count; 842 843 count = lkp->lk_count; 844 cpu_ccfence(); 845 846 if (td && td != LK_KERNTHREAD && td != LK_NOTHREAD) 847 p = td->td_proc; 848 else 849 p = NULL; 850 851 if (count & LKC_EXCL) { 852 kprintf(" lock type %s: EXCLUS (count %08x) by td %p pid %d", 853 lkp->lk_wmesg, count, td, 854 p ? p->p_pid : -99); 855 } else if (count & LKC_MASK) { 856 kprintf(" lock type %s: SHARED (count %08x)", 857 lkp->lk_wmesg, count); 858 } else { 859 kprintf(" lock type %s: NOTHELD", lkp->lk_wmesg); 860 } 861 if (count & (LKC_EXREQ|LKC_SHREQ)) 862 kprintf(" with waiters\n"); 863 else 864 kprintf("\n"); 865 } 866 867 void 868 lock_sysinit(struct lock_args *arg) 869 { 870 lockinit(arg->la_lock, arg->la_desc, 0, arg->la_flags); 871 } 872 873 #ifdef DEBUG_CANCEL_LOCKS 874 875 static 876 int 877 sysctl_cancel_lock(SYSCTL_HANDLER_ARGS) 878 { 879 int error; 880 881 if (req->newptr) { 882 lockmgr(&sysctllock, LK_RELEASE); 883 lockmgr(&cancel_lk, LK_EXCLUSIVE); 884 kprintf("x"); 885 error = tsleep(&error, PCATCH, "canmas", hz * 5); 886 lockmgr(&cancel_lk, LK_CANCEL_BEG); 887 kprintf("y"); 888 error = tsleep(&error, PCATCH, "canmas", hz * 5); 889 kprintf("z"); 890 lockmgr(&cancel_lk, LK_RELEASE); 891 lockmgr(&sysctllock, LK_EXCLUSIVE); 892 SYSCTL_OUT(req, &error, sizeof(error)); 893 } 894 error = 0; 895 896 return error; 897 } 898 899 static 900 int 901 sysctl_cancel_test(SYSCTL_HANDLER_ARGS) 902 { 903 int error; 904 905 if (req->newptr) { 906 error = lockmgr(&cancel_lk, LK_EXCLUSIVE|LK_CANCELABLE); 907 if (error == 0) 908 lockmgr(&cancel_lk, LK_RELEASE); 909 SYSCTL_OUT(req, &error, sizeof(error)); 910 kprintf("test %d\n", error); 911 } 912 913 return 0; 914 } 915 916 #endif 917