1 /* 2 * Copyright (c) 1995 3 * The Regents of the University of California. All rights reserved. 4 * 5 * Copyright (C) 1997 6 * John S. Dyson. All rights reserved. 7 * 8 * This code contains ideas from software contributed to Berkeley by 9 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 10 * System project at Carnegie-Mellon University. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the University of 23 * California, Berkeley and its contributors. 24 * 4. Neither the name of the University nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 41 * $FreeBSD: src/sys/kern/kern_lock.c,v 1.31.2.3 2001/12/25 01:44:44 dillon Exp $ 42 * $DragonFly: src/sys/kern/kern_lock.c,v 1.23 2006/08/08 03:52:40 dillon Exp $ 43 */ 44 45 #include "opt_lint.h" 46 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/kernel.h> 50 #include <sys/proc.h> 51 #include <sys/lock.h> 52 #include <sys/sysctl.h> 53 #include <sys/spinlock.h> 54 #include <sys/thread2.h> 55 #include <sys/spinlock2.h> 56 57 /* 58 * 0: no warnings, 1: warnings, 2: panic 59 */ 60 static int lockmgr_from_int = 1; 61 SYSCTL_INT(_debug, OID_AUTO, lockmgr_from_int, CTLFLAG_RW, &lockmgr_from_int, 0, ""); 62 63 /* 64 * Locking primitives implementation. 65 * Locks provide shared/exclusive sychronization. 66 */ 67 68 #ifdef SIMPLELOCK_DEBUG 69 #define COUNT(td, x) (td)->td_locks += (x) 70 #else 71 #define COUNT(td, x) 72 #endif 73 74 #define LOCK_WAIT_TIME 100 75 #define LOCK_SAMPLE_WAIT 7 76 77 #if defined(DIAGNOSTIC) 78 #define LOCK_INLINE 79 #else 80 #define LOCK_INLINE __inline 81 #endif 82 83 #define LK_ALL (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | \ 84 LK_SHARE_NONZERO | LK_WAIT_NONZERO) 85 86 static int acquire(struct lock *lkp, int extflags, int wanted); 87 88 static LOCK_INLINE void 89 sharelock(struct lock *lkp, int incr) { 90 lkp->lk_flags |= LK_SHARE_NONZERO; 91 lkp->lk_sharecount += incr; 92 } 93 94 static LOCK_INLINE void 95 shareunlock(struct lock *lkp, int decr) { 96 97 KASSERT(lkp->lk_sharecount >= decr, ("shareunlock: count < decr")); 98 99 if (lkp->lk_sharecount == decr) { 100 lkp->lk_flags &= ~LK_SHARE_NONZERO; 101 if (lkp->lk_flags & (LK_WANT_UPGRADE | LK_WANT_EXCL)) { 102 wakeup(lkp); 103 } 104 lkp->lk_sharecount = 0; 105 } else { 106 lkp->lk_sharecount -= decr; 107 } 108 } 109 110 /* 111 * lock acquisition helper routine. Called with the lock's spinlock held. 112 */ 113 static int 114 acquire(struct lock *lkp, int extflags, int wanted) 115 { 116 int error; 117 118 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & wanted)) { 119 return EBUSY; 120 } 121 122 while ((lkp->lk_flags & wanted) != 0) { 123 lkp->lk_flags |= LK_WAIT_NONZERO; 124 lkp->lk_waitcount++; 125 126 /* 127 * Atomic spinlock release/sleep/reacquire. 128 */ 129 error = msleep(lkp, &lkp->lk_spinlock, 130 ((extflags & LK_PCATCH) ? PCATCH : 0), 131 lkp->lk_wmesg, 132 ((extflags & LK_TIMELOCK) ? lkp->lk_timo : 0)); 133 if (lkp->lk_waitcount == 1) { 134 lkp->lk_flags &= ~LK_WAIT_NONZERO; 135 lkp->lk_waitcount = 0; 136 } else { 137 lkp->lk_waitcount--; 138 } 139 if (error) 140 return error; 141 if (extflags & LK_SLEEPFAIL) 142 return ENOLCK; 143 } 144 return 0; 145 } 146 147 /* 148 * Set, change, or release a lock. 149 * 150 * Shared requests increment the shared count. Exclusive requests set the 151 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 152 * accepted shared locks and shared-to-exclusive upgrades to go away. 153 * 154 * A spinlock is held for most of the procedure. We must not do anything 155 * fancy while holding the spinlock. 156 */ 157 int 158 #ifndef DEBUG_LOCKS 159 lockmgr(struct lock *lkp, u_int flags) 160 #else 161 debuglockmgr(struct lock *lkp, u_int flags, 162 const char *name, const char *file, int line) 163 #endif 164 { 165 thread_t td; 166 int error; 167 int extflags; 168 static int didpanic; 169 170 error = 0; 171 172 if (lockmgr_from_int && mycpu->gd_intr_nesting_level && 173 (flags & LK_NOWAIT) == 0 && 174 (flags & LK_TYPE_MASK) != LK_RELEASE && didpanic == 0) { 175 #ifndef DEBUG_LOCKS 176 if (lockmgr_from_int == 2) { 177 didpanic = 1; 178 panic( 179 "lockmgr %s from %p: called from interrupt", 180 lkp->lk_wmesg, ((int **)&lkp)[-1]); 181 didpanic = 0; 182 } else { 183 printf( 184 "lockmgr %s from %p: called from interrupt\n", 185 lkp->lk_wmesg, ((int **)&lkp)[-1]); 186 } 187 #else 188 if (lockmgr_from_int == 2) { 189 didpanic = 1; 190 panic( 191 "lockmgr %s from %s:%d: called from interrupt", 192 lkp->lk_wmesg, file, line); 193 didpanic = 0; 194 } else { 195 printf( 196 "lockmgr %s from %s:%d: called from interrupt\n", 197 lkp->lk_wmesg, file, line); 198 } 199 #endif 200 } 201 202 spin_lock_wr(&lkp->lk_spinlock); 203 204 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 205 td = curthread; 206 207 switch (flags & LK_TYPE_MASK) { 208 case LK_SHARED: 209 /* 210 * If we are not the exclusive lock holder, we have to block 211 * while there is an exclusive lock holder or while an 212 * exclusive lock request or upgrade request is in progress. 213 * 214 * However, if P_DEADLKTREAT is set, we override exclusive 215 * lock requests or upgrade requests ( but not the exclusive 216 * lock itself ). 217 */ 218 if (lkp->lk_lockholder != td) { 219 if (td->td_flags & TDF_DEADLKTREAT) { 220 error = acquire( 221 lkp, 222 extflags, 223 LK_HAVE_EXCL 224 ); 225 } else { 226 error = acquire( 227 lkp, 228 extflags, 229 LK_HAVE_EXCL | LK_WANT_EXCL | 230 LK_WANT_UPGRADE 231 ); 232 } 233 if (error) 234 break; 235 sharelock(lkp, 1); 236 COUNT(td, 1); 237 break; 238 } 239 /* 240 * We hold an exclusive lock, so downgrade it to shared. 241 * An alternative would be to fail with EDEADLK. 242 */ 243 sharelock(lkp, 1); 244 COUNT(td, 1); 245 /* fall into downgrade */ 246 247 case LK_DOWNGRADE: 248 if (lkp->lk_lockholder != td || lkp->lk_exclusivecount == 0) { 249 spin_unlock_wr(&lkp->lk_spinlock); 250 panic("lockmgr: not holding exclusive lock"); 251 } 252 sharelock(lkp, lkp->lk_exclusivecount); 253 lkp->lk_exclusivecount = 0; 254 lkp->lk_flags &= ~LK_HAVE_EXCL; 255 lkp->lk_lockholder = LK_NOTHREAD; 256 if (lkp->lk_waitcount) 257 wakeup((void *)lkp); 258 break; 259 260 case LK_EXCLUPGRADE: 261 /* 262 * If another process is ahead of us to get an upgrade, 263 * then we want to fail rather than have an intervening 264 * exclusive access. 265 */ 266 if (lkp->lk_flags & LK_WANT_UPGRADE) { 267 shareunlock(lkp, 1); 268 COUNT(td, -1); 269 error = EBUSY; 270 break; 271 } 272 /* fall into normal upgrade */ 273 274 case LK_UPGRADE: 275 /* 276 * Upgrade a shared lock to an exclusive one. If another 277 * shared lock has already requested an upgrade to an 278 * exclusive lock, our shared lock is released and an 279 * exclusive lock is requested (which will be granted 280 * after the upgrade). If we return an error, the file 281 * will always be unlocked. 282 */ 283 if ((lkp->lk_lockholder == td) || (lkp->lk_sharecount <= 0)) { 284 spin_unlock_wr(&lkp->lk_spinlock); 285 panic("lockmgr: upgrade exclusive lock"); 286 } 287 shareunlock(lkp, 1); 288 COUNT(td, -1); 289 /* 290 * If we are just polling, check to see if we will block. 291 */ 292 if ((extflags & LK_NOWAIT) && 293 ((lkp->lk_flags & LK_WANT_UPGRADE) || 294 lkp->lk_sharecount > 1)) { 295 error = EBUSY; 296 break; 297 } 298 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 299 /* 300 * We are first shared lock to request an upgrade, so 301 * request upgrade and wait for the shared count to 302 * drop to zero, then take exclusive lock. 303 */ 304 lkp->lk_flags |= LK_WANT_UPGRADE; 305 error = acquire(lkp, extflags, LK_SHARE_NONZERO); 306 lkp->lk_flags &= ~LK_WANT_UPGRADE; 307 308 if (error) 309 break; 310 lkp->lk_flags |= LK_HAVE_EXCL; 311 lkp->lk_lockholder = td; 312 if (lkp->lk_exclusivecount != 0) { 313 spin_unlock_wr(&lkp->lk_spinlock); 314 panic("lockmgr: non-zero exclusive count"); 315 } 316 lkp->lk_exclusivecount = 1; 317 #if defined(DEBUG_LOCKS) 318 lkp->lk_filename = file; 319 lkp->lk_lineno = line; 320 lkp->lk_lockername = name; 321 #endif 322 COUNT(td, 1); 323 break; 324 } 325 /* 326 * Someone else has requested upgrade. Release our shared 327 * lock, awaken upgrade requestor if we are the last shared 328 * lock, then request an exclusive lock. 329 */ 330 if ( (lkp->lk_flags & (LK_SHARE_NONZERO|LK_WAIT_NONZERO)) == 331 LK_WAIT_NONZERO) 332 wakeup((void *)lkp); 333 /* fall into exclusive request */ 334 335 case LK_EXCLUSIVE: 336 if (lkp->lk_lockholder == td && td != LK_KERNTHREAD) { 337 /* 338 * Recursive lock. 339 */ 340 if ((extflags & (LK_NOWAIT | LK_CANRECURSE)) == 0) { 341 spin_unlock_wr(&lkp->lk_spinlock); 342 panic("lockmgr: locking against myself"); 343 } 344 if ((extflags & LK_CANRECURSE) != 0) { 345 lkp->lk_exclusivecount++; 346 COUNT(td, 1); 347 break; 348 } 349 } 350 /* 351 * If we are just polling, check to see if we will sleep. 352 */ 353 if ((extflags & LK_NOWAIT) && 354 (lkp->lk_flags & (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO))) { 355 error = EBUSY; 356 break; 357 } 358 /* 359 * Try to acquire the want_exclusive flag. 360 */ 361 error = acquire(lkp, extflags, (LK_HAVE_EXCL | LK_WANT_EXCL)); 362 if (error) 363 break; 364 lkp->lk_flags |= LK_WANT_EXCL; 365 /* 366 * Wait for shared locks and upgrades to finish. 367 */ 368 error = acquire(lkp, extflags, LK_WANT_UPGRADE | LK_SHARE_NONZERO); 369 lkp->lk_flags &= ~LK_WANT_EXCL; 370 if (error) 371 break; 372 lkp->lk_flags |= LK_HAVE_EXCL; 373 lkp->lk_lockholder = td; 374 if (lkp->lk_exclusivecount != 0) { 375 spin_unlock_wr(&lkp->lk_spinlock); 376 panic("lockmgr: non-zero exclusive count"); 377 } 378 lkp->lk_exclusivecount = 1; 379 #if defined(DEBUG_LOCKS) 380 lkp->lk_filename = file; 381 lkp->lk_lineno = line; 382 lkp->lk_lockername = name; 383 #endif 384 COUNT(td, 1); 385 break; 386 387 case LK_RELEASE: 388 if (lkp->lk_exclusivecount != 0) { 389 if (lkp->lk_lockholder != td && 390 lkp->lk_lockholder != LK_KERNTHREAD) { 391 spin_unlock_wr(&lkp->lk_spinlock); 392 panic("lockmgr: pid %d, not %s thr %p unlocking", 393 (td->td_proc ? td->td_proc->p_pid : -99), 394 "exclusive lock holder", 395 lkp->lk_lockholder); 396 } 397 if (lkp->lk_lockholder != LK_KERNTHREAD) { 398 COUNT(td, -1); 399 } 400 if (lkp->lk_exclusivecount == 1) { 401 lkp->lk_flags &= ~LK_HAVE_EXCL; 402 lkp->lk_lockholder = LK_NOTHREAD; 403 lkp->lk_exclusivecount = 0; 404 } else { 405 lkp->lk_exclusivecount--; 406 } 407 } else if (lkp->lk_flags & LK_SHARE_NONZERO) { 408 shareunlock(lkp, 1); 409 COUNT(td, -1); 410 } 411 if (lkp->lk_flags & LK_WAIT_NONZERO) 412 wakeup((void *)lkp); 413 break; 414 415 default: 416 spin_unlock_wr(&lkp->lk_spinlock); 417 panic("lockmgr: unknown locktype request %d", 418 flags & LK_TYPE_MASK); 419 /* NOTREACHED */ 420 } 421 spin_unlock_wr(&lkp->lk_spinlock); 422 return (error); 423 } 424 425 void 426 lockmgr_kernproc(struct lock *lp) 427 { 428 struct thread *td = curthread; 429 430 if (lp->lk_lockholder != LK_KERNTHREAD) { 431 KASSERT(lp->lk_lockholder == td, 432 ("lockmgr_kernproc: lock not owned by curthread %p", td)); 433 COUNT(td, -1); 434 lp->lk_lockholder = LK_KERNTHREAD; 435 } 436 } 437 438 /* 439 * Initialize a lock; required before use. 440 */ 441 void 442 lockinit(struct lock *lkp, char *wmesg, int timo, int flags) 443 { 444 spin_init(&lkp->lk_spinlock); 445 lkp->lk_flags = (flags & LK_EXTFLG_MASK); 446 lkp->lk_sharecount = 0; 447 lkp->lk_waitcount = 0; 448 lkp->lk_exclusivecount = 0; 449 lkp->lk_wmesg = wmesg; 450 lkp->lk_timo = timo; 451 lkp->lk_lockholder = LK_NOTHREAD; 452 } 453 454 /* 455 * Reinitialize a lock that is being reused for a different purpose, but 456 * which may have pending (blocked) threads sitting on it. The caller 457 * must already hold the interlock. 458 */ 459 void 460 lockreinit(struct lock *lkp, char *wmesg, int timo, int flags) 461 { 462 spin_lock_wr(&lkp->lk_spinlock); 463 lkp->lk_flags = (lkp->lk_flags & ~LK_EXTFLG_MASK) | 464 (flags & LK_EXTFLG_MASK); 465 lkp->lk_wmesg = wmesg; 466 lkp->lk_timo = timo; 467 spin_unlock_wr(&lkp->lk_spinlock); 468 } 469 470 /* 471 * Determine the status of a lock. 472 */ 473 int 474 lockstatus(struct lock *lkp, struct thread *td) 475 { 476 int lock_type = 0; 477 478 spin_lock_wr(&lkp->lk_spinlock); 479 if (lkp->lk_exclusivecount != 0) { 480 if (td == NULL || lkp->lk_lockholder == td) 481 lock_type = LK_EXCLUSIVE; 482 else 483 lock_type = LK_EXCLOTHER; 484 } else if (lkp->lk_sharecount != 0) { 485 lock_type = LK_SHARED; 486 } 487 spin_unlock_wr(&lkp->lk_spinlock); 488 return (lock_type); 489 } 490 491 /* 492 * Determine the number of holders of a lock. 493 * 494 * The non-blocking version can usually be used for assertions. 495 */ 496 int 497 lockcount(struct lock *lkp) 498 { 499 int count; 500 501 spin_lock_wr(&lkp->lk_spinlock); 502 count = lkp->lk_exclusivecount + lkp->lk_sharecount; 503 spin_unlock_wr(&lkp->lk_spinlock); 504 return (count); 505 } 506 507 int 508 lockcountnb(struct lock *lkp) 509 { 510 return (lkp->lk_exclusivecount + lkp->lk_sharecount); 511 } 512 513 /* 514 * Print out information about state of a lock. Used by VOP_PRINT 515 * routines to display status about contained locks. 516 */ 517 void 518 lockmgr_printinfo(struct lock *lkp) 519 { 520 struct thread *td = lkp->lk_lockholder; 521 struct proc *p; 522 523 if (td && td != LK_KERNTHREAD && td != LK_NOTHREAD) 524 p = td->td_proc; 525 else 526 p = NULL; 527 528 if (lkp->lk_sharecount) 529 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 530 lkp->lk_sharecount); 531 else if (lkp->lk_flags & LK_HAVE_EXCL) 532 printf(" lock type %s: EXCL (count %d) by td %p pid %d", 533 lkp->lk_wmesg, lkp->lk_exclusivecount, td, 534 p ? p->p_pid : -99); 535 if (lkp->lk_waitcount > 0) 536 printf(" with %d pending", lkp->lk_waitcount); 537 } 538 539