1 /* $OpenBSD: kern_rwlock.c,v 1.15 2009/08/13 23:12:15 blambert Exp $ */ 2 3 /* 4 * Copyright (c) 2002, 2003 Artur Grabowski <art@openbsd.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 17 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY 18 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 19 * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 20 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/proc.h> 31 #include <sys/rwlock.h> 32 #include <sys/limits.h> 33 34 #include <machine/lock.h> 35 36 /* XXX - temporary measure until proc0 is properly aligned */ 37 #define RW_PROC(p) (((long)p) & ~RWLOCK_MASK) 38 39 /* 40 * Magic wand for lock operations. Every operation checks if certain 41 * flags are set and if they aren't, it increments the lock with some 42 * value (that might need some computing in a few cases). If the operation 43 * fails, we need to set certain flags while waiting for the lock. 44 * 45 * RW_WRITE The lock must be completely empty. We increment it with 46 * RWLOCK_WRLOCK and the proc pointer of the holder. 47 * Sets RWLOCK_WAIT|RWLOCK_WRWANT while waiting. 48 * RW_READ RWLOCK_WRLOCK|RWLOCK_WRWANT may not be set. We increment 49 * with RWLOCK_READ_INCR. RWLOCK_WAIT while waiting. 50 */ 51 static const struct rwlock_op { 52 unsigned long inc; 53 unsigned long check; 54 unsigned long wait_set; 55 long proc_mult; 56 int wait_prio; 57 } rw_ops[] = { 58 { /* RW_WRITE */ 59 RWLOCK_WRLOCK, 60 ULONG_MAX, 61 RWLOCK_WAIT | RWLOCK_WRWANT, 62 1, 63 PLOCK - 4 64 }, 65 { /* RW_READ */ 66 RWLOCK_READ_INCR, 67 RWLOCK_WRLOCK, 68 RWLOCK_WAIT, 69 0, 70 PLOCK 71 }, 72 { /* RW_DOWNGRADE */ 73 RWLOCK_READ_INCR - RWLOCK_WRLOCK, 74 0, 75 0, 76 -1, 77 PLOCK 78 }, 79 }; 80 81 #ifndef __HAVE_MD_RWLOCK 82 /* 83 * Simple cases that should be in MD code and atomic. 84 */ 85 void 86 rw_enter_read(struct rwlock *rwl) 87 { 88 unsigned long owner = rwl->rwl_owner; 89 90 if (__predict_false((owner & RWLOCK_WRLOCK) || 91 rw_cas(&rwl->rwl_owner, owner, owner + RWLOCK_READ_INCR))) 92 rw_enter(rwl, RW_READ); 93 } 94 95 void 96 rw_enter_write(struct rwlock *rwl) 97 { 98 struct proc *p = curproc; 99 100 if (__predict_false(rw_cas(&rwl->rwl_owner, 0, 101 RW_PROC(p) | RWLOCK_WRLOCK))) 102 rw_enter(rwl, RW_WRITE); 103 } 104 105 void 106 rw_exit_read(struct rwlock *rwl) 107 { 108 unsigned long owner = rwl->rwl_owner; 109 110 if (__predict_false((owner & RWLOCK_WAIT) || 111 rw_cas(&rwl->rwl_owner, owner, owner - RWLOCK_READ_INCR))) 112 rw_exit(rwl); 113 } 114 115 void 116 rw_exit_write(struct rwlock *rwl) 117 { 118 unsigned long owner = rwl->rwl_owner; 119 120 if (__predict_false((owner & RWLOCK_WAIT) || 121 rw_cas(&rwl->rwl_owner, owner, 0))) 122 rw_exit(rwl); 123 } 124 125 #ifndef rw_cas 126 int 127 rw_cas(volatile unsigned long *p, unsigned long o, unsigned long n) 128 { 129 if (*p != o) 130 return (1); 131 *p = n; 132 133 return (0); 134 } 135 #endif 136 137 #endif 138 139 #ifdef DIAGNOSTIC 140 /* 141 * Put the diagnostic functions here to keep the main code free 142 * from ifdef clutter. 143 */ 144 static void 145 rw_enter_diag(struct rwlock *rwl, int flags) 146 { 147 switch (flags & RW_OPMASK) { 148 case RW_WRITE: 149 case RW_READ: 150 if (RW_PROC(curproc) == RW_PROC(rwl->rwl_owner)) 151 panic("rw_enter: %s locking against myself", 152 rwl->rwl_name); 153 break; 154 case RW_DOWNGRADE: 155 /* 156 * If we're downgrading, we must hold the write lock. 157 */ 158 if ((rwl->rwl_owner & RWLOCK_WRLOCK) == 0) 159 panic("rw_enter: %s downgrade of non-write lock", 160 rwl->rwl_name); 161 if (RW_PROC(curproc) != RW_PROC(rwl->rwl_owner)) 162 panic("rw_enter: %s downgrade, not holder", 163 rwl->rwl_name); 164 break; 165 166 default: 167 panic("rw_enter: unknown op 0x%x", flags); 168 } 169 } 170 171 #else 172 #define rw_enter_diag(r, f) 173 #endif 174 175 void 176 rw_init(struct rwlock *rwl, const char *name) 177 { 178 rwl->rwl_owner = 0; 179 rwl->rwl_name = name; 180 } 181 182 int 183 rw_enter(struct rwlock *rwl, int flags) 184 { 185 const struct rwlock_op *op; 186 struct sleep_state sls; 187 unsigned long inc, o; 188 int error; 189 190 op = &rw_ops[flags & RW_OPMASK]; 191 192 inc = op->inc + RW_PROC(curproc) * op->proc_mult; 193 retry: 194 while (__predict_false(((o = rwl->rwl_owner) & op->check) != 0)) { 195 unsigned long set = o | op->wait_set; 196 int do_sleep; 197 198 rw_enter_diag(rwl, flags); 199 200 if (flags & RW_NOSLEEP) 201 return (EBUSY); 202 203 sleep_setup(&sls, rwl, op->wait_prio, rwl->rwl_name); 204 if (flags & RW_INTR) 205 sleep_setup_signal(&sls, op->wait_prio | PCATCH); 206 207 do_sleep = !rw_cas(&rwl->rwl_owner, o, set); 208 209 sleep_finish(&sls, do_sleep); 210 if ((flags & RW_INTR) && 211 (error = sleep_finish_signal(&sls)) != 0) 212 return (error); 213 if (flags & RW_SLEEPFAIL) 214 return (EAGAIN); 215 } 216 217 if (__predict_false(rw_cas(&rwl->rwl_owner, o, o + inc))) 218 goto retry; 219 220 /* 221 * If old lock had RWLOCK_WAIT and RWLOCK_WRLOCK set, it means we 222 * downgraded a write lock and had possible read waiter, wake them 223 * to let them retry the lock. 224 */ 225 if (__predict_false((o & (RWLOCK_WRLOCK|RWLOCK_WAIT)) == 226 (RWLOCK_WRLOCK|RWLOCK_WAIT))) 227 wakeup(rwl); 228 229 return (0); 230 } 231 232 void 233 rw_exit(struct rwlock *rwl) 234 { 235 unsigned long owner = rwl->rwl_owner; 236 int wrlock = owner & RWLOCK_WRLOCK; 237 unsigned long set; 238 239 do { 240 owner = rwl->rwl_owner; 241 if (wrlock) 242 set = 0; 243 else 244 set = (owner - RWLOCK_READ_INCR) & 245 ~(RWLOCK_WAIT|RWLOCK_WRWANT); 246 } while (rw_cas(&rwl->rwl_owner, owner, set)); 247 248 if (owner & RWLOCK_WAIT) 249 wakeup(rwl); 250 } 251 252 void 253 rw_assert_wrlock(struct rwlock *rwl) 254 { 255 if (!(rwl->rwl_owner & RWLOCK_WRLOCK)) 256 panic("%s: lock not held", rwl->rwl_name); 257 258 if (RWLOCK_OWNER(rwl) != (struct proc *)((long)curproc & ~RWLOCK_MASK)) 259 panic("%s: lock not held by this process", rwl->rwl_name); 260 } 261 262 void 263 rw_assert_rdlock(struct rwlock *rwl) 264 { 265 if (!RWLOCK_OWNER(rwl) || (rwl->rwl_owner & RWLOCK_WRLOCK)) 266 panic("%s: lock not shared", rwl->rwl_name); 267 } 268 269 void 270 rw_assert_unlocked(struct rwlock *rwl) 271 { 272 if (rwl->rwl_owner != 0L) 273 panic("%s: lock held", rwl->rwl_name); 274 } 275