1 /* 2 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 * 28 */ 29 30 #include <sys/param.h> 31 #include <sys/kernel.h> 32 #include <sys/lock.h> 33 #include <sys/mutex.h> 34 #include <sys/proc.h> 35 #include <sys/signalvar.h> 36 #include <sys/sysent.h> 37 #include <sys/systm.h> 38 #include <sys/sysproto.h> 39 #include <sys/thr.h> 40 #include <sys/umtx.h> 41 42 #define UMTX_LOCK() mtx_lock(&umtx_lock); 43 #define UMTX_UNLOCK() mtx_unlock(&umtx_lock); 44 45 struct mtx umtx_lock; 46 47 MTX_SYSINIT(umtx, &umtx_lock, "User-land mutex lock", MTX_DEF); 48 49 int 50 _umtx_lock(struct thread *td, struct _umtx_lock_args *uap) 51 /* struct umtx *umtx */ 52 { 53 struct umtx *umtx; 54 struct thread *blocked; 55 intptr_t owner; 56 intptr_t old; 57 int error; 58 59 error = 0; 60 61 /* 62 * Care must be exercised when dealing with this structure. It 63 * can fault on any access. 64 */ 65 umtx = uap->umtx; 66 67 UMTX_LOCK(); 68 69 for (;;) { 70 /* 71 * Try the uncontested case. This should be done in userland. 72 */ 73 owner = casuptr((intptr_t *)&umtx->u_owner, 74 UMTX_UNOWNED, (intptr_t)td); 75 76 /* The acquire succeeded. */ 77 if (owner == UMTX_UNOWNED) { 78 error = 0; 79 goto out; 80 } 81 82 /* The address was invalid. */ 83 if (owner == -1) { 84 error = EFAULT; 85 goto out; 86 } 87 88 if (owner & UMTX_CONTESTED) 89 break; 90 91 /* 92 * Set the contested bit so that a release in user space 93 * knows to use the system call for unlock. If this fails 94 * either some one else has acquired the lock or it has been 95 * released. 96 */ 97 old = casuptr((intptr_t *)&umtx->u_owner, owner, 98 owner | UMTX_CONTESTED); 99 100 /* We set the contested bit. */ 101 if (old == owner) 102 break; 103 104 /* The address was invalid. */ 105 if (old == -1) { 106 error = EFAULT; 107 goto out; 108 } 109 /* We didn't set the contested bit, try again. */ 110 } 111 112 /* 113 * We are now protected from further races via umtx_lock. 114 * If userland messes with their mutex without using cmpset 115 * they will deadlock themselves but they will still be 116 * killable via signals. 117 */ 118 119 if ((owner = fuword(&umtx->u_blocked)) == -1) { 120 error = EFAULT; 121 goto out; 122 } 123 124 if (owner == UMTX_UNOWNED) { 125 if (suword(&umtx->u_blocked, (long)td) == -1) { 126 error = EFAULT; 127 goto out; 128 } 129 /* 130 * Other blocked threads will reside here. 131 */ 132 STAILQ_INIT(&td->td_umtxq); 133 } else { 134 FOREACH_THREAD_IN_PROC(td->td_proc, blocked) 135 if (blocked == (struct thread *)(owner)) 136 break; 137 138 if (blocked == NULL) { 139 error = EINVAL; 140 goto out; 141 } 142 /* 143 * Insert us onto the end of the TAILQ. 144 */ 145 STAILQ_INSERT_TAIL(&blocked->td_umtxq, td, td_umtx); 146 } 147 148 for (;;) { 149 /* 150 * Sleep until we can acquire the lock. We must still deliver 151 * signals so that they are not deferred until we acquire the 152 * lock which may be never. The threads actual priority is 153 * used to maintain proper ordering. 154 */ 155 156 error = msleep(&td->td_umtx, &umtx_lock, 157 td->td_priority | PCATCH, "umtx", 0); 158 159 /* 160 * When we are woken up we need to see if we now own the lock 161 * even if a signal was delivered. 162 */ 163 if ((owner = fuword(&umtx->u_owner)) == -1) { 164 error = EFAULT; 165 break; 166 } 167 owner &= ~UMTX_CONTESTED; 168 if ((struct thread *)owner == td) { 169 error = 0; 170 break; 171 } 172 173 /* 174 * We may have signals to deliver. 175 */ 176 if (error) 177 break; 178 } 179 180 out: 181 UMTX_UNLOCK(); 182 183 return (error); 184 } 185 186 int 187 _umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap) 188 /* struct umtx *umtx */ 189 { 190 struct thread *td0; 191 struct umtx *umtx; 192 intptr_t owner; 193 intptr_t blocked; 194 intptr_t old; 195 int error; 196 197 error = 0; 198 umtx = uap->umtx; 199 200 UMTX_LOCK(); 201 202 /* 203 * Make sure we own this mtx. 204 * 205 * XXX Need a {fu,su}ptr this is not correct on arch where 206 * sizeof(intptr_t) != sizeof(long). 207 */ 208 if ((owner = fuword(&umtx->u_owner)) == -1) { 209 error = EFAULT; 210 goto out; 211 } 212 if ((struct thread *)(owner & ~UMTX_CONTESTED) != td) { 213 error = EPERM; 214 goto out; 215 } 216 /* 217 * If we own it but it isn't contested then we can just release and 218 * return. 219 */ 220 if ((owner & UMTX_CONTESTED) == 0) { 221 owner = casuptr((intptr_t *)&umtx->u_owner, 222 (intptr_t)td, UMTX_UNOWNED); 223 224 if (owner == -1) 225 error = EFAULT; 226 /* 227 * If this failed someone modified the memory without going 228 * through this api. 229 */ 230 else if (owner != (intptr_t)td) 231 error = EINVAL; 232 else 233 error = 0; 234 235 goto out; 236 } 237 238 /* 239 * Since we own the mutex and the proc lock we are free to inspect 240 * the blocked queue. It must have one valid entry since the 241 * CONTESTED bit was set. 242 */ 243 blocked = fuword(&umtx->u_blocked); 244 if (blocked == -1) { 245 error = EFAULT; 246 goto out; 247 } 248 if (blocked == 0) { 249 error = EINVAL; 250 goto out; 251 } 252 253 FOREACH_THREAD_IN_PROC(td->td_proc, td0) 254 if (td0 == (struct thread *)blocked) 255 break; 256 257 if (td0 == NULL) { 258 error = EINVAL; 259 goto out; 260 } 261 262 if (!STAILQ_EMPTY(&td0->td_umtxq)) { 263 struct thread *next; 264 265 blocked |= UMTX_CONTESTED; 266 next = STAILQ_FIRST(&td0->td_umtxq); 267 if (suword(&umtx->u_blocked, (long)next) == -1) { 268 error = EFAULT; 269 goto out; 270 } 271 STAILQ_REMOVE_HEAD(&td0->td_umtxq, td_umtx); 272 273 /* 274 * Switch the queue over to the next blocked thread. 275 */ 276 if (!STAILQ_EMPTY(&td0->td_umtxq)) { 277 next->td_umtxq = td0->td_umtxq; 278 STAILQ_INIT(&td0->td_umtxq); 279 } else 280 STAILQ_INIT(&next->td_umtxq); 281 } else { 282 if (suword(&umtx->u_blocked, UMTX_UNOWNED) == -1) { 283 error = EFAULT; 284 goto out; 285 } 286 } 287 /* 288 * Now directly assign this mutex to the first thread that was 289 * blocked on it. 290 */ 291 old = casuptr((intptr_t *)&umtx->u_owner, owner, blocked); 292 293 /* 294 * This will only happen if someone modifies the lock without going 295 * through this api. 296 */ 297 if (old != owner) { 298 error = EINVAL; 299 goto out; 300 } 301 if (old == -1) { 302 error = EFAULT; 303 goto out; 304 } 305 /* Success. */ 306 error = 0; 307 wakeup(&td0->td_umtx); 308 309 out: 310 UMTX_UNLOCK(); 311 312 return (error); 313 } 314