1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 1998 Alex Nash 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <errno.h> 33 #include <limits.h> 34 #include <stdlib.h> 35 36 #include "namespace.h" 37 #include <pthread.h> 38 #include "un-namespace.h" 39 #include "thr_private.h" 40 41 _Static_assert(sizeof(struct pthread_rwlock) <= PAGE_SIZE, 42 "pthread_rwlock is too large for off-page"); 43 44 __weak_reference(_pthread_rwlock_destroy, pthread_rwlock_destroy); 45 __weak_reference(_pthread_rwlock_init, pthread_rwlock_init); 46 __weak_reference(_pthread_rwlock_rdlock, pthread_rwlock_rdlock); 47 __weak_reference(_pthread_rwlock_timedrdlock, pthread_rwlock_timedrdlock); 48 __weak_reference(_pthread_rwlock_tryrdlock, pthread_rwlock_tryrdlock); 49 __weak_reference(_pthread_rwlock_trywrlock, pthread_rwlock_trywrlock); 50 __weak_reference(_pthread_rwlock_unlock, pthread_rwlock_unlock); 51 __weak_reference(_pthread_rwlock_wrlock, pthread_rwlock_wrlock); 52 __weak_reference(_pthread_rwlock_timedwrlock, pthread_rwlock_timedwrlock); 53 54 static int init_static(struct pthread *thread, pthread_rwlock_t *rwlock); 55 static int init_rwlock(pthread_rwlock_t *rwlock, pthread_rwlock_t *rwlock_out); 56 57 static int __always_inline 58 check_and_init_rwlock(pthread_rwlock_t *rwlock, pthread_rwlock_t *rwlock_out) 59 { 60 if (__predict_false(*rwlock == THR_PSHARED_PTR || 61 *rwlock <= THR_RWLOCK_DESTROYED)) 62 return (init_rwlock(rwlock, rwlock_out)); 63 *rwlock_out = *rwlock; 64 return (0); 65 } 66 67 static int __noinline 68 init_rwlock(pthread_rwlock_t *rwlock, pthread_rwlock_t *rwlock_out) 69 { 70 pthread_rwlock_t prwlock; 71 int ret; 72 73 if (*rwlock == THR_PSHARED_PTR) { 74 prwlock = __thr_pshared_offpage(rwlock, 0); 75 if (prwlock == NULL) 76 return (EINVAL); 77 } else if ((prwlock = *rwlock) <= THR_RWLOCK_DESTROYED) { 78 if (prwlock == THR_RWLOCK_INITIALIZER) { 79 ret = init_static(_get_curthread(), rwlock); 80 if (ret != 0) 81 return (ret); 82 } else if (prwlock == THR_RWLOCK_DESTROYED) { 83 return (EINVAL); 84 } 85 prwlock = *rwlock; 86 } 87 *rwlock_out = prwlock; 88 return (0); 89 } 90 91 static int 92 rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr) 93 { 94 pthread_rwlock_t prwlock; 95 96 if (attr == NULL || *attr == NULL || 97 (*attr)->pshared == PTHREAD_PROCESS_PRIVATE) { 98 prwlock = calloc(1, sizeof(struct pthread_rwlock)); 99 if (prwlock == NULL) 100 return (ENOMEM); 101 *rwlock = prwlock; 102 } else { 103 prwlock = __thr_pshared_offpage(rwlock, 1); 104 if (prwlock == NULL) 105 return (EFAULT); 106 prwlock->lock.rw_flags |= USYNC_PROCESS_SHARED; 107 *rwlock = THR_PSHARED_PTR; 108 } 109 return (0); 110 } 111 112 int 113 _pthread_rwlock_destroy (pthread_rwlock_t *rwlock) 114 { 115 pthread_rwlock_t prwlock; 116 int ret; 117 118 prwlock = *rwlock; 119 if (prwlock == THR_RWLOCK_INITIALIZER) 120 ret = 0; 121 else if (prwlock == THR_RWLOCK_DESTROYED) 122 ret = EINVAL; 123 else if (prwlock == THR_PSHARED_PTR) { 124 *rwlock = THR_RWLOCK_DESTROYED; 125 __thr_pshared_destroy(rwlock); 126 ret = 0; 127 } else { 128 *rwlock = THR_RWLOCK_DESTROYED; 129 free(prwlock); 130 ret = 0; 131 } 132 return (ret); 133 } 134 135 static int 136 init_static(struct pthread *thread, pthread_rwlock_t *rwlock) 137 { 138 int ret; 139 140 THR_LOCK_ACQUIRE(thread, &_rwlock_static_lock); 141 142 if (*rwlock == THR_RWLOCK_INITIALIZER) 143 ret = rwlock_init(rwlock, NULL); 144 else 145 ret = 0; 146 147 THR_LOCK_RELEASE(thread, &_rwlock_static_lock); 148 149 return (ret); 150 } 151 152 int 153 _pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr) 154 { 155 156 *rwlock = NULL; 157 return (rwlock_init(rwlock, attr)); 158 } 159 160 static int 161 rwlock_rdlock_common(pthread_rwlock_t *rwlock, const struct timespec *abstime) 162 { 163 struct pthread *curthread = _get_curthread(); 164 pthread_rwlock_t prwlock; 165 int flags; 166 int ret; 167 168 ret = check_and_init_rwlock(rwlock, &prwlock); 169 if (ret != 0) 170 return (ret); 171 172 if (curthread->rdlock_count) { 173 /* 174 * To avoid having to track all the rdlocks held by 175 * a thread or all of the threads that hold a rdlock, 176 * we keep a simple count of all the rdlocks held by 177 * a thread. If a thread holds any rdlocks it is 178 * possible that it is attempting to take a recursive 179 * rdlock. If there are blocked writers and precedence 180 * is given to them, then that would result in the thread 181 * deadlocking. So allowing a thread to take the rdlock 182 * when it already has one or more rdlocks avoids the 183 * deadlock. I hope the reader can follow that logic ;-) 184 */ 185 flags = URWLOCK_PREFER_READER; 186 } else { 187 flags = 0; 188 } 189 190 /* 191 * POSIX said the validity of the abstimeout parameter need 192 * not be checked if the lock can be immediately acquired. 193 */ 194 ret = _thr_rwlock_tryrdlock(&prwlock->lock, flags); 195 if (ret == 0) { 196 curthread->rdlock_count++; 197 return (ret); 198 } 199 200 if (__predict_false(abstime && 201 (abstime->tv_nsec >= 1000000000 || abstime->tv_nsec < 0))) 202 return (EINVAL); 203 204 for (;;) { 205 /* goto kernel and lock it */ 206 ret = __thr_rwlock_rdlock(&prwlock->lock, flags, abstime); 207 if (ret != EINTR) 208 break; 209 210 /* if interrupted, try to lock it in userland again. */ 211 if (_thr_rwlock_tryrdlock(&prwlock->lock, flags) == 0) { 212 ret = 0; 213 break; 214 } 215 } 216 if (ret == 0) 217 curthread->rdlock_count++; 218 return (ret); 219 } 220 221 int 222 _pthread_rwlock_rdlock (pthread_rwlock_t *rwlock) 223 { 224 return (rwlock_rdlock_common(rwlock, NULL)); 225 } 226 227 int 228 _pthread_rwlock_timedrdlock(pthread_rwlock_t * __restrict rwlock, 229 const struct timespec * __restrict abstime) 230 { 231 return (rwlock_rdlock_common(rwlock, abstime)); 232 } 233 234 int 235 _pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock) 236 { 237 struct pthread *curthread = _get_curthread(); 238 pthread_rwlock_t prwlock; 239 int flags; 240 int ret; 241 242 ret = check_and_init_rwlock(rwlock, &prwlock); 243 if (ret != 0) 244 return (ret); 245 246 if (curthread->rdlock_count) { 247 /* 248 * To avoid having to track all the rdlocks held by 249 * a thread or all of the threads that hold a rdlock, 250 * we keep a simple count of all the rdlocks held by 251 * a thread. If a thread holds any rdlocks it is 252 * possible that it is attempting to take a recursive 253 * rdlock. If there are blocked writers and precedence 254 * is given to them, then that would result in the thread 255 * deadlocking. So allowing a thread to take the rdlock 256 * when it already has one or more rdlocks avoids the 257 * deadlock. I hope the reader can follow that logic ;-) 258 */ 259 flags = URWLOCK_PREFER_READER; 260 } else { 261 flags = 0; 262 } 263 264 ret = _thr_rwlock_tryrdlock(&prwlock->lock, flags); 265 if (ret == 0) 266 curthread->rdlock_count++; 267 return (ret); 268 } 269 270 int 271 _pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock) 272 { 273 struct pthread *curthread = _get_curthread(); 274 pthread_rwlock_t prwlock; 275 int ret; 276 277 ret = check_and_init_rwlock(rwlock, &prwlock); 278 if (ret != 0) 279 return (ret); 280 281 ret = _thr_rwlock_trywrlock(&prwlock->lock); 282 if (ret == 0) 283 prwlock->owner = TID(curthread); 284 return (ret); 285 } 286 287 static int 288 rwlock_wrlock_common (pthread_rwlock_t *rwlock, const struct timespec *abstime) 289 { 290 struct pthread *curthread = _get_curthread(); 291 pthread_rwlock_t prwlock; 292 int ret; 293 294 ret = check_and_init_rwlock(rwlock, &prwlock); 295 if (ret != 0) 296 return (ret); 297 298 /* 299 * POSIX said the validity of the abstimeout parameter need 300 * not be checked if the lock can be immediately acquired. 301 */ 302 ret = _thr_rwlock_trywrlock(&prwlock->lock); 303 if (ret == 0) { 304 prwlock->owner = TID(curthread); 305 return (ret); 306 } 307 308 if (__predict_false(abstime && 309 (abstime->tv_nsec >= 1000000000 || abstime->tv_nsec < 0))) 310 return (EINVAL); 311 312 for (;;) { 313 /* goto kernel and lock it */ 314 ret = __thr_rwlock_wrlock(&prwlock->lock, abstime); 315 if (ret == 0) { 316 prwlock->owner = TID(curthread); 317 break; 318 } 319 320 if (ret != EINTR) 321 break; 322 323 /* if interrupted, try to lock it in userland again. */ 324 if (_thr_rwlock_trywrlock(&prwlock->lock) == 0) { 325 ret = 0; 326 prwlock->owner = TID(curthread); 327 break; 328 } 329 } 330 return (ret); 331 } 332 333 int 334 _pthread_rwlock_wrlock (pthread_rwlock_t *rwlock) 335 { 336 return (rwlock_wrlock_common (rwlock, NULL)); 337 } 338 339 int 340 _pthread_rwlock_timedwrlock(pthread_rwlock_t * __restrict rwlock, 341 const struct timespec * __restrict abstime) 342 { 343 return (rwlock_wrlock_common (rwlock, abstime)); 344 } 345 346 int 347 _pthread_rwlock_unlock(pthread_rwlock_t *rwlock) 348 { 349 struct pthread *curthread = _get_curthread(); 350 pthread_rwlock_t prwlock; 351 int ret; 352 int32_t state; 353 354 if (*rwlock == THR_PSHARED_PTR) { 355 prwlock = __thr_pshared_offpage(rwlock, 0); 356 if (prwlock == NULL) 357 return (EINVAL); 358 } else { 359 prwlock = *rwlock; 360 } 361 362 if (__predict_false(prwlock <= THR_RWLOCK_DESTROYED)) 363 return (EINVAL); 364 365 state = prwlock->lock.rw_state; 366 if (state & URWLOCK_WRITE_OWNER) { 367 if (__predict_false(prwlock->owner != TID(curthread))) 368 return (EPERM); 369 prwlock->owner = 0; 370 } 371 372 ret = _thr_rwlock_unlock(&prwlock->lock); 373 if (ret == 0 && (state & URWLOCK_WRITE_OWNER) == 0) 374 curthread->rdlock_count--; 375 376 return (ret); 377 } 378