1 /* 2 * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by John Birrell. 16 * 4. Neither the name of the author nor the names of any co-contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * $FreeBSD: src/lib/libpthread/thread/thr_mutex.c,v 1.46 2004/10/31 05:03:50 green Exp $ 33 * $DragonFly: src/lib/libthread_xu/thread/thr_mutex.c,v 1.7 2005/12/18 11:02:05 davidxu Exp $ 34 */ 35 36 #include <machine/tls.h> 37 38 #include <stdlib.h> 39 #include <errno.h> 40 #include <string.h> 41 #include <sys/param.h> 42 #include <sys/queue.h> 43 #include <pthread.h> 44 #include "thr_private.h" 45 46 #if defined(_PTHREADS_INVARIANTS) 47 #define MUTEX_INIT_LINK(m) do { \ 48 (m)->m_qe.tqe_prev = NULL; \ 49 (m)->m_qe.tqe_next = NULL; \ 50 } while (0) 51 #define MUTEX_ASSERT_IS_OWNED(m) do { \ 52 if ((m)->m_qe.tqe_prev == NULL) \ 53 PANIC("mutex is not on list"); \ 54 } while (0) 55 #define MUTEX_ASSERT_NOT_OWNED(m) do { \ 56 if (((m)->m_qe.tqe_prev != NULL) || \ 57 ((m)->m_qe.tqe_next != NULL)) \ 58 PANIC("mutex is on list"); \ 59 } while (0) 60 #define THR_ASSERT_NOT_IN_SYNCQ(thr) do { \ 61 THR_ASSERT(((thr)->sflags & THR_FLAGS_IN_SYNCQ) == 0, \ 62 "thread in syncq when it shouldn't be."); \ 63 } while (0); 64 #else 65 #define MUTEX_INIT_LINK(m) 66 #define MUTEX_ASSERT_IS_OWNED(m) 67 #define MUTEX_ASSERT_NOT_OWNED(m) 68 #define THR_ASSERT_NOT_IN_SYNCQ(thr) 69 #endif 70 71 #define THR_IN_MUTEXQ(thr) (((thr)->sflags & THR_FLAGS_IN_SYNCQ) != 0) 72 #define MUTEX_DESTROY(m) do { \ 73 free(m); \ 74 } while (0) 75 76 77 /* 78 * Prototypes 79 */ 80 static long mutex_handoff(struct pthread *, struct pthread_mutex *); 81 static int mutex_self_trylock(struct pthread *, pthread_mutex_t); 82 static int mutex_self_lock(struct pthread *, pthread_mutex_t, 83 const struct timespec *abstime); 84 static int mutex_unlock_common(pthread_mutex_t *, int); 85 static void mutex_priority_adjust(struct pthread *, pthread_mutex_t); 86 static void mutex_rescan_owned (struct pthread *, struct pthread *, 87 struct pthread_mutex *); 88 #if 0 89 static pthread_t mutex_queue_deq(pthread_mutex_t); 90 #endif 91 static void mutex_queue_remove(pthread_mutex_t, pthread_t); 92 static void mutex_queue_enq(pthread_mutex_t, pthread_t); 93 94 __weak_reference(__pthread_mutex_init, pthread_mutex_init); 95 __weak_reference(__pthread_mutex_lock, pthread_mutex_lock); 96 __weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock); 97 __weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock); 98 99 /* Single underscore versions provided for libc internal usage: */ 100 /* No difference between libc and application usage of these: */ 101 __weak_reference(_pthread_mutex_destroy, pthread_mutex_destroy); 102 __weak_reference(_pthread_mutex_unlock, pthread_mutex_unlock); 103 104 static int 105 mutex_init(pthread_mutex_t *mutex, 106 const pthread_mutexattr_t *mutex_attr, int private) 107 { 108 struct pthread_mutex *pmutex; 109 enum pthread_mutextype type; 110 int protocol; 111 int ceiling; 112 int flags; 113 int ret = 0; 114 115 /* Check if default mutex attributes: */ 116 if (mutex_attr == NULL || *mutex_attr == NULL) { 117 /* Default to a (error checking) POSIX mutex: */ 118 type = PTHREAD_MUTEX_ERRORCHECK; 119 protocol = PTHREAD_PRIO_NONE; 120 ceiling = THR_MAX_PRIORITY; 121 flags = 0; 122 } 123 124 /* Check mutex type: */ 125 else if (((*mutex_attr)->m_type < PTHREAD_MUTEX_ERRORCHECK) || 126 ((*mutex_attr)->m_type >= MUTEX_TYPE_MAX)) 127 /* Return an invalid argument error: */ 128 ret = EINVAL; 129 130 /* Check mutex protocol: */ 131 else if (((*mutex_attr)->m_protocol < PTHREAD_PRIO_NONE) || 132 ((*mutex_attr)->m_protocol > PTHREAD_PRIO_PROTECT)) 133 /* Return an invalid argument error: */ 134 ret = EINVAL; 135 136 else { 137 /* Use the requested mutex type and protocol: */ 138 type = (*mutex_attr)->m_type; 139 protocol = (*mutex_attr)->m_protocol; 140 ceiling = (*mutex_attr)->m_ceiling; 141 flags = (*mutex_attr)->m_flags; 142 } 143 144 /* Check no errors so far: */ 145 if (ret == 0) { 146 if ((pmutex = (pthread_mutex_t) 147 malloc(sizeof(struct pthread_mutex))) == NULL) { 148 ret = ENOMEM; 149 } else { 150 _thr_umtx_init(&pmutex->m_lock); 151 /* Set the mutex flags: */ 152 pmutex->m_flags = flags; 153 154 /* Process according to mutex type: */ 155 switch (type) { 156 /* case PTHREAD_MUTEX_DEFAULT: */ 157 case PTHREAD_MUTEX_ERRORCHECK: 158 case PTHREAD_MUTEX_NORMAL: 159 /* Nothing to do here. */ 160 break; 161 162 /* Single UNIX Spec 2 recursive mutex: */ 163 case PTHREAD_MUTEX_RECURSIVE: 164 /* Reset the mutex count: */ 165 pmutex->m_count = 0; 166 break; 167 168 /* Trap invalid mutex types: */ 169 default: 170 /* Return an invalid argument error: */ 171 ret = EINVAL; 172 break; 173 } 174 if (ret == 0) { 175 /* Initialise the rest of the mutex: */ 176 TAILQ_INIT(&pmutex->m_queue); 177 pmutex->m_flags |= MUTEX_FLAGS_INITED; 178 if (private) 179 pmutex->m_flags |= MUTEX_FLAGS_PRIVATE; 180 pmutex->m_owner = NULL; 181 pmutex->m_type = type; 182 pmutex->m_protocol = protocol; 183 pmutex->m_refcount = 0; 184 if (protocol == PTHREAD_PRIO_PROTECT) 185 pmutex->m_prio = ceiling; 186 else 187 pmutex->m_prio = -1; 188 pmutex->m_saved_prio = 0; 189 MUTEX_INIT_LINK(pmutex); 190 *mutex = pmutex; 191 } else { 192 /* Free the mutex lock structure: */ 193 MUTEX_DESTROY(pmutex); 194 *mutex = NULL; 195 } 196 } 197 } 198 /* Return the completion status: */ 199 return (ret); 200 } 201 202 static int 203 init_static(struct pthread *thread, pthread_mutex_t *mutex) 204 { 205 int ret; 206 207 THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); 208 209 if (*mutex == NULL) 210 ret = mutex_init(mutex, NULL, 0); 211 else 212 ret = 0; 213 214 THR_LOCK_RELEASE(thread, &_mutex_static_lock); 215 216 return (ret); 217 } 218 219 static int 220 init_static_private(struct pthread *thread, pthread_mutex_t *mutex) 221 { 222 int ret; 223 224 THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); 225 226 if (*mutex == NULL) 227 ret = mutex_init(mutex, NULL, 1); 228 else 229 ret = 0; 230 231 THR_LOCK_RELEASE(thread, &_mutex_static_lock); 232 233 return (ret); 234 } 235 236 int 237 _pthread_mutex_init(pthread_mutex_t *mutex, 238 const pthread_mutexattr_t *mutex_attr) 239 { 240 return mutex_init(mutex, mutex_attr, 1); 241 } 242 243 int 244 __pthread_mutex_init(pthread_mutex_t *mutex, 245 const pthread_mutexattr_t *mutex_attr) 246 { 247 return mutex_init(mutex, mutex_attr, 0); 248 } 249 250 int 251 _mutex_reinit(pthread_mutex_t *mutex) 252 { 253 _thr_umtx_init(&(*mutex)->m_lock); 254 TAILQ_INIT(&(*mutex)->m_queue); 255 MUTEX_INIT_LINK(*mutex); 256 (*mutex)->m_owner = NULL; 257 (*mutex)->m_count = 0; 258 (*mutex)->m_refcount = 0; 259 (*mutex)->m_prio = 0; 260 (*mutex)->m_saved_prio = 0; 261 return (0); 262 } 263 264 void 265 _mutex_fork(struct pthread *curthread) 266 { 267 struct pthread_mutex *m; 268 269 TAILQ_FOREACH(m, &curthread->mutexq, m_qe) 270 m->m_lock = UMTX_LOCKED; 271 272 /* Clear contender for priority mutexes */ 273 TAILQ_FOREACH(m, &curthread->pri_mutexq, m_qe) { 274 /* clear another thread locked us */ 275 _thr_umtx_init(&m->m_lock); 276 TAILQ_INIT(&m->m_queue); 277 } 278 } 279 280 int 281 _pthread_mutex_destroy(pthread_mutex_t *mutex) 282 { 283 struct pthread *curthread = tls_get_curthread(); 284 pthread_mutex_t m; 285 int ret = 0; 286 287 if (mutex == NULL || *mutex == NULL) 288 ret = EINVAL; 289 else { 290 /* 291 * Try to lock the mutex structure, we only need to 292 * try once, if failed, the mutex is in used. 293 */ 294 ret = THR_UMTX_TRYLOCK(curthread, &(*mutex)->m_lock); 295 if (ret) 296 return (ret); 297 298 /* 299 * Check mutex other fields to see if this mutex is 300 * in use. Mostly for prority mutex types, or there 301 * are condition variables referencing it. 302 */ 303 if (((*mutex)->m_owner != NULL) || 304 (TAILQ_FIRST(&(*mutex)->m_queue) != NULL) || 305 ((*mutex)->m_refcount != 0)) { 306 THR_UMTX_UNLOCK(curthread, &(*mutex)->m_lock); 307 ret = EBUSY; 308 } else { 309 /* 310 * Save a pointer to the mutex so it can be free'd 311 * and set the caller's pointer to NULL: 312 */ 313 m = *mutex; 314 *mutex = NULL; 315 316 /* Unlock the mutex structure: */ 317 _thr_umtx_unlock(&m->m_lock, curthread->tid); 318 319 /* 320 * Free the memory allocated for the mutex 321 * structure: 322 */ 323 MUTEX_ASSERT_NOT_OWNED(m); 324 MUTEX_DESTROY(m); 325 } 326 } 327 328 /* Return the completion status: */ 329 return (ret); 330 } 331 332 static int 333 mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex) 334 { 335 int ret = 0; 336 337 THR_ASSERT((mutex != NULL) && (*mutex != NULL), 338 "Uninitialized mutex in mutex_trylock_common"); 339 340 /* Short cut for simple mutex. */ 341 if ((*mutex)->m_protocol == PTHREAD_PRIO_NONE) { 342 ret = THR_UMTX_TRYLOCK(curthread, &(*mutex)->m_lock); 343 if (ret == 0) { 344 (*mutex)->m_owner = curthread; 345 /* Add to the list of owned mutexes: */ 346 MUTEX_ASSERT_NOT_OWNED(*mutex); 347 TAILQ_INSERT_TAIL(&curthread->mutexq, 348 (*mutex), m_qe); 349 } else if ((*mutex)->m_owner == curthread) { 350 ret = mutex_self_trylock(curthread, *mutex); 351 } /* else {} */ 352 353 return (ret); 354 } 355 356 /* Code for priority mutex */ 357 358 /* Lock the mutex structure: */ 359 THR_LOCK_ACQUIRE(curthread, &(*mutex)->m_lock); 360 361 /* 362 * If the mutex was statically allocated, properly 363 * initialize the tail queue. 364 */ 365 if (((*mutex)->m_flags & MUTEX_FLAGS_INITED) == 0) { 366 TAILQ_INIT(&(*mutex)->m_queue); 367 MUTEX_INIT_LINK(*mutex); 368 (*mutex)->m_flags |= MUTEX_FLAGS_INITED; 369 } 370 371 /* Process according to mutex type: */ 372 switch ((*mutex)->m_protocol) { 373 /* POSIX priority inheritence mutex: */ 374 case PTHREAD_PRIO_INHERIT: 375 /* Check if this mutex is not locked: */ 376 if ((*mutex)->m_owner == NULL) { 377 /* Lock the mutex for the running thread: */ 378 (*mutex)->m_owner = curthread; 379 380 THR_LOCK(curthread); 381 /* Track number of priority mutexes owned: */ 382 curthread->priority_mutex_count++; 383 384 /* 385 * The mutex takes on the attributes of the 386 * running thread when there are no waiters. 387 */ 388 (*mutex)->m_prio = curthread->active_priority; 389 (*mutex)->m_saved_prio = 390 curthread->inherited_priority; 391 curthread->inherited_priority = (*mutex)->m_prio; 392 THR_UNLOCK(curthread); 393 394 /* Add to the list of owned mutexes: */ 395 MUTEX_ASSERT_NOT_OWNED(*mutex); 396 TAILQ_INSERT_TAIL(&curthread->pri_mutexq, 397 (*mutex), m_qe); 398 } else if ((*mutex)->m_owner == curthread) 399 ret = mutex_self_trylock(curthread, *mutex); 400 else 401 /* Return a busy error: */ 402 ret = EBUSY; 403 break; 404 405 /* POSIX priority protection mutex: */ 406 case PTHREAD_PRIO_PROTECT: 407 /* Check for a priority ceiling violation: */ 408 if (curthread->active_priority > (*mutex)->m_prio) 409 ret = EINVAL; 410 411 /* Check if this mutex is not locked: */ 412 else if ((*mutex)->m_owner == NULL) { 413 /* Lock the mutex for the running thread: */ 414 (*mutex)->m_owner = curthread; 415 416 THR_LOCK(curthread); 417 /* Track number of priority mutexes owned: */ 418 curthread->priority_mutex_count++; 419 420 /* 421 * The running thread inherits the ceiling 422 * priority of the mutex and executes at that 423 * priority. 424 */ 425 curthread->active_priority = (*mutex)->m_prio; 426 (*mutex)->m_saved_prio = 427 curthread->inherited_priority; 428 curthread->inherited_priority = 429 (*mutex)->m_prio; 430 THR_UNLOCK(curthread); 431 /* Add to the list of owned mutexes: */ 432 MUTEX_ASSERT_NOT_OWNED(*mutex); 433 TAILQ_INSERT_TAIL(&curthread->pri_mutexq, 434 (*mutex), m_qe); 435 } else if ((*mutex)->m_owner == curthread) 436 ret = mutex_self_trylock(curthread, *mutex); 437 else 438 /* Return a busy error: */ 439 ret = EBUSY; 440 break; 441 442 /* Trap invalid mutex types: */ 443 default: 444 /* Return an invalid argument error: */ 445 ret = EINVAL; 446 break; 447 } 448 449 /* Unlock the mutex structure: */ 450 THR_LOCK_RELEASE(curthread, &(*mutex)->m_lock); 451 452 /* Return the completion status: */ 453 return (ret); 454 } 455 456 int 457 __pthread_mutex_trylock(pthread_mutex_t *mutex) 458 { 459 struct pthread *curthread = tls_get_curthread(); 460 int ret = 0; 461 462 /* 463 * If the mutex is statically initialized, perform the dynamic 464 * initialization: 465 */ 466 if ((*mutex != NULL) || 467 ((ret = init_static(curthread, mutex)) == 0)) 468 ret = mutex_trylock_common(curthread, mutex); 469 470 return (ret); 471 } 472 473 int 474 _pthread_mutex_trylock(pthread_mutex_t *mutex) 475 { 476 struct pthread *curthread = tls_get_curthread(); 477 int ret = 0; 478 479 /* 480 * If the mutex is statically initialized, perform the dynamic 481 * initialization marking the mutex private (delete safe): 482 */ 483 if ((*mutex != NULL) || 484 ((ret = init_static_private(curthread, mutex)) == 0)) 485 ret = mutex_trylock_common(curthread, mutex); 486 487 return (ret); 488 } 489 490 static int 491 mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m, 492 const struct timespec * abstime) 493 { 494 struct timespec ts, ts2; 495 long cycle; 496 int ret = 0; 497 498 THR_ASSERT((m != NULL) && (*m != NULL), 499 "Uninitialized mutex in mutex_lock_common"); 500 501 if (abstime != NULL && (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || 502 abstime->tv_nsec >= 1000000000)) 503 return (EINVAL); 504 505 /* Short cut for simple mutex. */ 506 507 if ((*m)->m_protocol == PTHREAD_PRIO_NONE) { 508 /* Default POSIX mutex: */ 509 ret = THR_UMTX_TRYLOCK(curthread, &(*m)->m_lock); 510 if (ret == 0) { 511 (*m)->m_owner = curthread; 512 /* Add to the list of owned mutexes: */ 513 MUTEX_ASSERT_NOT_OWNED(*m); 514 TAILQ_INSERT_TAIL(&curthread->mutexq, 515 (*m), m_qe); 516 } else if ((*m)->m_owner == curthread) { 517 ret = mutex_self_lock(curthread, *m, abstime); 518 } else { 519 if (abstime == NULL) { 520 THR_UMTX_LOCK(curthread, &(*m)->m_lock); 521 ret = 0; 522 } else { 523 clock_gettime(CLOCK_REALTIME, &ts); 524 TIMESPEC_SUB(&ts2, abstime, &ts); 525 ret = THR_UMTX_TIMEDLOCK(curthread, 526 &(*m)->m_lock, &ts2); 527 /* 528 * Timed out wait is not restarted if 529 * it was interrupted, not worth to do it. 530 */ 531 if (ret == EINTR) 532 ret = ETIMEDOUT; 533 } 534 if (ret == 0) { 535 (*m)->m_owner = curthread; 536 /* Add to the list of owned mutexes: */ 537 MUTEX_ASSERT_NOT_OWNED(*m); 538 TAILQ_INSERT_TAIL(&curthread->mutexq, 539 (*m), m_qe); 540 } 541 } 542 return (ret); 543 } 544 545 /* Code for priority mutex */ 546 547 /* 548 * Enter a loop waiting to become the mutex owner. We need a 549 * loop in case the waiting thread is interrupted by a signal 550 * to execute a signal handler. It is not (currently) possible 551 * to remain in the waiting queue while running a handler. 552 * Instead, the thread is interrupted and backed out of the 553 * waiting queue prior to executing the signal handler. 554 */ 555 do { 556 /* Lock the mutex structure: */ 557 THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); 558 559 /* 560 * If the mutex was statically allocated, properly 561 * initialize the tail queue. 562 */ 563 if (((*m)->m_flags & MUTEX_FLAGS_INITED) == 0) { 564 TAILQ_INIT(&(*m)->m_queue); 565 (*m)->m_flags |= MUTEX_FLAGS_INITED; 566 MUTEX_INIT_LINK(*m); 567 } 568 569 /* Process according to mutex type: */ 570 switch ((*m)->m_protocol) { 571 /* POSIX priority inheritence mutex: */ 572 case PTHREAD_PRIO_INHERIT: 573 /* Check if this mutex is not locked: */ 574 if ((*m)->m_owner == NULL) { 575 /* Lock the mutex for this thread: */ 576 (*m)->m_owner = curthread; 577 578 THR_LOCK(curthread); 579 /* Track number of priority mutexes owned: */ 580 curthread->priority_mutex_count++; 581 582 /* 583 * The mutex takes on attributes of the 584 * running thread when there are no waiters. 585 * Make sure the thread's scheduling lock is 586 * held while priorities are adjusted. 587 */ 588 (*m)->m_prio = curthread->active_priority; 589 (*m)->m_saved_prio = 590 curthread->inherited_priority; 591 curthread->inherited_priority = (*m)->m_prio; 592 THR_UNLOCK(curthread); 593 594 /* Add to the list of owned mutexes: */ 595 MUTEX_ASSERT_NOT_OWNED(*m); 596 TAILQ_INSERT_TAIL(&curthread->pri_mutexq, 597 (*m), m_qe); 598 599 /* Unlock the mutex structure: */ 600 THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 601 } else if ((*m)->m_owner == curthread) { 602 ret = mutex_self_lock(curthread, *m, abstime); 603 604 /* Unlock the mutex structure: */ 605 THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 606 } else { 607 /* 608 * Join the queue of threads waiting to lock 609 * the mutex and save a pointer to the mutex. 610 */ 611 mutex_queue_enq(*m, curthread); 612 curthread->data.mutex = *m; 613 614 if (curthread->active_priority > (*m)->m_prio) 615 /* Adjust priorities: */ 616 mutex_priority_adjust(curthread, *m); 617 618 THR_LOCK(curthread); 619 cycle = curthread->cycle; 620 THR_UNLOCK(curthread); 621 622 /* Unlock the mutex structure: */ 623 THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 624 625 clock_gettime(CLOCK_REALTIME, &ts); 626 TIMESPEC_SUB(&ts2, abstime, &ts); 627 ret = _thr_umtx_wait(&curthread->cycle, cycle, 628 &ts2, CLOCK_REALTIME); 629 if (ret == EINTR) 630 ret = 0; 631 632 if (THR_IN_MUTEXQ(curthread)) { 633 THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); 634 mutex_queue_remove(*m, curthread); 635 THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 636 } 637 /* 638 * Only clear these after assuring the 639 * thread is dequeued. 640 */ 641 curthread->data.mutex = NULL; 642 } 643 break; 644 645 /* POSIX priority protection mutex: */ 646 case PTHREAD_PRIO_PROTECT: 647 /* Check for a priority ceiling violation: */ 648 if (curthread->active_priority > (*m)->m_prio) { 649 /* Unlock the mutex structure: */ 650 THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 651 ret = EINVAL; 652 } 653 /* Check if this mutex is not locked: */ 654 else if ((*m)->m_owner == NULL) { 655 /* 656 * Lock the mutex for the running 657 * thread: 658 */ 659 (*m)->m_owner = curthread; 660 661 THR_LOCK(curthread); 662 /* Track number of priority mutexes owned: */ 663 curthread->priority_mutex_count++; 664 665 /* 666 * The running thread inherits the ceiling 667 * priority of the mutex and executes at that 668 * priority. Make sure the thread's 669 * scheduling lock is held while priorities 670 * are adjusted. 671 */ 672 curthread->active_priority = (*m)->m_prio; 673 (*m)->m_saved_prio = 674 curthread->inherited_priority; 675 curthread->inherited_priority = (*m)->m_prio; 676 THR_UNLOCK(curthread); 677 678 /* Add to the list of owned mutexes: */ 679 MUTEX_ASSERT_NOT_OWNED(*m); 680 TAILQ_INSERT_TAIL(&curthread->pri_mutexq, 681 (*m), m_qe); 682 683 /* Unlock the mutex structure: */ 684 THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 685 } else if ((*m)->m_owner == curthread) { 686 ret = mutex_self_lock(curthread, *m, abstime); 687 688 /* Unlock the mutex structure: */ 689 THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 690 } else { 691 /* 692 * Join the queue of threads waiting to lock 693 * the mutex and save a pointer to the mutex. 694 */ 695 mutex_queue_enq(*m, curthread); 696 curthread->data.mutex = *m; 697 698 /* Clear any previous error: */ 699 curthread->error = 0; 700 701 THR_LOCK(curthread); 702 cycle = curthread->cycle; 703 THR_UNLOCK(curthread); 704 705 /* Unlock the mutex structure: */ 706 THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 707 708 clock_gettime(CLOCK_REALTIME, &ts); 709 TIMESPEC_SUB(&ts2, abstime, &ts); 710 ret = _thr_umtx_wait(&curthread->cycle, cycle, 711 &ts2, CLOCK_REALTIME); 712 if (ret == EINTR) 713 ret = 0; 714 715 curthread->data.mutex = NULL; 716 if (THR_IN_MUTEXQ(curthread)) { 717 THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); 718 mutex_queue_remove(*m, curthread); 719 THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 720 } 721 /* 722 * Only clear these after assuring the 723 * thread is dequeued. 724 */ 725 curthread->data.mutex = NULL; 726 727 /* 728 * The threads priority may have changed while 729 * waiting for the mutex causing a ceiling 730 * violation. 731 */ 732 ret = curthread->error; 733 curthread->error = 0; 734 } 735 break; 736 737 /* Trap invalid mutex types: */ 738 default: 739 /* Unlock the mutex structure: */ 740 THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 741 742 /* Return an invalid argument error: */ 743 ret = EINVAL; 744 break; 745 } 746 747 } while (((*m)->m_owner != curthread) && (ret == 0)); 748 749 /* Return the completion status: */ 750 return (ret); 751 } 752 753 int 754 __pthread_mutex_lock(pthread_mutex_t *m) 755 { 756 struct pthread *curthread; 757 int ret = 0; 758 759 _thr_check_init(); 760 761 curthread = tls_get_curthread(); 762 763 /* 764 * If the mutex is statically initialized, perform the dynamic 765 * initialization: 766 */ 767 if ((*m != NULL) || ((ret = init_static(curthread, m)) == 0)) 768 ret = mutex_lock_common(curthread, m, NULL); 769 770 return (ret); 771 } 772 773 int 774 _pthread_mutex_lock(pthread_mutex_t *m) 775 { 776 struct pthread *curthread; 777 int ret = 0; 778 779 _thr_check_init(); 780 781 curthread = tls_get_curthread(); 782 783 /* 784 * If the mutex is statically initialized, perform the dynamic 785 * initialization marking it private (delete safe): 786 */ 787 if ((*m != NULL) || 788 ((ret = init_static_private(curthread, m)) == 0)) 789 ret = mutex_lock_common(curthread, m, NULL); 790 791 return (ret); 792 } 793 794 int 795 __pthread_mutex_timedlock(pthread_mutex_t *m, 796 const struct timespec *abs_timeout) 797 { 798 struct pthread *curthread; 799 int ret = 0; 800 801 _thr_check_init(); 802 803 curthread = tls_get_curthread(); 804 805 /* 806 * If the mutex is statically initialized, perform the dynamic 807 * initialization: 808 */ 809 if ((*m != NULL) || ((ret = init_static(curthread, m)) == 0)) 810 ret = mutex_lock_common(curthread, m, abs_timeout); 811 812 return (ret); 813 } 814 815 int 816 _pthread_mutex_timedlock(pthread_mutex_t *m, 817 const struct timespec *abs_timeout) 818 { 819 struct pthread *curthread; 820 int ret = 0; 821 822 _thr_check_init(); 823 824 curthread = tls_get_curthread(); 825 826 /* 827 * If the mutex is statically initialized, perform the dynamic 828 * initialization marking it private (delete safe): 829 */ 830 if ((*m != NULL) || 831 ((ret = init_static_private(curthread, m)) == 0)) 832 ret = mutex_lock_common(curthread, m, abs_timeout); 833 834 return (ret); 835 } 836 837 int 838 _pthread_mutex_unlock(pthread_mutex_t *m) 839 { 840 return (mutex_unlock_common(m, /* add reference */ 0)); 841 } 842 843 int 844 _mutex_cv_unlock(pthread_mutex_t *m) 845 { 846 return (mutex_unlock_common(m, /* add reference */ 1)); 847 } 848 849 int 850 _mutex_cv_lock(pthread_mutex_t *m) 851 { 852 int ret; 853 854 if ((ret = _pthread_mutex_lock(m)) == 0) 855 (*m)->m_refcount--; 856 return (ret); 857 } 858 859 static int 860 mutex_self_trylock(struct pthread *curthread, pthread_mutex_t m) 861 { 862 int ret; 863 864 switch (m->m_type) { 865 /* case PTHREAD_MUTEX_DEFAULT: */ 866 case PTHREAD_MUTEX_ERRORCHECK: 867 case PTHREAD_MUTEX_NORMAL: 868 ret = EBUSY; 869 break; 870 871 case PTHREAD_MUTEX_RECURSIVE: 872 /* Increment the lock count: */ 873 if (m->m_count + 1 > 0) { 874 m->m_count++; 875 ret = 0; 876 } else 877 ret = EAGAIN; 878 break; 879 880 default: 881 /* Trap invalid mutex types; */ 882 ret = EINVAL; 883 } 884 885 return (ret); 886 } 887 888 static int 889 mutex_self_lock(struct pthread *curthread, pthread_mutex_t m, 890 const struct timespec *abstime) 891 { 892 struct timespec ts1, ts2; 893 int ret; 894 895 switch (m->m_type) { 896 /* case PTHREAD_MUTEX_DEFAULT: */ 897 case PTHREAD_MUTEX_ERRORCHECK: 898 if (abstime) { 899 clock_gettime(CLOCK_REALTIME, &ts1); 900 TIMESPEC_SUB(&ts2, abstime, &ts1); 901 __sys_nanosleep(&ts2, NULL); 902 ret = ETIMEDOUT; 903 } else { 904 /* 905 * POSIX specifies that mutexes should return 906 * EDEADLK if a recursive lock is detected. 907 */ 908 ret = EDEADLK; 909 } 910 break; 911 912 case PTHREAD_MUTEX_NORMAL: 913 /* 914 * What SS2 define as a 'normal' mutex. Intentionally 915 * deadlock on attempts to get a lock you already own. 916 */ 917 ret = 0; 918 if (m->m_protocol != PTHREAD_PRIO_NONE) { 919 /* Unlock the mutex structure: */ 920 THR_LOCK_RELEASE(curthread, &m->m_lock); 921 } 922 if (abstime) { 923 clock_gettime(CLOCK_REALTIME, &ts1); 924 TIMESPEC_SUB(&ts2, abstime, &ts1); 925 __sys_nanosleep(&ts2, NULL); 926 ret = ETIMEDOUT; 927 } else { 928 ts1.tv_sec = 30; 929 ts1.tv_nsec = 0; 930 for (;;) 931 __sys_nanosleep(&ts1, NULL); 932 } 933 break; 934 935 case PTHREAD_MUTEX_RECURSIVE: 936 /* Increment the lock count: */ 937 if (m->m_count + 1 > 0) { 938 m->m_count++; 939 ret = 0; 940 } else 941 ret = EAGAIN; 942 break; 943 944 default: 945 /* Trap invalid mutex types; */ 946 ret = EINVAL; 947 } 948 949 return (ret); 950 } 951 952 static int 953 mutex_unlock_common(pthread_mutex_t *m, int add_reference) 954 { 955 struct pthread *curthread = tls_get_curthread(); 956 long tid = -1; 957 int ret = 0; 958 959 if (m == NULL || *m == NULL) 960 ret = EINVAL; 961 else { 962 /* Short cut for simple mutex. */ 963 964 if ((*m)->m_protocol == PTHREAD_PRIO_NONE) { 965 /* 966 * Check if the running thread is not the owner of the 967 * mutex: 968 */ 969 if (__predict_false((*m)->m_owner != curthread)) { 970 ret = EPERM; 971 } else if (__predict_false( 972 (*m)->m_type == PTHREAD_MUTEX_RECURSIVE && 973 (*m)->m_count > 0)) { 974 /* Decrement the count: */ 975 (*m)->m_count--; 976 if (add_reference) 977 (*m)->m_refcount++; 978 } else { 979 /* 980 * Clear the count in case this is a recursive 981 * mutex. 982 */ 983 (*m)->m_count = 0; 984 (*m)->m_owner = NULL; 985 /* Remove the mutex from the threads queue. */ 986 MUTEX_ASSERT_IS_OWNED(*m); 987 TAILQ_REMOVE(&curthread->mutexq, (*m), m_qe); 988 MUTEX_INIT_LINK(*m); 989 if (add_reference) 990 (*m)->m_refcount++; 991 /* 992 * Hand off the mutex to the next waiting 993 * thread. 994 */ 995 _thr_umtx_unlock(&(*m)->m_lock, curthread->tid); 996 } 997 return (ret); 998 } 999 1000 /* Code for priority mutex */ 1001 1002 /* Lock the mutex structure: */ 1003 THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); 1004 1005 /* Process according to mutex type: */ 1006 switch ((*m)->m_protocol) { 1007 /* POSIX priority inheritence mutex: */ 1008 case PTHREAD_PRIO_INHERIT: 1009 /* 1010 * Check if the running thread is not the owner of the 1011 * mutex: 1012 */ 1013 if ((*m)->m_owner != curthread) 1014 ret = EPERM; 1015 else if (((*m)->m_type == PTHREAD_MUTEX_RECURSIVE) && 1016 ((*m)->m_count > 0)) 1017 /* Decrement the count: */ 1018 (*m)->m_count--; 1019 else { 1020 /* 1021 * Clear the count in case this is recursive 1022 * mutex. 1023 */ 1024 (*m)->m_count = 0; 1025 1026 /* 1027 * Restore the threads inherited priority and 1028 * recompute the active priority (being careful 1029 * not to override changes in the threads base 1030 * priority subsequent to locking the mutex). 1031 */ 1032 THR_LOCK(curthread); 1033 curthread->inherited_priority = 1034 (*m)->m_saved_prio; 1035 curthread->active_priority = 1036 MAX(curthread->inherited_priority, 1037 curthread->base_priority); 1038 1039 /* 1040 * This thread now owns one less priority mutex. 1041 */ 1042 curthread->priority_mutex_count--; 1043 THR_UNLOCK(curthread); 1044 1045 /* Remove the mutex from the threads queue. */ 1046 MUTEX_ASSERT_IS_OWNED(*m); 1047 TAILQ_REMOVE(&(*m)->m_owner->pri_mutexq, 1048 (*m), m_qe); 1049 MUTEX_INIT_LINK(*m); 1050 1051 /* 1052 * Hand off the mutex to the next waiting 1053 * thread: 1054 */ 1055 tid = mutex_handoff(curthread, *m); 1056 } 1057 break; 1058 1059 /* POSIX priority ceiling mutex: */ 1060 case PTHREAD_PRIO_PROTECT: 1061 /* 1062 * Check if the running thread is not the owner of the 1063 * mutex: 1064 */ 1065 if ((*m)->m_owner != curthread) 1066 ret = EPERM; 1067 else if (((*m)->m_type == PTHREAD_MUTEX_RECURSIVE) && 1068 ((*m)->m_count > 0)) 1069 /* Decrement the count: */ 1070 (*m)->m_count--; 1071 else { 1072 /* 1073 * Clear the count in case this is a recursive 1074 * mutex. 1075 */ 1076 (*m)->m_count = 0; 1077 1078 /* 1079 * Restore the threads inherited priority and 1080 * recompute the active priority (being careful 1081 * not to override changes in the threads base 1082 * priority subsequent to locking the mutex). 1083 */ 1084 THR_LOCK(curthread); 1085 curthread->inherited_priority = 1086 (*m)->m_saved_prio; 1087 curthread->active_priority = 1088 MAX(curthread->inherited_priority, 1089 curthread->base_priority); 1090 1091 /* 1092 * This thread now owns one less priority mutex. 1093 */ 1094 curthread->priority_mutex_count--; 1095 THR_UNLOCK(curthread); 1096 1097 /* Remove the mutex from the threads queue. */ 1098 MUTEX_ASSERT_IS_OWNED(*m); 1099 TAILQ_REMOVE(&(*m)->m_owner->pri_mutexq, 1100 (*m), m_qe); 1101 MUTEX_INIT_LINK(*m); 1102 1103 /* 1104 * Hand off the mutex to the next waiting 1105 * thread: 1106 */ 1107 tid = mutex_handoff(curthread, *m); 1108 } 1109 break; 1110 1111 /* Trap invalid mutex types: */ 1112 default: 1113 /* Return an invalid argument error: */ 1114 ret = EINVAL; 1115 break; 1116 } 1117 1118 if ((ret == 0) && (add_reference != 0)) 1119 /* Increment the reference count: */ 1120 (*m)->m_refcount++; 1121 1122 /* Unlock the mutex structure: */ 1123 THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 1124 } 1125 1126 /* Return the completion status: */ 1127 return (ret); 1128 } 1129 1130 1131 /* 1132 * This function is called when a change in base priority occurs for 1133 * a thread that is holding or waiting for a priority protection or 1134 * inheritence mutex. A change in a threads base priority can effect 1135 * changes to active priorities of other threads and to the ordering 1136 * of mutex locking by waiting threads. 1137 * 1138 * This must be called without the target thread's scheduling lock held. 1139 */ 1140 void 1141 _mutex_notify_priochange(struct pthread *curthread, struct pthread *pthread, 1142 int propagate_prio) 1143 { 1144 struct pthread_mutex *m; 1145 1146 /* Adjust the priorites of any owned priority mutexes: */ 1147 if (pthread->priority_mutex_count > 0) { 1148 /* 1149 * Rescan the mutexes owned by this thread and correct 1150 * their priorities to account for this threads change 1151 * in priority. This has the side effect of changing 1152 * the threads active priority. 1153 * 1154 * Be sure to lock the first mutex in the list of owned 1155 * mutexes. This acts as a barrier against another 1156 * simultaneous call to change the threads priority 1157 * and from the owning thread releasing the mutex. 1158 */ 1159 m = TAILQ_FIRST(&pthread->pri_mutexq); 1160 if (m != NULL) { 1161 THR_LOCK_ACQUIRE(curthread, &m->m_lock); 1162 /* 1163 * Make sure the thread still owns the lock. 1164 */ 1165 if (m == TAILQ_FIRST(&pthread->pri_mutexq)) 1166 mutex_rescan_owned(curthread, pthread, 1167 /* rescan all owned */ NULL); 1168 THR_LOCK_RELEASE(curthread, &m->m_lock); 1169 } 1170 } 1171 1172 /* 1173 * If this thread is waiting on a priority inheritence mutex, 1174 * check for priority adjustments. A change in priority can 1175 * also cause a ceiling violation(*) for a thread waiting on 1176 * a priority protection mutex; we don't perform the check here 1177 * as it is done in pthread_mutex_unlock. 1178 * 1179 * (*) It should be noted that a priority change to a thread 1180 * _after_ taking and owning a priority ceiling mutex 1181 * does not affect ownership of that mutex; the ceiling 1182 * priority is only checked before mutex ownership occurs. 1183 */ 1184 if (propagate_prio != 0) { 1185 /* 1186 * Lock the thread's scheduling queue. This is a bit 1187 * convoluted; the "in synchronization queue flag" can 1188 * only be cleared with both the thread's scheduling and 1189 * mutex locks held. The thread's pointer to the wanted 1190 * mutex is guaranteed to be valid during this time. 1191 */ 1192 THR_THREAD_LOCK(curthread, pthread); 1193 1194 if (((pthread->sflags & THR_FLAGS_IN_SYNCQ) == 0) || 1195 ((m = pthread->data.mutex) == NULL)) 1196 THR_THREAD_UNLOCK(curthread, pthread); 1197 else { 1198 /* 1199 * This thread is currently waiting on a mutex; unlock 1200 * the scheduling queue lock and lock the mutex. We 1201 * can't hold both at the same time because the locking 1202 * order could cause a deadlock. 1203 */ 1204 THR_THREAD_UNLOCK(curthread, pthread); 1205 THR_LOCK_ACQUIRE(curthread, &m->m_lock); 1206 1207 /* 1208 * Check to make sure this thread is still in the 1209 * same state (the lock above can yield the CPU to 1210 * another thread or the thread may be running on 1211 * another CPU). 1212 */ 1213 if (((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) && 1214 (pthread->data.mutex == m)) { 1215 /* 1216 * Remove and reinsert this thread into 1217 * the list of waiting threads to preserve 1218 * decreasing priority order. 1219 */ 1220 mutex_queue_remove(m, pthread); 1221 mutex_queue_enq(m, pthread); 1222 1223 if (m->m_protocol == PTHREAD_PRIO_INHERIT) 1224 /* Adjust priorities: */ 1225 mutex_priority_adjust(curthread, m); 1226 } 1227 1228 /* Unlock the mutex structure: */ 1229 THR_LOCK_RELEASE(curthread, &m->m_lock); 1230 } 1231 } 1232 } 1233 1234 /* 1235 * Called when a new thread is added to the mutex waiting queue or 1236 * when a threads priority changes that is already in the mutex 1237 * waiting queue. 1238 * 1239 * This must be called with the mutex locked by the current thread. 1240 */ 1241 static void 1242 mutex_priority_adjust(struct pthread *curthread, pthread_mutex_t mutex) 1243 { 1244 pthread_mutex_t m = mutex; 1245 struct pthread *pthread_next, *pthread = mutex->m_owner; 1246 int done, temp_prio; 1247 1248 /* 1249 * Calculate the mutex priority as the maximum of the highest 1250 * active priority of any waiting threads and the owning threads 1251 * active priority(*). 1252 * 1253 * (*) Because the owning threads current active priority may 1254 * reflect priority inherited from this mutex (and the mutex 1255 * priority may have changed) we must recalculate the active 1256 * priority based on the threads saved inherited priority 1257 * and its base priority. 1258 */ 1259 pthread_next = TAILQ_FIRST(&m->m_queue); /* should never be NULL */ 1260 temp_prio = MAX(pthread_next->active_priority, 1261 MAX(m->m_saved_prio, pthread->base_priority)); 1262 1263 /* See if this mutex really needs adjusting: */ 1264 if (temp_prio == m->m_prio) 1265 /* No need to propagate the priority: */ 1266 return; 1267 1268 /* Set new priority of the mutex: */ 1269 m->m_prio = temp_prio; 1270 1271 /* 1272 * Don't unlock the mutex passed in as an argument. It is 1273 * expected to be locked and unlocked by the caller. 1274 */ 1275 done = 1; 1276 do { 1277 /* 1278 * Save the threads priority before rescanning the 1279 * owned mutexes: 1280 */ 1281 temp_prio = pthread->active_priority; 1282 1283 /* 1284 * Fix the priorities for all mutexes held by the owning 1285 * thread since taking this mutex. This also has a 1286 * potential side-effect of changing the threads priority. 1287 * 1288 * At this point the mutex is locked by the current thread. 1289 * The owning thread can't release the mutex until it is 1290 * unlocked, so we should be able to safely walk its list 1291 * of owned mutexes. 1292 */ 1293 mutex_rescan_owned(curthread, pthread, m); 1294 1295 /* 1296 * If this isn't the first time through the loop, 1297 * the current mutex needs to be unlocked. 1298 */ 1299 if (done == 0) 1300 THR_LOCK_RELEASE(curthread, &m->m_lock); 1301 1302 /* Assume we're done unless told otherwise: */ 1303 done = 1; 1304 1305 /* 1306 * If the thread is currently waiting on a mutex, check 1307 * to see if the threads new priority has affected the 1308 * priority of the mutex. 1309 */ 1310 if ((temp_prio != pthread->active_priority) && 1311 ((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) && 1312 ((m = pthread->data.mutex) != NULL) && 1313 (m->m_protocol == PTHREAD_PRIO_INHERIT)) { 1314 /* Lock the mutex structure: */ 1315 THR_LOCK_ACQUIRE(curthread, &m->m_lock); 1316 1317 /* 1318 * Make sure the thread is still waiting on the 1319 * mutex: 1320 */ 1321 if (((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) && 1322 (m == pthread->data.mutex)) { 1323 /* 1324 * The priority for this thread has changed. 1325 * Remove and reinsert this thread into the 1326 * list of waiting threads to preserve 1327 * decreasing priority order. 1328 */ 1329 mutex_queue_remove(m, pthread); 1330 mutex_queue_enq(m, pthread); 1331 1332 /* 1333 * Grab the waiting thread with highest 1334 * priority: 1335 */ 1336 pthread_next = TAILQ_FIRST(&m->m_queue); 1337 1338 /* 1339 * Calculate the mutex priority as the maximum 1340 * of the highest active priority of any 1341 * waiting threads and the owning threads 1342 * active priority. 1343 */ 1344 temp_prio = MAX(pthread_next->active_priority, 1345 MAX(m->m_saved_prio, 1346 m->m_owner->base_priority)); 1347 1348 if (temp_prio != m->m_prio) { 1349 /* 1350 * The priority needs to be propagated 1351 * to the mutex this thread is waiting 1352 * on and up to the owner of that mutex. 1353 */ 1354 m->m_prio = temp_prio; 1355 pthread = m->m_owner; 1356 1357 /* We're not done yet: */ 1358 done = 0; 1359 } 1360 } 1361 /* Only release the mutex if we're done: */ 1362 if (done != 0) 1363 THR_LOCK_RELEASE(curthread, &m->m_lock); 1364 } 1365 } while (done == 0); 1366 } 1367 1368 static void 1369 mutex_rescan_owned(struct pthread *curthread, struct pthread *pthread, 1370 struct pthread_mutex *mutex) 1371 { 1372 struct pthread_mutex *m; 1373 struct pthread *pthread_next; 1374 int active_prio, inherited_prio; 1375 1376 /* 1377 * Start walking the mutexes the thread has taken since 1378 * taking this mutex. 1379 */ 1380 if (mutex == NULL) { 1381 /* 1382 * A null mutex means start at the beginning of the owned 1383 * mutex list. 1384 */ 1385 m = TAILQ_FIRST(&pthread->pri_mutexq); 1386 1387 /* There is no inherited priority yet. */ 1388 inherited_prio = 0; 1389 } else { 1390 /* 1391 * The caller wants to start after a specific mutex. It 1392 * is assumed that this mutex is a priority inheritence 1393 * mutex and that its priority has been correctly 1394 * calculated. 1395 */ 1396 m = TAILQ_NEXT(mutex, m_qe); 1397 1398 /* Start inheriting priority from the specified mutex. */ 1399 inherited_prio = mutex->m_prio; 1400 } 1401 active_prio = MAX(inherited_prio, pthread->base_priority); 1402 1403 for (; m != NULL; m = TAILQ_NEXT(m, m_qe)) { 1404 /* 1405 * We only want to deal with priority inheritence 1406 * mutexes. This might be optimized by only placing 1407 * priority inheritence mutexes into the owned mutex 1408 * list, but it may prove to be useful having all 1409 * owned mutexes in this list. Consider a thread 1410 * exiting while holding mutexes... 1411 */ 1412 if (m->m_protocol == PTHREAD_PRIO_INHERIT) { 1413 /* 1414 * Fix the owners saved (inherited) priority to 1415 * reflect the priority of the previous mutex. 1416 */ 1417 m->m_saved_prio = inherited_prio; 1418 1419 if ((pthread_next = TAILQ_FIRST(&m->m_queue)) != NULL) 1420 /* Recalculate the priority of the mutex: */ 1421 m->m_prio = MAX(active_prio, 1422 pthread_next->active_priority); 1423 else 1424 m->m_prio = active_prio; 1425 1426 /* Recalculate new inherited and active priorities: */ 1427 inherited_prio = m->m_prio; 1428 active_prio = MAX(m->m_prio, pthread->base_priority); 1429 } 1430 } 1431 1432 /* 1433 * Fix the threads inherited priority and recalculate its 1434 * active priority. 1435 */ 1436 pthread->inherited_priority = inherited_prio; 1437 active_prio = MAX(inherited_prio, pthread->base_priority); 1438 1439 if (active_prio != pthread->active_priority) { 1440 /* Lock the thread's scheduling queue: */ 1441 THR_THREAD_LOCK(curthread, pthread); 1442 1443 /* if ((pthread->flags & THR_FLAGS_IN_RUNQ) == 0) */ 1444 if (1) { 1445 /* 1446 * This thread is not in a run queue. Just set 1447 * its active priority. 1448 */ 1449 pthread->active_priority = active_prio; 1450 } 1451 else { 1452 /* 1453 * This thread is in a run queue. Remove it from 1454 * the queue before changing its priority: 1455 */ 1456 /* THR_RUNQ_REMOVE(pthread);*/ 1457 /* 1458 * POSIX states that if the priority is being 1459 * lowered, the thread must be inserted at the 1460 * head of the queue for its priority if it owns 1461 * any priority protection or inheritence mutexes. 1462 */ 1463 if ((active_prio < pthread->active_priority) && 1464 (pthread->priority_mutex_count > 0)) { 1465 /* Set the new active priority. */ 1466 pthread->active_priority = active_prio; 1467 /* THR_RUNQ_INSERT_HEAD(pthread); */ 1468 } else { 1469 /* Set the new active priority. */ 1470 pthread->active_priority = active_prio; 1471 /* THR_RUNQ_INSERT_TAIL(pthread);*/ 1472 } 1473 } 1474 THR_THREAD_UNLOCK(curthread, pthread); 1475 } 1476 } 1477 1478 void 1479 _mutex_unlock_private(pthread_t pthread) 1480 { 1481 struct pthread_mutex *m, *m_next; 1482 1483 for (m = TAILQ_FIRST(&pthread->pri_mutexq); m != NULL; m = m_next) { 1484 m_next = TAILQ_NEXT(m, m_qe); 1485 if ((m->m_flags & MUTEX_FLAGS_PRIVATE) != 0) 1486 pthread_mutex_unlock(&m); 1487 } 1488 } 1489 1490 /* 1491 * Dequeue a waiting thread from the head of a mutex queue in descending 1492 * priority order. 1493 * 1494 * In order to properly dequeue a thread from the mutex queue and 1495 * make it runnable without the possibility of errant wakeups, it 1496 * is necessary to lock the thread's scheduling queue while also 1497 * holding the mutex lock. 1498 */ 1499 static long 1500 mutex_handoff(struct pthread *curthread, struct pthread_mutex *mutex) 1501 { 1502 struct pthread *pthread; 1503 long tid = -1; 1504 1505 /* Keep dequeueing until we find a valid thread: */ 1506 mutex->m_owner = NULL; 1507 pthread = TAILQ_FIRST(&mutex->m_queue); 1508 while (pthread != NULL) { 1509 /* Take the thread's scheduling lock: */ 1510 THR_THREAD_LOCK(curthread, pthread); 1511 1512 /* Remove the thread from the mutex queue: */ 1513 TAILQ_REMOVE(&mutex->m_queue, pthread, sqe); 1514 pthread->sflags &= ~THR_FLAGS_IN_SYNCQ; 1515 1516 /* 1517 * Only exit the loop if the thread hasn't been 1518 * cancelled. 1519 */ 1520 switch (mutex->m_protocol) { 1521 case PTHREAD_PRIO_NONE: 1522 /* 1523 * Assign the new owner and add the mutex to the 1524 * thread's list of owned mutexes. 1525 */ 1526 mutex->m_owner = pthread; 1527 TAILQ_INSERT_TAIL(&pthread->pri_mutexq, mutex, m_qe); 1528 break; 1529 1530 case PTHREAD_PRIO_INHERIT: 1531 /* 1532 * Assign the new owner and add the mutex to the 1533 * thread's list of owned mutexes. 1534 */ 1535 mutex->m_owner = pthread; 1536 TAILQ_INSERT_TAIL(&pthread->pri_mutexq, mutex, m_qe); 1537 1538 /* Track number of priority mutexes owned: */ 1539 pthread->priority_mutex_count++; 1540 1541 /* 1542 * Set the priority of the mutex. Since our waiting 1543 * threads are in descending priority order, the 1544 * priority of the mutex becomes the active priority 1545 * of the thread we just dequeued. 1546 */ 1547 mutex->m_prio = pthread->active_priority; 1548 1549 /* Save the owning threads inherited priority: */ 1550 mutex->m_saved_prio = pthread->inherited_priority; 1551 1552 /* 1553 * The owning threads inherited priority now becomes 1554 * his active priority (the priority of the mutex). 1555 */ 1556 pthread->inherited_priority = mutex->m_prio; 1557 break; 1558 1559 case PTHREAD_PRIO_PROTECT: 1560 if (pthread->active_priority > mutex->m_prio) { 1561 /* 1562 * Either the mutex ceiling priority has 1563 * been lowered and/or this threads priority 1564 * has been raised subsequent to the thread 1565 * being queued on the waiting list. 1566 */ 1567 pthread->error = EINVAL; 1568 } 1569 else { 1570 /* 1571 * Assign the new owner and add the mutex 1572 * to the thread's list of owned mutexes. 1573 */ 1574 mutex->m_owner = pthread; 1575 TAILQ_INSERT_TAIL(&pthread->pri_mutexq, 1576 mutex, m_qe); 1577 1578 /* Track number of priority mutexes owned: */ 1579 pthread->priority_mutex_count++; 1580 1581 /* 1582 * Save the owning threads inherited 1583 * priority: 1584 */ 1585 mutex->m_saved_prio = 1586 pthread->inherited_priority; 1587 1588 /* 1589 * The owning thread inherits the ceiling 1590 * priority of the mutex and executes at 1591 * that priority: 1592 */ 1593 pthread->inherited_priority = mutex->m_prio; 1594 pthread->active_priority = mutex->m_prio; 1595 1596 } 1597 break; 1598 } 1599 1600 /* Make the thread runnable and unlock the scheduling queue: */ 1601 pthread->cycle++; 1602 _thr_umtx_wake(&pthread->cycle, 1); 1603 1604 THR_THREAD_UNLOCK(curthread, pthread); 1605 if (mutex->m_owner == pthread) 1606 /* We're done; a valid owner was found. */ 1607 break; 1608 else 1609 /* Get the next thread from the waiting queue: */ 1610 pthread = TAILQ_NEXT(pthread, sqe); 1611 } 1612 1613 if ((pthread == NULL) && (mutex->m_protocol == PTHREAD_PRIO_INHERIT)) 1614 /* This mutex has no priority: */ 1615 mutex->m_prio = 0; 1616 return (tid); 1617 } 1618 1619 #if 0 1620 /* 1621 * Dequeue a waiting thread from the head of a mutex queue in descending 1622 * priority order. 1623 */ 1624 static pthread_t 1625 mutex_queue_deq(struct pthread_mutex *mutex) 1626 { 1627 pthread_t pthread; 1628 1629 while ((pthread = TAILQ_FIRST(&mutex->m_queue)) != NULL) { 1630 TAILQ_REMOVE(&mutex->m_queue, pthread, sqe); 1631 pthread->sflags &= ~THR_FLAGS_IN_SYNCQ; 1632 } 1633 1634 return (pthread); 1635 } 1636 #endif 1637 1638 /* 1639 * Remove a waiting thread from a mutex queue in descending priority order. 1640 */ 1641 static void 1642 mutex_queue_remove(pthread_mutex_t mutex, pthread_t pthread) 1643 { 1644 if ((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) { 1645 TAILQ_REMOVE(&mutex->m_queue, pthread, sqe); 1646 pthread->sflags &= ~THR_FLAGS_IN_SYNCQ; 1647 } 1648 } 1649 1650 /* 1651 * Enqueue a waiting thread to a queue in descending priority order. 1652 */ 1653 static void 1654 mutex_queue_enq(pthread_mutex_t mutex, pthread_t pthread) 1655 { 1656 pthread_t tid = TAILQ_LAST(&mutex->m_queue, mutex_head); 1657 1658 THR_ASSERT_NOT_IN_SYNCQ(pthread); 1659 /* 1660 * For the common case of all threads having equal priority, 1661 * we perform a quick check against the priority of the thread 1662 * at the tail of the queue. 1663 */ 1664 if ((tid == NULL) || (pthread->active_priority <= tid->active_priority)) 1665 TAILQ_INSERT_TAIL(&mutex->m_queue, pthread, sqe); 1666 else { 1667 tid = TAILQ_FIRST(&mutex->m_queue); 1668 while (pthread->active_priority <= tid->active_priority) 1669 tid = TAILQ_NEXT(tid, sqe); 1670 TAILQ_INSERT_BEFORE(tid, pthread, sqe); 1671 } 1672 pthread->sflags |= THR_FLAGS_IN_SYNCQ; 1673 } 1674