1 /* $NetBSD: pthread.c,v 1.182 2023/03/24 14:18:18 joerg Exp $ */
2
3 /*-
4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020
5 * The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by Nathan J. Williams and Andrew Doran.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 #include <sys/cdefs.h>
34 __RCSID("$NetBSD: pthread.c,v 1.182 2023/03/24 14:18:18 joerg Exp $");
35
36 #define __EXPOSE_STACK 1
37
38 /* Need to use libc-private names for atomic operations. */
39 #include "../../common/lib/libc/atomic/atomic_op_namespace.h"
40
41 #include <sys/param.h>
42 #include <sys/exec_elf.h>
43 #include <sys/mman.h>
44 #include <sys/lwp.h>
45 #include <sys/lwpctl.h>
46 #include <sys/resource.h>
47 #include <sys/sysctl.h>
48 #include <sys/tls.h>
49 #include <uvm/uvm_param.h>
50
51 #include <assert.h>
52 #include <dlfcn.h>
53 #include <err.h>
54 #include <errno.h>
55 #include <lwp.h>
56 #include <signal.h>
57 #include <stdio.h>
58 #include <stdlib.h>
59 #include <stddef.h>
60 #include <string.h>
61 #include <syslog.h>
62 #include <ucontext.h>
63 #include <unistd.h>
64 #include <sched.h>
65
66 #include "atexit.h"
67 #include "pthread.h"
68 #include "pthread_int.h"
69 #include "pthread_makelwp.h"
70 #include "reentrant.h"
71
72 __BEGIN_DECLS
73 void _malloc_thread_cleanup(void) __weak;
74 __END_DECLS
75
76 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER;
77 static rb_tree_t pthread__alltree;
78
79 static signed int pthread__cmp(void *, const void *, const void *);
80
81 static const rb_tree_ops_t pthread__alltree_ops = {
82 .rbto_compare_nodes = pthread__cmp,
83 .rbto_compare_key = pthread__cmp,
84 .rbto_node_offset = offsetof(struct __pthread_st, pt_alltree),
85 .rbto_context = NULL
86 };
87
88 static void pthread__create_tramp(void *);
89 static void pthread__initthread(pthread_t);
90 static void pthread__scrubthread(pthread_t, char *, int);
91 static void pthread__initmain(pthread_t *);
92 static void pthread__reap(pthread_t);
93
94 void pthread__init(void);
95
96 int pthread__started;
97 int __uselibcstub = 1;
98 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER;
99 pthread_queue_t pthread__deadqueue;
100 pthread_queue_t pthread__allqueue;
101
102 static pthread_attr_t pthread_default_attr;
103 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE };
104
105 enum {
106 DIAGASSERT_ABORT = 1<<0,
107 DIAGASSERT_STDERR = 1<<1,
108 DIAGASSERT_SYSLOG = 1<<2
109 };
110
111 static int pthread__diagassert;
112
113 int pthread__concurrency;
114 int pthread__nspins;
115 size_t pthread__unpark_max = PTHREAD__UNPARK_MAX;
116 int pthread__dbg; /* set by libpthread_dbg if active */
117
118 /*
119 * We have to initialize the pthread_stack* variables here because
120 * mutexes are used before pthread_init() and thus pthread__initmain()
121 * are called. Since mutexes only save the stack pointer and not a
122 * pointer to the thread data, it is safe to change the mapping from
123 * stack pointer to thread data afterwards.
124 */
125 size_t pthread__stacksize;
126 size_t pthread__guardsize;
127 size_t pthread__pagesize;
128 static struct __pthread_st *pthread__main;
129 static size_t __pthread_st_size;
130
131 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *);
132
133 __strong_alias(__libc_thr_self,pthread_self)
134 __strong_alias(__libc_thr_create,pthread_create)
135 __strong_alias(__libc_thr_exit,pthread_exit)
136 __strong_alias(__libc_thr_errno,pthread__errno)
137 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate)
138 __strong_alias(__libc_thr_equal,pthread_equal)
139 __strong_alias(__libc_thr_init,pthread__init)
140
141 /*
142 * Static library kludge. Place a reference to a symbol any library
143 * file which does not already have a reference here.
144 */
145 extern int pthread__cancel_stub_binder;
146
147 void *pthread__static_lib_binder[] = {
148 &pthread__cancel_stub_binder,
149 pthread_cond_init,
150 pthread_mutex_init,
151 pthread_rwlock_init,
152 pthread_barrier_init,
153 pthread_key_create,
154 pthread_setspecific,
155 };
156
157 #define NHASHLOCK 64
158
159 static union hashlock {
160 pthread_mutex_t mutex;
161 char pad[64];
162 } hashlocks[NHASHLOCK] __aligned(64);
163
164 static void
pthread__prefork(void)165 pthread__prefork(void)
166 {
167 pthread_mutex_lock(&pthread__deadqueue_lock);
168 }
169
170 static void
pthread__fork_parent(void)171 pthread__fork_parent(void)
172 {
173 pthread_mutex_unlock(&pthread__deadqueue_lock);
174 }
175
176 static void
pthread__fork_child(void)177 pthread__fork_child(void)
178 {
179 struct __pthread_st *self = pthread__self();
180
181 pthread_mutex_init(&pthread__deadqueue_lock, NULL);
182
183 /* lwpctl state is not copied across fork. */
184 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
185 err(EXIT_FAILURE, "_lwp_ctl");
186 }
187 self->pt_lid = _lwp_self();
188 }
189
190 /*
191 * This needs to be started by the library loading code, before main()
192 * gets to run, for various things that use the state of the initial thread
193 * to work properly (thread-specific data is an application-visible example;
194 * spinlock counts for mutexes is an internal example).
195 */
196 void
pthread__init(void)197 pthread__init(void)
198 {
199 pthread_t first;
200 char *p;
201 int mib[2];
202 unsigned int value;
203 ssize_t slen;
204 size_t len;
205 extern int __isthreaded;
206
207 /*
208 * Allocate pthread_keys descriptors before
209 * resetting __uselibcstub because otherwise
210 * malloc() will call pthread_keys_create()
211 * while pthread_keys descriptors are not
212 * yet allocated.
213 */
214 pthread__main = pthread_tsd_init(&__pthread_st_size);
215 if (pthread__main == NULL)
216 err(EXIT_FAILURE, "Cannot allocate pthread storage");
217
218 __uselibcstub = 0;
219
220 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE);
221 pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF);
222
223 mib[0] = CTL_VM;
224 mib[1] = VM_THREAD_GUARD_SIZE;
225 len = sizeof(value);
226 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
227 pthread__guardsize = value;
228 else
229 pthread__guardsize = pthread__pagesize;
230
231 /* Initialize locks first; they're needed elsewhere. */
232 pthread__lockprim_init();
233 for (int i = 0; i < NHASHLOCK; i++) {
234 pthread_mutex_init(&hashlocks[i].mutex, NULL);
235 }
236
237 /* Fetch parameters. */
238 slen = _lwp_unpark_all(NULL, 0, NULL);
239 if (slen < 0)
240 err(EXIT_FAILURE, "_lwp_unpark_all");
241 if ((size_t)slen < pthread__unpark_max)
242 pthread__unpark_max = slen;
243
244 /* Basic data structure setup */
245 pthread_attr_init(&pthread_default_attr);
246 PTQ_INIT(&pthread__allqueue);
247 PTQ_INIT(&pthread__deadqueue);
248
249 rb_tree_init(&pthread__alltree, &pthread__alltree_ops);
250
251 /* Create the thread structure corresponding to main() */
252 pthread__initmain(&first);
253 pthread__initthread(first);
254 pthread__scrubthread(first, NULL, 0);
255
256 first->pt_lid = _lwp_self();
257 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq);
258 (void)rb_tree_insert_node(&pthread__alltree, first);
259
260 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) {
261 err(EXIT_FAILURE, "_lwp_ctl");
262 }
263
264 /* Start subsystems */
265 PTHREAD_MD_INIT
266
267 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) {
268 switch (*p) {
269 case 'a':
270 pthread__diagassert |= DIAGASSERT_ABORT;
271 break;
272 case 'A':
273 pthread__diagassert &= ~DIAGASSERT_ABORT;
274 break;
275 case 'e':
276 pthread__diagassert |= DIAGASSERT_STDERR;
277 break;
278 case 'E':
279 pthread__diagassert &= ~DIAGASSERT_STDERR;
280 break;
281 case 'l':
282 pthread__diagassert |= DIAGASSERT_SYSLOG;
283 break;
284 case 'L':
285 pthread__diagassert &= ~DIAGASSERT_SYSLOG;
286 break;
287 }
288 }
289
290 /* Tell libc that we're here and it should role-play accordingly. */
291 pthread_atfork(pthread__prefork, pthread__fork_parent, pthread__fork_child);
292 __isthreaded = 1;
293 }
294
295 /* General-purpose thread data structure sanitization. */
296 /* ARGSUSED */
297 static void
pthread__initthread(pthread_t t)298 pthread__initthread(pthread_t t)
299 {
300
301 t->pt_self = t;
302 t->pt_magic = PT_MAGIC;
303 t->pt_sleepobj = NULL;
304 t->pt_havespecific = 0;
305 t->pt_lwpctl = &pthread__dummy_lwpctl;
306
307 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops));
308 pthread_mutex_init(&t->pt_lock, NULL);
309 PTQ_INIT(&t->pt_cleanup_stack);
310 }
311
312 static void
pthread__scrubthread(pthread_t t,char * name,int flags)313 pthread__scrubthread(pthread_t t, char *name, int flags)
314 {
315
316 t->pt_state = PT_STATE_RUNNING;
317 t->pt_exitval = NULL;
318 t->pt_flags = flags;
319 t->pt_cancel = 0;
320 t->pt_errno = 0;
321 t->pt_name = name;
322 t->pt_lid = 0;
323 }
324
325 static int
pthread__getstack(pthread_t newthread,const pthread_attr_t * attr)326 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr)
327 {
328 void *stackbase, *stackbase2, *redzone;
329 size_t stacksize, guardsize;
330 bool allocated;
331
332 if (attr != NULL) {
333 pthread_attr_getstack(attr, &stackbase, &stacksize);
334 pthread_attr_getguardsize(attr, &guardsize);
335 } else {
336 stackbase = NULL;
337 stacksize = 0;
338 guardsize = pthread__guardsize;
339 }
340 if (stacksize == 0)
341 stacksize = pthread__stacksize;
342
343 if (newthread->pt_stack_allocated) {
344 if (stackbase == NULL &&
345 newthread->pt_stack.ss_size == stacksize &&
346 newthread->pt_guardsize == guardsize)
347 return 0;
348 stackbase2 = newthread->pt_stack.ss_sp;
349 #ifndef __MACHINE_STACK_GROWS_UP
350 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize;
351 #endif
352 munmap(stackbase2,
353 newthread->pt_stack.ss_size + newthread->pt_guardsize);
354 newthread->pt_stack.ss_sp = NULL;
355 newthread->pt_stack.ss_size = 0;
356 newthread->pt_guardsize = 0;
357 newthread->pt_stack_allocated = false;
358 }
359
360 newthread->pt_stack_allocated = false;
361
362 if (stackbase == NULL) {
363 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1;
364 guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1;
365 stackbase = mmap(NULL, stacksize + guardsize,
366 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0);
367 if (stackbase == MAP_FAILED)
368 return ENOMEM;
369 allocated = true;
370 } else {
371 allocated = false;
372 }
373 #ifdef __MACHINE_STACK_GROWS_UP
374 redzone = (char *)stackbase + stacksize;
375 stackbase2 = (char *)stackbase;
376 #else
377 redzone = (char *)stackbase;
378 stackbase2 = (char *)stackbase + guardsize;
379 #endif
380 if (allocated && guardsize &&
381 mprotect(redzone, guardsize, PROT_NONE) == -1) {
382 munmap(stackbase, stacksize + guardsize);
383 return EPERM;
384 }
385 newthread->pt_stack.ss_size = stacksize;
386 newthread->pt_stack.ss_sp = stackbase2;
387 newthread->pt_guardsize = guardsize;
388 newthread->pt_stack_allocated = allocated;
389 return 0;
390 }
391
392 int
pthread_create(pthread_t * thread,const pthread_attr_t * attr,void * (* startfunc)(void *),void * arg)393 pthread_create(pthread_t *thread, const pthread_attr_t *attr,
394 void *(*startfunc)(void *), void *arg)
395 {
396 pthread_t newthread;
397 pthread_attr_t nattr;
398 struct pthread_attr_private *p;
399 char * volatile name;
400 unsigned long flag;
401 void *private_area;
402 int ret;
403
404 if (__predict_false(__uselibcstub)) {
405 pthread__errorfunc(__FILE__, __LINE__, __func__,
406 "pthread_create() requires linking with -lpthread");
407 return __libc_thr_create_stub(thread, attr, startfunc, arg);
408 }
409
410 if (attr == NULL)
411 nattr = pthread_default_attr;
412 else if (attr->pta_magic == PT_ATTR_MAGIC)
413 nattr = *attr;
414 else
415 return EINVAL;
416
417 if (!pthread__started) {
418 /*
419 * Force the _lwp_park symbol to be resolved before we
420 * begin any activity that might rely on concurrent
421 * wakeups.
422 *
423 * This is necessary because rtld itself uses _lwp_park
424 * and _lwp_unpark internally for its own locking: If
425 * we wait to resolve _lwp_park until there is an
426 * _lwp_unpark from another thread pending in the
427 * current lwp (for example, pthread_mutex_unlock or
428 * pthread_cond_signal), rtld's internal use of
429 * _lwp_park might consume the pending unpark. The
430 * result is a deadlock where libpthread and rtld have
431 * both correctly used _lwp_park and _lwp_unpark for
432 * themselves, but rtld has consumed the wakeup meant
433 * for libpthread so it is lost to libpthread.
434 *
435 * For the very first thread, before pthread__started
436 * is set to true, pthread__self()->pt_lid should have
437 * been initialized in pthread__init by the time we get
438 * here to the correct lid so we go to sleep and wake
439 * ourselves at the same time as a no-op.
440 */
441 _lwp_park(CLOCK_REALTIME, 0, NULL, pthread__self()->pt_lid,
442 NULL, NULL);
443 }
444
445 pthread__started = 1;
446
447 /* Fetch misc. attributes from the attr structure. */
448 name = NULL;
449 if ((p = nattr.pta_private) != NULL)
450 if (p->ptap_name[0] != '\0')
451 if ((name = strdup(p->ptap_name)) == NULL)
452 return ENOMEM;
453
454 newthread = NULL;
455
456 /*
457 * Try to reclaim a dead thread.
458 */
459 if (!PTQ_EMPTY(&pthread__deadqueue)) {
460 pthread_mutex_lock(&pthread__deadqueue_lock);
461 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) {
462 /* Still busily exiting, or finished? */
463 if (newthread->pt_lwpctl->lc_curcpu ==
464 LWPCTL_CPU_EXITED)
465 break;
466 }
467 if (newthread)
468 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq);
469 pthread_mutex_unlock(&pthread__deadqueue_lock);
470 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
471 if (newthread && newthread->pt_tls) {
472 _rtld_tls_free(newthread->pt_tls);
473 newthread->pt_tls = NULL;
474 }
475 #endif
476 }
477
478 /*
479 * If necessary set up a stack, allocate space for a pthread_st,
480 * and initialize it.
481 */
482 if (newthread == NULL) {
483 newthread = calloc(1, __pthread_st_size);
484 if (newthread == NULL) {
485 free(name);
486 return ENOMEM;
487 }
488 newthread->pt_stack_allocated = false;
489
490 if (pthread__getstack(newthread, attr)) {
491 free(newthread);
492 free(name);
493 return ENOMEM;
494 }
495
496 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
497 newthread->pt_tls = NULL;
498 #endif
499
500 /* Add to list of all threads. */
501 pthread_rwlock_wrlock(&pthread__alltree_lock);
502 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq);
503 (void)rb_tree_insert_node(&pthread__alltree, newthread);
504 pthread_rwlock_unlock(&pthread__alltree_lock);
505
506 /* Will be reset by the thread upon exit. */
507 pthread__initthread(newthread);
508 } else {
509 if (pthread__getstack(newthread, attr)) {
510 pthread_mutex_lock(&pthread__deadqueue_lock);
511 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq);
512 pthread_mutex_unlock(&pthread__deadqueue_lock);
513 return ENOMEM;
514 }
515 }
516
517 /*
518 * Create the new LWP.
519 */
520 pthread__scrubthread(newthread, name, nattr.pta_flags);
521 newthread->pt_func = startfunc;
522 newthread->pt_arg = arg;
523 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
524 private_area = newthread->pt_tls = _rtld_tls_allocate();
525 newthread->pt_tls->tcb_pthread = newthread;
526 #else
527 private_area = newthread;
528 #endif
529
530 flag = 0;
531 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 ||
532 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0)
533 flag |= LWP_SUSPENDED;
534 if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0)
535 flag |= LWP_DETACHED;
536
537 ret = pthread__makelwp(pthread__create_tramp, newthread, private_area,
538 newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size,
539 flag, &newthread->pt_lid);
540 if (ret != 0) {
541 ret = errno;
542 pthread_mutex_lock(&newthread->pt_lock);
543 /* Will unlock and free name. */
544 pthread__reap(newthread);
545 return ret;
546 }
547
548 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) {
549 if (p != NULL) {
550 (void)pthread_setschedparam(newthread, p->ptap_policy,
551 &p->ptap_sp);
552 }
553 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) {
554 (void)_lwp_continue(newthread->pt_lid);
555 }
556 }
557
558 *thread = newthread;
559
560 return 0;
561 }
562
563
564 __dead static void
pthread__create_tramp(void * cookie)565 pthread__create_tramp(void *cookie)
566 {
567 pthread_t self;
568 void *retval;
569 void *junk __unused;
570
571 self = cookie;
572
573 /*
574 * Throw away some stack in a feeble attempt to reduce cache
575 * thrash. May help for SMT processors. XXX We should not
576 * be allocating stacks on fixed 2MB boundaries. Needs a
577 * thread register or decent thread local storage.
578 */
579 junk = alloca(((unsigned)self->pt_lid & 7) << 8);
580
581 if (self->pt_name != NULL) {
582 pthread_mutex_lock(&self->pt_lock);
583 if (self->pt_name != NULL)
584 (void)_lwp_setname(0, self->pt_name);
585 pthread_mutex_unlock(&self->pt_lock);
586 }
587
588 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
589 err(EXIT_FAILURE, "_lwp_ctl");
590 }
591
592 retval = (*self->pt_func)(self->pt_arg);
593
594 pthread_exit(retval);
595
596 /*NOTREACHED*/
597 pthread__abort();
598 }
599
600 int
pthread_suspend_np(pthread_t thread)601 pthread_suspend_np(pthread_t thread)
602 {
603 pthread_t self;
604
605 pthread__error(EINVAL, "Invalid thread",
606 thread->pt_magic == PT_MAGIC);
607
608 self = pthread__self();
609 if (self == thread) {
610 return EDEADLK;
611 }
612 if (pthread__find(thread) != 0)
613 return ESRCH;
614 if (_lwp_suspend(thread->pt_lid) == 0)
615 return 0;
616 return errno;
617 }
618
619 int
pthread_resume_np(pthread_t thread)620 pthread_resume_np(pthread_t thread)
621 {
622
623 pthread__error(EINVAL, "Invalid thread",
624 thread->pt_magic == PT_MAGIC);
625
626 if (pthread__find(thread) != 0)
627 return ESRCH;
628 if (_lwp_continue(thread->pt_lid) == 0)
629 return 0;
630 return errno;
631 }
632
633 void
pthread_exit(void * retval)634 pthread_exit(void *retval)
635 {
636 pthread_t self;
637 struct pt_clean_t *cleanup;
638
639 if (__predict_false(__uselibcstub)) {
640 __libc_thr_exit_stub(retval);
641 goto out;
642 }
643
644 self = pthread__self();
645
646 /* Disable cancellability. */
647 pthread_mutex_lock(&self->pt_lock);
648 self->pt_flags |= PT_FLAG_CS_DISABLED;
649 self->pt_cancel = 0;
650 pthread_mutex_unlock(&self->pt_lock);
651
652 /* Call any cancellation cleanup handlers */
653 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
654 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
655 cleanup = PTQ_FIRST(&self->pt_cleanup_stack);
656 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next);
657 (*cleanup->ptc_cleanup)(cleanup->ptc_arg);
658 }
659 }
660
661 __cxa_thread_run_atexit();
662
663 /* Perform cleanup of thread-specific data */
664 pthread__destroy_tsd(self);
665
666 if (_malloc_thread_cleanup)
667 _malloc_thread_cleanup();
668
669 /*
670 * Signal our exit. Our stack and pthread_t won't be reused until
671 * pthread_create() can see from kernel info that this LWP is gone.
672 */
673 pthread_mutex_lock(&self->pt_lock);
674 self->pt_exitval = retval;
675 if (self->pt_flags & PT_FLAG_DETACHED) {
676 /* pthread__reap() will drop the lock. */
677 pthread__reap(self);
678 _lwp_exit();
679 } else {
680 self->pt_state = PT_STATE_ZOMBIE;
681 pthread_mutex_unlock(&self->pt_lock);
682 /* Note: name will be freed by the joiner. */
683 _lwp_exit();
684 }
685
686 out:
687 /*NOTREACHED*/
688 pthread__abort();
689 exit(1);
690 }
691
692
693 int
pthread_join(pthread_t thread,void ** valptr)694 pthread_join(pthread_t thread, void **valptr)
695 {
696 pthread_t self;
697
698 pthread__error(EINVAL, "Invalid thread",
699 thread->pt_magic == PT_MAGIC);
700
701 self = pthread__self();
702
703 if (pthread__find(thread) != 0)
704 return ESRCH;
705
706 if (thread == self)
707 return EDEADLK;
708
709 /* IEEE Std 1003.1 says pthread_join() never returns EINTR. */
710 for (;;) {
711 pthread__testcancel(self);
712 if (_lwp_wait(thread->pt_lid, NULL) == 0)
713 break;
714 if (errno != EINTR)
715 return errno;
716 }
717
718 /*
719 * Don't test for cancellation again. The spec is that if
720 * cancelled, pthread_join() must not have succeeded.
721 */
722 pthread_mutex_lock(&thread->pt_lock);
723 if (thread->pt_state != PT_STATE_ZOMBIE) {
724 pthread__errorfunc(__FILE__, __LINE__, __func__,
725 "not a zombie");
726 }
727 if (valptr != NULL)
728 *valptr = thread->pt_exitval;
729
730 /* pthread__reap() will drop the lock. */
731 pthread__reap(thread);
732 return 0;
733 }
734
735 static void
pthread__reap(pthread_t thread)736 pthread__reap(pthread_t thread)
737 {
738 char *name;
739
740 name = thread->pt_name;
741 thread->pt_name = NULL;
742 thread->pt_state = PT_STATE_DEAD;
743 pthread_mutex_unlock(&thread->pt_lock);
744
745 pthread_mutex_lock(&pthread__deadqueue_lock);
746 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq);
747 pthread_mutex_unlock(&pthread__deadqueue_lock);
748
749 if (name != NULL)
750 free(name);
751 }
752
753 int
pthread_equal(pthread_t t1,pthread_t t2)754 pthread_equal(pthread_t t1, pthread_t t2)
755 {
756
757 if (__predict_false(__uselibcstub))
758 return __libc_thr_equal_stub(t1, t2);
759
760 pthread__error(0, "Invalid thread",
761 (t1 != NULL) && (t1->pt_magic == PT_MAGIC));
762
763 pthread__error(0, "Invalid thread",
764 (t2 != NULL) && (t2->pt_magic == PT_MAGIC));
765
766 /* Nothing special here. */
767 return (t1 == t2);
768 }
769
770
771 int
pthread_detach(pthread_t thread)772 pthread_detach(pthread_t thread)
773 {
774 int error;
775
776 pthread__error(EINVAL, "Invalid thread",
777 thread->pt_magic == PT_MAGIC);
778
779 if (pthread__find(thread) != 0)
780 return ESRCH;
781
782 pthread_mutex_lock(&thread->pt_lock);
783 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) {
784 error = EINVAL;
785 } else {
786 error = _lwp_detach(thread->pt_lid);
787 if (error == 0)
788 thread->pt_flags |= PT_FLAG_DETACHED;
789 else
790 error = errno;
791 }
792 if (thread->pt_state == PT_STATE_ZOMBIE) {
793 /* pthread__reap() will drop the lock. */
794 pthread__reap(thread);
795 } else
796 pthread_mutex_unlock(&thread->pt_lock);
797 return error;
798 }
799
800
801 int
pthread_getname_np(pthread_t thread,char * name,size_t len)802 pthread_getname_np(pthread_t thread, char *name, size_t len)
803 {
804
805 pthread__error(EINVAL, "Invalid thread",
806 thread->pt_magic == PT_MAGIC);
807
808 if (pthread__find(thread) != 0)
809 return ESRCH;
810
811 pthread_mutex_lock(&thread->pt_lock);
812 if (thread->pt_name == NULL)
813 name[0] = '\0';
814 else
815 strlcpy(name, thread->pt_name, len);
816 pthread_mutex_unlock(&thread->pt_lock);
817
818 return 0;
819 }
820
821
822 int
pthread_setname_np(pthread_t thread,const char * name,void * arg)823 pthread_setname_np(pthread_t thread, const char *name, void *arg)
824 {
825 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP];
826 int namelen;
827
828 pthread__error(EINVAL, "Invalid thread",
829 thread->pt_magic == PT_MAGIC);
830
831 if (pthread__find(thread) != 0)
832 return ESRCH;
833
834 namelen = snprintf(newname, sizeof(newname), name, arg);
835 if (namelen >= PTHREAD_MAX_NAMELEN_NP)
836 return EINVAL;
837
838 cp = strdup(newname);
839 if (cp == NULL)
840 return ENOMEM;
841
842 pthread_mutex_lock(&thread->pt_lock);
843 oldname = thread->pt_name;
844 thread->pt_name = cp;
845 (void)_lwp_setname(thread->pt_lid, cp);
846 pthread_mutex_unlock(&thread->pt_lock);
847
848 if (oldname != NULL)
849 free(oldname);
850
851 return 0;
852 }
853
854
855 pthread_t
pthread_self(void)856 pthread_self(void)
857 {
858 if (__predict_false(__uselibcstub))
859 return (pthread_t)__libc_thr_self_stub();
860
861 return pthread__self();
862 }
863
864
865 int
pthread_cancel(pthread_t thread)866 pthread_cancel(pthread_t thread)
867 {
868
869 pthread__error(EINVAL, "Invalid thread",
870 thread->pt_magic == PT_MAGIC);
871
872 if (pthread__find(thread) != 0)
873 return ESRCH;
874 pthread_mutex_lock(&thread->pt_lock);
875 thread->pt_flags |= PT_FLAG_CS_PENDING;
876 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) {
877 thread->pt_cancel = 1;
878 pthread_mutex_unlock(&thread->pt_lock);
879 _lwp_wakeup(thread->pt_lid);
880 } else
881 pthread_mutex_unlock(&thread->pt_lock);
882
883 return 0;
884 }
885
886
887 int
pthread_setcancelstate(int state,int * oldstate)888 pthread_setcancelstate(int state, int *oldstate)
889 {
890 pthread_t self;
891 int retval;
892
893 if (__predict_false(__uselibcstub))
894 return __libc_thr_setcancelstate_stub(state, oldstate);
895
896 self = pthread__self();
897 retval = 0;
898
899 pthread_mutex_lock(&self->pt_lock);
900
901 if (oldstate != NULL) {
902 if (self->pt_flags & PT_FLAG_CS_DISABLED)
903 *oldstate = PTHREAD_CANCEL_DISABLE;
904 else
905 *oldstate = PTHREAD_CANCEL_ENABLE;
906 }
907
908 if (state == PTHREAD_CANCEL_DISABLE) {
909 self->pt_flags |= PT_FLAG_CS_DISABLED;
910 if (self->pt_cancel) {
911 self->pt_flags |= PT_FLAG_CS_PENDING;
912 self->pt_cancel = 0;
913 }
914 } else if (state == PTHREAD_CANCEL_ENABLE) {
915 self->pt_flags &= ~PT_FLAG_CS_DISABLED;
916 /*
917 * If a cancellation was requested while cancellation
918 * was disabled, note that fact for future
919 * cancellation tests.
920 */
921 if (self->pt_flags & PT_FLAG_CS_PENDING) {
922 self->pt_cancel = 1;
923 /* This is not a deferred cancellation point. */
924 if (self->pt_flags & PT_FLAG_CS_ASYNC) {
925 pthread_mutex_unlock(&self->pt_lock);
926 pthread__cancelled();
927 }
928 }
929 } else
930 retval = EINVAL;
931
932 pthread_mutex_unlock(&self->pt_lock);
933
934 return retval;
935 }
936
937
938 int
pthread_setcanceltype(int type,int * oldtype)939 pthread_setcanceltype(int type, int *oldtype)
940 {
941 pthread_t self;
942 int retval;
943
944 self = pthread__self();
945 retval = 0;
946
947 pthread_mutex_lock(&self->pt_lock);
948
949 if (oldtype != NULL) {
950 if (self->pt_flags & PT_FLAG_CS_ASYNC)
951 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS;
952 else
953 *oldtype = PTHREAD_CANCEL_DEFERRED;
954 }
955
956 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) {
957 self->pt_flags |= PT_FLAG_CS_ASYNC;
958 if (self->pt_cancel) {
959 pthread_mutex_unlock(&self->pt_lock);
960 pthread__cancelled();
961 }
962 } else if (type == PTHREAD_CANCEL_DEFERRED)
963 self->pt_flags &= ~PT_FLAG_CS_ASYNC;
964 else
965 retval = EINVAL;
966
967 pthread_mutex_unlock(&self->pt_lock);
968
969 return retval;
970 }
971
972
973 void
pthread_testcancel(void)974 pthread_testcancel(void)
975 {
976 pthread_t self;
977
978 self = pthread__self();
979 if (self->pt_cancel)
980 pthread__cancelled();
981 }
982
983
984 /*
985 * POSIX requires that certain functions return an error rather than
986 * invoking undefined behavior even when handed completely bogus
987 * pthread_t values, e.g. stack garbage.
988 */
989 int
pthread__find(pthread_t id)990 pthread__find(pthread_t id)
991 {
992 pthread_t target;
993 int error;
994
995 pthread_rwlock_rdlock(&pthread__alltree_lock);
996 target = rb_tree_find_node(&pthread__alltree, id);
997 error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH;
998 pthread_rwlock_unlock(&pthread__alltree_lock);
999
1000 return error;
1001 }
1002
1003
1004 void
pthread__testcancel(pthread_t self)1005 pthread__testcancel(pthread_t self)
1006 {
1007
1008 if (self->pt_cancel)
1009 pthread__cancelled();
1010 }
1011
1012
1013 void
pthread__cancelled(void)1014 pthread__cancelled(void)
1015 {
1016
1017 pthread_exit(PTHREAD_CANCELED);
1018 }
1019
1020
1021 void
pthread__cleanup_push(void (* cleanup)(void *),void * arg,void * store)1022 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store)
1023 {
1024 pthread_t self;
1025 struct pt_clean_t *entry;
1026
1027 self = pthread__self();
1028 entry = store;
1029 entry->ptc_cleanup = cleanup;
1030 entry->ptc_arg = arg;
1031 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next);
1032 }
1033
1034
1035 void
pthread__cleanup_pop(int ex,void * store)1036 pthread__cleanup_pop(int ex, void *store)
1037 {
1038 pthread_t self;
1039 struct pt_clean_t *entry;
1040
1041 self = pthread__self();
1042 entry = store;
1043
1044 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next);
1045 if (ex)
1046 (*entry->ptc_cleanup)(entry->ptc_arg);
1047 }
1048
1049
1050 int *
pthread__errno(void)1051 pthread__errno(void)
1052 {
1053 pthread_t self;
1054
1055 if (__predict_false(__uselibcstub)) {
1056 pthread__errorfunc(__FILE__, __LINE__, __func__,
1057 "pthread__errno() requires linking with -lpthread");
1058 return __libc_thr_errno_stub();
1059 }
1060
1061 self = pthread__self();
1062
1063 return &(self->pt_errno);
1064 }
1065
1066 ssize_t _sys_write(int, const void *, size_t);
1067
1068 void
pthread__assertfunc(const char * file,int line,const char * function,const char * expr)1069 pthread__assertfunc(const char *file, int line, const char *function,
1070 const char *expr)
1071 {
1072 char buf[1024];
1073 int len;
1074
1075 /*
1076 * snprintf_ss should not acquire any locks, or we could
1077 * end up deadlocked if the assert caller held locks.
1078 */
1079 len = snprintf_ss(buf, 1024,
1080 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n",
1081 expr, file, line,
1082 function ? ", function \"" : "",
1083 function ? function : "",
1084 function ? "\"" : "");
1085
1086 _sys_write(STDERR_FILENO, buf, (size_t)len);
1087 (void)raise(SIGABRT);
1088 _exit(1);
1089 }
1090
1091
1092 void
pthread__errorfunc(const char * file,int line,const char * function,const char * msg,...)1093 pthread__errorfunc(const char *file, int line, const char *function,
1094 const char *msg, ...)
1095 {
1096 char buf[1024];
1097 char buf2[1024];
1098 size_t len;
1099 va_list ap;
1100
1101 if (pthread__diagassert == 0)
1102 return;
1103
1104 va_start(ap, msg);
1105 vsnprintf_ss(buf2, sizeof(buf2), msg, ap);
1106 va_end(ap);
1107
1108 /*
1109 * snprintf_ss should not acquire any locks, or we could
1110 * end up deadlocked if the assert caller held locks.
1111 */
1112 len = snprintf_ss(buf, sizeof(buf),
1113 "%s: Error detected by libpthread: %s.\n"
1114 "Detected by file \"%s\", line %d%s%s%s.\n"
1115 "See pthread(3) for information.\n",
1116 getprogname(), buf2, file, line,
1117 function ? ", function \"" : "",
1118 function ? function : "",
1119 function ? "\"" : "");
1120
1121 if (pthread__diagassert & DIAGASSERT_STDERR)
1122 _sys_write(STDERR_FILENO, buf, len);
1123
1124 if (pthread__diagassert & DIAGASSERT_SYSLOG)
1125 syslog(LOG_DEBUG | LOG_USER, "%s", buf);
1126
1127 if (pthread__diagassert & DIAGASSERT_ABORT) {
1128 (void)_lwp_kill(_lwp_self(), SIGABRT);
1129 _exit(1);
1130 }
1131 }
1132
1133 /*
1134 * Thread park/unpark operations. The kernel operations are
1135 * modelled after a brief description from "Multithreading in
1136 * the Solaris Operating Environment":
1137 *
1138 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf
1139 */
1140
1141 int
pthread__park(pthread_t self,pthread_mutex_t * lock,pthread_queue_t * queue,const struct timespec * abstime,int cancelpt)1142 pthread__park(pthread_t self, pthread_mutex_t *lock,
1143 pthread_queue_t *queue, const struct timespec *abstime,
1144 int cancelpt)
1145 {
1146 int rv, error;
1147
1148 pthread_mutex_unlock(lock);
1149
1150 /*
1151 * Wait until we are awoken by a pending unpark operation,
1152 * a signal, an unpark posted after we have gone asleep,
1153 * or an expired timeout.
1154 *
1155 * It is fine to test the value of pt_sleepobj without
1156 * holding any locks, because:
1157 *
1158 * o Only the blocking thread (this thread) ever sets it
1159 * to a non-NULL value.
1160 *
1161 * o Other threads may set it NULL, but if they do so they
1162 * must also make this thread return from _lwp_park.
1163 *
1164 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system
1165 * calls and all make use of spinlocks in the kernel. So
1166 * these system calls act as full memory barriers.
1167 */
1168 rv = 0;
1169 do {
1170 /*
1171 * If we deferred unparking a thread, arrange to
1172 * have _lwp_park() restart it before blocking.
1173 */
1174 error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME,
1175 __UNCONST(abstime), 0, NULL, NULL);
1176 if (error != 0) {
1177 switch (rv = errno) {
1178 case EINTR:
1179 case EALREADY:
1180 rv = 0;
1181 break;
1182 case ETIMEDOUT:
1183 break;
1184 default:
1185 pthread__errorfunc(__FILE__, __LINE__,
1186 __func__, "_lwp_park failed: %d", errno);
1187 break;
1188 }
1189 }
1190 /* Check for cancellation. */
1191 if (cancelpt && self->pt_cancel)
1192 rv = EINTR;
1193 } while (self->pt_sleepobj != NULL && rv == 0);
1194 return rv;
1195 }
1196
1197 void
pthread__unpark(pthread_queue_t * queue,pthread_t self,pthread_mutex_t * interlock)1198 pthread__unpark(pthread_queue_t *queue, pthread_t self,
1199 pthread_mutex_t *interlock)
1200 {
1201 pthread_t target;
1202
1203 target = PTQ_FIRST(queue);
1204 target->pt_sleepobj = NULL;
1205 PTQ_REMOVE(queue, target, pt_sleep);
1206 (void)_lwp_unpark(target->pt_lid, NULL);
1207 }
1208
1209 void
pthread__unpark_all(pthread_queue_t * queue,pthread_t self,pthread_mutex_t * interlock)1210 pthread__unpark_all(pthread_queue_t *queue, pthread_t self,
1211 pthread_mutex_t *interlock)
1212 {
1213 lwpid_t lids[PTHREAD__UNPARK_MAX];
1214 const size_t mlid = pthread__unpark_max;
1215 pthread_t target;
1216 size_t nlid = 0;
1217
1218 PTQ_FOREACH(target, queue, pt_sleep) {
1219 if (nlid == mlid) {
1220 (void)_lwp_unpark_all(lids, nlid, NULL);
1221 nlid = 0;
1222 }
1223 target->pt_sleepobj = NULL;
1224 lids[nlid++] = target->pt_lid;
1225 }
1226 PTQ_INIT(queue);
1227 if (nlid == 1) {
1228 (void)_lwp_unpark(lids[0], NULL);
1229 } else if (nlid > 1) {
1230 (void)_lwp_unpark_all(lids, nlid, NULL);
1231 }
1232 }
1233
1234 #undef OOPS
1235
1236 static void
pthread__initmainstack(void)1237 pthread__initmainstack(void)
1238 {
1239 struct rlimit slimit;
1240 const AuxInfo *aux;
1241 size_t size, len;
1242 int mib[2];
1243 unsigned int value;
1244
1245 _DIAGASSERT(_dlauxinfo() != NULL);
1246
1247 if (getrlimit(RLIMIT_STACK, &slimit) == -1)
1248 err(EXIT_FAILURE,
1249 "Couldn't get stack resource consumption limits");
1250 size = slimit.rlim_cur;
1251 pthread__main->pt_stack.ss_size = size;
1252 pthread__main->pt_guardsize = pthread__pagesize;
1253
1254 mib[0] = CTL_VM;
1255 mib[1] = VM_GUARD_SIZE;
1256 len = sizeof(value);
1257 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
1258 pthread__main->pt_guardsize = value;
1259
1260 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) {
1261 if (aux->a_type == AT_STACKBASE) {
1262 #ifdef __MACHINE_STACK_GROWS_UP
1263 pthread__main->pt_stack.ss_sp = (void *)aux->a_v;
1264 #else
1265 pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size;
1266 #endif
1267 break;
1268 }
1269 }
1270 pthread__copy_tsd(pthread__main);
1271 }
1272
1273 /*
1274 * Set up the slightly special stack for the "initial" thread, which
1275 * runs on the normal system stack, and thus gets slightly different
1276 * treatment.
1277 */
1278 static void
pthread__initmain(pthread_t * newt)1279 pthread__initmain(pthread_t *newt)
1280 {
1281 char *value;
1282
1283 pthread__initmainstack();
1284
1285 value = pthread__getenv("PTHREAD_STACKSIZE");
1286 if (value != NULL) {
1287 pthread__stacksize = atoi(value) * 1024;
1288 if (pthread__stacksize > pthread__main->pt_stack.ss_size)
1289 pthread__stacksize = pthread__main->pt_stack.ss_size;
1290 }
1291 if (pthread__stacksize == 0)
1292 pthread__stacksize = pthread__main->pt_stack.ss_size;
1293 pthread__stacksize += pthread__pagesize - 1;
1294 pthread__stacksize &= ~(pthread__pagesize - 1);
1295 if (pthread__stacksize < 4 * pthread__pagesize)
1296 errx(1, "Stacksize limit is too low, minimum %zd kbyte.",
1297 4 * pthread__pagesize / 1024);
1298
1299 *newt = pthread__main;
1300 #if defined(_PTHREAD_GETTCB_EXT)
1301 pthread__main->pt_tls = _PTHREAD_GETTCB_EXT();
1302 #elif defined(__HAVE___LWP_GETTCB_FAST)
1303 pthread__main->pt_tls = __lwp_gettcb_fast();
1304 #else
1305 pthread__main->pt_tls = _lwp_getprivate();
1306 #endif
1307 pthread__main->pt_tls->tcb_pthread = pthread__main;
1308 }
1309
1310 static signed int
1311 /*ARGSUSED*/
pthread__cmp(void * ctx,const void * n1,const void * n2)1312 pthread__cmp(void *ctx, const void *n1, const void *n2)
1313 {
1314 const uintptr_t p1 = (const uintptr_t)n1;
1315 const uintptr_t p2 = (const uintptr_t)n2;
1316
1317 if (p1 < p2)
1318 return -1;
1319 if (p1 > p2)
1320 return 1;
1321 return 0;
1322 }
1323
1324 /* Because getenv() wants to use locks. */
1325 char *
pthread__getenv(const char * name)1326 pthread__getenv(const char *name)
1327 {
1328 extern char **environ;
1329 size_t l_name, offset;
1330
1331 if (issetugid())
1332 return (NULL);
1333
1334 l_name = strlen(name);
1335 for (offset = 0; environ[offset] != NULL; offset++) {
1336 if (strncmp(name, environ[offset], l_name) == 0 &&
1337 environ[offset][l_name] == '=') {
1338 return environ[offset] + l_name + 1;
1339 }
1340 }
1341
1342 return NULL;
1343 }
1344
1345 pthread_mutex_t *
pthread__hashlock(volatile const void * p)1346 pthread__hashlock(volatile const void *p)
1347 {
1348 uintptr_t v;
1349
1350 v = (uintptr_t)p;
1351 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex;
1352 }
1353
1354 int
pthread__checkpri(int pri)1355 pthread__checkpri(int pri)
1356 {
1357 static int havepri;
1358 static long min, max;
1359
1360 if (!havepri) {
1361 min = sysconf(_SC_SCHED_PRI_MIN);
1362 max = sysconf(_SC_SCHED_PRI_MAX);
1363 havepri = 1;
1364 }
1365 return (pri < min || pri > max) ? EINVAL : 0;
1366 }
1367