xref: /freebsd/sys/kern/kern_thread.c (revision e17f5b1d)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>.
5  *  All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice(s), this list of conditions and the following disclaimer as
12  *    the first lines of this file unmodified other than the possible
13  *    addition of one or more copyright notices.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice(s), this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
19  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
22  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
28  * DAMAGE.
29  */
30 
31 #include "opt_witness.h"
32 #include "opt_hwpmc_hooks.h"
33 
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
36 
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/kernel.h>
40 #include <sys/lock.h>
41 #include <sys/mutex.h>
42 #include <sys/proc.h>
43 #include <sys/epoch.h>
44 #include <sys/rangelock.h>
45 #include <sys/resourcevar.h>
46 #include <sys/sdt.h>
47 #include <sys/smp.h>
48 #include <sys/sched.h>
49 #include <sys/sleepqueue.h>
50 #include <sys/selinfo.h>
51 #include <sys/syscallsubr.h>
52 #include <sys/sysent.h>
53 #include <sys/turnstile.h>
54 #include <sys/ktr.h>
55 #include <sys/rwlock.h>
56 #include <sys/umtx.h>
57 #include <sys/vmmeter.h>
58 #include <sys/cpuset.h>
59 #ifdef	HWPMC_HOOKS
60 #include <sys/pmckern.h>
61 #endif
62 
63 #include <security/audit/audit.h>
64 
65 #include <vm/vm.h>
66 #include <vm/vm_extern.h>
67 #include <vm/uma.h>
68 #include <sys/eventhandler.h>
69 
70 /*
71  * Asserts below verify the stability of struct thread and struct proc
72  * layout, as exposed by KBI to modules.  On head, the KBI is allowed
73  * to drift, change to the structures must be accompanied by the
74  * assert update.
75  *
76  * On the stable branches after KBI freeze, conditions must not be
77  * violated.  Typically new fields are moved to the end of the
78  * structures.
79  */
80 #ifdef __amd64__
81 _Static_assert(offsetof(struct thread, td_flags) == 0xfc,
82     "struct thread KBI td_flags");
83 _Static_assert(offsetof(struct thread, td_pflags) == 0x104,
84     "struct thread KBI td_pflags");
85 _Static_assert(offsetof(struct thread, td_frame) == 0x4a8,
86     "struct thread KBI td_frame");
87 _Static_assert(offsetof(struct thread, td_emuldata) == 0x6b0,
88     "struct thread KBI td_emuldata");
89 _Static_assert(offsetof(struct proc, p_flag) == 0xb0,
90     "struct proc KBI p_flag");
91 _Static_assert(offsetof(struct proc, p_pid) == 0xbc,
92     "struct proc KBI p_pid");
93 _Static_assert(offsetof(struct proc, p_filemon) == 0x3b8,
94     "struct proc KBI p_filemon");
95 _Static_assert(offsetof(struct proc, p_comm) == 0x3d0,
96     "struct proc KBI p_comm");
97 _Static_assert(offsetof(struct proc, p_emuldata) == 0x4b0,
98     "struct proc KBI p_emuldata");
99 #endif
100 #ifdef __i386__
101 _Static_assert(offsetof(struct thread, td_flags) == 0x98,
102     "struct thread KBI td_flags");
103 _Static_assert(offsetof(struct thread, td_pflags) == 0xa0,
104     "struct thread KBI td_pflags");
105 _Static_assert(offsetof(struct thread, td_frame) == 0x304,
106     "struct thread KBI td_frame");
107 _Static_assert(offsetof(struct thread, td_emuldata) == 0x348,
108     "struct thread KBI td_emuldata");
109 _Static_assert(offsetof(struct proc, p_flag) == 0x68,
110     "struct proc KBI p_flag");
111 _Static_assert(offsetof(struct proc, p_pid) == 0x74,
112     "struct proc KBI p_pid");
113 _Static_assert(offsetof(struct proc, p_filemon) == 0x268,
114     "struct proc KBI p_filemon");
115 _Static_assert(offsetof(struct proc, p_comm) == 0x27c,
116     "struct proc KBI p_comm");
117 _Static_assert(offsetof(struct proc, p_emuldata) == 0x308,
118     "struct proc KBI p_emuldata");
119 #endif
120 
121 SDT_PROVIDER_DECLARE(proc);
122 SDT_PROBE_DEFINE(proc, , , lwp__exit);
123 
124 /*
125  * thread related storage.
126  */
127 static uma_zone_t thread_zone;
128 
129 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads);
130 static struct mtx zombie_lock;
131 MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN);
132 
133 static void thread_zombie(struct thread *);
134 static int thread_unsuspend_one(struct thread *td, struct proc *p,
135     bool boundary);
136 
137 #define TID_BUFFER_SIZE	1024
138 
139 struct mtx tid_lock;
140 static struct unrhdr *tid_unrhdr;
141 static lwpid_t tid_buffer[TID_BUFFER_SIZE];
142 static int tid_head, tid_tail;
143 static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash");
144 
145 struct	tidhashhead *tidhashtbl;
146 u_long	tidhash;
147 struct	rwlock tidhash_lock;
148 
149 EVENTHANDLER_LIST_DEFINE(thread_ctor);
150 EVENTHANDLER_LIST_DEFINE(thread_dtor);
151 EVENTHANDLER_LIST_DEFINE(thread_init);
152 EVENTHANDLER_LIST_DEFINE(thread_fini);
153 
154 static lwpid_t
155 tid_alloc(void)
156 {
157 	lwpid_t	tid;
158 
159 	tid = alloc_unr(tid_unrhdr);
160 	if (tid != -1)
161 		return (tid);
162 	mtx_lock(&tid_lock);
163 	if (tid_head == tid_tail) {
164 		mtx_unlock(&tid_lock);
165 		return (-1);
166 	}
167 	tid = tid_buffer[tid_head];
168 	tid_head = (tid_head + 1) % TID_BUFFER_SIZE;
169 	mtx_unlock(&tid_lock);
170 	return (tid);
171 }
172 
173 static void
174 tid_free(lwpid_t tid)
175 {
176 	lwpid_t tmp_tid = -1;
177 
178 	mtx_lock(&tid_lock);
179 	if ((tid_tail + 1) % TID_BUFFER_SIZE == tid_head) {
180 		tmp_tid = tid_buffer[tid_head];
181 		tid_head = (tid_head + 1) % TID_BUFFER_SIZE;
182 	}
183 	tid_buffer[tid_tail] = tid;
184 	tid_tail = (tid_tail + 1) % TID_BUFFER_SIZE;
185 	mtx_unlock(&tid_lock);
186 	if (tmp_tid != -1)
187 		free_unr(tid_unrhdr, tmp_tid);
188 }
189 
190 /*
191  * Prepare a thread for use.
192  */
193 static int
194 thread_ctor(void *mem, int size, void *arg, int flags)
195 {
196 	struct thread	*td;
197 
198 	td = (struct thread *)mem;
199 	td->td_state = TDS_INACTIVE;
200 	td->td_lastcpu = td->td_oncpu = NOCPU;
201 
202 	td->td_tid = tid_alloc();
203 
204 	/*
205 	 * Note that td_critnest begins life as 1 because the thread is not
206 	 * running and is thereby implicitly waiting to be on the receiving
207 	 * end of a context switch.
208 	 */
209 	td->td_critnest = 1;
210 	td->td_lend_user_pri = PRI_MAX;
211 	EVENTHANDLER_DIRECT_INVOKE(thread_ctor, td);
212 #ifdef AUDIT
213 	audit_thread_alloc(td);
214 #endif
215 	umtx_thread_alloc(td);
216 	return (0);
217 }
218 
219 /*
220  * Reclaim a thread after use.
221  */
222 static void
223 thread_dtor(void *mem, int size, void *arg)
224 {
225 	struct thread *td;
226 
227 	td = (struct thread *)mem;
228 
229 #ifdef INVARIANTS
230 	/* Verify that this thread is in a safe state to free. */
231 	switch (td->td_state) {
232 	case TDS_INHIBITED:
233 	case TDS_RUNNING:
234 	case TDS_CAN_RUN:
235 	case TDS_RUNQ:
236 		/*
237 		 * We must never unlink a thread that is in one of
238 		 * these states, because it is currently active.
239 		 */
240 		panic("bad state for thread unlinking");
241 		/* NOTREACHED */
242 	case TDS_INACTIVE:
243 		break;
244 	default:
245 		panic("bad thread state");
246 		/* NOTREACHED */
247 	}
248 #endif
249 #ifdef AUDIT
250 	audit_thread_free(td);
251 #endif
252 	/* Free all OSD associated to this thread. */
253 	osd_thread_exit(td);
254 	td_softdep_cleanup(td);
255 	MPASS(td->td_su == NULL);
256 
257 	EVENTHANDLER_DIRECT_INVOKE(thread_dtor, td);
258 	tid_free(td->td_tid);
259 }
260 
261 /*
262  * Initialize type-stable parts of a thread (when newly created).
263  */
264 static int
265 thread_init(void *mem, int size, int flags)
266 {
267 	struct thread *td;
268 
269 	td = (struct thread *)mem;
270 
271 	td->td_sleepqueue = sleepq_alloc();
272 	td->td_turnstile = turnstile_alloc();
273 	td->td_rlqe = NULL;
274 	EVENTHANDLER_DIRECT_INVOKE(thread_init, td);
275 	umtx_thread_init(td);
276 	td->td_kstack = 0;
277 	td->td_sel = NULL;
278 	return (0);
279 }
280 
281 /*
282  * Tear down type-stable parts of a thread (just before being discarded).
283  */
284 static void
285 thread_fini(void *mem, int size)
286 {
287 	struct thread *td;
288 
289 	td = (struct thread *)mem;
290 	EVENTHANDLER_DIRECT_INVOKE(thread_fini, td);
291 	rlqentry_free(td->td_rlqe);
292 	turnstile_free(td->td_turnstile);
293 	sleepq_free(td->td_sleepqueue);
294 	umtx_thread_fini(td);
295 	seltdfini(td);
296 }
297 
298 /*
299  * For a newly created process,
300  * link up all the structures and its initial threads etc.
301  * called from:
302  * {arch}/{arch}/machdep.c   {arch}_init(), init386() etc.
303  * proc_dtor() (should go away)
304  * proc_init()
305  */
306 void
307 proc_linkup0(struct proc *p, struct thread *td)
308 {
309 	TAILQ_INIT(&p->p_threads);	     /* all threads in proc */
310 	proc_linkup(p, td);
311 }
312 
313 void
314 proc_linkup(struct proc *p, struct thread *td)
315 {
316 
317 	sigqueue_init(&p->p_sigqueue, p);
318 	p->p_ksi = ksiginfo_alloc(1);
319 	if (p->p_ksi != NULL) {
320 		/* XXX p_ksi may be null if ksiginfo zone is not ready */
321 		p->p_ksi->ksi_flags = KSI_EXT | KSI_INS;
322 	}
323 	LIST_INIT(&p->p_mqnotifier);
324 	p->p_numthreads = 0;
325 	thread_link(td, p);
326 }
327 
328 /*
329  * Initialize global thread allocation resources.
330  */
331 void
332 threadinit(void)
333 {
334 	uint32_t flags;
335 
336 	mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF);
337 
338 	/*
339 	 * pid_max cannot be greater than PID_MAX.
340 	 * leave one number for thread0.
341 	 */
342 	tid_unrhdr = new_unrhdr(PID_MAX + 2, INT_MAX, &tid_lock);
343 
344 	flags = UMA_ZONE_NOFREE;
345 #ifdef __aarch64__
346 	/*
347 	 * Force thread structures to be allocated from the direct map.
348 	 * Otherwise, superpage promotions and demotions may temporarily
349 	 * invalidate thread structure mappings.  For most dynamically allocated
350 	 * structures this is not a problem, but translation faults cannot be
351 	 * handled without accessing curthread.
352 	 */
353 	flags |= UMA_ZONE_CONTIG;
354 #endif
355 	thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
356 	    thread_ctor, thread_dtor, thread_init, thread_fini,
357 	    32 - 1, flags);
358 	tidhashtbl = hashinit(maxproc / 2, M_TIDHASH, &tidhash);
359 	rw_init(&tidhash_lock, "tidhash");
360 }
361 
362 /*
363  * Place an unused thread on the zombie list.
364  * Use the slpq as that must be unused by now.
365  */
366 void
367 thread_zombie(struct thread *td)
368 {
369 	mtx_lock_spin(&zombie_lock);
370 	TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq);
371 	mtx_unlock_spin(&zombie_lock);
372 }
373 
374 /*
375  * Release a thread that has exited after cpu_throw().
376  */
377 void
378 thread_stash(struct thread *td)
379 {
380 	atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1);
381 	thread_zombie(td);
382 }
383 
384 /*
385  * Reap zombie resources.
386  */
387 void
388 thread_reap(void)
389 {
390 	struct thread *td_first, *td_next;
391 
392 	/*
393 	 * Don't even bother to lock if none at this instant,
394 	 * we really don't care about the next instant.
395 	 */
396 	if (!TAILQ_EMPTY(&zombie_threads)) {
397 		mtx_lock_spin(&zombie_lock);
398 		td_first = TAILQ_FIRST(&zombie_threads);
399 		if (td_first)
400 			TAILQ_INIT(&zombie_threads);
401 		mtx_unlock_spin(&zombie_lock);
402 		while (td_first) {
403 			td_next = TAILQ_NEXT(td_first, td_slpq);
404 			thread_cow_free(td_first);
405 			thread_free(td_first);
406 			td_first = td_next;
407 		}
408 	}
409 }
410 
411 /*
412  * Allocate a thread.
413  */
414 struct thread *
415 thread_alloc(int pages)
416 {
417 	struct thread *td;
418 
419 	thread_reap(); /* check if any zombies to get */
420 
421 	td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK);
422 	KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack"));
423 	if (!vm_thread_new(td, pages)) {
424 		uma_zfree(thread_zone, td);
425 		return (NULL);
426 	}
427 	cpu_thread_alloc(td);
428 	return (td);
429 }
430 
431 int
432 thread_alloc_stack(struct thread *td, int pages)
433 {
434 
435 	KASSERT(td->td_kstack == 0,
436 	    ("thread_alloc_stack called on a thread with kstack"));
437 	if (!vm_thread_new(td, pages))
438 		return (0);
439 	cpu_thread_alloc(td);
440 	return (1);
441 }
442 
443 /*
444  * Deallocate a thread.
445  */
446 void
447 thread_free(struct thread *td)
448 {
449 
450 	lock_profile_thread_exit(td);
451 	if (td->td_cpuset)
452 		cpuset_rel(td->td_cpuset);
453 	td->td_cpuset = NULL;
454 	cpu_thread_free(td);
455 	if (td->td_kstack != 0)
456 		vm_thread_dispose(td);
457 	callout_drain(&td->td_slpcallout);
458 	uma_zfree(thread_zone, td);
459 }
460 
461 void
462 thread_cow_get_proc(struct thread *newtd, struct proc *p)
463 {
464 
465 	PROC_LOCK_ASSERT(p, MA_OWNED);
466 	newtd->td_realucred = crcowget(p->p_ucred);
467 	newtd->td_ucred = newtd->td_realucred;
468 	newtd->td_limit = lim_hold(p->p_limit);
469 	newtd->td_cowgen = p->p_cowgen;
470 }
471 
472 void
473 thread_cow_get(struct thread *newtd, struct thread *td)
474 {
475 
476 	MPASS(td->td_realucred == td->td_ucred);
477 	newtd->td_realucred = crcowget(td->td_realucred);
478 	newtd->td_ucred = newtd->td_realucred;
479 	newtd->td_limit = lim_hold(td->td_limit);
480 	newtd->td_cowgen = td->td_cowgen;
481 }
482 
483 void
484 thread_cow_free(struct thread *td)
485 {
486 
487 	if (td->td_realucred != NULL)
488 		crcowfree(td);
489 	if (td->td_limit != NULL)
490 		lim_free(td->td_limit);
491 }
492 
493 void
494 thread_cow_update(struct thread *td)
495 {
496 	struct proc *p;
497 	struct ucred *oldcred;
498 	struct plimit *oldlimit;
499 
500 	p = td->td_proc;
501 	oldlimit = NULL;
502 	PROC_LOCK(p);
503 	oldcred = crcowsync();
504 	if (td->td_limit != p->p_limit) {
505 		oldlimit = td->td_limit;
506 		td->td_limit = lim_hold(p->p_limit);
507 	}
508 	td->td_cowgen = p->p_cowgen;
509 	PROC_UNLOCK(p);
510 	if (oldcred != NULL)
511 		crfree(oldcred);
512 	if (oldlimit != NULL)
513 		lim_free(oldlimit);
514 }
515 
516 /*
517  * Discard the current thread and exit from its context.
518  * Always called with scheduler locked.
519  *
520  * Because we can't free a thread while we're operating under its context,
521  * push the current thread into our CPU's deadthread holder. This means
522  * we needn't worry about someone else grabbing our context before we
523  * do a cpu_throw().
524  */
525 void
526 thread_exit(void)
527 {
528 	uint64_t runtime, new_switchtime;
529 	struct thread *td;
530 	struct thread *td2;
531 	struct proc *p;
532 	int wakeup_swapper;
533 
534 	td = curthread;
535 	p = td->td_proc;
536 
537 	PROC_SLOCK_ASSERT(p, MA_OWNED);
538 	mtx_assert(&Giant, MA_NOTOWNED);
539 
540 	PROC_LOCK_ASSERT(p, MA_OWNED);
541 	KASSERT(p != NULL, ("thread exiting without a process"));
542 	CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td,
543 	    (long)p->p_pid, td->td_name);
544 	SDT_PROBE0(proc, , , lwp__exit);
545 	KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending"));
546 
547 	/*
548 	 * drop FPU & debug register state storage, or any other
549 	 * architecture specific resources that
550 	 * would not be on a new untouched process.
551 	 */
552 	cpu_thread_exit(td);
553 
554 	/*
555 	 * The last thread is left attached to the process
556 	 * So that the whole bundle gets recycled. Skip
557 	 * all this stuff if we never had threads.
558 	 * EXIT clears all sign of other threads when
559 	 * it goes to single threading, so the last thread always
560 	 * takes the short path.
561 	 */
562 	if (p->p_flag & P_HADTHREADS) {
563 		if (p->p_numthreads > 1) {
564 			atomic_add_int(&td->td_proc->p_exitthreads, 1);
565 			thread_unlink(td);
566 			td2 = FIRST_THREAD_IN_PROC(p);
567 			sched_exit_thread(td2, td);
568 
569 			/*
570 			 * The test below is NOT true if we are the
571 			 * sole exiting thread. P_STOPPED_SINGLE is unset
572 			 * in exit1() after it is the only survivor.
573 			 */
574 			if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
575 				if (p->p_numthreads == p->p_suspcount) {
576 					thread_lock(p->p_singlethread);
577 					wakeup_swapper = thread_unsuspend_one(
578 						p->p_singlethread, p, false);
579 					if (wakeup_swapper)
580 						kick_proc0();
581 				}
582 			}
583 
584 			PCPU_SET(deadthread, td);
585 		} else {
586 			/*
587 			 * The last thread is exiting.. but not through exit()
588 			 */
589 			panic ("thread_exit: Last thread exiting on its own");
590 		}
591 	}
592 #ifdef	HWPMC_HOOKS
593 	/*
594 	 * If this thread is part of a process that is being tracked by hwpmc(4),
595 	 * inform the module of the thread's impending exit.
596 	 */
597 	if (PMC_PROC_IS_USING_PMCS(td->td_proc)) {
598 		PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
599 		PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_THR_EXIT, NULL);
600 	} else if (PMC_SYSTEM_SAMPLING_ACTIVE())
601 		PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_THR_EXIT_LOG, NULL);
602 #endif
603 	PROC_UNLOCK(p);
604 	PROC_STATLOCK(p);
605 	thread_lock(td);
606 	PROC_SUNLOCK(p);
607 
608 	/* Do the same timestamp bookkeeping that mi_switch() would do. */
609 	new_switchtime = cpu_ticks();
610 	runtime = new_switchtime - PCPU_GET(switchtime);
611 	td->td_runtime += runtime;
612 	td->td_incruntime += runtime;
613 	PCPU_SET(switchtime, new_switchtime);
614 	PCPU_SET(switchticks, ticks);
615 	VM_CNT_INC(v_swtch);
616 
617 	/* Save our resource usage in our process. */
618 	td->td_ru.ru_nvcsw++;
619 	ruxagg_locked(p, td);
620 	rucollect(&p->p_ru, &td->td_ru);
621 	PROC_STATUNLOCK(p);
622 
623 	td->td_state = TDS_INACTIVE;
624 #ifdef WITNESS
625 	witness_thread_exit(td);
626 #endif
627 	CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td);
628 	sched_throw(td);
629 	panic("I'm a teapot!");
630 	/* NOTREACHED */
631 }
632 
633 /*
634  * Do any thread specific cleanups that may be needed in wait()
635  * called with Giant, proc and schedlock not held.
636  */
637 void
638 thread_wait(struct proc *p)
639 {
640 	struct thread *td;
641 
642 	mtx_assert(&Giant, MA_NOTOWNED);
643 	KASSERT(p->p_numthreads == 1, ("multiple threads in thread_wait()"));
644 	KASSERT(p->p_exitthreads == 0, ("p_exitthreads leaking"));
645 	td = FIRST_THREAD_IN_PROC(p);
646 	/* Lock the last thread so we spin until it exits cpu_throw(). */
647 	thread_lock(td);
648 	thread_unlock(td);
649 	lock_profile_thread_exit(td);
650 	cpuset_rel(td->td_cpuset);
651 	td->td_cpuset = NULL;
652 	cpu_thread_clean(td);
653 	thread_cow_free(td);
654 	callout_drain(&td->td_slpcallout);
655 	thread_reap();	/* check for zombie threads etc. */
656 }
657 
658 /*
659  * Link a thread to a process.
660  * set up anything that needs to be initialized for it to
661  * be used by the process.
662  */
663 void
664 thread_link(struct thread *td, struct proc *p)
665 {
666 
667 	/*
668 	 * XXX This can't be enabled because it's called for proc0 before
669 	 * its lock has been created.
670 	 * PROC_LOCK_ASSERT(p, MA_OWNED);
671 	 */
672 	td->td_state    = TDS_INACTIVE;
673 	td->td_proc     = p;
674 	td->td_flags    = TDF_INMEM;
675 
676 	LIST_INIT(&td->td_contested);
677 	LIST_INIT(&td->td_lprof[0]);
678 	LIST_INIT(&td->td_lprof[1]);
679 #ifdef EPOCH_TRACE
680 	SLIST_INIT(&td->td_epochs);
681 #endif
682 	sigqueue_init(&td->td_sigqueue, p);
683 	callout_init(&td->td_slpcallout, 1);
684 	TAILQ_INSERT_TAIL(&p->p_threads, td, td_plist);
685 	p->p_numthreads++;
686 }
687 
688 /*
689  * Called from:
690  *  thread_exit()
691  */
692 void
693 thread_unlink(struct thread *td)
694 {
695 	struct proc *p = td->td_proc;
696 
697 	PROC_LOCK_ASSERT(p, MA_OWNED);
698 #ifdef EPOCH_TRACE
699 	MPASS(SLIST_EMPTY(&td->td_epochs));
700 #endif
701 
702 	TAILQ_REMOVE(&p->p_threads, td, td_plist);
703 	p->p_numthreads--;
704 	/* could clear a few other things here */
705 	/* Must  NOT clear links to proc! */
706 }
707 
708 static int
709 calc_remaining(struct proc *p, int mode)
710 {
711 	int remaining;
712 
713 	PROC_LOCK_ASSERT(p, MA_OWNED);
714 	PROC_SLOCK_ASSERT(p, MA_OWNED);
715 	if (mode == SINGLE_EXIT)
716 		remaining = p->p_numthreads;
717 	else if (mode == SINGLE_BOUNDARY)
718 		remaining = p->p_numthreads - p->p_boundary_count;
719 	else if (mode == SINGLE_NO_EXIT || mode == SINGLE_ALLPROC)
720 		remaining = p->p_numthreads - p->p_suspcount;
721 	else
722 		panic("calc_remaining: wrong mode %d", mode);
723 	return (remaining);
724 }
725 
726 static int
727 remain_for_mode(int mode)
728 {
729 
730 	return (mode == SINGLE_ALLPROC ? 0 : 1);
731 }
732 
733 static int
734 weed_inhib(int mode, struct thread *td2, struct proc *p)
735 {
736 	int wakeup_swapper;
737 
738 	PROC_LOCK_ASSERT(p, MA_OWNED);
739 	PROC_SLOCK_ASSERT(p, MA_OWNED);
740 	THREAD_LOCK_ASSERT(td2, MA_OWNED);
741 
742 	wakeup_swapper = 0;
743 
744 	/*
745 	 * Since the thread lock is dropped by the scheduler we have
746 	 * to retry to check for races.
747 	 */
748 restart:
749 	switch (mode) {
750 	case SINGLE_EXIT:
751 		if (TD_IS_SUSPENDED(td2)) {
752 			wakeup_swapper |= thread_unsuspend_one(td2, p, true);
753 			thread_lock(td2);
754 			goto restart;
755 		}
756 		if (TD_CAN_ABORT(td2)) {
757 			wakeup_swapper |= sleepq_abort(td2, EINTR);
758 			return (wakeup_swapper);
759 		}
760 		break;
761 	case SINGLE_BOUNDARY:
762 	case SINGLE_NO_EXIT:
763 		if (TD_IS_SUSPENDED(td2) &&
764 		    (td2->td_flags & TDF_BOUNDARY) == 0) {
765 			wakeup_swapper |= thread_unsuspend_one(td2, p, false);
766 			thread_lock(td2);
767 			goto restart;
768 		}
769 		if (TD_CAN_ABORT(td2)) {
770 			wakeup_swapper |= sleepq_abort(td2, ERESTART);
771 			return (wakeup_swapper);
772 		}
773 		break;
774 	case SINGLE_ALLPROC:
775 		/*
776 		 * ALLPROC suspend tries to avoid spurious EINTR for
777 		 * threads sleeping interruptable, by suspending the
778 		 * thread directly, similarly to sig_suspend_threads().
779 		 * Since such sleep is not performed at the user
780 		 * boundary, TDF_BOUNDARY flag is not set, and TDF_ALLPROCSUSP
781 		 * is used to avoid immediate un-suspend.
782 		 */
783 		if (TD_IS_SUSPENDED(td2) && (td2->td_flags & (TDF_BOUNDARY |
784 		    TDF_ALLPROCSUSP)) == 0) {
785 			wakeup_swapper |= thread_unsuspend_one(td2, p, false);
786 			thread_lock(td2);
787 			goto restart;
788 		}
789 		if (TD_CAN_ABORT(td2)) {
790 			if ((td2->td_flags & TDF_SBDRY) == 0) {
791 				thread_suspend_one(td2);
792 				td2->td_flags |= TDF_ALLPROCSUSP;
793 			} else {
794 				wakeup_swapper |= sleepq_abort(td2, ERESTART);
795 				return (wakeup_swapper);
796 			}
797 		}
798 		break;
799 	default:
800 		break;
801 	}
802 	thread_unlock(td2);
803 	return (wakeup_swapper);
804 }
805 
806 /*
807  * Enforce single-threading.
808  *
809  * Returns 1 if the caller must abort (another thread is waiting to
810  * exit the process or similar). Process is locked!
811  * Returns 0 when you are successfully the only thread running.
812  * A process has successfully single threaded in the suspend mode when
813  * There are no threads in user mode. Threads in the kernel must be
814  * allowed to continue until they get to the user boundary. They may even
815  * copy out their return values and data before suspending. They may however be
816  * accelerated in reaching the user boundary as we will wake up
817  * any sleeping threads that are interruptable. (PCATCH).
818  */
819 int
820 thread_single(struct proc *p, int mode)
821 {
822 	struct thread *td;
823 	struct thread *td2;
824 	int remaining, wakeup_swapper;
825 
826 	td = curthread;
827 	KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY ||
828 	    mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT,
829 	    ("invalid mode %d", mode));
830 	/*
831 	 * If allowing non-ALLPROC singlethreading for non-curproc
832 	 * callers, calc_remaining() and remain_for_mode() should be
833 	 * adjusted to also account for td->td_proc != p.  For now
834 	 * this is not implemented because it is not used.
835 	 */
836 	KASSERT((mode == SINGLE_ALLPROC && td->td_proc != p) ||
837 	    (mode != SINGLE_ALLPROC && td->td_proc == p),
838 	    ("mode %d proc %p curproc %p", mode, p, td->td_proc));
839 	mtx_assert(&Giant, MA_NOTOWNED);
840 	PROC_LOCK_ASSERT(p, MA_OWNED);
841 
842 	if ((p->p_flag & P_HADTHREADS) == 0 && mode != SINGLE_ALLPROC)
843 		return (0);
844 
845 	/* Is someone already single threading? */
846 	if (p->p_singlethread != NULL && p->p_singlethread != td)
847 		return (1);
848 
849 	if (mode == SINGLE_EXIT) {
850 		p->p_flag |= P_SINGLE_EXIT;
851 		p->p_flag &= ~P_SINGLE_BOUNDARY;
852 	} else {
853 		p->p_flag &= ~P_SINGLE_EXIT;
854 		if (mode == SINGLE_BOUNDARY)
855 			p->p_flag |= P_SINGLE_BOUNDARY;
856 		else
857 			p->p_flag &= ~P_SINGLE_BOUNDARY;
858 	}
859 	if (mode == SINGLE_ALLPROC)
860 		p->p_flag |= P_TOTAL_STOP;
861 	p->p_flag |= P_STOPPED_SINGLE;
862 	PROC_SLOCK(p);
863 	p->p_singlethread = td;
864 	remaining = calc_remaining(p, mode);
865 	while (remaining != remain_for_mode(mode)) {
866 		if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE)
867 			goto stopme;
868 		wakeup_swapper = 0;
869 		FOREACH_THREAD_IN_PROC(p, td2) {
870 			if (td2 == td)
871 				continue;
872 			thread_lock(td2);
873 			td2->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK;
874 			if (TD_IS_INHIBITED(td2)) {
875 				wakeup_swapper |= weed_inhib(mode, td2, p);
876 #ifdef SMP
877 			} else if (TD_IS_RUNNING(td2) && td != td2) {
878 				forward_signal(td2);
879 				thread_unlock(td2);
880 #endif
881 			} else
882 				thread_unlock(td2);
883 		}
884 		if (wakeup_swapper)
885 			kick_proc0();
886 		remaining = calc_remaining(p, mode);
887 
888 		/*
889 		 * Maybe we suspended some threads.. was it enough?
890 		 */
891 		if (remaining == remain_for_mode(mode))
892 			break;
893 
894 stopme:
895 		/*
896 		 * Wake us up when everyone else has suspended.
897 		 * In the mean time we suspend as well.
898 		 */
899 		thread_suspend_switch(td, p);
900 		remaining = calc_remaining(p, mode);
901 	}
902 	if (mode == SINGLE_EXIT) {
903 		/*
904 		 * Convert the process to an unthreaded process.  The
905 		 * SINGLE_EXIT is called by exit1() or execve(), in
906 		 * both cases other threads must be retired.
907 		 */
908 		KASSERT(p->p_numthreads == 1, ("Unthreading with >1 threads"));
909 		p->p_singlethread = NULL;
910 		p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_HADTHREADS);
911 
912 		/*
913 		 * Wait for any remaining threads to exit cpu_throw().
914 		 */
915 		while (p->p_exitthreads != 0) {
916 			PROC_SUNLOCK(p);
917 			PROC_UNLOCK(p);
918 			sched_relinquish(td);
919 			PROC_LOCK(p);
920 			PROC_SLOCK(p);
921 		}
922 	} else if (mode == SINGLE_BOUNDARY) {
923 		/*
924 		 * Wait until all suspended threads are removed from
925 		 * the processors.  The thread_suspend_check()
926 		 * increments p_boundary_count while it is still
927 		 * running, which makes it possible for the execve()
928 		 * to destroy vmspace while our other threads are
929 		 * still using the address space.
930 		 *
931 		 * We lock the thread, which is only allowed to
932 		 * succeed after context switch code finished using
933 		 * the address space.
934 		 */
935 		FOREACH_THREAD_IN_PROC(p, td2) {
936 			if (td2 == td)
937 				continue;
938 			thread_lock(td2);
939 			KASSERT((td2->td_flags & TDF_BOUNDARY) != 0,
940 			    ("td %p not on boundary", td2));
941 			KASSERT(TD_IS_SUSPENDED(td2),
942 			    ("td %p is not suspended", td2));
943 			thread_unlock(td2);
944 		}
945 	}
946 	PROC_SUNLOCK(p);
947 	return (0);
948 }
949 
950 bool
951 thread_suspend_check_needed(void)
952 {
953 	struct proc *p;
954 	struct thread *td;
955 
956 	td = curthread;
957 	p = td->td_proc;
958 	PROC_LOCK_ASSERT(p, MA_OWNED);
959 	return (P_SHOULDSTOP(p) || ((p->p_flag & P_TRACED) != 0 &&
960 	    (td->td_dbgflags & TDB_SUSPEND) != 0));
961 }
962 
963 /*
964  * Called in from locations that can safely check to see
965  * whether we have to suspend or at least throttle for a
966  * single-thread event (e.g. fork).
967  *
968  * Such locations include userret().
969  * If the "return_instead" argument is non zero, the thread must be able to
970  * accept 0 (caller may continue), or 1 (caller must abort) as a result.
971  *
972  * The 'return_instead' argument tells the function if it may do a
973  * thread_exit() or suspend, or whether the caller must abort and back
974  * out instead.
975  *
976  * If the thread that set the single_threading request has set the
977  * P_SINGLE_EXIT bit in the process flags then this call will never return
978  * if 'return_instead' is false, but will exit.
979  *
980  * P_SINGLE_EXIT | return_instead == 0| return_instead != 0
981  *---------------+--------------------+---------------------
982  *       0       | returns 0          |   returns 0 or 1
983  *               | when ST ends       |   immediately
984  *---------------+--------------------+---------------------
985  *       1       | thread exits       |   returns 1
986  *               |                    |  immediately
987  * 0 = thread_exit() or suspension ok,
988  * other = return error instead of stopping the thread.
989  *
990  * While a full suspension is under effect, even a single threading
991  * thread would be suspended if it made this call (but it shouldn't).
992  * This call should only be made from places where
993  * thread_exit() would be safe as that may be the outcome unless
994  * return_instead is set.
995  */
996 int
997 thread_suspend_check(int return_instead)
998 {
999 	struct thread *td;
1000 	struct proc *p;
1001 	int wakeup_swapper;
1002 
1003 	td = curthread;
1004 	p = td->td_proc;
1005 	mtx_assert(&Giant, MA_NOTOWNED);
1006 	PROC_LOCK_ASSERT(p, MA_OWNED);
1007 	while (thread_suspend_check_needed()) {
1008 		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
1009 			KASSERT(p->p_singlethread != NULL,
1010 			    ("singlethread not set"));
1011 			/*
1012 			 * The only suspension in action is a
1013 			 * single-threading. Single threader need not stop.
1014 			 * It is safe to access p->p_singlethread unlocked
1015 			 * because it can only be set to our address by us.
1016 			 */
1017 			if (p->p_singlethread == td)
1018 				return (0);	/* Exempt from stopping. */
1019 		}
1020 		if ((p->p_flag & P_SINGLE_EXIT) && return_instead)
1021 			return (EINTR);
1022 
1023 		/* Should we goto user boundary if we didn't come from there? */
1024 		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
1025 		    (p->p_flag & P_SINGLE_BOUNDARY) && return_instead)
1026 			return (ERESTART);
1027 
1028 		/*
1029 		 * Ignore suspend requests if they are deferred.
1030 		 */
1031 		if ((td->td_flags & TDF_SBDRY) != 0) {
1032 			KASSERT(return_instead,
1033 			    ("TDF_SBDRY set for unsafe thread_suspend_check"));
1034 			KASSERT((td->td_flags & (TDF_SEINTR | TDF_SERESTART)) !=
1035 			    (TDF_SEINTR | TDF_SERESTART),
1036 			    ("both TDF_SEINTR and TDF_SERESTART"));
1037 			return (TD_SBDRY_INTR(td) ? TD_SBDRY_ERRNO(td) : 0);
1038 		}
1039 
1040 		/*
1041 		 * If the process is waiting for us to exit,
1042 		 * this thread should just suicide.
1043 		 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE.
1044 		 */
1045 		if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) {
1046 			PROC_UNLOCK(p);
1047 
1048 			/*
1049 			 * Allow Linux emulation layer to do some work
1050 			 * before thread suicide.
1051 			 */
1052 			if (__predict_false(p->p_sysent->sv_thread_detach != NULL))
1053 				(p->p_sysent->sv_thread_detach)(td);
1054 			umtx_thread_exit(td);
1055 			kern_thr_exit(td);
1056 			panic("stopped thread did not exit");
1057 		}
1058 
1059 		PROC_SLOCK(p);
1060 		thread_stopped(p);
1061 		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
1062 			if (p->p_numthreads == p->p_suspcount + 1) {
1063 				thread_lock(p->p_singlethread);
1064 				wakeup_swapper = thread_unsuspend_one(
1065 				    p->p_singlethread, p, false);
1066 				if (wakeup_swapper)
1067 					kick_proc0();
1068 			}
1069 		}
1070 		PROC_UNLOCK(p);
1071 		thread_lock(td);
1072 		/*
1073 		 * When a thread suspends, it just
1074 		 * gets taken off all queues.
1075 		 */
1076 		thread_suspend_one(td);
1077 		if (return_instead == 0) {
1078 			p->p_boundary_count++;
1079 			td->td_flags |= TDF_BOUNDARY;
1080 		}
1081 		PROC_SUNLOCK(p);
1082 		mi_switch(SW_INVOL | SWT_SUSPEND);
1083 		PROC_LOCK(p);
1084 	}
1085 	return (0);
1086 }
1087 
1088 /*
1089  * Check for possible stops and suspensions while executing a
1090  * casueword or similar transiently failing operation.
1091  *
1092  * The sleep argument controls whether the function can handle a stop
1093  * request itself or it should return ERESTART and the request is
1094  * proceed at the kernel/user boundary in ast.
1095  *
1096  * Typically, when retrying due to casueword(9) failure (rv == 1), we
1097  * should handle the stop requests there, with exception of cases when
1098  * the thread owns a kernel resource, for instance busied the umtx
1099  * key, or when functions return immediately if thread_check_susp()
1100  * returned non-zero.  On the other hand, retrying the whole lock
1101  * operation, we better not stop there but delegate the handling to
1102  * ast.
1103  *
1104  * If the request is for thread termination P_SINGLE_EXIT, we cannot
1105  * handle it at all, and simply return EINTR.
1106  */
1107 int
1108 thread_check_susp(struct thread *td, bool sleep)
1109 {
1110 	struct proc *p;
1111 	int error;
1112 
1113 	/*
1114 	 * The check for TDF_NEEDSUSPCHK is racy, but it is enough to
1115 	 * eventually break the lockstep loop.
1116 	 */
1117 	if ((td->td_flags & TDF_NEEDSUSPCHK) == 0)
1118 		return (0);
1119 	error = 0;
1120 	p = td->td_proc;
1121 	PROC_LOCK(p);
1122 	if (p->p_flag & P_SINGLE_EXIT)
1123 		error = EINTR;
1124 	else if (P_SHOULDSTOP(p) ||
1125 	    ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND)))
1126 		error = sleep ? thread_suspend_check(0) : ERESTART;
1127 	PROC_UNLOCK(p);
1128 	return (error);
1129 }
1130 
1131 void
1132 thread_suspend_switch(struct thread *td, struct proc *p)
1133 {
1134 
1135 	KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
1136 	PROC_LOCK_ASSERT(p, MA_OWNED);
1137 	PROC_SLOCK_ASSERT(p, MA_OWNED);
1138 	/*
1139 	 * We implement thread_suspend_one in stages here to avoid
1140 	 * dropping the proc lock while the thread lock is owned.
1141 	 */
1142 	if (p == td->td_proc) {
1143 		thread_stopped(p);
1144 		p->p_suspcount++;
1145 	}
1146 	PROC_UNLOCK(p);
1147 	thread_lock(td);
1148 	td->td_flags &= ~TDF_NEEDSUSPCHK;
1149 	TD_SET_SUSPENDED(td);
1150 	sched_sleep(td, 0);
1151 	PROC_SUNLOCK(p);
1152 	DROP_GIANT();
1153 	mi_switch(SW_VOL | SWT_SUSPEND);
1154 	PICKUP_GIANT();
1155 	PROC_LOCK(p);
1156 	PROC_SLOCK(p);
1157 }
1158 
1159 void
1160 thread_suspend_one(struct thread *td)
1161 {
1162 	struct proc *p;
1163 
1164 	p = td->td_proc;
1165 	PROC_SLOCK_ASSERT(p, MA_OWNED);
1166 	THREAD_LOCK_ASSERT(td, MA_OWNED);
1167 	KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
1168 	p->p_suspcount++;
1169 	td->td_flags &= ~TDF_NEEDSUSPCHK;
1170 	TD_SET_SUSPENDED(td);
1171 	sched_sleep(td, 0);
1172 }
1173 
1174 static int
1175 thread_unsuspend_one(struct thread *td, struct proc *p, bool boundary)
1176 {
1177 
1178 	THREAD_LOCK_ASSERT(td, MA_OWNED);
1179 	KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended"));
1180 	TD_CLR_SUSPENDED(td);
1181 	td->td_flags &= ~TDF_ALLPROCSUSP;
1182 	if (td->td_proc == p) {
1183 		PROC_SLOCK_ASSERT(p, MA_OWNED);
1184 		p->p_suspcount--;
1185 		if (boundary && (td->td_flags & TDF_BOUNDARY) != 0) {
1186 			td->td_flags &= ~TDF_BOUNDARY;
1187 			p->p_boundary_count--;
1188 		}
1189 	}
1190 	return (setrunnable(td, 0));
1191 }
1192 
1193 /*
1194  * Allow all threads blocked by single threading to continue running.
1195  */
1196 void
1197 thread_unsuspend(struct proc *p)
1198 {
1199 	struct thread *td;
1200 	int wakeup_swapper;
1201 
1202 	PROC_LOCK_ASSERT(p, MA_OWNED);
1203 	PROC_SLOCK_ASSERT(p, MA_OWNED);
1204 	wakeup_swapper = 0;
1205 	if (!P_SHOULDSTOP(p)) {
1206                 FOREACH_THREAD_IN_PROC(p, td) {
1207 			thread_lock(td);
1208 			if (TD_IS_SUSPENDED(td)) {
1209 				wakeup_swapper |= thread_unsuspend_one(td, p,
1210 				    true);
1211 			} else
1212 				thread_unlock(td);
1213 		}
1214 	} else if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
1215 	    p->p_numthreads == p->p_suspcount) {
1216 		/*
1217 		 * Stopping everything also did the job for the single
1218 		 * threading request. Now we've downgraded to single-threaded,
1219 		 * let it continue.
1220 		 */
1221 		if (p->p_singlethread->td_proc == p) {
1222 			thread_lock(p->p_singlethread);
1223 			wakeup_swapper = thread_unsuspend_one(
1224 			    p->p_singlethread, p, false);
1225 		}
1226 	}
1227 	if (wakeup_swapper)
1228 		kick_proc0();
1229 }
1230 
1231 /*
1232  * End the single threading mode..
1233  */
1234 void
1235 thread_single_end(struct proc *p, int mode)
1236 {
1237 	struct thread *td;
1238 	int wakeup_swapper;
1239 
1240 	KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY ||
1241 	    mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT,
1242 	    ("invalid mode %d", mode));
1243 	PROC_LOCK_ASSERT(p, MA_OWNED);
1244 	KASSERT((mode == SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) != 0) ||
1245 	    (mode != SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) == 0),
1246 	    ("mode %d does not match P_TOTAL_STOP", mode));
1247 	KASSERT(mode == SINGLE_ALLPROC || p->p_singlethread == curthread,
1248 	    ("thread_single_end from other thread %p %p",
1249 	    curthread, p->p_singlethread));
1250 	KASSERT(mode != SINGLE_BOUNDARY ||
1251 	    (p->p_flag & P_SINGLE_BOUNDARY) != 0,
1252 	    ("mis-matched SINGLE_BOUNDARY flags %x", p->p_flag));
1253 	p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY |
1254 	    P_TOTAL_STOP);
1255 	PROC_SLOCK(p);
1256 	p->p_singlethread = NULL;
1257 	wakeup_swapper = 0;
1258 	/*
1259 	 * If there are other threads they may now run,
1260 	 * unless of course there is a blanket 'stop order'
1261 	 * on the process. The single threader must be allowed
1262 	 * to continue however as this is a bad place to stop.
1263 	 */
1264 	if (p->p_numthreads != remain_for_mode(mode) && !P_SHOULDSTOP(p)) {
1265                 FOREACH_THREAD_IN_PROC(p, td) {
1266 			thread_lock(td);
1267 			if (TD_IS_SUSPENDED(td)) {
1268 				wakeup_swapper |= thread_unsuspend_one(td, p,
1269 				    mode == SINGLE_BOUNDARY);
1270 			} else
1271 				thread_unlock(td);
1272 		}
1273 	}
1274 	KASSERT(mode != SINGLE_BOUNDARY || p->p_boundary_count == 0,
1275 	    ("inconsistent boundary count %d", p->p_boundary_count));
1276 	PROC_SUNLOCK(p);
1277 	if (wakeup_swapper)
1278 		kick_proc0();
1279 }
1280 
1281 struct thread *
1282 thread_find(struct proc *p, lwpid_t tid)
1283 {
1284 	struct thread *td;
1285 
1286 	PROC_LOCK_ASSERT(p, MA_OWNED);
1287 	FOREACH_THREAD_IN_PROC(p, td) {
1288 		if (td->td_tid == tid)
1289 			break;
1290 	}
1291 	return (td);
1292 }
1293 
1294 /* Locate a thread by number; return with proc lock held. */
1295 struct thread *
1296 tdfind(lwpid_t tid, pid_t pid)
1297 {
1298 #define RUN_THRESH	16
1299 	struct thread *td;
1300 	int run = 0;
1301 
1302 	td = curthread;
1303 	if (td->td_tid == tid) {
1304 		if (pid != -1 && td->td_proc->p_pid != pid)
1305 			return (NULL);
1306 		PROC_LOCK(td->td_proc);
1307 		return (td);
1308 	}
1309 
1310 	rw_rlock(&tidhash_lock);
1311 	LIST_FOREACH(td, TIDHASH(tid), td_hash) {
1312 		if (td->td_tid == tid) {
1313 			if (pid != -1 && td->td_proc->p_pid != pid) {
1314 				td = NULL;
1315 				break;
1316 			}
1317 			PROC_LOCK(td->td_proc);
1318 			if (td->td_proc->p_state == PRS_NEW) {
1319 				PROC_UNLOCK(td->td_proc);
1320 				td = NULL;
1321 				break;
1322 			}
1323 			if (run > RUN_THRESH) {
1324 				if (rw_try_upgrade(&tidhash_lock)) {
1325 					LIST_REMOVE(td, td_hash);
1326 					LIST_INSERT_HEAD(TIDHASH(td->td_tid),
1327 						td, td_hash);
1328 					rw_wunlock(&tidhash_lock);
1329 					return (td);
1330 				}
1331 			}
1332 			break;
1333 		}
1334 		run++;
1335 	}
1336 	rw_runlock(&tidhash_lock);
1337 	return (td);
1338 }
1339 
1340 void
1341 tidhash_add(struct thread *td)
1342 {
1343 	rw_wlock(&tidhash_lock);
1344 	LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash);
1345 	rw_wunlock(&tidhash_lock);
1346 }
1347 
1348 void
1349 tidhash_remove(struct thread *td)
1350 {
1351 	rw_wlock(&tidhash_lock);
1352 	LIST_REMOVE(td, td_hash);
1353 	rw_wunlock(&tidhash_lock);
1354 }
1355