1 /* $NetBSD: lwp.h,v 1.206 2020/04/10 17:16:21 ad Exp $ */ 2 3 /* 4 * Copyright (c) 2001, 2006, 2007, 2008, 2009, 2010, 2019, 2020 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Nathan J. Williams and Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #ifndef _SYS_LWP_H_ 34 #define _SYS_LWP_H_ 35 36 #if defined(_KERNEL) || defined(_KMEMUSER) 37 38 #include <sys/param.h> 39 #include <sys/time.h> 40 #include <sys/queue.h> 41 #include <sys/callout.h> 42 #include <sys/kcpuset.h> 43 #include <sys/mutex.h> 44 #include <sys/condvar.h> 45 #include <sys/signalvar.h> 46 #include <sys/sched.h> 47 #include <sys/specificdata.h> 48 #include <sys/syncobj.h> 49 #include <sys/resource.h> 50 51 #if defined(_KERNEL) 52 struct lwp; 53 /* forward declare this for <machine/cpu.h> so it can get l_cpu. */ 54 static __inline struct cpu_info *lwp_getcpu(struct lwp *); 55 #include <machine/cpu.h> /* curcpu() and cpu_info */ 56 #ifdef _KERNEL_OPT 57 #include "opt_kcov.h" 58 #include "opt_kmsan.h" 59 #endif 60 #endif 61 62 #include <machine/proc.h> /* Machine-dependent proc substruct. */ 63 64 /* 65 * Lightweight process. Field markings and the corresponding locks: 66 * 67 * a: proc_lock 68 * c: condition variable interlock, passed to cv_wait() 69 * l: *l_mutex 70 * p: l_proc->p_lock 71 * s: spc_mutex, which may or may not be referenced by l_mutex 72 * S: l_selcluster->sc_lock 73 * (: unlocked, stable 74 * !: unlocked, may only be reliably accessed by the LWP itself 75 * 76 * Fields are clustered together by usage (to increase the likelihood 77 * of cache hits) and by size (to reduce dead space in the structure). 78 */ 79 80 #include <sys/pcu.h> 81 82 struct lockdebug; 83 struct sysent; 84 85 struct lwp { 86 /* Scheduling and overall state. */ 87 TAILQ_ENTRY(lwp) l_runq; /* s: run queue */ 88 union { 89 void * info; /* s: scheduler-specific structure */ 90 u_int timeslice; /* l: time-quantum for SCHED_M2 */ 91 } l_sched; 92 struct cpu_info *volatile l_cpu;/* s: CPU we're on if LSONPROC */ 93 kmutex_t * volatile l_mutex; /* l: ptr to mutex on sched state */ 94 void *l_addr; /* l: PCB address; use lwp_getpcb() */ 95 struct mdlwp l_md; /* l: machine-dependent fields. */ 96 struct bintime l_rtime; /* l: real time */ 97 struct bintime l_stime; /* l: start time (while ONPROC) */ 98 int l_flag; /* l: misc flag values */ 99 u_int l_swtime; /* l: time swapped in or out */ 100 u_int l_rticks; /* l: Saved start time of run */ 101 u_int l_rticksum; /* l: Sum of ticks spent running */ 102 u_int l_slpticks; /* l: Saved start time of sleep */ 103 u_int l_slpticksum; /* l: Sum of ticks spent sleeping */ 104 int l_biglocks; /* l: biglock count before sleep */ 105 short l_stat; /* l: overall LWP status */ 106 short l_class; /* l: scheduling class */ 107 int l_kpriority; /* !: has kernel priority boost */ 108 pri_t l_kpribase; /* !: kernel priority base level */ 109 pri_t l_priority; /* l: scheduler priority */ 110 pri_t l_inheritedprio;/* l: inherited priority */ 111 pri_t l_protectprio; /* l: for PTHREAD_PRIO_PROTECT */ 112 pri_t l_auxprio; /* l: max(inherit,protect) priority */ 113 int l_protectdepth; /* l: for PTHREAD_PRIO_PROTECT */ 114 u_int l_cpticks; /* (: Ticks of CPU time */ 115 psetid_t l_psid; /* l: assigned processor-set ID */ 116 fixpt_t l_pctcpu; /* p: %cpu during l_swtime */ 117 fixpt_t l_estcpu; /* l: cpu time for SCHED_4BSD */ 118 volatile uint64_t l_ncsw; /* l: total context switches */ 119 volatile uint64_t l_nivcsw; /* l: involuntary context switches */ 120 SLIST_HEAD(, turnstile) l_pi_lenders; /* l: ts lending us priority */ 121 struct cpu_info *l_target_cpu; /* l: target CPU to migrate */ 122 struct lwpctl *l_lwpctl; /* p: lwpctl block kernel address */ 123 struct lcpage *l_lcpage; /* p: lwpctl containing page */ 124 kcpuset_t *l_affinity; /* l: CPU set for affinity */ 125 126 /* Synchronisation. */ 127 struct turnstile *l_ts; /* l: current turnstile */ 128 struct syncobj *l_syncobj; /* l: sync object operations set */ 129 LIST_ENTRY(lwp) l_sleepchain; /* l: sleep queue */ 130 wchan_t l_wchan; /* l: sleep address */ 131 const char *l_wmesg; /* l: reason for sleep */ 132 struct sleepq *l_sleepq; /* l: current sleep queue */ 133 callout_t l_timeout_ch; /* !: callout for tsleep */ 134 kcondvar_t l_waitcv; /* a: vfork() wait */ 135 u_int l_slptime; /* l: time since last blocked */ 136 bool l_vforkwaiting; /* a: vfork() waiting */ 137 138 /* User-space synchronization. */ 139 uintptr_t l___reserved; /* reserved for future use */ 140 /* 141 * The global thread ID has special locking and access 142 * considerations. Because many LWPs may never need one, 143 * global thread IDs are allocated lazily in lwp_gettid(). 144 * l___tid is not bean to be accessed directly unless 145 * the accessor has specific knowledge that doing so 146 * is safe. l___tid is only assigned by the LWP itself. 147 * Once assigned, it is stable until the LWP exits. 148 * An LWP assigns its own thread ID unlocked before it 149 * reaches visibility to the rest of the system, and 150 * can access its own thread ID unlocked. But once 151 * published, it must hold the proc's lock to change 152 * the value. 153 */ 154 lwpid_t l___tid; /* p: global thread id */ 155 156 #if PCU_UNIT_COUNT > 0 157 struct cpu_info * volatile l_pcu_cpu[PCU_UNIT_COUNT]; 158 uint32_t l_pcu_valid; 159 #endif 160 161 /* Process level and global state, misc. */ 162 lwpid_t l_lid; /* (: LWP identifier; local to proc */ 163 LIST_ENTRY(lwp) l_list; /* a: entry on list of all LWPs */ 164 void *l_ctxlink; /* p: uc_link {get,set}context */ 165 struct proc *l_proc; /* p: parent process */ 166 LIST_ENTRY(lwp) l_sibling; /* p: entry on proc's list of LWPs */ 167 char *l_name; /* (: name, optional */ 168 lwpid_t l_waiter; /* p: first LWP waiting on us */ 169 lwpid_t l_waitingfor; /* p: specific LWP we are waiting on */ 170 int l_prflag; /* p: process level flags */ 171 u_int l_refcnt; /* p: reference count on this LWP */ 172 173 /* State of select() or poll(). */ 174 int l_selflag; /* S: polling state flags */ 175 int l_selret; /* S: return value of select/poll */ 176 SLIST_HEAD(,selinfo) l_selwait; /* S: descriptors waited on */ 177 uintptr_t l_selrec; /* !: argument for selrecord() */ 178 struct selcluster *l_selcluster;/* !: associated cluster data */ 179 void * l_selbits; /* (: select() bit-field */ 180 size_t l_selni; /* (: size of a single bit-field */ 181 182 /* Signals. */ 183 int l_sigrestore; /* p: need to restore old sig mask */ 184 sigset_t l_sigwaitset; /* p: signals being waited for */ 185 kcondvar_t l_sigcv; /* p: for sigsuspend() */ 186 struct ksiginfo *l_sigwaited; /* p: delivered signals from set */ 187 sigpend_t *l_sigpendset; /* p: XXX issignal()/postsig() baton */ 188 LIST_ENTRY(lwp) l_sigwaiter; /* p: chain on list of waiting LWPs */ 189 stack_t l_sigstk; /* p: sp & on stack state variable */ 190 sigset_t l_sigmask; /* p: signal mask */ 191 sigpend_t l_sigpend; /* p: signals to this LWP */ 192 sigset_t l_sigoldmask; /* p: mask for sigpause */ 193 194 /* Private data. */ 195 specificdata_reference 196 l_specdataref; /* !: subsystem lwp-specific data */ 197 struct timespec l_ktrcsw; /* !: for ktrace CSW trace XXX */ 198 void *l_private; /* !: svr4-style lwp-private data */ 199 struct lwp *l_switchto; /* !: mi_switch: switch to this LWP */ 200 struct kauth_cred *l_cred; /* !: cached credentials */ 201 struct filedesc *l_fd; /* !: cached copy of proc::p_fd */ 202 void *l_emuldata; /* !: kernel lwp-private data */ 203 struct fstrans_lwp_info *l_fstrans; /* (: fstrans private data */ 204 u_short l_shlocks; /* !: lockdebug: shared locks held */ 205 u_short l_exlocks; /* !: lockdebug: excl. locks held */ 206 u_short l_psrefs; /* !: count of psref held */ 207 u_short l_blcnt; /* !: count of kernel_lock held */ 208 volatile int l_nopreempt; /* !: don't preempt me! */ 209 volatile u_int l_dopreempt; /* s: kernel preemption pending */ 210 int l_pflag; /* !: LWP private flags */ 211 int l_dupfd; /* !: side return from cloning devs XXX */ 212 const struct sysent * volatile l_sysent;/* !: currently active syscall */ 213 struct rusage l_ru; /* !: accounting information */ 214 uint64_t l_pfailtime; /* !: for kernel preemption */ 215 uintptr_t l_pfailaddr; /* !: for kernel preemption */ 216 uintptr_t l_pfaillock; /* !: for kernel preemption */ 217 _TAILQ_HEAD(,struct lockdebug,volatile) l_ld_locks;/* !: locks held by LWP */ 218 volatile void *l_ld_wanted; /* !: lock currently wanted by LWP */ 219 uintptr_t l_rwcallsite; /* !: rwlock actual callsite */ 220 int l_tcgen; /* !: for timecounter removal */ 221 222 /* These are only used by 'options SYSCALL_TIMES'. */ 223 uint32_t l_syscall_time; /* !: time epoch for current syscall */ 224 uint64_t *l_syscall_counter; /* !: counter for current process */ 225 226 struct kdtrace_thread *l_dtrace; /* (: DTrace-specific data. */ 227 228 #ifdef KMSAN 229 void *l_kmsan; /* !: KMSAN private data. */ 230 #endif 231 #ifdef KCOV 232 void *l_kcov; /* !: KCOV private data. */ 233 #endif 234 }; 235 236 /* 237 * UAREA_PCB_OFFSET: an offset of PCB structure in the uarea. MD code may 238 * define it in <machine/proc.h>, to indicate a different uarea layout. 239 */ 240 #ifndef UAREA_PCB_OFFSET 241 #define UAREA_PCB_OFFSET 0 242 #endif 243 244 LIST_HEAD(lwplist, lwp); /* A list of LWPs. */ 245 246 #ifdef _KERNEL 247 extern struct lwplist alllwp; /* List of all LWPs. */ 248 extern lwp_t lwp0; /* LWP for proc0. */ 249 extern int maxlwp __read_mostly; /* max number of lwps */ 250 #ifndef MAXLWP 251 #define MAXLWP 2048 252 #endif 253 #ifndef __HAVE_CPU_MAXLWP 254 #define cpu_maxlwp() MAXLWP 255 #endif 256 #endif 257 258 #endif /* _KERNEL || _KMEMUSER */ 259 260 /* 261 * These flags are kept in l_flag, and they are modified only with the LWP 262 * locked. 263 */ 264 #define LW_IDLE 0x00000001 /* Idle lwp. */ 265 #define LW_LWPCTL 0x00000002 /* Adjust lwpctl in userret */ 266 #define LW_STIMO 0x00000040 /* Sleep timed out */ 267 #define LW_SINTR 0x00000080 /* Sleep is interruptible. */ 268 #define LW_SYSTEM 0x00000200 /* Kernel thread */ 269 #define LW_DBGSUSPEND 0x00010000 /* Suspend by debugger */ 270 #define LW_WSUSPEND 0x00020000 /* Suspend before return to user */ 271 #define LW_BATCH 0x00040000 /* LWP tends to hog CPU */ 272 #define LW_WCORE 0x00080000 /* Stop for core dump on return to user */ 273 #define LW_WEXIT 0x00100000 /* Exit before return to user */ 274 #define LW_PENDSIG 0x01000000 /* Pending signal for us */ 275 #define LW_CANCELLED 0x02000000 /* tsleep should not sleep */ 276 #define LW_WREBOOT 0x08000000 /* System is rebooting, please suspend */ 277 #define LW_UNPARKED 0x10000000 /* Unpark op pending */ 278 #define LW_RUMP_CLEAR 0x40000000 /* Clear curlwp in RUMP scheduler */ 279 #define LW_RUMP_QEXIT 0x80000000 /* LWP should exit ASAP */ 280 281 /* 282 * The second set of flags is kept in l_pflag, and they are modified only by 283 * the LWP itself, or modified when it's known the LWP cannot be running. 284 * LP_RUNNING is typically updated with the LWP locked, but not always in 285 * the case of soft interrupt handlers. 286 */ 287 #define LP_KTRACTIVE 0x00000001 /* Executing ktrace operation */ 288 #define LP_KTRCSW 0x00000002 /* ktrace context switch marker */ 289 #define LP_KTRCSWUSER 0x00000004 /* ktrace context switch marker */ 290 #define LP_PIDLID 0x00000008 /* free LID from PID space on exit */ 291 #define LP_OWEUPC 0x00000010 /* Owe user profiling tick */ 292 #define LP_MPSAFE 0x00000020 /* Starts life without kernel_lock */ 293 #define LP_INTR 0x00000040 /* Soft interrupt handler */ 294 #define LP_SYSCTLWRITE 0x00000080 /* sysctl write lock held */ 295 #define LP_MUSTJOIN 0x00000100 /* Must join kthread on exit */ 296 #define LP_SINGLESTEP 0x00000400 /* Single step thread in ptrace(2) */ 297 #define LP_TIMEINTR 0x00010000 /* Time this soft interrupt */ 298 #define LP_PREEMPTING 0x00020000 /* mi_switch called involuntarily */ 299 #define LP_RUNNING 0x20000000 /* Active on a CPU */ 300 #define LP_TELEPORT 0x40000000 /* Teleport to new CPU on preempt() */ 301 #define LP_BOUND 0x80000000 /* Bound to a CPU */ 302 303 /* 304 * The third set of flags is kept in l_prflag and they are modified only 305 * with p_lock held. 306 */ 307 #define LPR_DETACHED 0x00800000 /* Won't be waited for. */ 308 #define LPR_CRMOD 0x00000100 /* Credentials modified */ 309 #define LPR_DRAINING 0x80000000 /* Draining references before exiting */ 310 311 /* 312 * Mask indicating that there is "exceptional" work to be done on return to 313 * user. 314 */ 315 #define LW_USERRET \ 316 (LW_WEXIT | LW_PENDSIG | LW_WREBOOT | LW_WSUSPEND | LW_WCORE | LW_LWPCTL) 317 318 /* 319 * Status values. 320 * 321 * A note about LSRUN and LSONPROC: LSRUN indicates that a process is 322 * runnable but *not* yet running, i.e. is on a run queue. LSONPROC 323 * indicates that the process is actually executing on a CPU, i.e. 324 * it is no longer on a run queue. 325 * 326 * These values are set in stone and must not be reused with future changes. 327 */ 328 #define LSIDL 1 /* Process being created by fork. */ 329 #define LSRUN 2 /* Currently runnable. */ 330 #define LSSLEEP 3 /* Sleeping on an address. */ 331 #define LSSTOP 4 /* Process debugging or suspension. */ 332 #define LSZOMB 5 /* Awaiting collection by parent. */ 333 /* define LSDEAD 6 Process is almost a zombie. (removed in 5.0) */ 334 #define LSONPROC 7 /* Process is currently on a CPU. */ 335 #define LSSUSPENDED 8 /* Not running, not signalable. */ 336 337 #if defined(_KERNEL) || defined(_KMEMUSER) 338 static __inline void * 339 lwp_getpcb(struct lwp *l) 340 { 341 342 return l->l_addr; 343 } 344 #endif /* _KERNEL || _KMEMUSER */ 345 346 #ifdef _KERNEL 347 #define LWP_CACHE_CREDS(l, p) \ 348 do { \ 349 (void)p; \ 350 if (__predict_false((l)->l_prflag & LPR_CRMOD)) \ 351 lwp_update_creds(l); \ 352 } while (/* CONSTCOND */ 0) 353 354 void lwpinit(void); 355 void lwp0_init(void); 356 357 void lwp_startup(lwp_t *, lwp_t *); 358 void startlwp(void *); 359 360 int lwp_locked(lwp_t *, kmutex_t *); 361 kmutex_t *lwp_setlock(lwp_t *, kmutex_t *); 362 void lwp_unlock_to(lwp_t *, kmutex_t *); 363 int lwp_trylock(lwp_t *); 364 void lwp_addref(lwp_t *); 365 void lwp_delref(lwp_t *); 366 void lwp_delref2(lwp_t *); 367 bool lwp_drainrefs(lwp_t *); 368 bool lwp_alive(lwp_t *); 369 lwp_t *lwp_find_first(proc_t *); 370 371 void lwp_renumber(lwp_t *, lwpid_t); 372 int lwp_wait(lwp_t *, lwpid_t, lwpid_t *, bool); 373 void lwp_continue(lwp_t *); 374 void lwp_unsleep(lwp_t *, bool); 375 void lwp_unstop(lwp_t *); 376 void lwp_exit(lwp_t *); 377 int lwp_suspend(lwp_t *, lwp_t *); 378 int lwp_create1(lwp_t *, const void *, size_t, u_long, lwpid_t *); 379 void lwp_start(lwp_t *, int); 380 void lwp_update_creds(lwp_t *); 381 void lwp_migrate(lwp_t *, struct cpu_info *); 382 lwp_t * lwp_find2(pid_t, lwpid_t); 383 lwp_t * lwp_find(proc_t *, int); 384 void lwp_userret(lwp_t *); 385 void lwp_need_userret(lwp_t *); 386 void lwp_free(lwp_t *, bool, bool); 387 uint64_t lwp_pctr(void); 388 int lwp_setprivate(lwp_t *, void *); 389 int do_lwp_create(lwp_t *, void *, u_long, lwp_t **, const sigset_t *, 390 const stack_t *); 391 392 lwpid_t lwp_gettid(void); 393 lwp_t * lwp_getref_tid(lwpid_t); 394 void lwp_thread_cleanup(lwp_t *); 395 396 void lwpinit_specificdata(void); 397 int lwp_specific_key_create(specificdata_key_t *, specificdata_dtor_t); 398 void lwp_specific_key_delete(specificdata_key_t); 399 void lwp_initspecific(lwp_t *); 400 void lwp_finispecific(lwp_t *); 401 void *lwp_getspecific(specificdata_key_t); 402 #if defined(_LWP_API_PRIVATE) 403 void *_lwp_getspecific_by_lwp(lwp_t *, specificdata_key_t); 404 #endif 405 void lwp_setspecific(specificdata_key_t, void *); 406 void lwp_setspecific_by_lwp(lwp_t *, specificdata_key_t, void *); 407 408 /* Syscalls. */ 409 int lwp_park(clockid_t, int, struct timespec *); 410 int lwp_unpark(const lwpid_t *, const u_int); 411 412 /* DDB. */ 413 void lwp_whatis(uintptr_t, void (*)(const char *, ...) __printflike(1, 2)); 414 415 /* 416 * Lock an LWP. XXX _MODULE 417 */ 418 static __inline void 419 lwp_lock(lwp_t *l) 420 { 421 kmutex_t *old = l->l_mutex; 422 423 /* 424 * Note: mutex_spin_enter() will have posted a read barrier. 425 * Re-test l->l_mutex. If it has changed, we need to try again. 426 */ 427 mutex_spin_enter(old); 428 while (__predict_false(l->l_mutex != old)) { 429 mutex_spin_exit(old); 430 old = l->l_mutex; 431 mutex_spin_enter(old); 432 } 433 } 434 435 /* 436 * Unlock an LWP. XXX _MODULE 437 */ 438 static __inline void 439 lwp_unlock(lwp_t *l) 440 { 441 mutex_spin_exit(l->l_mutex); 442 } 443 444 static __inline void 445 lwp_changepri(lwp_t *l, pri_t pri) 446 { 447 KASSERT(mutex_owned(l->l_mutex)); 448 449 if (l->l_priority == pri) 450 return; 451 452 (*l->l_syncobj->sobj_changepri)(l, pri); 453 KASSERT(l->l_priority == pri); 454 } 455 456 static __inline void 457 lwp_lendpri(lwp_t *l, pri_t pri) 458 { 459 KASSERT(mutex_owned(l->l_mutex)); 460 461 (*l->l_syncobj->sobj_lendpri)(l, pri); 462 KASSERT(l->l_inheritedprio == pri); 463 } 464 465 static __inline pri_t 466 lwp_eprio(lwp_t *l) 467 { 468 pri_t pri; 469 470 pri = l->l_priority; 471 if ((l->l_flag & LW_SYSTEM) == 0 && l->l_kpriority && pri < PRI_KERNEL) 472 pri = (pri >> 1) + l->l_kpribase; 473 return MAX(l->l_auxprio, pri); 474 } 475 476 int lwp_create(lwp_t *, struct proc *, vaddr_t, int, void *, size_t, 477 void (*)(void *), void *, lwp_t **, int, const sigset_t *, const stack_t *); 478 479 /* 480 * XXX _MODULE 481 * We should provide real stubs for the below that modules can use. 482 */ 483 484 static __inline void 485 spc_lock(struct cpu_info *ci) 486 { 487 mutex_spin_enter(ci->ci_schedstate.spc_mutex); 488 } 489 490 static __inline void 491 spc_unlock(struct cpu_info *ci) 492 { 493 mutex_spin_exit(ci->ci_schedstate.spc_mutex); 494 } 495 496 static __inline void 497 spc_dlock(struct cpu_info *ci1, struct cpu_info *ci2) 498 { 499 struct schedstate_percpu *spc1 = &ci1->ci_schedstate; 500 struct schedstate_percpu *spc2 = &ci2->ci_schedstate; 501 502 KASSERT(ci1 != ci2); 503 if (ci1 < ci2) { 504 mutex_spin_enter(spc1->spc_mutex); 505 mutex_spin_enter(spc2->spc_mutex); 506 } else { 507 mutex_spin_enter(spc2->spc_mutex); 508 mutex_spin_enter(spc1->spc_mutex); 509 } 510 } 511 512 /* 513 * Allow machine-dependent code to override curlwp in <machine/cpu.h> for 514 * its own convenience. Otherwise, we declare it as appropriate. 515 */ 516 #if !defined(curlwp) 517 #if defined(MULTIPROCESSOR) 518 #define curlwp curcpu()->ci_curlwp /* Current running LWP */ 519 #else 520 extern struct lwp *curlwp; /* Current running LWP */ 521 #endif /* MULTIPROCESSOR */ 522 #endif /* ! curlwp */ 523 #define curproc (curlwp->l_proc) 524 525 /* 526 * This provide a way for <machine/cpu.h> to get l_cpu for curlwp before 527 * struct lwp is defined. 528 */ 529 static __inline struct cpu_info * 530 lwp_getcpu(struct lwp *l) 531 { 532 return l->l_cpu; 533 } 534 535 static __inline bool 536 CURCPU_IDLE_P(void) 537 { 538 struct cpu_info *ci = curcpu(); 539 return ci->ci_onproc == ci->ci_data.cpu_idlelwp; 540 } 541 542 /* 543 * Disable and re-enable preemption. Only for low-level kernel 544 * use. Device drivers and anything that could potentially be 545 * compiled as a module should use kpreempt_disable() and 546 * kpreempt_enable(). 547 */ 548 static __inline void 549 KPREEMPT_DISABLE(lwp_t *l) 550 { 551 552 KASSERT(l == curlwp); 553 l->l_nopreempt++; 554 __insn_barrier(); 555 } 556 557 static __inline void 558 KPREEMPT_ENABLE(lwp_t *l) 559 { 560 561 KASSERT(l == curlwp); 562 KASSERT(l->l_nopreempt > 0); 563 __insn_barrier(); 564 if (--l->l_nopreempt != 0) 565 return; 566 __insn_barrier(); 567 if (__predict_false(l->l_dopreempt)) 568 kpreempt(0); 569 __insn_barrier(); 570 } 571 572 /* For lwp::l_dopreempt */ 573 #define DOPREEMPT_ACTIVE 0x01 574 #define DOPREEMPT_COUNTED 0x02 575 576 /* 577 * Prevent curlwp from migrating between CPUs between curlwp_bind and 578 * curlwp_bindx. One use case is psref(9) that has a contract that 579 * forbids migrations. 580 */ 581 static __inline int 582 curlwp_bind(void) 583 { 584 int bound; 585 586 bound = curlwp->l_pflag & LP_BOUND; 587 curlwp->l_pflag |= LP_BOUND; 588 __insn_barrier(); 589 590 return bound; 591 } 592 593 static __inline void 594 curlwp_bindx(int bound) 595 { 596 597 KASSERT(curlwp->l_pflag & LP_BOUND); 598 __insn_barrier(); 599 curlwp->l_pflag ^= bound ^ LP_BOUND; 600 } 601 602 #endif /* _KERNEL */ 603 604 /* Flags for _lwp_create(), as per Solaris. */ 605 #define LWP_DETACHED 0x00000040 606 #define LWP_SUSPENDED 0x00000080 607 608 /* Kernel-internal flags for LWP creation. */ 609 #define LWP_PIDLID 0x40000000 610 #define LWP_VFORK 0x80000000 611 612 #endif /* !_SYS_LWP_H_ */ 613