1 /*
2 * SYS/THREAD2.H
3 *
4 * Implements inline procedure support for the LWKT subsystem.
5 *
6 * Generally speaking these routines only operate on threads associated
7 * with the current cpu. For example, a higher priority thread pending
8 * on a different cpu will not be immediately scheduled by a yield() on
9 * this cpu.
10 */
11
12 #ifndef _SYS_THREAD2_H_
13 #define _SYS_THREAD2_H_
14
15 #ifndef _KERNEL
16 #error "This file should not be included by userland programs."
17 #endif
18
19 /*
20 * Userland will have its own globaldata which it includes prior to this.
21 */
22 #ifndef _SYS_SYSTM_H_
23 #include <sys/systm.h>
24 #endif
25 #ifndef _SYS_GLOBALDATA_H_
26 #include <sys/globaldata.h>
27 #endif
28 #ifndef _SYS_CPUMASK_H_
29 #include <sys/cpumask.h>
30 #endif
31 #include <machine/cpufunc.h>
32
33 /*
34 * Don't let GCC reorder critical section count adjustments, because it
35 * will BLOW US UP if it does.
36 */
37 static __inline void
crit_enter_raw(thread_t td)38 crit_enter_raw(thread_t td)
39 {
40 cpu_ccfence();
41 ++td->td_critcount;
42 cpu_ccfence();
43 }
44
45 static __inline void
crit_exit_raw(thread_t td)46 crit_exit_raw(thread_t td)
47 {
48 cpu_ccfence();
49 --td->td_critcount;
50 cpu_ccfence();
51 }
52
53 /*
54 * Is a token held either by the specified thread or held shared?
55 *
56 * We can't inexpensively validate the thread for a shared token
57 * without iterating td->td_toks, so this isn't a perfect test.
58 */
59 static __inline int
_lwkt_token_held_any(lwkt_token_t tok,thread_t td)60 _lwkt_token_held_any(lwkt_token_t tok, thread_t td)
61 {
62 long count = tok->t_count;
63
64 cpu_ccfence();
65 if (tok->t_ref >= &td->td_toks_base && tok->t_ref < td->td_toks_stop)
66 return TRUE;
67 if ((count & TOK_EXCLUSIVE) == 0 &&
68 (count & ~(TOK_EXCLUSIVE|TOK_EXCLREQ))) {
69 return TRUE;
70 }
71 return FALSE;
72 }
73
74 /*
75 * Is a token held by the specified thread?
76 */
77 static __inline int
_lwkt_token_held_excl(lwkt_token_t tok,thread_t td)78 _lwkt_token_held_excl(lwkt_token_t tok, thread_t td)
79 {
80 return ((tok->t_ref >= &td->td_toks_base &&
81 tok->t_ref < td->td_toks_stop));
82 }
83
84 /*
85 * Critical section debugging
86 */
87 #ifdef DEBUG_CRIT_SECTIONS
88 #define __DEBUG_CRIT_ARG__ const char *id
89 #define __DEBUG_CRIT_ADD_ARG__ , const char *id
90 #define __DEBUG_CRIT_PASS_ARG__ , id
91 #define __DEBUG_CRIT_ENTER(td) _debug_crit_enter((td), id)
92 #define __DEBUG_CRIT_EXIT(td) _debug_crit_exit((td), id)
93 #define crit_enter() _crit_enter(mycpu, __func__)
94 #define crit_enter_id(id) _crit_enter(mycpu, id)
95 #define crit_enter_gd(curgd) _crit_enter((curgd), __func__)
96 #define crit_enter_quick(curtd) _crit_enter_quick((curtd), __func__)
97 #define crit_enter_hard() _crit_enter_hard(mycpu, __func__)
98 #define crit_enter_hard_gd(curgd) _crit_enter_hard((curgd), __func__)
99 #define crit_exit() _crit_exit(mycpu, __func__)
100 #define crit_exit_id(id) _crit_exit(mycpu, id)
101 #define crit_exit_gd(curgd) _crit_exit((curgd), __func__)
102 #define crit_exit_quick(curtd) _crit_exit_quick((curtd), __func__)
103 #define crit_exit_hard() _crit_exit_hard(mycpu, __func__)
104 #define crit_exit_hard_gd(curgd) _crit_exit_hard((curgd), __func__)
105 #define crit_exit_noyield(curtd) _crit_exit_noyield((curtd),__func__)
106 #else
107 #define __DEBUG_CRIT_ARG__ void
108 #define __DEBUG_CRIT_ADD_ARG__
109 #define __DEBUG_CRIT_PASS_ARG__
110 #define __DEBUG_CRIT_ENTER(td)
111 #define __DEBUG_CRIT_EXIT(td)
112 #define crit_enter() _crit_enter(mycpu)
113 #define crit_enter_id(id) _crit_enter(mycpu)
114 #define crit_enter_gd(curgd) _crit_enter((curgd))
115 #define crit_enter_quick(curtd) _crit_enter_quick((curtd))
116 #define crit_enter_hard() _crit_enter_hard(mycpu)
117 #define crit_enter_hard_gd(curgd) _crit_enter_hard((curgd))
118 #define crit_exit() crit_exit_wrapper()
119 #define crit_exit_id(id) _crit_exit(mycpu)
120 #define crit_exit_gd(curgd) _crit_exit((curgd))
121 #define crit_exit_quick(curtd) _crit_exit_quick((curtd))
122 #define crit_exit_hard() _crit_exit_hard(mycpu)
123 #define crit_exit_hard_gd(curgd) _crit_exit_hard((curgd))
124 #define crit_exit_noyield(curtd) _crit_exit_noyield((curtd))
125 #endif
126
127 void crit_exit_wrapper(__DEBUG_CRIT_ARG__);
128
129 /*
130 * Track crit_enter()/crit_exit() pairs and warn on mismatches.
131 */
132 #ifdef DEBUG_CRIT_SECTIONS
133
134 static __inline void
_debug_crit_enter(thread_t td,const char * id)135 _debug_crit_enter(thread_t td, const char *id)
136 {
137 int wi = td->td_crit_debug_index;
138
139 td->td_crit_debug_array[wi & CRIT_DEBUG_ARRAY_MASK] = id;
140 ++td->td_crit_debug_index;
141 }
142
143 static __inline void
_debug_crit_exit(thread_t td,const char * id)144 _debug_crit_exit(thread_t td, const char *id)
145 {
146 const char *gid;
147 int wi;
148
149 wi = td->td_crit_debug_index - 1;
150 if ((gid = td->td_crit_debug_array[wi & CRIT_DEBUG_ARRAY_MASK]) != id) {
151 if (td->td_in_crit_report == 0) {
152 td->td_in_crit_report = 1;
153 kprintf("crit_exit(%s) expected id %s\n", id, gid);
154 td->td_in_crit_report = 0;
155 }
156 }
157 --td->td_crit_debug_index;
158 }
159
160 #endif
161
162 /*
163 * Critical sections prevent preemption, but allowing explicit blocking
164 * and thread switching. Any interrupt occuring while in a critical
165 * section is made pending and returns immediately. Interrupts are not
166 * physically disabled.
167 *
168 * Hard critical sections prevent preemption and disallow any blocking
169 * or thread switching, and in addition will assert on any blockable
170 * operation (acquire token not already held, lockmgr, mutex ops, or
171 * splz). Spinlocks can still be used in hard sections.
172 *
173 * All critical section routines only operate on the current thread.
174 * Passed gd or td arguments are simply optimizations when mycpu or
175 * curthread is already available to the caller.
176 */
177
178 /*
179 * crit_enter
180 */
181 static __inline void
_crit_enter_quick(thread_t td __DEBUG_CRIT_ADD_ARG__)182 _crit_enter_quick(thread_t td __DEBUG_CRIT_ADD_ARG__)
183 {
184 crit_enter_raw(td);
185 __DEBUG_CRIT_ENTER(td);
186 }
187
188 static __inline void
_crit_enter(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)189 _crit_enter(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)
190 {
191 _crit_enter_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
192 }
193
194 static __inline void
_crit_enter_hard(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)195 _crit_enter_hard(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)
196 {
197 _crit_enter_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
198 ++gd->gd_intr_nesting_level;
199 cpu_ccfence();
200 }
201
202
203 /*
204 * crit_exit*()
205 *
206 * NOTE: Conditionalizing just gd_reqflags, a case which is virtually
207 * never true regardless of crit_count, should result in 100%
208 * optimal code execution. We don't check crit_count because
209 * it just bloats the inline and does not improve performance.
210 *
211 * NOTE: This can produce a considerable amount of code despite the
212 * relatively few lines of code so the non-debug case typically
213 * just wraps it in a real function, crit_exit_wrapper().
214 */
215 static __inline void
_crit_exit_noyield(thread_t td __DEBUG_CRIT_ADD_ARG__)216 _crit_exit_noyield(thread_t td __DEBUG_CRIT_ADD_ARG__)
217 {
218 __DEBUG_CRIT_EXIT(td);
219 crit_exit_raw(td);
220 #ifdef INVARIANTS
221 if (__predict_false(td->td_critcount < 0))
222 crit_panic();
223 #endif
224 }
225
226 static __inline void
_crit_exit_quick(thread_t td __DEBUG_CRIT_ADD_ARG__)227 _crit_exit_quick(thread_t td __DEBUG_CRIT_ADD_ARG__)
228 {
229 _crit_exit_noyield(td __DEBUG_CRIT_PASS_ARG__);
230 if (__predict_false(td->td_gd->gd_reqflags & RQF_IDLECHECK_MASK))
231 lwkt_maybe_splz(td);
232 }
233
234 static __inline void
_crit_exit(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)235 _crit_exit(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)
236 {
237 _crit_exit_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
238 }
239
240 static __inline void
_crit_exit_hard(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)241 _crit_exit_hard(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)
242 {
243 cpu_ccfence();
244 --gd->gd_intr_nesting_level;
245 _crit_exit_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
246 }
247
248 static __inline int
crit_test(thread_t td)249 crit_test(thread_t td)
250 {
251 return(td->td_critcount);
252 }
253
254 /*
255 * Return whether any threads are runnable.
256 */
257 static __inline int
lwkt_runnable(void)258 lwkt_runnable(void)
259 {
260 return (TAILQ_FIRST(&mycpu->gd_tdrunq) != NULL);
261 }
262
263 static __inline int
lwkt_getpri(thread_t td)264 lwkt_getpri(thread_t td)
265 {
266 return(td->td_pri);
267 }
268
269 static __inline int
lwkt_getpri_self(void)270 lwkt_getpri_self(void)
271 {
272 return(lwkt_getpri(curthread));
273 }
274
275 /*
276 * Reduce our priority in preparation for a return to userland. If
277 * our passive release function was still in place, our priority was
278 * never raised and does not need to be reduced.
279 *
280 * See also lwkt_passive_release() and platform/blah/trap.c
281 */
282 static __inline void
lwkt_passive_recover(thread_t td)283 lwkt_passive_recover(thread_t td)
284 {
285 #ifndef NO_LWKT_SPLIT_USERPRI
286 if (td->td_release == NULL)
287 lwkt_setpri_self(TDPRI_USER_NORM);
288 td->td_release = NULL;
289 #endif
290 }
291
292 /*
293 * cpusync support
294 */
295 static __inline void
lwkt_cpusync_init(lwkt_cpusync_t cs,cpumask_t mask,cpusync_func_t func,void * data)296 lwkt_cpusync_init(lwkt_cpusync_t cs, cpumask_t mask,
297 cpusync_func_t func, void *data)
298 {
299 cs->cs_mask = mask;
300 /* cs->cs_mack = 0; handled by _interlock */
301 cs->cs_func = func;
302 cs->cs_data = data;
303 }
304
305 /*
306 * IPIQ messaging wrappers. IPIQ remote functions are passed three arguments:
307 * a void * pointer, an integer, and a pointer to the trap frame (or NULL if
308 * the trap frame is not known). However, we wish to provide opaque
309 * interfaces for simpler callbacks... the basic IPI messaging function as
310 * used by the kernel takes a single argument.
311 */
312 static __inline int
lwkt_send_ipiq(globaldata_t target,ipifunc1_t func,void * arg)313 lwkt_send_ipiq(globaldata_t target, ipifunc1_t func, void *arg)
314 {
315 return(lwkt_send_ipiq3(target, (ipifunc3_t)func, arg, 0));
316 }
317
318 static __inline int
lwkt_send_ipiq2(globaldata_t target,ipifunc2_t func,void * arg1,int arg2)319 lwkt_send_ipiq2(globaldata_t target, ipifunc2_t func, void *arg1, int arg2)
320 {
321 return(lwkt_send_ipiq3(target, (ipifunc3_t)func, arg1, arg2));
322 }
323
324 static __inline int
lwkt_send_ipiq_mask(cpumask_t mask,ipifunc1_t func,void * arg)325 lwkt_send_ipiq_mask(cpumask_t mask, ipifunc1_t func, void *arg)
326 {
327 return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg, 0));
328 }
329
330 static __inline int
lwkt_send_ipiq2_mask(cpumask_t mask,ipifunc2_t func,void * arg1,int arg2)331 lwkt_send_ipiq2_mask(cpumask_t mask, ipifunc2_t func, void *arg1, int arg2)
332 {
333 return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg1, arg2));
334 }
335
336 static __inline int
lwkt_send_ipiq_passive(globaldata_t target,ipifunc1_t func,void * arg)337 lwkt_send_ipiq_passive(globaldata_t target, ipifunc1_t func, void *arg)
338 {
339 return(lwkt_send_ipiq3_passive(target, (ipifunc3_t)func, arg, 0));
340 }
341
342 static __inline int
lwkt_send_ipiq2_passive(globaldata_t target,ipifunc2_t func,void * arg1,int arg2)343 lwkt_send_ipiq2_passive(globaldata_t target, ipifunc2_t func,
344 void *arg1, int arg2)
345 {
346 return(lwkt_send_ipiq3_passive(target, (ipifunc3_t)func, arg1, arg2));
347 }
348
349 static __inline int
lwkt_send_ipiq_bycpu(int dcpu,ipifunc1_t func,void * arg)350 lwkt_send_ipiq_bycpu(int dcpu, ipifunc1_t func, void *arg)
351 {
352 return(lwkt_send_ipiq3_bycpu(dcpu, (ipifunc3_t)func, arg, 0));
353 }
354
355 static __inline int
lwkt_send_ipiq2_bycpu(int dcpu,ipifunc2_t func,void * arg1,int arg2)356 lwkt_send_ipiq2_bycpu(int dcpu, ipifunc2_t func, void *arg1, int arg2)
357 {
358 return(lwkt_send_ipiq3_bycpu(dcpu, (ipifunc3_t)func, arg1, arg2));
359 }
360
361 static __inline int
lwkt_need_ipiq_process(globaldata_t gd)362 lwkt_need_ipiq_process(globaldata_t gd)
363 {
364 lwkt_ipiq_t ipiq;
365
366 if (CPUMASK_TESTNZERO(gd->gd_ipimask))
367 return 1;
368
369 ipiq = &gd->gd_cpusyncq;
370 return (ipiq->ip_rindex != ipiq->ip_windex);
371 }
372
373 #endif /* _SYS_THREAD2_H_ */
374