1 /* 2 * SYS/THREAD2.H 3 * 4 * Implements inline procedure support for the LWKT subsystem. 5 * 6 * Generally speaking these routines only operate on threads associated 7 * with the current cpu. For example, a higher priority thread pending 8 * on a different cpu will not be immediately scheduled by a yield() on 9 * this cpu. 10 * 11 * $DragonFly: src/sys/sys/thread2.h,v 1.28 2006/12/23 00:27:03 swildner Exp $ 12 */ 13 14 #ifndef _SYS_THREAD2_H_ 15 #define _SYS_THREAD2_H_ 16 17 #ifndef _KERNEL 18 19 #error "This file should not be included by userland programs." 20 21 #else 22 23 /* 24 * Userland will have its own globaldata which it includes prior to this. 25 */ 26 #ifndef _SYS_SYSTM_H_ 27 #include <sys/systm.h> 28 #endif 29 #ifndef _SYS_GLOBALDATA_H_ 30 #include <sys/globaldata.h> 31 #endif 32 #ifndef _MACHINE_CPUFUNC_H_ 33 #include <machine/cpufunc.h> 34 #endif 35 36 /* 37 * Critical section debugging 38 */ 39 #ifdef DEBUG_CRIT_SECTIONS 40 #define __DEBUG_CRIT_ARG__ const char *id 41 #define __DEBUG_CRIT_ADD_ARG__ , const char *id 42 #define __DEBUG_CRIT_PASS_ARG__ , id 43 #define __DEBUG_CRIT_ENTER(td) _debug_crit_enter((td), id) 44 #define __DEBUG_CRIT_EXIT(td) _debug_crit_exit((td), id) 45 #define crit_enter() _crit_enter(mycpu, __FUNCTION__) 46 #define crit_enter_id(id) _crit_enter(mycpu, id) 47 #define crit_enter_gd(curgd) _crit_enter((curgd), __FUNCTION__) 48 #define crit_enter_quick(curtd) _crit_enter_quick((curtd), __FUNCTION__) 49 #define crit_enter_hard() _crit_enter_hard(mycpu, __FUNCTION__) 50 #define crit_enter_hard_gd(curgd) _crit_enter_hard((curgd), __FUNCTION__) 51 #define crit_exit() _crit_exit(mycpu, __FUNCTION__) 52 #define crit_exit_id(id) _crit_exit(mycpu, id) 53 #define crit_exit_gd(curgd) _crit_exit((curgd), __FUNCTION__) 54 #define crit_exit_quick(curtd) _crit_exit_quick((curtd), __FUNCTION__) 55 #define crit_exit_hard() _crit_exit_hard(mycpu, __FUNCTION__) 56 #define crit_exit_hard_gd(curgd) _crit_exit_hard((curgd), __FUNCTION__) 57 #define crit_exit_noyield(curtd) _crit_exit_noyield((curtd),__FUNCTION__) 58 #else 59 #define __DEBUG_CRIT_ARG__ void 60 #define __DEBUG_CRIT_ADD_ARG__ 61 #define __DEBUG_CRIT_PASS_ARG__ 62 #define __DEBUG_CRIT_ENTER(td) 63 #define __DEBUG_CRIT_EXIT(td) 64 #define crit_enter() _crit_enter(mycpu) 65 #define crit_enter_id(id) _crit_enter(mycpu) 66 #define crit_enter_gd(curgd) _crit_enter((curgd)) 67 #define crit_enter_quick(curtd) _crit_enter_quick((curtd)) 68 #define crit_enter_hard() _crit_enter_hard(mycpu) 69 #define crit_enter_hard_gd(curgd) _crit_enter_hard((curgd)) 70 #define crit_exit() crit_exit_wrapper() 71 #define crit_exit_id(id) _crit_exit(mycpu) 72 #define crit_exit_gd(curgd) _crit_exit((curgd)) 73 #define crit_exit_quick(curtd) _crit_exit_quick((curtd)) 74 #define crit_exit_hard() _crit_exit_hard(mycpu) 75 #define crit_exit_hard_gd(curgd) _crit_exit_hard((curgd)) 76 #define crit_exit_noyield(curtd) _crit_exit_noyield((curtd)) 77 #endif 78 79 extern void crit_exit_wrapper(__DEBUG_CRIT_ARG__); 80 81 /* 82 * Track crit_enter()/crit_exit() pairs and warn on mismatches. 83 */ 84 #ifdef DEBUG_CRIT_SECTIONS 85 86 static __inline void 87 _debug_crit_enter(thread_t td, const char *id) 88 { 89 int wi = td->td_crit_debug_index; 90 91 td->td_crit_debug_array[wi & CRIT_DEBUG_ARRAY_MASK] = id; 92 ++td->td_crit_debug_index; 93 } 94 95 static __inline void 96 _debug_crit_exit(thread_t td, const char *id) 97 { 98 const char *gid; 99 int wi; 100 101 wi = td->td_crit_debug_index - 1; 102 if ((gid = td->td_crit_debug_array[wi & CRIT_DEBUG_ARRAY_MASK]) != id) { 103 if (td->td_in_crit_report == 0) { 104 td->td_in_crit_report = 1; 105 kprintf("crit_exit(%s) expected id %s\n", id, gid); 106 td->td_in_crit_report = 0; 107 } 108 } 109 --td->td_crit_debug_index; 110 } 111 112 #endif 113 114 /* 115 * Critical sections prevent preemption, but allowing explicit blocking 116 * and thread switching. Any interrupt occuring while in a critical 117 * section is made pending and returns immediately. Interrupts are not 118 * physically disabled. 119 * 120 * Hard critical sections prevent preemption and disallow any blocking 121 * or thread switching, and in addition will assert on any blockable 122 * operation (acquire token not already held, lockmgr, mutex ops, or 123 * splz). Spinlocks can still be used in hard sections. 124 * 125 * All critical section routines only operate on the current thread. 126 * Passed gd or td arguments are simply optimizations when mycpu or 127 * curthread is already available to the caller. 128 */ 129 130 /* 131 * crit_enter 132 */ 133 static __inline void 134 _crit_enter_quick(thread_t td __DEBUG_CRIT_ADD_ARG__) 135 { 136 ++td->td_critcount; 137 __DEBUG_CRIT_ENTER(td); 138 cpu_ccfence(); 139 } 140 141 static __inline void 142 _crit_enter(globaldata_t gd __DEBUG_CRIT_ADD_ARG__) 143 { 144 _crit_enter_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__); 145 } 146 147 static __inline void 148 _crit_enter_hard(globaldata_t gd __DEBUG_CRIT_ADD_ARG__) 149 { 150 _crit_enter_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__); 151 ++gd->gd_intr_nesting_level; 152 } 153 154 155 /* 156 * crit_exit*() 157 * 158 * NOTE: Conditionalizing just gd_reqflags, a case which is virtually 159 * never true regardless of crit_count, should result in 100% 160 * optimal code execution. We don't check crit_count because 161 * it just bloats the inline and does not improve performance. 162 * 163 * NOTE: This can produce a considerable amount of code despite the 164 * relatively few lines of code so the non-debug case typically 165 * just wraps it in a real function, crit_exit_wrapper(). 166 */ 167 static __inline void 168 _crit_exit_noyield(thread_t td __DEBUG_CRIT_ADD_ARG__) 169 { 170 __DEBUG_CRIT_EXIT(td); 171 --td->td_critcount; 172 #ifdef INVARIANTS 173 if (__predict_false(td->td_critcount < 0)) 174 crit_panic(); 175 #endif 176 cpu_ccfence(); /* prevent compiler reordering */ 177 } 178 179 static __inline void 180 _crit_exit_quick(thread_t td __DEBUG_CRIT_ADD_ARG__) 181 { 182 _crit_exit_noyield(td __DEBUG_CRIT_PASS_ARG__); 183 if (__predict_false(td->td_gd->gd_reqflags & RQF_IDLECHECK_MASK)) 184 lwkt_maybe_splz(td); 185 } 186 187 static __inline void 188 _crit_exit(globaldata_t gd __DEBUG_CRIT_ADD_ARG__) 189 { 190 _crit_exit_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__); 191 } 192 193 static __inline void 194 _crit_exit_hard(globaldata_t gd __DEBUG_CRIT_ADD_ARG__) 195 { 196 --gd->gd_intr_nesting_level; 197 _crit_exit_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__); 198 } 199 200 static __inline int 201 crit_test(thread_t td) 202 { 203 return(td->td_critcount); 204 } 205 206 /* 207 * Return whether any threads are runnable, whether they meet mp_lock 208 * requirements or not. 209 */ 210 static __inline int 211 lwkt_runnable(void) 212 { 213 return (TAILQ_FIRST(&mycpu->gd_tdrunq) != NULL); 214 } 215 216 static __inline int 217 lwkt_getpri(thread_t td) 218 { 219 return(td->td_pri); 220 } 221 222 static __inline int 223 lwkt_getpri_self(void) 224 { 225 return(lwkt_getpri(curthread)); 226 } 227 228 /* 229 * Reduce our priority in preparation for a return to userland. If 230 * our passive release function was still in place, our priority was 231 * never raised and does not need to be reduced. 232 * 233 * See also lwkt_passive_release() and platform/blah/trap.c 234 */ 235 static __inline void 236 lwkt_passive_recover(thread_t td) 237 { 238 if (td->td_release == NULL) 239 lwkt_setpri_self(TDPRI_USER_NORM); 240 td->td_release = NULL; 241 } 242 243 #ifdef SMP 244 245 /* 246 * IPIQ messaging wrappers. IPIQ remote functions are passed three arguments: 247 * a void * pointer, an integer, and a pointer to the trap frame (or NULL if 248 * the trap frame is not known). However, we wish to provide opaque 249 * interfaces for simpler callbacks... the basic IPI messaging function as 250 * used by the kernel takes a single argument. 251 */ 252 static __inline int 253 lwkt_send_ipiq(globaldata_t target, ipifunc1_t func, void *arg) 254 { 255 return(lwkt_send_ipiq3(target, (ipifunc3_t)func, arg, 0)); 256 } 257 258 static __inline int 259 lwkt_send_ipiq2(globaldata_t target, ipifunc2_t func, void *arg1, int arg2) 260 { 261 return(lwkt_send_ipiq3(target, (ipifunc3_t)func, arg1, arg2)); 262 } 263 264 static __inline int 265 lwkt_send_ipiq_mask(u_int32_t mask, ipifunc1_t func, void *arg) 266 { 267 return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg, 0)); 268 } 269 270 static __inline int 271 lwkt_send_ipiq2_mask(u_int32_t mask, ipifunc2_t func, void *arg1, int arg2) 272 { 273 return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg1, arg2)); 274 } 275 276 static __inline int 277 lwkt_send_ipiq_nowait(globaldata_t target, ipifunc1_t func, void *arg) 278 { 279 return(lwkt_send_ipiq3_nowait(target, (ipifunc3_t)func, arg, 0)); 280 } 281 282 static __inline int 283 lwkt_send_ipiq2_nowait(globaldata_t target, ipifunc2_t func, 284 void *arg1, int arg2) 285 { 286 return(lwkt_send_ipiq3_nowait(target, (ipifunc3_t)func, arg1, arg2)); 287 } 288 289 static __inline int 290 lwkt_send_ipiq_passive(globaldata_t target, ipifunc1_t func, void *arg) 291 { 292 return(lwkt_send_ipiq3_passive(target, (ipifunc3_t)func, arg, 0)); 293 } 294 295 static __inline int 296 lwkt_send_ipiq2_passive(globaldata_t target, ipifunc2_t func, 297 void *arg1, int arg2) 298 { 299 return(lwkt_send_ipiq3_passive(target, (ipifunc3_t)func, arg1, arg2)); 300 } 301 302 static __inline int 303 lwkt_send_ipiq_bycpu(int dcpu, ipifunc1_t func, void *arg) 304 { 305 return(lwkt_send_ipiq3_bycpu(dcpu, (ipifunc3_t)func, arg, 0)); 306 } 307 308 static __inline int 309 lwkt_send_ipiq2_bycpu(int dcpu, ipifunc2_t func, void *arg1, int arg2) 310 { 311 return(lwkt_send_ipiq3_bycpu(dcpu, (ipifunc3_t)func, arg1, arg2)); 312 } 313 314 #endif /* SMP */ 315 #endif /* _KERNEL */ 316 #endif /* _SYS_THREAD2_H_ */ 317 318