79 #define ipiq_stat(gd)	ipiq_stats_percpu[(gd)->gd_cpuid]  argument
188     struct globaldata *gd = mycpu;  in lwkt_send_ipiq3()  local
190     logipiq(send_norm, func, arg1, arg2, gd, target);  in lwkt_send_ipiq3()
192     if (target == gd) {  in lwkt_send_ipiq3()
194 	logipiq(send_end, func, arg1, arg2, gd, target);  in lwkt_send_ipiq3()
198     ++gd->gd_intr_nesting_level;  in lwkt_send_ipiq3()
200     if (gd->gd_intr_nesting_level > 20)  in lwkt_send_ipiq3()
204     ++ipiq_stat(gd).ipiq_count;  in lwkt_send_ipiq3()
205     ip = &gd->gd_ipiq[target->gd_cpuid];  in lwkt_send_ipiq3()
224     if (gd->gd_processing_ipiq == 0) {  in lwkt_send_ipiq3()
241 	++ipiq_stat(gd).ipiq_fifofull;  in lwkt_send_ipiq3()
262 				gd->gd_cpuid, target->gd_cpuid, repeating,  in lwkt_send_ipiq3()
266 			ATOMIC_CPUMASK_ORBIT(target->gd_ipimask, gd->gd_cpuid);  in lwkt_send_ipiq3()
271 				gd->gd_cpuid, target->gd_cpuid, repeating);  in lwkt_send_ipiq3()
308     ATOMIC_CPUMASK_ORBIT(target->gd_ipimask, gd->gd_cpuid);  in lwkt_send_ipiq3()
314 	logipiq(cpu_send, func, arg1, arg2, gd, target);  in lwkt_send_ipiq3()
317 	++ipiq_stat(gd).ipiq_avoided;  in lwkt_send_ipiq3()
321     --gd->gd_intr_nesting_level;  in lwkt_send_ipiq3()
323     logipiq(send_end, func, arg1, arg2, gd, target);  in lwkt_send_ipiq3()
345     struct globaldata *gd = mycpu;  in lwkt_send_ipiq3_passive()  local
347     KKASSERT(target != gd);  in lwkt_send_ipiq3_passive()
348     crit_enter_gd(gd);  in lwkt_send_ipiq3_passive()
349     ++gd->gd_intr_nesting_level;  in lwkt_send_ipiq3_passive()
350     ip = &gd->gd_ipiq[target->gd_cpuid];  in lwkt_send_ipiq3_passive()
360 	--gd->gd_intr_nesting_level;  in lwkt_send_ipiq3_passive()
361 	crit_exit_gd(gd);  in lwkt_send_ipiq3_passive()
368     logipiq(send_pasv, func, arg1, arg2, gd, target);  in lwkt_send_ipiq3_passive()
369     ++ipiq_stat(gd).ipiq_count;  in lwkt_send_ipiq3_passive()
370     ++ipiq_stat(gd).ipiq_passive;  in lwkt_send_ipiq3_passive()
381     ATOMIC_CPUMASK_ORBIT(target->gd_ipimask, gd->gd_cpuid);  in lwkt_send_ipiq3_passive()
382     --gd->gd_intr_nesting_level;  in lwkt_send_ipiq3_passive()
389     logipiq(send_end, func, arg1, arg2, gd, target);  in lwkt_send_ipiq3_passive()
547     globaldata_t gd = mycpu;  in lwkt_process_ipiq()  local
553     ++gd->gd_processing_ipiq;  in lwkt_process_ipiq()
555     mask = gd->gd_ipimask;  in lwkt_process_ipiq()
559 	if (n != gd->gd_cpuid) {  in lwkt_process_ipiq()
563 		ip += gd->gd_cpuid;  in lwkt_process_ipiq()
571 		ATOMIC_CPUMASK_NANDBIT(gd->gd_ipimask, n);  in lwkt_process_ipiq()
573 			ATOMIC_CPUMASK_ORBIT(gd->gd_ipimask, n);  in lwkt_process_ipiq()
584     if (lwkt_process_ipiq_core(gd, &gd->gd_cpusyncq, NULL, 0)) {  in lwkt_process_ipiq()
585 	if (gd->gd_curthread->td_cscount == 0)  in lwkt_process_ipiq()
593     --gd->gd_processing_ipiq;  in lwkt_process_ipiq()
599     globaldata_t gd = mycpu;  in lwkt_process_ipiq_frame()  local
605     ++gd->gd_processing_ipiq;  in lwkt_process_ipiq_frame()
607     mask = gd->gd_ipimask;  in lwkt_process_ipiq_frame()
611 	if (n != gd->gd_cpuid) {  in lwkt_process_ipiq_frame()
615 		ip += gd->gd_cpuid;  in lwkt_process_ipiq_frame()
623 		ATOMIC_CPUMASK_NANDBIT(gd->gd_ipimask, n);  in lwkt_process_ipiq_frame()
625 			ATOMIC_CPUMASK_ORBIT(gd->gd_ipimask, n);  in lwkt_process_ipiq_frame()
630     if (gd->gd_cpusyncq.ip_rindex != gd->gd_cpusyncq.ip_windex) {  in lwkt_process_ipiq_frame()
631 	if (lwkt_process_ipiq_core(gd, &gd->gd_cpusyncq, frame, 0)) {  in lwkt_process_ipiq_frame()
632 	    if (gd->gd_curthread->td_cscount == 0)  in lwkt_process_ipiq_frame()
637     --gd->gd_processing_ipiq;  in lwkt_process_ipiq_frame()
654     globaldata_t gd = mycpu;  in lwkt_process_ipiq_nested()  local
661     ++gd->gd_processing_ipiq;  in lwkt_process_ipiq_nested()
663     mask = gd->gd_ipimask;  in lwkt_process_ipiq_nested()
667 	if (n != gd->gd_cpuid) {  in lwkt_process_ipiq_nested()
676 		ip += gd->gd_cpuid;  in lwkt_process_ipiq_nested()
691     if (lwkt_process_ipiq_core(gd, &gd->gd_cpusyncq, NULL, 0)) {  in lwkt_process_ipiq_nested()
692 	if (gd->gd_curthread->td_cscount == 0)  in lwkt_process_ipiq_nested()
696     --gd->gd_processing_ipiq;  in lwkt_process_ipiq_nested()
846     globaldata_t gd = mycpu;  in lwkt_cpusync_interlock()  local
855     CPUMASK_ANDMASK(mask, gd->gd_other_cpus);  in lwkt_cpusync_interlock()
862 	++ipiq_stat(gd).ipiq_cscount;  in lwkt_cpusync_interlock()
863 	++gd->gd_curthread->td_cscount;  in lwkt_cpusync_interlock()
886     globaldata_t gd = mycpu;  in lwkt_cpusync_deinterlock()  local
902     if (cs->cs_func && CPUMASK_TESTBIT(cs->cs_mask, gd->gd_cpuid))  in lwkt_cpusync_deinterlock()
919 	--gd->gd_curthread->td_cscount;  in lwkt_cpusync_deinterlock()
937     globaldata_t gd = mycpu;  in lwkt_cpusync_quick()  local
944     CPUMASK_ANDMASK(mask, gd->gd_other_cpus);  in lwkt_cpusync_quick()
951 	++ipiq_stat(gd).ipiq_cscount;  in lwkt_cpusync_quick()
952 	++gd->gd_curthread->td_cscount;  in lwkt_cpusync_quick()
969 	--gd->gd_curthread->td_cscount;  in lwkt_cpusync_quick()
972     if (cs->cs_func && CPUMASK_TESTBIT(cs->cs_mask, gd->gd_cpuid))  in lwkt_cpusync_quick()
987     globaldata_t gd = mycpu;  in lwkt_cpusync_remote1()  local
989     ATOMIC_CPUMASK_ORBIT(cs->cs_mack, gd->gd_cpuid);  in lwkt_cpusync_remote1()
1002     globaldata_t gd = mycpu;  in lwkt_cpusync_remote2()  local
1004     if (CPUMASK_TESTMASK(cs->cs_mack, gd->gd_cpumask) == 0) {  in lwkt_cpusync_remote2()
1007 	ATOMIC_CPUMASK_ORBIT(cs->cs_mack, gd->gd_cpuid);  in lwkt_cpusync_remote2()
1024 	ip = &gd->gd_cpusyncq;  in lwkt_cpusync_remote2()
1035 			gd->gd_cpuid,  in lwkt_cpusync_remote2()
1062 	struct globaldata *gd;  in lwkt_ipiq_latency_testfunc()  local
1075 	gd = mycpu;  in lwkt_ipiq_latency_testfunc()
1076 	lat = &lwkt_ipiq_latency_logs[gd->gd_cpuid];  in lwkt_ipiq_latency_testfunc()
1093 	struct globaldata *gd;  in lwkt_ipiq_latency_test()  local
1108 	gd = globaldata_find(cpu);  in lwkt_ipiq_latency_test()
1111 	lwkt_send_ipiq(gd, lwkt_ipiq_latency_testfunc, NULL);  in lwkt_ipiq_latency_test()