1 /* 2 * Copyright (c) 2003 Matthew Dillon <dillon@backplane.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * Each cpu in a system has its own self-contained light weight kernel 27 * thread scheduler, which means that generally speaking we only need 28 * to use a critical section to avoid problems. Foreign thread 29 * scheduling is queued via (async) IPIs. 30 * 31 * $DragonFly: src/sys/kern/lwkt_thread.c,v 1.28 2003/07/25 05:51:19 dillon Exp $ 32 */ 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/proc.h> 38 #include <sys/rtprio.h> 39 #include <sys/queue.h> 40 #include <sys/thread2.h> 41 #include <sys/sysctl.h> 42 #include <sys/kthread.h> 43 #include <machine/cpu.h> 44 #include <sys/lock.h> 45 46 #include <vm/vm.h> 47 #include <vm/vm_param.h> 48 #include <vm/vm_kern.h> 49 #include <vm/vm_object.h> 50 #include <vm/vm_page.h> 51 #include <vm/vm_map.h> 52 #include <vm/vm_pager.h> 53 #include <vm/vm_extern.h> 54 #include <vm/vm_zone.h> 55 56 #include <machine/stdarg.h> 57 #include <machine/ipl.h> 58 #ifdef SMP 59 #include <machine/smp.h> 60 #endif 61 62 static int untimely_switch = 0; 63 SYSCTL_INT(_lwkt, OID_AUTO, untimely_switch, CTLFLAG_RW, &untimely_switch, 0, ""); 64 #ifdef INVARIANTS 65 static int token_debug = 0; 66 SYSCTL_INT(_lwkt, OID_AUTO, token_debug, CTLFLAG_RW, &token_debug, 0, ""); 67 #endif 68 static quad_t switch_count = 0; 69 SYSCTL_QUAD(_lwkt, OID_AUTO, switch_count, CTLFLAG_RW, &switch_count, 0, ""); 70 static quad_t preempt_hit = 0; 71 SYSCTL_QUAD(_lwkt, OID_AUTO, preempt_hit, CTLFLAG_RW, &preempt_hit, 0, ""); 72 static quad_t preempt_miss = 0; 73 SYSCTL_QUAD(_lwkt, OID_AUTO, preempt_miss, CTLFLAG_RW, &preempt_miss, 0, ""); 74 static quad_t preempt_weird = 0; 75 SYSCTL_QUAD(_lwkt, OID_AUTO, preempt_weird, CTLFLAG_RW, &preempt_weird, 0, ""); 76 static quad_t ipiq_count = 0; 77 SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_count, CTLFLAG_RW, &ipiq_count, 0, ""); 78 static quad_t ipiq_fifofull = 0; 79 SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_fifofull, CTLFLAG_RW, &ipiq_fifofull, 0, ""); 80 81 /* 82 * These helper procedures handle the runq, they can only be called from 83 * within a critical section. 84 */ 85 static __inline 86 void 87 _lwkt_dequeue(thread_t td) 88 { 89 if (td->td_flags & TDF_RUNQ) { 90 int nq = td->td_pri & TDPRI_MASK; 91 struct globaldata *gd = mycpu; 92 93 td->td_flags &= ~TDF_RUNQ; 94 TAILQ_REMOVE(&gd->gd_tdrunq[nq], td, td_threadq); 95 /* runqmask is passively cleaned up by the switcher */ 96 } 97 } 98 99 static __inline 100 void 101 _lwkt_enqueue(thread_t td) 102 { 103 if ((td->td_flags & TDF_RUNQ) == 0) { 104 int nq = td->td_pri & TDPRI_MASK; 105 struct globaldata *gd = mycpu; 106 107 td->td_flags |= TDF_RUNQ; 108 TAILQ_INSERT_TAIL(&gd->gd_tdrunq[nq], td, td_threadq); 109 gd->gd_runqmask |= 1 << nq; 110 } 111 } 112 113 static __inline 114 int 115 _lwkt_wantresched(thread_t ntd, thread_t cur) 116 { 117 return((ntd->td_pri & TDPRI_MASK) > (cur->td_pri & TDPRI_MASK)); 118 } 119 120 /* 121 * LWKTs operate on a per-cpu basis 122 * 123 * WARNING! Called from early boot, 'mycpu' may not work yet. 124 */ 125 void 126 lwkt_gdinit(struct globaldata *gd) 127 { 128 int i; 129 130 for (i = 0; i < sizeof(gd->gd_tdrunq)/sizeof(gd->gd_tdrunq[0]); ++i) 131 TAILQ_INIT(&gd->gd_tdrunq[i]); 132 gd->gd_runqmask = 0; 133 TAILQ_INIT(&gd->gd_tdallq); 134 } 135 136 /* 137 * Initialize a thread wait structure prior to first use. 138 * 139 * NOTE! called from low level boot code, we cannot do anything fancy! 140 */ 141 void 142 lwkt_init_wait(lwkt_wait_t w) 143 { 144 TAILQ_INIT(&w->wa_waitq); 145 } 146 147 /* 148 * Create a new thread. The thread must be associated with a process context 149 * or LWKT start address before it can be scheduled. 150 * 151 * If you intend to create a thread without a process context this function 152 * does everything except load the startup and switcher function. 153 */ 154 thread_t 155 lwkt_alloc_thread(struct thread *td) 156 { 157 void *stack; 158 int flags = 0; 159 160 if (td == NULL) { 161 crit_enter(); 162 if (mycpu->gd_tdfreecount > 0) { 163 --mycpu->gd_tdfreecount; 164 td = TAILQ_FIRST(&mycpu->gd_tdfreeq); 165 KASSERT(td != NULL && (td->td_flags & TDF_RUNNING) == 0, 166 ("lwkt_alloc_thread: unexpected NULL or corrupted td")); 167 TAILQ_REMOVE(&mycpu->gd_tdfreeq, td, td_threadq); 168 crit_exit(); 169 stack = td->td_kstack; 170 flags = td->td_flags & (TDF_ALLOCATED_STACK|TDF_ALLOCATED_THREAD); 171 } else { 172 crit_exit(); 173 td = zalloc(thread_zone); 174 td->td_kstack = NULL; 175 flags |= TDF_ALLOCATED_THREAD; 176 } 177 } 178 if ((stack = td->td_kstack) == NULL) { 179 stack = (void *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE); 180 flags |= TDF_ALLOCATED_STACK; 181 } 182 lwkt_init_thread(td, stack, flags, mycpu); 183 return(td); 184 } 185 186 /* 187 * Initialize a preexisting thread structure. This function is used by 188 * lwkt_alloc_thread() and also used to initialize the per-cpu idlethread. 189 * 190 * NOTE! called from low level boot code, we cannot do anything fancy! 191 * Only the low level boot code will call this function with gd != mycpu. 192 */ 193 void 194 lwkt_init_thread(thread_t td, void *stack, int flags, struct globaldata *gd) 195 { 196 bzero(td, sizeof(struct thread)); 197 td->td_kstack = stack; 198 td->td_flags |= flags; 199 td->td_gd = gd; 200 td->td_pri = TDPRI_CRIT; 201 lwkt_init_port(&td->td_msgport, td); 202 pmap_init_thread(td); 203 crit_enter(); 204 TAILQ_INSERT_TAIL(&gd->gd_tdallq, td, td_allq); 205 crit_exit(); 206 } 207 208 void 209 lwkt_set_comm(thread_t td, const char *ctl, ...) 210 { 211 va_list va; 212 213 va_start(va, ctl); 214 vsnprintf(td->td_comm, sizeof(td->td_comm), ctl, va); 215 va_end(va); 216 } 217 218 void 219 lwkt_hold(thread_t td) 220 { 221 ++td->td_refs; 222 } 223 224 void 225 lwkt_rele(thread_t td) 226 { 227 KKASSERT(td->td_refs > 0); 228 --td->td_refs; 229 } 230 231 void 232 lwkt_wait_free(thread_t td) 233 { 234 while (td->td_refs) 235 tsleep(td, 0, "tdreap", hz); 236 } 237 238 void 239 lwkt_free_thread(thread_t td) 240 { 241 struct globaldata *gd = mycpu; 242 243 KASSERT((td->td_flags & TDF_RUNNING) == 0, 244 ("lwkt_free_thread: did not exit! %p", td)); 245 246 crit_enter(); 247 TAILQ_REMOVE(&gd->gd_tdallq, td, td_allq); 248 if (gd->gd_tdfreecount < CACHE_NTHREADS && 249 (td->td_flags & TDF_ALLOCATED_THREAD) 250 ) { 251 ++gd->gd_tdfreecount; 252 TAILQ_INSERT_HEAD(&gd->gd_tdfreeq, td, td_threadq); 253 crit_exit(); 254 } else { 255 crit_exit(); 256 if (td->td_kstack && (td->td_flags & TDF_ALLOCATED_STACK)) { 257 kmem_free(kernel_map, 258 (vm_offset_t)td->td_kstack, UPAGES * PAGE_SIZE); 259 /* gd invalid */ 260 td->td_kstack = NULL; 261 } 262 if (td->td_flags & TDF_ALLOCATED_THREAD) 263 zfree(thread_zone, td); 264 } 265 } 266 267 268 /* 269 * Switch to the next runnable lwkt. If no LWKTs are runnable then 270 * switch to the idlethread. Switching must occur within a critical 271 * section to avoid races with the scheduling queue. 272 * 273 * We always have full control over our cpu's run queue. Other cpus 274 * that wish to manipulate our queue must use the cpu_*msg() calls to 275 * talk to our cpu, so a critical section is all that is needed and 276 * the result is very, very fast thread switching. 277 * 278 * The LWKT scheduler uses a fixed priority model and round-robins at 279 * each priority level. User process scheduling is a totally 280 * different beast and LWKT priorities should not be confused with 281 * user process priorities. 282 * 283 * The MP lock may be out of sync with the thread's td_mpcount. lwkt_switch() 284 * cleans it up. Note that the td_switch() function cannot do anything that 285 * requires the MP lock since the MP lock will have already been setup for 286 * the target thread (not the current thread). 287 */ 288 289 void 290 lwkt_switch(void) 291 { 292 struct globaldata *gd; 293 thread_t td = curthread; 294 thread_t ntd; 295 #ifdef SMP 296 int mpheld; 297 #endif 298 299 if (mycpu->gd_intr_nesting_level && 300 td->td_preempted == NULL && panicstr == NULL 301 ) { 302 panic("lwkt_switch: cannot switch from within an interrupt, yet\n"); 303 } 304 305 /* 306 * Passive release (used to transition from user to kernel mode 307 * when we block or switch rather then when we enter the kernel). 308 * This function is NOT called if we are switching into a preemption 309 * or returning from a preemption. Typically this causes us to lose 310 * our P_CURPROC designation (if we have one) and become a true LWKT 311 * thread, and may also hand P_CURPROC to another process and schedule 312 * its thread. 313 */ 314 if (td->td_release) 315 td->td_release(td); 316 317 crit_enter(); 318 ++switch_count; 319 320 #ifdef SMP 321 /* 322 * td_mpcount cannot be used to determine if we currently hold the 323 * MP lock because get_mplock() will increment it prior to attempting 324 * to get the lock, and switch out if it can't. Look at the actual lock. 325 */ 326 mpheld = MP_LOCK_HELD(); 327 #endif 328 if ((ntd = td->td_preempted) != NULL) { 329 /* 330 * We had preempted another thread on this cpu, resume the preempted 331 * thread. This occurs transparently, whether the preempted thread 332 * was scheduled or not (it may have been preempted after descheduling 333 * itself). 334 * 335 * We have to setup the MP lock for the original thread after backing 336 * out the adjustment that was made to curthread when the original 337 * was preempted. 338 */ 339 KKASSERT(ntd->td_flags & TDF_PREEMPT_LOCK); 340 #ifdef SMP 341 if (ntd->td_mpcount && mpheld == 0) { 342 panic("MPLOCK NOT HELD ON RETURN: %p %p %d %d\n", 343 td, ntd, td->td_mpcount, ntd->td_mpcount); 344 } 345 if (ntd->td_mpcount) { 346 td->td_mpcount -= ntd->td_mpcount; 347 KKASSERT(td->td_mpcount >= 0); 348 } 349 #endif 350 ntd->td_flags |= TDF_PREEMPT_DONE; 351 /* YYY release mp lock on switchback if original doesn't need it */ 352 } else { 353 /* 354 * Priority queue / round-robin at each priority. Note that user 355 * processes run at a fixed, low priority and the user process 356 * scheduler deals with interactions between user processes 357 * by scheduling and descheduling them from the LWKT queue as 358 * necessary. 359 * 360 * We have to adjust the MP lock for the target thread. If we 361 * need the MP lock and cannot obtain it we try to locate a 362 * thread that does not need the MP lock. 363 */ 364 gd = mycpu; 365 again: 366 if (gd->gd_runqmask) { 367 int nq = bsrl(gd->gd_runqmask); 368 if ((ntd = TAILQ_FIRST(&gd->gd_tdrunq[nq])) == NULL) { 369 gd->gd_runqmask &= ~(1 << nq); 370 goto again; 371 } 372 #ifdef SMP 373 if (ntd->td_mpcount && mpheld == 0 && !cpu_try_mplock()) { 374 /* 375 * Target needs MP lock and we couldn't get it, try 376 * to locate a thread which does not need the MP lock 377 * to run. If we cannot locate a thread spin in idle. 378 */ 379 u_int32_t rqmask = gd->gd_runqmask; 380 while (rqmask) { 381 TAILQ_FOREACH(ntd, &gd->gd_tdrunq[nq], td_threadq) { 382 if (ntd->td_mpcount == 0) 383 break; 384 } 385 if (ntd) 386 break; 387 rqmask &= ~(1 << nq); 388 nq = bsrl(rqmask); 389 } 390 if (ntd == NULL) { 391 ntd = &gd->gd_idlethread; 392 ntd->td_flags |= TDF_IDLE_NOHLT; 393 } else { 394 TAILQ_REMOVE(&gd->gd_tdrunq[nq], ntd, td_threadq); 395 TAILQ_INSERT_TAIL(&gd->gd_tdrunq[nq], ntd, td_threadq); 396 } 397 } else { 398 TAILQ_REMOVE(&gd->gd_tdrunq[nq], ntd, td_threadq); 399 TAILQ_INSERT_TAIL(&gd->gd_tdrunq[nq], ntd, td_threadq); 400 } 401 #else 402 TAILQ_REMOVE(&gd->gd_tdrunq[nq], ntd, td_threadq); 403 TAILQ_INSERT_TAIL(&gd->gd_tdrunq[nq], ntd, td_threadq); 404 #endif 405 } else { 406 /* 407 * Nothing to run but we may still need the BGL to deal with 408 * pending interrupts, spin in idle if so. 409 */ 410 ntd = &gd->gd_idlethread; 411 if (gd->gd_reqflags) 412 ntd->td_flags |= TDF_IDLE_NOHLT; 413 } 414 } 415 KASSERT(ntd->td_pri >= TDPRI_CRIT, 416 ("priority problem in lwkt_switch %d %d", td->td_pri, ntd->td_pri)); 417 418 /* 419 * Do the actual switch. If the new target does not need the MP lock 420 * and we are holding it, release the MP lock. If the new target requires 421 * the MP lock we have already acquired it for the target. 422 */ 423 #ifdef SMP 424 if (ntd->td_mpcount == 0 ) { 425 if (MP_LOCK_HELD()) 426 cpu_rel_mplock(); 427 } else { 428 ASSERT_MP_LOCK_HELD(); 429 } 430 #endif 431 if (td != ntd) { 432 td->td_switch(ntd); 433 } 434 435 crit_exit(); 436 } 437 438 /* 439 * Switch if another thread has a higher priority. Do not switch to other 440 * threads at the same priority. 441 */ 442 void 443 lwkt_maybe_switch() 444 { 445 struct globaldata *gd = mycpu; 446 struct thread *td = gd->gd_curthread; 447 448 if ((td->td_pri & TDPRI_MASK) < bsrl(gd->gd_runqmask)) { 449 lwkt_switch(); 450 } 451 } 452 453 /* 454 * Request that the target thread preempt the current thread. Preemption 455 * only works under a specific set of conditions: 456 * 457 * - We are not preempting ourselves 458 * - The target thread is owned by the current cpu 459 * - We are not currently being preempted 460 * - The target is not currently being preempted 461 * - We are able to satisfy the target's MP lock requirements (if any). 462 * 463 * THE CALLER OF LWKT_PREEMPT() MUST BE IN A CRITICAL SECTION. Typically 464 * this is called via lwkt_schedule() through the td_preemptable callback. 465 * critpri is the managed critical priority that we should ignore in order 466 * to determine whether preemption is possible (aka usually just the crit 467 * priority of lwkt_schedule() itself). 468 * 469 * XXX at the moment we run the target thread in a critical section during 470 * the preemption in order to prevent the target from taking interrupts 471 * that *WE* can't. Preemption is strictly limited to interrupt threads 472 * and interrupt-like threads, outside of a critical section, and the 473 * preempted source thread will be resumed the instant the target blocks 474 * whether or not the source is scheduled (i.e. preemption is supposed to 475 * be as transparent as possible). 476 * 477 * The target thread inherits our MP count (added to its own) for the 478 * duration of the preemption in order to preserve the atomicy of the 479 * MP lock during the preemption. Therefore, any preempting targets must be 480 * careful in regards to MP assertions. Note that the MP count may be 481 * out of sync with the physical mp_lock. If we preempt we have to preserve 482 * the expected situation. 483 */ 484 void 485 lwkt_preempt(thread_t ntd, int critpri) 486 { 487 thread_t td = curthread; 488 #ifdef SMP 489 int mpheld; 490 int savecnt; 491 #endif 492 493 /* 494 * The caller has put us in a critical section. We can only preempt 495 * if the caller of the caller was not in a critical section (basically 496 * a local interrupt), as determined by the 'critpri' parameter. If 497 * we are unable to preempt 498 * 499 * YYY The target thread must be in a critical section (else it must 500 * inherit our critical section? I dunno yet). 501 */ 502 KASSERT(ntd->td_pri >= TDPRI_CRIT, ("BADCRIT0 %d", ntd->td_pri)); 503 504 need_resched(); 505 if (!_lwkt_wantresched(ntd, td)) { 506 ++preempt_miss; 507 return; 508 } 509 if ((td->td_pri & ~TDPRI_MASK) > critpri) { 510 ++preempt_miss; 511 return; 512 } 513 #ifdef SMP 514 if (ntd->td_gd != mycpu) { 515 ++preempt_miss; 516 return; 517 } 518 #endif 519 if (td == ntd || ((td->td_flags | ntd->td_flags) & TDF_PREEMPT_LOCK)) { 520 ++preempt_weird; 521 return; 522 } 523 if (ntd->td_preempted) { 524 ++preempt_hit; 525 return; 526 } 527 #ifdef SMP 528 /* 529 * note: an interrupt might have occured just as we were transitioning 530 * to the MP lock. In this case td_mpcount will be pre-disposed but 531 * not actually synchronized with the actual state of the lock. We 532 * can use it to imply an MP lock requirement for the preemption but 533 * we cannot use it to test whether we hold the MP lock or not. 534 */ 535 mpheld = MP_LOCK_HELD(); 536 if (mpheld && td->td_mpcount == 0) 537 panic("lwkt_preempt(): held and no count"); 538 savecnt = td->td_mpcount; 539 ntd->td_mpcount += td->td_mpcount; 540 if (mpheld == 0 && ntd->td_mpcount && !cpu_try_mplock()) { 541 ntd->td_mpcount -= td->td_mpcount; 542 ++preempt_miss; 543 return; 544 } 545 #endif 546 547 ++preempt_hit; 548 ntd->td_preempted = td; 549 td->td_flags |= TDF_PREEMPT_LOCK; 550 td->td_switch(ntd); 551 KKASSERT(ntd->td_preempted && (td->td_flags & TDF_PREEMPT_DONE)); 552 #ifdef SMP 553 KKASSERT(savecnt == td->td_mpcount); 554 if (mpheld == 0 && MP_LOCK_HELD()) 555 cpu_rel_mplock(); 556 else if (mpheld && !MP_LOCK_HELD()) 557 panic("lwkt_preempt(): MP lock was not held through"); 558 #endif 559 ntd->td_preempted = NULL; 560 td->td_flags &= ~(TDF_PREEMPT_LOCK|TDF_PREEMPT_DONE); 561 } 562 563 /* 564 * Yield our thread while higher priority threads are pending. This is 565 * typically called when we leave a critical section but it can be safely 566 * called while we are in a critical section. 567 * 568 * This function will not generally yield to equal priority threads but it 569 * can occur as a side effect. Note that lwkt_switch() is called from 570 * inside the critical section to pervent its own crit_exit() from reentering 571 * lwkt_yield_quick(). 572 * 573 * gd_reqflags indicates that *something* changed, e.g. an interrupt or softint 574 * came along but was blocked and made pending. 575 * 576 * (self contained on a per cpu basis) 577 */ 578 void 579 lwkt_yield_quick(void) 580 { 581 globaldata_t gd = mycpu; 582 thread_t td = gd->gd_curthread; 583 584 /* 585 * gd_reqflags is cleared in splz if the cpl is 0. If we were to clear 586 * it with a non-zero cpl then we might not wind up calling splz after 587 * a task switch when the critical section is exited even though the 588 * new task could accept the interrupt. YYY alternative is to have 589 * lwkt_switch() just call splz unconditionally. 590 * 591 * XXX from crit_exit() only called after last crit section is released. 592 * If called directly will run splz() even if in a critical section. 593 */ 594 if (gd->gd_reqflags) 595 splz(); 596 597 /* 598 * YYY enabling will cause wakeup() to task-switch, which really 599 * confused the old 4.x code. This is a good way to simulate 600 * preemption and MP without actually doing preemption or MP, because a 601 * lot of code assumes that wakeup() does not block. 602 */ 603 if (untimely_switch && gd->gd_intr_nesting_level == 0) { 604 crit_enter(); 605 /* 606 * YYY temporary hacks until we disassociate the userland scheduler 607 * from the LWKT scheduler. 608 */ 609 if (td->td_flags & TDF_RUNQ) { 610 lwkt_switch(); /* will not reenter yield function */ 611 } else { 612 lwkt_schedule_self(); /* make sure we are scheduled */ 613 lwkt_switch(); /* will not reenter yield function */ 614 lwkt_deschedule_self(); /* make sure we are descheduled */ 615 } 616 crit_exit_noyield(td); 617 } 618 } 619 620 /* 621 * This implements a normal yield which, unlike _quick, will yield to equal 622 * priority threads as well. Note that gd_reqflags tests will be handled by 623 * the crit_exit() call in lwkt_switch(). 624 * 625 * (self contained on a per cpu basis) 626 */ 627 void 628 lwkt_yield(void) 629 { 630 lwkt_schedule_self(); 631 lwkt_switch(); 632 } 633 634 /* 635 * Schedule a thread to run. As the current thread we can always safely 636 * schedule ourselves, and a shortcut procedure is provided for that 637 * function. 638 * 639 * (non-blocking, self contained on a per cpu basis) 640 */ 641 void 642 lwkt_schedule_self(void) 643 { 644 thread_t td = curthread; 645 646 crit_enter(); 647 KASSERT(td->td_wait == NULL, ("lwkt_schedule_self(): td_wait not NULL!")); 648 _lwkt_enqueue(td); 649 if (td->td_proc && td->td_proc->p_stat == SSLEEP) 650 panic("SCHED SELF PANIC"); 651 crit_exit(); 652 } 653 654 /* 655 * Generic schedule. Possibly schedule threads belonging to other cpus and 656 * deal with threads that might be blocked on a wait queue. 657 * 658 * YYY this is one of the best places to implement load balancing code. 659 * Load balancing can be accomplished by requesting other sorts of actions 660 * for the thread in question. 661 */ 662 void 663 lwkt_schedule(thread_t td) 664 { 665 #ifdef INVARIANTS 666 if ((td->td_flags & TDF_PREEMPT_LOCK) == 0 && td->td_proc 667 && td->td_proc->p_stat == SSLEEP 668 ) { 669 printf("PANIC schedule curtd = %p (%d %d) target %p (%d %d)\n", 670 curthread, 671 curthread->td_proc ? curthread->td_proc->p_pid : -1, 672 curthread->td_proc ? curthread->td_proc->p_stat : -1, 673 td, 674 td->td_proc ? curthread->td_proc->p_pid : -1, 675 td->td_proc ? curthread->td_proc->p_stat : -1 676 ); 677 panic("SCHED PANIC"); 678 } 679 #endif 680 crit_enter(); 681 if (td == curthread) { 682 _lwkt_enqueue(td); 683 } else { 684 lwkt_wait_t w; 685 686 /* 687 * If the thread is on a wait list we have to send our scheduling 688 * request to the owner of the wait structure. Otherwise we send 689 * the scheduling request to the cpu owning the thread. Races 690 * are ok, the target will forward the message as necessary (the 691 * message may chase the thread around before it finally gets 692 * acted upon). 693 * 694 * (remember, wait structures use stable storage) 695 */ 696 if ((w = td->td_wait) != NULL) { 697 if (lwkt_trytoken(&w->wa_token)) { 698 TAILQ_REMOVE(&w->wa_waitq, td, td_threadq); 699 --w->wa_count; 700 td->td_wait = NULL; 701 if (td->td_gd == mycpu) { 702 _lwkt_enqueue(td); 703 if (td->td_preemptable) { 704 td->td_preemptable(td, TDPRI_CRIT*2); /* YYY +token */ 705 } else if (_lwkt_wantresched(td, curthread)) { 706 need_resched(); 707 } 708 } else { 709 lwkt_send_ipiq(td->td_gd->gd_cpuid, (ipifunc_t)lwkt_schedule, td); 710 } 711 lwkt_reltoken(&w->wa_token); 712 } else { 713 lwkt_send_ipiq(w->wa_token.t_cpu, (ipifunc_t)lwkt_schedule, td); 714 } 715 } else { 716 /* 717 * If the wait structure is NULL and we own the thread, there 718 * is no race (since we are in a critical section). If we 719 * do not own the thread there might be a race but the 720 * target cpu will deal with it. 721 */ 722 if (td->td_gd == mycpu) { 723 _lwkt_enqueue(td); 724 if (td->td_preemptable) { 725 td->td_preemptable(td, TDPRI_CRIT); 726 } else if (_lwkt_wantresched(td, curthread)) { 727 need_resched(); 728 } 729 } else { 730 lwkt_send_ipiq(td->td_gd->gd_cpuid, (ipifunc_t)lwkt_schedule, td); 731 } 732 } 733 } 734 crit_exit(); 735 } 736 737 /* 738 * Managed acquisition. This code assumes that the MP lock is held for 739 * the tdallq operation and that the thread has been descheduled from its 740 * original cpu. We also have to wait for the thread to be entirely switched 741 * out on its original cpu (this is usually fast enough that we never loop) 742 * since the LWKT system does not have to hold the MP lock while switching 743 * and the target may have released it before switching. 744 */ 745 void 746 lwkt_acquire(thread_t td) 747 { 748 struct globaldata *gd; 749 750 gd = td->td_gd; 751 KKASSERT((td->td_flags & TDF_RUNQ) == 0); 752 while (td->td_flags & TDF_RUNNING) /* XXX spin */ 753 ; 754 if (gd != mycpu) { 755 crit_enter(); 756 TAILQ_REMOVE(&gd->gd_tdallq, td, td_allq); /* protected by BGL */ 757 gd = mycpu; 758 td->td_gd = gd; 759 TAILQ_INSERT_TAIL(&gd->gd_tdallq, td, td_allq); /* protected by BGL */ 760 crit_exit(); 761 } 762 } 763 764 /* 765 * Deschedule a thread. 766 * 767 * (non-blocking, self contained on a per cpu basis) 768 */ 769 void 770 lwkt_deschedule_self(void) 771 { 772 thread_t td = curthread; 773 774 crit_enter(); 775 KASSERT(td->td_wait == NULL, ("lwkt_schedule_self(): td_wait not NULL!")); 776 _lwkt_dequeue(td); 777 crit_exit(); 778 } 779 780 /* 781 * Generic deschedule. Descheduling threads other then your own should be 782 * done only in carefully controlled circumstances. Descheduling is 783 * asynchronous. 784 * 785 * This function may block if the cpu has run out of messages. 786 */ 787 void 788 lwkt_deschedule(thread_t td) 789 { 790 crit_enter(); 791 if (td == curthread) { 792 _lwkt_dequeue(td); 793 } else { 794 if (td->td_gd == mycpu) { 795 _lwkt_dequeue(td); 796 } else { 797 lwkt_send_ipiq(td->td_gd->gd_cpuid, (ipifunc_t)lwkt_deschedule, td); 798 } 799 } 800 crit_exit(); 801 } 802 803 /* 804 * Set the target thread's priority. This routine does not automatically 805 * switch to a higher priority thread, LWKT threads are not designed for 806 * continuous priority changes. Yield if you want to switch. 807 * 808 * We have to retain the critical section count which uses the high bits 809 * of the td_pri field. The specified priority may also indicate zero or 810 * more critical sections by adding TDPRI_CRIT*N. 811 */ 812 void 813 lwkt_setpri(thread_t td, int pri) 814 { 815 KKASSERT(pri >= 0); 816 KKASSERT(td->td_gd == mycpu); 817 crit_enter(); 818 if (td->td_flags & TDF_RUNQ) { 819 _lwkt_dequeue(td); 820 td->td_pri = (td->td_pri & ~TDPRI_MASK) + pri; 821 _lwkt_enqueue(td); 822 } else { 823 td->td_pri = (td->td_pri & ~TDPRI_MASK) + pri; 824 } 825 crit_exit(); 826 } 827 828 void 829 lwkt_setpri_self(int pri) 830 { 831 thread_t td = curthread; 832 833 KKASSERT(pri >= 0 && pri <= TDPRI_MAX); 834 crit_enter(); 835 if (td->td_flags & TDF_RUNQ) { 836 _lwkt_dequeue(td); 837 td->td_pri = (td->td_pri & ~TDPRI_MASK) + pri; 838 _lwkt_enqueue(td); 839 } else { 840 td->td_pri = (td->td_pri & ~TDPRI_MASK) + pri; 841 } 842 crit_exit(); 843 } 844 845 struct proc * 846 lwkt_preempted_proc(void) 847 { 848 thread_t td = curthread; 849 while (td->td_preempted) 850 td = td->td_preempted; 851 return(td->td_proc); 852 } 853 854 typedef struct lwkt_gettoken_req { 855 lwkt_token_t tok; 856 int cpu; 857 } lwkt_gettoken_req; 858 859 #if 0 860 861 /* 862 * This function deschedules the current thread and blocks on the specified 863 * wait queue. We obtain ownership of the wait queue in order to block 864 * on it. A generation number is used to interlock the wait queue in case 865 * it gets signalled while we are blocked waiting on the token. 866 * 867 * Note: alternatively we could dequeue our thread and then message the 868 * target cpu owning the wait queue. YYY implement as sysctl. 869 * 870 * Note: wait queue signals normally ping-pong the cpu as an optimization. 871 */ 872 873 void 874 lwkt_block(lwkt_wait_t w, const char *wmesg, int *gen) 875 { 876 thread_t td = curthread; 877 878 lwkt_gettoken(&w->wa_token); 879 if (w->wa_gen == *gen) { 880 _lwkt_dequeue(td); 881 TAILQ_INSERT_TAIL(&w->wa_waitq, td, td_threadq); 882 ++w->wa_count; 883 td->td_wait = w; 884 td->td_wmesg = wmesg; 885 again: 886 lwkt_switch(); 887 lwkt_regettoken(&w->wa_token); 888 if (td->td_wmesg != NULL) { 889 _lwkt_dequeue(td); 890 goto again; 891 } 892 } 893 /* token might be lost, doesn't matter for gen update */ 894 *gen = w->wa_gen; 895 lwkt_reltoken(&w->wa_token); 896 } 897 898 /* 899 * Signal a wait queue. We gain ownership of the wait queue in order to 900 * signal it. Once a thread is removed from the wait queue we have to 901 * deal with the cpu owning the thread. 902 * 903 * Note: alternatively we could message the target cpu owning the wait 904 * queue. YYY implement as sysctl. 905 */ 906 void 907 lwkt_signal(lwkt_wait_t w, int count) 908 { 909 thread_t td; 910 int count; 911 912 lwkt_gettoken(&w->wa_token); 913 ++w->wa_gen; 914 if (count < 0) 915 count = w->wa_count; 916 while ((td = TAILQ_FIRST(&w->wa_waitq)) != NULL && count) { 917 --count; 918 --w->wa_count; 919 TAILQ_REMOVE(&w->wa_waitq, td, td_threadq); 920 td->td_wait = NULL; 921 td->td_wmesg = NULL; 922 if (td->td_gd == mycpu) { 923 _lwkt_enqueue(td); 924 } else { 925 lwkt_send_ipiq(td->td_gd->gd_cpuid, (ipifunc_t)lwkt_schedule, td); 926 } 927 lwkt_regettoken(&w->wa_token); 928 } 929 lwkt_reltoken(&w->wa_token); 930 } 931 932 #endif 933 934 /* 935 * Acquire ownership of a token 936 * 937 * Acquire ownership of a token. The token may have spl and/or critical 938 * section side effects, depending on its purpose. These side effects 939 * guarentee that you will maintain ownership of the token as long as you 940 * do not block. If you block you may lose access to the token (but you 941 * must still release it even if you lose your access to it). 942 * 943 * YYY for now we use a critical section to prevent IPIs from taking away 944 * a token, but do we really only need to disable IPIs ? 945 * 946 * YYY certain tokens could be made to act like mutexes when performance 947 * would be better (e.g. t_cpu == -1). This is not yet implemented. 948 * 949 * YYY the tokens replace 4.x's simplelocks for the most part, but this 950 * means that 4.x does not expect a switch so for now we cannot switch 951 * when waiting for an IPI to be returned. 952 * 953 * YYY If the token is owned by another cpu we may have to send an IPI to 954 * it and then block. The IPI causes the token to be given away to the 955 * requesting cpu, unless it has already changed hands. Since only the 956 * current cpu can give away a token it owns we do not need a memory barrier. 957 * This needs serious optimization. 958 */ 959 960 #ifdef SMP 961 962 static 963 void 964 lwkt_gettoken_remote(void *arg) 965 { 966 lwkt_gettoken_req *req = arg; 967 if (req->tok->t_cpu == mycpu->gd_cpuid) { 968 #ifdef INVARIANTS 969 if (token_debug) 970 printf("GT(%d,%d) ", req->tok->t_cpu, req->cpu); 971 #endif 972 req->tok->t_cpu = req->cpu; 973 req->tok->t_reqcpu = req->cpu; /* YYY leave owned by target cpu */ 974 /* else set reqcpu to point to current cpu for release */ 975 } 976 } 977 978 #endif 979 980 int 981 lwkt_gettoken(lwkt_token_t tok) 982 { 983 /* 984 * Prevent preemption so the token can't be taken away from us once 985 * we gain ownership of it. Use a synchronous request which might 986 * block. The request will be forwarded as necessary playing catchup 987 * to the token. 988 */ 989 990 crit_enter(); 991 #ifdef INVARIANTS 992 if (curthread->td_pri > 2000) { 993 curthread->td_pri = 1000; 994 panic("too HIGH!"); 995 } 996 #endif 997 #ifdef SMP 998 while (tok->t_cpu != mycpu->gd_cpuid) { 999 struct lwkt_gettoken_req req; 1000 int seq; 1001 int dcpu; 1002 1003 req.cpu = mycpu->gd_cpuid; 1004 req.tok = tok; 1005 dcpu = (volatile int)tok->t_cpu; 1006 KKASSERT(dcpu >= 0 && dcpu < ncpus); 1007 #ifdef INVARIANTS 1008 if (token_debug) 1009 printf("REQT%d ", dcpu); 1010 #endif 1011 seq = lwkt_send_ipiq(dcpu, lwkt_gettoken_remote, &req); 1012 lwkt_wait_ipiq(dcpu, seq); 1013 #ifdef INVARIANTS 1014 if (token_debug) 1015 printf("REQR%d ", tok->t_cpu); 1016 #endif 1017 } 1018 #endif 1019 /* 1020 * leave us in a critical section on return. This will be undone 1021 * by lwkt_reltoken(). Bump the generation number. 1022 */ 1023 return(++tok->t_gen); 1024 } 1025 1026 /* 1027 * Attempt to acquire ownership of a token. Returns 1 on success, 0 on 1028 * failure. 1029 */ 1030 int 1031 lwkt_trytoken(lwkt_token_t tok) 1032 { 1033 crit_enter(); 1034 #ifdef SMP 1035 if (tok->t_cpu != mycpu->gd_cpuid) { 1036 return(0); 1037 } 1038 #endif 1039 /* leave us in the critical section */ 1040 ++tok->t_gen; 1041 return(1); 1042 } 1043 1044 /* 1045 * Release your ownership of a token. Releases must occur in reverse 1046 * order to aquisitions, eventually so priorities can be unwound properly 1047 * like SPLs. At the moment the actual implemention doesn't care. 1048 * 1049 * We can safely hand a token that we own to another cpu without notifying 1050 * it, but once we do we can't get it back without requesting it (unless 1051 * the other cpu hands it back to us before we check). 1052 * 1053 * We might have lost the token, so check that. 1054 */ 1055 void 1056 lwkt_reltoken(lwkt_token_t tok) 1057 { 1058 if (tok->t_cpu == mycpu->gd_cpuid) { 1059 tok->t_cpu = tok->t_reqcpu; 1060 } 1061 crit_exit(); 1062 } 1063 1064 /* 1065 * Reacquire a token that might have been lost and compare and update the 1066 * generation number. 0 is returned if the generation has not changed 1067 * (nobody else obtained the token while we were blocked, on this cpu or 1068 * any other cpu). 1069 * 1070 * This function returns with the token re-held whether the generation 1071 * number changed or not. 1072 */ 1073 int 1074 lwkt_gentoken(lwkt_token_t tok, int *gen) 1075 { 1076 if (lwkt_regettoken(tok) == *gen) { 1077 return(0); 1078 } else { 1079 *gen = tok->t_gen; 1080 return(-1); 1081 } 1082 } 1083 1084 1085 /* 1086 * Re-acquire a token that might have been lost. Returns the generation 1087 * number of the token. 1088 */ 1089 int 1090 lwkt_regettoken(lwkt_token_t tok) 1091 { 1092 /* assert we are in a critical section */ 1093 if (tok->t_cpu != mycpu->gd_cpuid) { 1094 #ifdef SMP 1095 while (tok->t_cpu != mycpu->gd_cpuid) { 1096 struct lwkt_gettoken_req req; 1097 int seq; 1098 int dcpu; 1099 1100 req.cpu = mycpu->gd_cpuid; 1101 req.tok = tok; 1102 dcpu = (volatile int)tok->t_cpu; 1103 KKASSERT(dcpu >= 0 && dcpu < ncpus); 1104 #ifdef INVARIANTS 1105 if (token_debug) 1106 printf("REQT%d ", dcpu); 1107 #endif 1108 seq = lwkt_send_ipiq(dcpu, lwkt_gettoken_remote, &req); 1109 lwkt_wait_ipiq(dcpu, seq); 1110 #ifdef INVARIATNS 1111 if (token_debug) 1112 printf("REQR%d ", tok->t_cpu); 1113 #endif 1114 } 1115 #endif 1116 ++tok->t_gen; 1117 } 1118 return(tok->t_gen); 1119 } 1120 1121 void 1122 lwkt_inittoken(lwkt_token_t tok) 1123 { 1124 /* 1125 * Zero structure and set cpu owner and reqcpu to cpu 0. 1126 */ 1127 bzero(tok, sizeof(*tok)); 1128 } 1129 1130 /* 1131 * Create a kernel process/thread/whatever. It shares it's address space 1132 * with proc0 - ie: kernel only. 1133 * 1134 * XXX should be renamed to lwkt_create() 1135 * 1136 * The thread will be entered with the MP lock held. 1137 */ 1138 int 1139 lwkt_create(void (*func)(void *), void *arg, 1140 struct thread **tdp, thread_t template, int tdflags, 1141 const char *fmt, ...) 1142 { 1143 thread_t td; 1144 va_list ap; 1145 1146 td = lwkt_alloc_thread(template); 1147 if (tdp) 1148 *tdp = td; 1149 cpu_set_thread_handler(td, kthread_exit, func, arg); 1150 td->td_flags |= TDF_VERBOSE | tdflags; 1151 #ifdef SMP 1152 td->td_mpcount = 1; 1153 #endif 1154 1155 /* 1156 * Set up arg0 for 'ps' etc 1157 */ 1158 va_start(ap, fmt); 1159 vsnprintf(td->td_comm, sizeof(td->td_comm), fmt, ap); 1160 va_end(ap); 1161 1162 /* 1163 * Schedule the thread to run 1164 */ 1165 if ((td->td_flags & TDF_STOPREQ) == 0) 1166 lwkt_schedule(td); 1167 else 1168 td->td_flags &= ~TDF_STOPREQ; 1169 return 0; 1170 } 1171 1172 /* 1173 * Destroy an LWKT thread. Warning! This function is not called when 1174 * a process exits, cpu_proc_exit() directly calls cpu_thread_exit() and 1175 * uses a different reaping mechanism. 1176 */ 1177 void 1178 lwkt_exit(void) 1179 { 1180 thread_t td = curthread; 1181 1182 if (td->td_flags & TDF_VERBOSE) 1183 printf("kthread %p %s has exited\n", td, td->td_comm); 1184 crit_enter(); 1185 lwkt_deschedule_self(); 1186 ++mycpu->gd_tdfreecount; 1187 TAILQ_INSERT_TAIL(&mycpu->gd_tdfreeq, td, td_threadq); 1188 cpu_thread_exit(); 1189 } 1190 1191 /* 1192 * Create a kernel process/thread/whatever. It shares it's address space 1193 * with proc0 - ie: kernel only. 5.x compatible. 1194 */ 1195 int 1196 kthread_create(void (*func)(void *), void *arg, 1197 struct thread **tdp, const char *fmt, ...) 1198 { 1199 thread_t td; 1200 va_list ap; 1201 1202 td = lwkt_alloc_thread(NULL); 1203 if (tdp) 1204 *tdp = td; 1205 cpu_set_thread_handler(td, kthread_exit, func, arg); 1206 td->td_flags |= TDF_VERBOSE; 1207 #ifdef SMP 1208 td->td_mpcount = 1; 1209 #endif 1210 1211 /* 1212 * Set up arg0 for 'ps' etc 1213 */ 1214 va_start(ap, fmt); 1215 vsnprintf(td->td_comm, sizeof(td->td_comm), fmt, ap); 1216 va_end(ap); 1217 1218 /* 1219 * Schedule the thread to run 1220 */ 1221 lwkt_schedule(td); 1222 return 0; 1223 } 1224 1225 void 1226 crit_panic(void) 1227 { 1228 thread_t td = curthread; 1229 int lpri = td->td_pri; 1230 1231 td->td_pri = 0; 1232 panic("td_pri is/would-go negative! %p %d", td, lpri); 1233 } 1234 1235 /* 1236 * Destroy an LWKT thread. Warning! This function is not called when 1237 * a process exits, cpu_proc_exit() directly calls cpu_thread_exit() and 1238 * uses a different reaping mechanism. 1239 * 1240 * XXX duplicates lwkt_exit() 1241 */ 1242 void 1243 kthread_exit(void) 1244 { 1245 lwkt_exit(); 1246 } 1247 1248 #ifdef SMP 1249 1250 /* 1251 * Send a function execution request to another cpu. The request is queued 1252 * on the cpu<->cpu ipiq matrix. Each cpu owns a unique ipiq FIFO for every 1253 * possible target cpu. The FIFO can be written. 1254 * 1255 * YYY If the FIFO fills up we have to enable interrupts and process the 1256 * IPIQ while waiting for it to empty or we may deadlock with another cpu. 1257 * Create a CPU_*() function to do this! 1258 * 1259 * Must be called from a critical section. 1260 */ 1261 int 1262 lwkt_send_ipiq(int dcpu, ipifunc_t func, void *arg) 1263 { 1264 lwkt_ipiq_t ip; 1265 int windex; 1266 struct globaldata *gd = mycpu; 1267 1268 if (dcpu == gd->gd_cpuid) { 1269 func(arg); 1270 return(0); 1271 } 1272 crit_enter(); 1273 ++gd->gd_intr_nesting_level; 1274 #ifdef INVARIANTS 1275 if (gd->gd_intr_nesting_level > 20) 1276 panic("lwkt_send_ipiq: TOO HEAVILY NESTED!"); 1277 #endif 1278 KKASSERT(curthread->td_pri >= TDPRI_CRIT); 1279 KKASSERT(dcpu >= 0 && dcpu < ncpus); 1280 ++ipiq_count; 1281 ip = &gd->gd_ipiq[dcpu]; 1282 1283 /* 1284 * We always drain before the FIFO becomes full so it should never 1285 * become full. We need to leave enough entries to deal with 1286 * reentrancy. 1287 */ 1288 KKASSERT(ip->ip_windex - ip->ip_rindex != MAXCPUFIFO); 1289 windex = ip->ip_windex & MAXCPUFIFO_MASK; 1290 ip->ip_func[windex] = func; 1291 ip->ip_arg[windex] = arg; 1292 /* YYY memory barrier */ 1293 ++ip->ip_windex; 1294 if (ip->ip_windex - ip->ip_rindex > MAXCPUFIFO / 2) { 1295 unsigned int eflags = read_eflags(); 1296 cpu_enable_intr(); 1297 ++ipiq_fifofull; 1298 while (ip->ip_windex - ip->ip_rindex > MAXCPUFIFO / 4) { 1299 KKASSERT(ip->ip_windex - ip->ip_rindex != MAXCPUFIFO - 1); 1300 lwkt_process_ipiq(); 1301 } 1302 write_eflags(eflags); 1303 } 1304 --gd->gd_intr_nesting_level; 1305 cpu_send_ipiq(dcpu); /* issues memory barrier if appropriate */ 1306 crit_exit(); 1307 return(ip->ip_windex); 1308 } 1309 1310 /* 1311 * Send a message to several target cpus. Typically used for scheduling. 1312 */ 1313 void 1314 lwkt_send_ipiq_mask(u_int32_t mask, ipifunc_t func, void *arg) 1315 { 1316 int cpuid; 1317 1318 while (mask) { 1319 cpuid = bsfl(mask); 1320 lwkt_send_ipiq(cpuid, func, arg); 1321 mask &= ~(1 << cpuid); 1322 } 1323 } 1324 1325 /* 1326 * Wait for the remote cpu to finish processing a function. 1327 * 1328 * YYY we have to enable interrupts and process the IPIQ while waiting 1329 * for it to empty or we may deadlock with another cpu. Create a CPU_*() 1330 * function to do this! YYY we really should 'block' here. 1331 * 1332 * Must be called from a critical section. Thsi routine may be called 1333 * from an interrupt (for example, if an interrupt wakes a foreign thread 1334 * up). 1335 */ 1336 void 1337 lwkt_wait_ipiq(int dcpu, int seq) 1338 { 1339 lwkt_ipiq_t ip; 1340 int maxc = 100000000; 1341 1342 if (dcpu != mycpu->gd_cpuid) { 1343 KKASSERT(dcpu >= 0 && dcpu < ncpus); 1344 ip = &mycpu->gd_ipiq[dcpu]; 1345 if ((int)(ip->ip_xindex - seq) < 0) { 1346 unsigned int eflags = read_eflags(); 1347 cpu_enable_intr(); 1348 while ((int)(ip->ip_xindex - seq) < 0) { 1349 lwkt_process_ipiq(); 1350 if (--maxc == 0) 1351 printf("LWKT_WAIT_IPIQ WARNING! %d wait %d (%d)\n", mycpu->gd_cpuid, dcpu, ip->ip_xindex - seq); 1352 if (maxc < -1000000) 1353 panic("LWKT_WAIT_IPIQ"); 1354 } 1355 write_eflags(eflags); 1356 } 1357 } 1358 } 1359 1360 /* 1361 * Called from IPI interrupt (like a fast interrupt), which has placed 1362 * us in a critical section. The MP lock may or may not be held. 1363 * May also be called from doreti or splz, or be reentrantly called 1364 * indirectly through the ip_func[] we run. 1365 */ 1366 void 1367 lwkt_process_ipiq(void) 1368 { 1369 int n; 1370 int cpuid = mycpu->gd_cpuid; 1371 1372 for (n = 0; n < ncpus; ++n) { 1373 lwkt_ipiq_t ip; 1374 int ri; 1375 1376 if (n == cpuid) 1377 continue; 1378 ip = globaldata_find(n)->gd_ipiq; 1379 if (ip == NULL) 1380 continue; 1381 ip = &ip[cpuid]; 1382 1383 /* 1384 * Note: xindex is only updated after we are sure the function has 1385 * finished execution. Beware lwkt_process_ipiq() reentrancy! The 1386 * function may send an IPI which may block/drain. 1387 */ 1388 while (ip->ip_rindex != ip->ip_windex) { 1389 ri = ip->ip_rindex & MAXCPUFIFO_MASK; 1390 ++ip->ip_rindex; 1391 ip->ip_func[ri](ip->ip_arg[ri]); 1392 /* YYY memory barrier */ 1393 ip->ip_xindex = ip->ip_rindex; 1394 } 1395 } 1396 } 1397 1398 #else 1399 1400 int 1401 lwkt_send_ipiq(int dcpu, ipifunc_t func, void *arg) 1402 { 1403 panic("lwkt_send_ipiq: UP box! (%d,%p,%p)", dcpu, func, arg); 1404 return(0); /* NOT REACHED */ 1405 } 1406 1407 void 1408 lwkt_wait_ipiq(int dcpu, int seq) 1409 { 1410 panic("lwkt_wait_ipiq: UP box! (%d,%d)", dcpu, seq); 1411 } 1412 1413 #endif 1414