1 /* 2 * PROJECT: ReactOS Kernel 3 * LICENSE: GPL - See COPYING in the top level directory 4 * FILE: ntoskrnl/include/ke_x.h 5 * PURPOSE: Internal Inlined Functions for the Kernel 6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org) 7 */ 8 9 #ifndef _M_ARM 10 FORCEINLINE 11 KPROCESSOR_MODE 12 KeGetPreviousMode(VOID) 13 { 14 /* Return the current mode */ 15 return KeGetCurrentThread()->PreviousMode; 16 } 17 #endif 18 19 // 20 // Enters a Guarded Region 21 // 22 #define KeEnterGuardedRegion() \ 23 { \ 24 PKTHREAD _Thread = KeGetCurrentThread(); \ 25 \ 26 /* Sanity checks */ \ 27 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \ 28 ASSERT(_Thread == KeGetCurrentThread()); \ 29 ASSERT((_Thread->SpecialApcDisable <= 0) && \ 30 (_Thread->SpecialApcDisable != -32768)); \ 31 \ 32 /* Disable Special APCs */ \ 33 _Thread->SpecialApcDisable--; \ 34 } 35 36 // 37 // Leaves a Guarded Region 38 // 39 #define KeLeaveGuardedRegion() \ 40 { \ 41 PKTHREAD _Thread = KeGetCurrentThread(); \ 42 \ 43 /* Sanity checks */ \ 44 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \ 45 ASSERT(_Thread == KeGetCurrentThread()); \ 46 ASSERT(_Thread->SpecialApcDisable < 0); \ 47 \ 48 /* Leave region and check if APCs are OK now */ \ 49 if (!(++_Thread->SpecialApcDisable)) \ 50 { \ 51 /* Check for Kernel APCs on the list */ \ 52 if (!IsListEmpty(&_Thread->ApcState. \ 53 ApcListHead[KernelMode])) \ 54 { \ 55 /* Check for APC Delivery */ \ 56 KiCheckForKernelApcDelivery(); \ 57 } \ 58 } \ 59 } 60 61 // 62 // Enters a Critical Region 63 // 64 #define KeEnterCriticalRegion() \ 65 { \ 66 PKTHREAD _Thread = KeGetCurrentThread(); \ 67 \ 68 /* Sanity checks */ \ 69 ASSERT(_Thread == KeGetCurrentThread()); \ 70 ASSERT((_Thread->KernelApcDisable <= 0) && \ 71 (_Thread->KernelApcDisable != -32768)); \ 72 \ 73 /* Disable Kernel APCs */ \ 74 _Thread->KernelApcDisable--; \ 75 } 76 77 // 78 // Leaves a Critical Region 79 // 80 #define KeLeaveCriticalRegion() \ 81 { \ 82 PKTHREAD _Thread = KeGetCurrentThread(); \ 83 \ 84 /* Sanity checks */ \ 85 ASSERT(_Thread == KeGetCurrentThread()); \ 86 ASSERT(_Thread->KernelApcDisable < 0); \ 87 \ 88 /* Enable Kernel APCs */ \ 89 _Thread->KernelApcDisable++; \ 90 \ 91 /* Check if Kernel APCs are now enabled */ \ 92 if (!(_Thread->KernelApcDisable)) \ 93 { \ 94 /* Check if we need to request an APC Delivery */ \ 95 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \ 96 !(_Thread->SpecialApcDisable)) \ 97 { \ 98 /* Check for the right environment */ \ 99 KiCheckForKernelApcDelivery(); \ 100 } \ 101 } \ 102 } 103 104 #ifndef CONFIG_SMP 105 106 // 107 // This routine protects against multiple CPU acquires, it's meaningless on UP. 108 // 109 FORCEINLINE 110 VOID 111 KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object) 112 { 113 UNREFERENCED_PARAMETER(Object); 114 } 115 116 // 117 // This routine protects against multiple CPU acquires, it's meaningless on UP. 118 // 119 FORCEINLINE 120 VOID 121 KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object) 122 { 123 UNREFERENCED_PARAMETER(Object); 124 } 125 126 FORCEINLINE 127 KIRQL 128 KiAcquireDispatcherLock(VOID) 129 { 130 /* Raise to DPC level */ 131 return KeRaiseIrqlToDpcLevel(); 132 } 133 134 FORCEINLINE 135 VOID 136 KiReleaseDispatcherLock(IN KIRQL OldIrql) 137 { 138 /* Just exit the dispatcher */ 139 KiExitDispatcher(OldIrql); 140 } 141 142 FORCEINLINE 143 VOID 144 KiAcquireDispatcherLockAtDpcLevel(VOID) 145 { 146 /* This is a no-op at DPC Level for UP systems */ 147 return; 148 } 149 150 FORCEINLINE 151 VOID 152 KiReleaseDispatcherLockFromDpcLevel(VOID) 153 { 154 /* This is a no-op at DPC Level for UP systems */ 155 return; 156 } 157 158 // 159 // This routine makes the thread deferred ready on the boot CPU. 160 // 161 FORCEINLINE 162 VOID 163 KiInsertDeferredReadyList(IN PKTHREAD Thread) 164 { 165 /* Set the thread to deferred state and boot CPU */ 166 Thread->State = DeferredReady; 167 Thread->DeferredProcessor = 0; 168 169 /* Make the thread ready immediately */ 170 KiDeferredReadyThread(Thread); 171 } 172 173 FORCEINLINE 174 VOID 175 KiRescheduleThread(IN BOOLEAN NewThread, 176 IN ULONG Cpu) 177 { 178 /* This is meaningless on UP systems */ 179 UNREFERENCED_PARAMETER(NewThread); 180 UNREFERENCED_PARAMETER(Cpu); 181 } 182 183 // 184 // This routine protects against multiple CPU acquires, it's meaningless on UP. 185 // 186 FORCEINLINE 187 VOID 188 KiSetThreadSwapBusy(IN PKTHREAD Thread) 189 { 190 UNREFERENCED_PARAMETER(Thread); 191 } 192 193 // 194 // This routine protects against multiple CPU acquires, it's meaningless on UP. 195 // 196 FORCEINLINE 197 VOID 198 KiAcquirePrcbLock(IN PKPRCB Prcb) 199 { 200 UNREFERENCED_PARAMETER(Prcb); 201 } 202 203 // 204 // This routine protects against multiple CPU acquires, it's meaningless on UP. 205 // 206 FORCEINLINE 207 VOID 208 KiReleasePrcbLock(IN PKPRCB Prcb) 209 { 210 UNREFERENCED_PARAMETER(Prcb); 211 } 212 213 // 214 // This routine protects against multiple CPU acquires, it's meaningless on UP. 215 // 216 FORCEINLINE 217 VOID 218 KiAcquireThreadLock(IN PKTHREAD Thread) 219 { 220 UNREFERENCED_PARAMETER(Thread); 221 } 222 223 // 224 // This routine protects against multiple CPU acquires, it's meaningless on UP. 225 // 226 FORCEINLINE 227 VOID 228 KiReleaseThreadLock(IN PKTHREAD Thread) 229 { 230 UNREFERENCED_PARAMETER(Thread); 231 } 232 233 // 234 // This routine protects against multiple CPU acquires, it's meaningless on UP. 235 // 236 FORCEINLINE 237 BOOLEAN 238 KiTryThreadLock(IN PKTHREAD Thread) 239 { 240 UNREFERENCED_PARAMETER(Thread); 241 return FALSE; 242 } 243 244 FORCEINLINE 245 VOID 246 KiCheckDeferredReadyList(IN PKPRCB Prcb) 247 { 248 /* There are no deferred ready lists on UP systems */ 249 UNREFERENCED_PARAMETER(Prcb); 250 } 251 252 FORCEINLINE 253 VOID 254 KiRequestApcInterrupt(IN BOOLEAN NeedApc, 255 IN UCHAR Processor) 256 { 257 /* We deliver instantly on UP */ 258 UNREFERENCED_PARAMETER(NeedApc); 259 UNREFERENCED_PARAMETER(Processor); 260 } 261 262 FORCEINLINE 263 PKSPIN_LOCK_QUEUE 264 KiAcquireTimerLock(IN ULONG Hand) 265 { 266 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 267 268 /* Nothing to do on UP */ 269 UNREFERENCED_PARAMETER(Hand); 270 return NULL; 271 } 272 273 FORCEINLINE 274 VOID 275 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue) 276 { 277 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 278 279 /* Nothing to do on UP */ 280 UNREFERENCED_PARAMETER(LockQueue); 281 } 282 283 #else 284 285 FORCEINLINE 286 VOID 287 KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object) 288 { 289 LONG OldValue; 290 291 /* Make sure we're at a safe level to touch the lock */ 292 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 293 294 /* Start acquire loop */ 295 do 296 { 297 /* Loop until the other CPU releases it */ 298 while (TRUE) 299 { 300 /* Check if it got released */ 301 OldValue = Object->Lock; 302 if ((OldValue & KOBJECT_LOCK_BIT) == 0) break; 303 304 /* Let the CPU know that this is a loop */ 305 YieldProcessor(); 306 } 307 308 /* Try acquiring the lock now */ 309 } while (InterlockedCompareExchange(&Object->Lock, 310 OldValue | KOBJECT_LOCK_BIT, 311 OldValue) != OldValue); 312 } 313 314 FORCEINLINE 315 VOID 316 KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object) 317 { 318 /* Make sure we're at a safe level to touch the lock */ 319 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 320 321 /* Release it */ 322 InterlockedAnd(&Object->Lock, ~KOBJECT_LOCK_BIT); 323 } 324 325 FORCEINLINE 326 KIRQL 327 KiAcquireDispatcherLock(VOID) 328 { 329 /* Raise to synchronization level and acquire the dispatcher lock */ 330 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock); 331 } 332 333 FORCEINLINE 334 VOID 335 KiReleaseDispatcherLock(IN KIRQL OldIrql) 336 { 337 /* First release the lock */ 338 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()-> 339 LockQueue[LockQueueDispatcherLock]); 340 341 /* Then exit the dispatcher */ 342 KiExitDispatcher(OldIrql); 343 } 344 345 FORCEINLINE 346 VOID 347 KiAcquireDispatcherLockAtDpcLevel(VOID) 348 { 349 /* Acquire the dispatcher lock */ 350 KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()-> 351 LockQueue[LockQueueDispatcherLock]); 352 } 353 354 FORCEINLINE 355 VOID 356 KiReleaseDispatcherLockFromDpcLevel(VOID) 357 { 358 /* Release the dispatcher lock */ 359 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()-> 360 LockQueue[LockQueueDispatcherLock]); 361 } 362 363 // 364 // This routine inserts a thread into the deferred ready list of the current CPU 365 // 366 FORCEINLINE 367 VOID 368 KiInsertDeferredReadyList(IN PKTHREAD Thread) 369 { 370 PKPRCB Prcb = KeGetCurrentPrcb(); 371 372 /* Set the thread to deferred state and CPU */ 373 Thread->State = DeferredReady; 374 Thread->DeferredProcessor = Prcb->Number; 375 376 /* Add it on the list */ 377 PushEntryList(&Prcb->DeferredReadyListHead, &Thread->SwapListEntry); 378 } 379 380 FORCEINLINE 381 VOID 382 KiRescheduleThread(IN BOOLEAN NewThread, 383 IN ULONG Cpu) 384 { 385 /* Check if a new thread needs to be scheduled on a different CPU */ 386 if ((NewThread) && !(KeGetPcr()->Number == Cpu)) 387 { 388 /* Send an IPI to request delivery */ 389 KiIpiSend(AFFINITY_MASK(Cpu), IPI_DPC); 390 } 391 } 392 393 // 394 // This routine sets the current thread in a swap busy state, which ensure that 395 // nobody else tries to swap it concurrently. 396 // 397 FORCEINLINE 398 VOID 399 KiSetThreadSwapBusy(IN PKTHREAD Thread) 400 { 401 /* Make sure nobody already set it */ 402 ASSERT(Thread->SwapBusy == FALSE); 403 404 /* Set it ourselves */ 405 Thread->SwapBusy = TRUE; 406 } 407 408 // 409 // This routine acquires the PRCB lock so that only one caller can touch 410 // volatile PRCB data. 411 // 412 // Since this is a simple optimized spin-lock, it must only be acquired 413 // at dispatcher level or higher! 414 // 415 FORCEINLINE 416 VOID 417 KiAcquirePrcbLock(IN PKPRCB Prcb) 418 { 419 /* Make sure we're at a safe level to touch the PRCB lock */ 420 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 421 422 /* Start acquire loop */ 423 for (;;) 424 { 425 /* Acquire the lock and break out if we acquired it first */ 426 if (!InterlockedExchange((PLONG)&Prcb->PrcbLock, 1)) break; 427 428 /* Loop until the other CPU releases it */ 429 do 430 { 431 /* Let the CPU know that this is a loop */ 432 YieldProcessor(); 433 } while (Prcb->PrcbLock); 434 } 435 } 436 437 // 438 // This routine releases the PRCB lock so that other callers can touch 439 // volatile PRCB data. 440 // 441 // Since this is a simple optimized spin-lock, it must be be only acquired 442 // at dispatcher level or higher! 443 // 444 FORCEINLINE 445 VOID 446 KiReleasePrcbLock(IN PKPRCB Prcb) 447 { 448 /* Make sure we are above dispatch and the lock is acquired! */ 449 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 450 ASSERT(Prcb->PrcbLock != 0); 451 452 /* Release it */ 453 InterlockedAnd((PLONG)&Prcb->PrcbLock, 0); 454 } 455 456 // 457 // This routine acquires the thread lock so that only one caller can touch 458 // volatile thread data. 459 // 460 // Since this is a simple optimized spin-lock, it must be be only acquired 461 // at dispatcher level or higher! 462 // 463 FORCEINLINE 464 VOID 465 KiAcquireThreadLock(IN PKTHREAD Thread) 466 { 467 /* Make sure we're at a safe level to touch the thread lock */ 468 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 469 470 /* Start acquire loop */ 471 for (;;) 472 { 473 /* Acquire the lock and break out if we acquired it first */ 474 if (!InterlockedExchange((PLONG)&Thread->ThreadLock, 1)) break; 475 476 /* Loop until the other CPU releases it */ 477 do 478 { 479 /* Let the CPU know that this is a loop */ 480 YieldProcessor(); 481 } while (Thread->ThreadLock); 482 } 483 } 484 485 // 486 // This routine releases the thread lock so that other callers can touch 487 // volatile thread data. 488 // 489 // Since this is a simple optimized spin-lock, it must be be only acquired 490 // at dispatcher level or higher! 491 // 492 FORCEINLINE 493 VOID 494 KiReleaseThreadLock(IN PKTHREAD Thread) 495 { 496 /* Make sure we are still above dispatch */ 497 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 498 499 /* Release it */ 500 InterlockedAnd((PLONG)&Thread->ThreadLock, 0); 501 } 502 503 FORCEINLINE 504 BOOLEAN 505 KiTryThreadLock(IN PKTHREAD Thread) 506 { 507 LONG Value; 508 509 /* If the lock isn't acquired, return false */ 510 if (!Thread->ThreadLock) return FALSE; 511 512 /* Otherwise, try to acquire it and check the result */ 513 Value = 1; 514 Value = InterlockedExchange((PLONG)&Thread->ThreadLock, Value); 515 516 /* Return the lock state */ 517 return (Value == TRUE); 518 } 519 520 FORCEINLINE 521 VOID 522 KiCheckDeferredReadyList(IN PKPRCB Prcb) 523 { 524 /* Scan the deferred ready lists if required */ 525 if (Prcb->DeferredReadyListHead.Next) KiProcessDeferredReadyList(Prcb); 526 } 527 528 FORCEINLINE 529 VOID 530 KiRequestApcInterrupt(IN BOOLEAN NeedApc, 531 IN UCHAR Processor) 532 { 533 /* Check if we need to request APC delivery */ 534 if (NeedApc) 535 { 536 /* Check if it's on another CPU */ 537 if (KeGetPcr()->Number != Processor) 538 { 539 /* Send an IPI to request delivery */ 540 KiIpiSend(AFFINITY_MASK(Processor), IPI_APC); 541 } 542 else 543 { 544 /* Request a software interrupt */ 545 HalRequestSoftwareInterrupt(APC_LEVEL); 546 } 547 } 548 } 549 550 FORCEINLINE 551 PKSPIN_LOCK_QUEUE 552 KiAcquireTimerLock(IN ULONG Hand) 553 { 554 PKSPIN_LOCK_QUEUE LockQueue; 555 ULONG LockIndex; 556 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 557 558 /* Get the lock index */ 559 LockIndex = Hand >> LOCK_QUEUE_TIMER_LOCK_SHIFT; 560 LockIndex &= (LOCK_QUEUE_TIMER_TABLE_LOCKS - 1); 561 562 /* Now get the lock */ 563 LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueueTimerTableLock + LockIndex]; 564 565 /* Acquire it and return */ 566 KeAcquireQueuedSpinLockAtDpcLevel(LockQueue); 567 return LockQueue; 568 } 569 570 FORCEINLINE 571 VOID 572 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue) 573 { 574 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 575 576 /* Release the lock */ 577 KeReleaseQueuedSpinLockFromDpcLevel(LockQueue); 578 } 579 580 #endif 581 582 FORCEINLINE 583 VOID 584 KiAcquireApcLock(IN PKTHREAD Thread, 585 IN PKLOCK_QUEUE_HANDLE Handle) 586 { 587 /* Acquire the lock and raise to synchronization level */ 588 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread->ApcQueueLock, Handle); 589 } 590 591 FORCEINLINE 592 VOID 593 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread, 594 IN PKLOCK_QUEUE_HANDLE Handle) 595 { 596 /* Acquire the lock */ 597 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread->ApcQueueLock, Handle); 598 } 599 600 FORCEINLINE 601 VOID 602 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread, 603 IN PKLOCK_QUEUE_HANDLE Handle) 604 { 605 /* Acquire the lock */ 606 KeAcquireInStackQueuedSpinLock(&Thread->ApcQueueLock, Handle); 607 } 608 609 FORCEINLINE 610 VOID 611 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle) 612 { 613 /* Release the lock */ 614 KeReleaseInStackQueuedSpinLock(Handle); 615 } 616 617 FORCEINLINE 618 VOID 619 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle) 620 { 621 /* Release the lock */ 622 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle); 623 } 624 625 FORCEINLINE 626 VOID 627 KiAcquireProcessLock(IN PKPROCESS Process, 628 IN PKLOCK_QUEUE_HANDLE Handle) 629 { 630 /* Acquire the lock and raise to synchronization level */ 631 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process->ProcessLock, Handle); 632 } 633 634 FORCEINLINE 635 VOID 636 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle) 637 { 638 /* Release the lock */ 639 KeReleaseInStackQueuedSpinLock(Handle); 640 } 641 642 FORCEINLINE 643 VOID 644 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle) 645 { 646 /* Release the lock */ 647 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle); 648 } 649 650 FORCEINLINE 651 VOID 652 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue, 653 IN PKLOCK_QUEUE_HANDLE DeviceLock) 654 { 655 /* Check if we were called from a threaded DPC */ 656 if (KeGetCurrentPrcb()->DpcThreadActive) 657 { 658 /* Lock the Queue, we're not at DPC level */ 659 KeAcquireInStackQueuedSpinLock(&DeviceQueue->Lock, DeviceLock); 660 } 661 else 662 { 663 /* We must be at DPC level, acquire the lock safely */ 664 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); 665 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue->Lock, 666 DeviceLock); 667 } 668 } 669 670 FORCEINLINE 671 VOID 672 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock) 673 { 674 /* Check if we were called from a threaded DPC */ 675 if (KeGetCurrentPrcb()->DpcThreadActive) 676 { 677 /* Unlock the Queue, we're not at DPC level */ 678 KeReleaseInStackQueuedSpinLock(DeviceLock); 679 } 680 else 681 { 682 /* We must be at DPC level, release the lock safely */ 683 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); 684 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock); 685 } 686 } 687 688 // 689 // Satisfies the wait of any dispatcher object 690 // 691 #define KiSatisfyObjectWait(Object, Thread) \ 692 { \ 693 /* Special case for Mutants */ \ 694 if ((Object)->Header.Type == MutantObject) \ 695 { \ 696 /* Decrease the Signal State */ \ 697 (Object)->Header.SignalState--; \ 698 \ 699 /* Check if it's now non-signaled */ \ 700 if (!(Object)->Header.SignalState) \ 701 { \ 702 /* Set the Owner Thread */ \ 703 (Object)->OwnerThread = Thread; \ 704 \ 705 /* Disable APCs if needed */ \ 706 Thread->KernelApcDisable = Thread->KernelApcDisable - \ 707 (Object)->ApcDisable; \ 708 \ 709 /* Check if it's abandoned */ \ 710 if ((Object)->Abandoned) \ 711 { \ 712 /* Unabandon it */ \ 713 (Object)->Abandoned = FALSE; \ 714 \ 715 /* Return Status */ \ 716 Thread->WaitStatus = STATUS_ABANDONED; \ 717 } \ 718 \ 719 /* Insert it into the Mutant List */ \ 720 InsertHeadList(Thread->MutantListHead.Blink, \ 721 &(Object)->MutantListEntry); \ 722 } \ 723 } \ 724 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \ 725 EventSynchronizationObject) \ 726 { \ 727 /* Synchronization Timers and Events just get un-signaled */ \ 728 (Object)->Header.SignalState = 0; \ 729 } \ 730 else if ((Object)->Header.Type == SemaphoreObject) \ 731 { \ 732 /* These ones can have multiple states, so we only decrease it */ \ 733 (Object)->Header.SignalState--; \ 734 } \ 735 } 736 737 // 738 // Satisfies the wait of a mutant dispatcher object 739 // 740 #define KiSatisfyMutantWait(Object, Thread) \ 741 { \ 742 /* Decrease the Signal State */ \ 743 (Object)->Header.SignalState--; \ 744 \ 745 /* Check if it's now non-signaled */ \ 746 if (!(Object)->Header.SignalState) \ 747 { \ 748 /* Set the Owner Thread */ \ 749 (Object)->OwnerThread = Thread; \ 750 \ 751 /* Disable APCs if needed */ \ 752 Thread->KernelApcDisable = Thread->KernelApcDisable - \ 753 (Object)->ApcDisable; \ 754 \ 755 /* Check if it's abandoned */ \ 756 if ((Object)->Abandoned) \ 757 { \ 758 /* Unabandon it */ \ 759 (Object)->Abandoned = FALSE; \ 760 \ 761 /* Return Status */ \ 762 Thread->WaitStatus = STATUS_ABANDONED; \ 763 } \ 764 \ 765 /* Insert it into the Mutant List */ \ 766 InsertHeadList(Thread->MutantListHead.Blink, \ 767 &(Object)->MutantListEntry); \ 768 } \ 769 } 770 771 // 772 // Satisfies the wait of any nonmutant dispatcher object 773 // 774 #define KiSatisfyNonMutantWait(Object) \ 775 { \ 776 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \ 777 EventSynchronizationObject) \ 778 { \ 779 /* Synchronization Timers and Events just get un-signaled */ \ 780 (Object)->Header.SignalState = 0; \ 781 } \ 782 else if ((Object)->Header.Type == SemaphoreObject) \ 783 { \ 784 /* These ones can have multiple states, so we only decrease it */ \ 785 (Object)->Header.SignalState--; \ 786 } \ 787 } 788 789 // 790 // Recalculates the due time 791 // 792 FORCEINLINE 793 PLARGE_INTEGER 794 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime, 795 IN PLARGE_INTEGER DueTime, 796 IN OUT PLARGE_INTEGER NewDueTime) 797 { 798 /* Don't do anything for absolute waits */ 799 if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime; 800 801 /* Otherwise, query the interrupt time and recalculate */ 802 NewDueTime->QuadPart = KeQueryInterruptTime(); 803 NewDueTime->QuadPart -= DueTime->QuadPart; 804 return NewDueTime; 805 } 806 807 // 808 // Determines whether a thread should be added to the wait list 809 // 810 FORCEINLINE 811 BOOLEAN 812 KiCheckThreadStackSwap(IN PKTHREAD Thread, 813 IN KPROCESSOR_MODE WaitMode) 814 { 815 /* Check the required conditions */ 816 if ((WaitMode != KernelMode) && 817 (Thread->EnableStackSwap) && 818 (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9))) 819 { 820 /* We are go for swap */ 821 return TRUE; 822 } 823 else 824 { 825 /* Don't swap the thread */ 826 return FALSE; 827 } 828 } 829 830 // 831 // Adds a thread to the wait list 832 // 833 #define KiAddThreadToWaitList(Thread, Swappable) \ 834 { \ 835 /* Make sure it's swappable */ \ 836 if (Swappable) \ 837 { \ 838 /* Insert it into the PRCB's List */ \ 839 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \ 840 &Thread->WaitListEntry); \ 841 } \ 842 } 843 844 // 845 // Checks if a wait in progress should be interrupted by APCs or an alertable 846 // state. 847 // 848 FORCEINLINE 849 NTSTATUS 850 KiCheckAlertability(IN PKTHREAD Thread, 851 IN BOOLEAN Alertable, 852 IN KPROCESSOR_MODE WaitMode) 853 { 854 /* Check if the wait is alertable */ 855 if (Alertable) 856 { 857 /* It is, first check if the thread is alerted in this mode */ 858 if (Thread->Alerted[WaitMode]) 859 { 860 /* It is, so bail out of the wait */ 861 Thread->Alerted[WaitMode] = FALSE; 862 return STATUS_ALERTED; 863 } 864 else if ((WaitMode != KernelMode) && 865 (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode]))) 866 { 867 /* It's isn't, but this is a user wait with queued user APCs */ 868 Thread->ApcState.UserApcPending = TRUE; 869 return STATUS_USER_APC; 870 } 871 else if (Thread->Alerted[KernelMode]) 872 { 873 /* It isn't that either, but we're alered in kernel mode */ 874 Thread->Alerted[KernelMode] = FALSE; 875 return STATUS_ALERTED; 876 } 877 } 878 else if ((WaitMode != KernelMode) && (Thread->ApcState.UserApcPending)) 879 { 880 /* Not alertable, but this is a user wait with pending user APCs */ 881 return STATUS_USER_APC; 882 } 883 884 /* Otherwise, we're fine */ 885 return STATUS_WAIT_0; 886 } 887 888 ULONG 889 FORCEINLINE 890 KiComputeTimerTableIndex(IN ULONGLONG DueTime) 891 { 892 return (DueTime / KeMaximumIncrement) & (TIMER_TABLE_SIZE - 1); 893 } 894 895 // 896 // Called from KiCompleteTimer, KiInsertTreeTimer, KeSetSystemTime 897 // to remove timer entries 898 // See Windows HPI blog for more information. 899 FORCEINLINE 900 VOID 901 KiRemoveEntryTimer(IN PKTIMER Timer) 902 { 903 ULONG Hand; 904 PKTIMER_TABLE_ENTRY TableEntry; 905 906 /* Remove the timer from the timer list and check if it's empty */ 907 Hand = Timer->Header.Hand; 908 if (RemoveEntryList(&Timer->TimerListEntry)) 909 { 910 /* Get the respective timer table entry */ 911 TableEntry = &KiTimerTableListHead[Hand]; 912 if (&TableEntry->Entry == TableEntry->Entry.Flink) 913 { 914 /* Set the entry to an infinite absolute time */ 915 TableEntry->Time.HighPart = 0xFFFFFFFF; 916 } 917 } 918 919 /* Clear the list entries on dbg builds so we can tell the timer is gone */ 920 #if DBG 921 Timer->TimerListEntry.Flink = NULL; 922 Timer->TimerListEntry.Blink = NULL; 923 #endif 924 } 925 926 // 927 // Called by Wait and Queue code to insert a timer for dispatching. 928 // Also called by KeSetTimerEx to insert a timer from the caller. 929 // 930 FORCEINLINE 931 VOID 932 KxInsertTimer(IN PKTIMER Timer, 933 IN ULONG Hand) 934 { 935 PKSPIN_LOCK_QUEUE LockQueue; 936 937 /* Acquire the lock and release the dispatcher lock */ 938 LockQueue = KiAcquireTimerLock(Hand); 939 KiReleaseDispatcherLockFromDpcLevel(); 940 941 /* Try to insert the timer */ 942 if (KiInsertTimerTable(Timer, Hand)) 943 { 944 /* Complete it */ 945 KiCompleteTimer(Timer, LockQueue); 946 } 947 else 948 { 949 /* Do nothing, just release the lock */ 950 KiReleaseTimerLock(LockQueue); 951 } 952 } 953 954 // 955 // Called by KeSetTimerEx and KiInsertTreeTimer to calculate Due Time 956 // See the Windows HPI Blog for more information 957 // 958 FORCEINLINE 959 BOOLEAN 960 KiComputeDueTime(IN PKTIMER Timer, 961 IN LARGE_INTEGER DueTime, 962 OUT PULONG Hand) 963 { 964 LARGE_INTEGER InterruptTime, SystemTime, DifferenceTime; 965 966 /* Convert to relative time if needed */ 967 Timer->Header.Absolute = FALSE; 968 if (DueTime.HighPart >= 0) 969 { 970 /* Get System Time */ 971 KeQuerySystemTime(&SystemTime); 972 973 /* Do the conversion */ 974 DifferenceTime.QuadPart = SystemTime.QuadPart - DueTime.QuadPart; 975 976 /* Make sure it hasn't already expired */ 977 Timer->Header.Absolute = TRUE; 978 if (DifferenceTime.HighPart >= 0) 979 { 980 /* Cancel everything */ 981 Timer->Header.SignalState = TRUE; 982 Timer->Header.Hand = 0; 983 Timer->DueTime.QuadPart = 0; 984 *Hand = 0; 985 return FALSE; 986 } 987 988 /* Set the time as Absolute */ 989 DueTime = DifferenceTime; 990 } 991 992 /* Get the Interrupt Time */ 993 InterruptTime.QuadPart = KeQueryInterruptTime(); 994 995 /* Recalculate due time */ 996 Timer->DueTime.QuadPart = InterruptTime.QuadPart - DueTime.QuadPart; 997 998 /* Get the handle */ 999 *Hand = KiComputeTimerTableIndex(Timer->DueTime.QuadPart); 1000 Timer->Header.Hand = (UCHAR)*Hand; 1001 Timer->Header.Inserted = TRUE; 1002 return TRUE; 1003 } 1004 1005 // 1006 // Called from Unlink and Queue Insert Code. 1007 // Also called by timer code when canceling an inserted timer. 1008 // Removes a timer from it's tree. 1009 // 1010 FORCEINLINE 1011 VOID 1012 KxRemoveTreeTimer(IN PKTIMER Timer) 1013 { 1014 ULONG Hand = Timer->Header.Hand; 1015 PKSPIN_LOCK_QUEUE LockQueue; 1016 PKTIMER_TABLE_ENTRY TimerEntry; 1017 1018 /* Acquire timer lock */ 1019 LockQueue = KiAcquireTimerLock(Hand); 1020 1021 /* Set the timer as non-inserted */ 1022 Timer->Header.Inserted = FALSE; 1023 1024 /* Remove it from the timer list */ 1025 if (RemoveEntryList(&Timer->TimerListEntry)) 1026 { 1027 /* Get the entry and check if it's empty */ 1028 TimerEntry = &KiTimerTableListHead[Hand]; 1029 if (IsListEmpty(&TimerEntry->Entry)) 1030 { 1031 /* Clear the time then */ 1032 TimerEntry->Time.HighPart = 0xFFFFFFFF; 1033 } 1034 } 1035 1036 /* Release the timer lock */ 1037 KiReleaseTimerLock(LockQueue); 1038 } 1039 1040 FORCEINLINE 1041 VOID 1042 KxSetTimerForThreadWait(IN PKTIMER Timer, 1043 IN LARGE_INTEGER Interval, 1044 OUT PULONG Hand) 1045 { 1046 ULONGLONG DueTime; 1047 LARGE_INTEGER InterruptTime, SystemTime, TimeDifference; 1048 1049 /* Check the timer's interval to see if it's absolute */ 1050 Timer->Header.Absolute = FALSE; 1051 if (Interval.HighPart >= 0) 1052 { 1053 /* Get the system time and calculate the relative time */ 1054 KeQuerySystemTime(&SystemTime); 1055 TimeDifference.QuadPart = SystemTime.QuadPart - Interval.QuadPart; 1056 Timer->Header.Absolute = TRUE; 1057 1058 /* Check if we've already expired */ 1059 if (TimeDifference.HighPart >= 0) 1060 { 1061 /* Reset everything */ 1062 Timer->DueTime.QuadPart = 0; 1063 *Hand = 0; 1064 Timer->Header.Hand = 0; 1065 return; 1066 } 1067 else 1068 { 1069 /* Update the interval */ 1070 Interval = TimeDifference; 1071 } 1072 } 1073 1074 /* Calculate the due time */ 1075 InterruptTime.QuadPart = KeQueryInterruptTime(); 1076 DueTime = InterruptTime.QuadPart - Interval.QuadPart; 1077 Timer->DueTime.QuadPart = DueTime; 1078 1079 /* Calculate the timer handle */ 1080 *Hand = KiComputeTimerTableIndex(DueTime); 1081 Timer->Header.Hand = (UCHAR)*Hand; 1082 } 1083 1084 #define KxDelayThreadWait() \ 1085 \ 1086 /* Setup the Wait Block */ \ 1087 Thread->WaitBlockList = TimerBlock; \ 1088 \ 1089 /* Setup the timer */ \ 1090 KxSetTimerForThreadWait(Timer, *Interval, &Hand); \ 1091 \ 1092 /* Save the due time for the caller */ \ 1093 DueTime.QuadPart = Timer->DueTime.QuadPart; \ 1094 \ 1095 /* Link the timer to this Wait Block */ \ 1096 TimerBlock->NextWaitBlock = TimerBlock; \ 1097 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \ 1098 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \ 1099 \ 1100 /* Clear wait status */ \ 1101 Thread->WaitStatus = STATUS_SUCCESS; \ 1102 \ 1103 /* Setup wait fields */ \ 1104 Thread->Alertable = Alertable; \ 1105 Thread->WaitReason = DelayExecution; \ 1106 Thread->WaitMode = WaitMode; \ 1107 \ 1108 /* Check if we can swap the thread's stack */ \ 1109 Thread->WaitListEntry.Flink = NULL; \ 1110 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ 1111 \ 1112 /* Set the wait time */ \ 1113 Thread->WaitTime = KeTickCount.LowPart; 1114 1115 #define KxMultiThreadWait() \ 1116 /* Link wait block array to the thread */ \ 1117 Thread->WaitBlockList = WaitBlockArray; \ 1118 \ 1119 /* Reset the index */ \ 1120 Index = 0; \ 1121 \ 1122 /* Loop wait blocks */ \ 1123 do \ 1124 { \ 1125 /* Fill out the wait block */ \ 1126 WaitBlock = &WaitBlockArray[Index]; \ 1127 WaitBlock->Object = Object[Index]; \ 1128 WaitBlock->WaitKey = (USHORT)Index; \ 1129 WaitBlock->WaitType = WaitType; \ 1130 WaitBlock->Thread = Thread; \ 1131 \ 1132 /* Link to next block */ \ 1133 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \ 1134 Index++; \ 1135 } while (Index < Count); \ 1136 \ 1137 /* Link the last block */ \ 1138 WaitBlock->NextWaitBlock = WaitBlockArray; \ 1139 \ 1140 /* Set default wait status */ \ 1141 Thread->WaitStatus = STATUS_WAIT_0; \ 1142 \ 1143 /* Check if we have a timer */ \ 1144 if (Timeout) \ 1145 { \ 1146 /* Link to the block */ \ 1147 TimerBlock->NextWaitBlock = WaitBlockArray; \ 1148 \ 1149 /* Setup the timer */ \ 1150 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \ 1151 \ 1152 /* Save the due time for the caller */ \ 1153 DueTime.QuadPart = Timer->DueTime.QuadPart; \ 1154 \ 1155 /* Initialize the list */ \ 1156 InitializeListHead(&Timer->Header.WaitListHead); \ 1157 } \ 1158 \ 1159 /* Set wait settings */ \ 1160 Thread->Alertable = Alertable; \ 1161 Thread->WaitMode = WaitMode; \ 1162 Thread->WaitReason = WaitReason; \ 1163 \ 1164 /* Check if we can swap the thread's stack */ \ 1165 Thread->WaitListEntry.Flink = NULL; \ 1166 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ 1167 \ 1168 /* Set the wait time */ \ 1169 Thread->WaitTime = KeTickCount.LowPart; 1170 1171 #define KxSingleThreadWait() \ 1172 /* Setup the Wait Block */ \ 1173 Thread->WaitBlockList = WaitBlock; \ 1174 WaitBlock->WaitKey = STATUS_SUCCESS; \ 1175 WaitBlock->Object = Object; \ 1176 WaitBlock->WaitType = WaitAny; \ 1177 \ 1178 /* Clear wait status */ \ 1179 Thread->WaitStatus = STATUS_SUCCESS; \ 1180 \ 1181 /* Check if we have a timer */ \ 1182 if (Timeout) \ 1183 { \ 1184 /* Setup the timer */ \ 1185 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \ 1186 \ 1187 /* Save the due time for the caller */ \ 1188 DueTime.QuadPart = Timer->DueTime.QuadPart; \ 1189 \ 1190 /* Pointer to timer block */ \ 1191 WaitBlock->NextWaitBlock = TimerBlock; \ 1192 TimerBlock->NextWaitBlock = WaitBlock; \ 1193 \ 1194 /* Link the timer to this Wait Block */ \ 1195 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \ 1196 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \ 1197 } \ 1198 else \ 1199 { \ 1200 /* No timer block, just ourselves */ \ 1201 WaitBlock->NextWaitBlock = WaitBlock; \ 1202 } \ 1203 \ 1204 /* Set wait settings */ \ 1205 Thread->Alertable = Alertable; \ 1206 Thread->WaitMode = WaitMode; \ 1207 Thread->WaitReason = WaitReason; \ 1208 \ 1209 /* Check if we can swap the thread's stack */ \ 1210 Thread->WaitListEntry.Flink = NULL; \ 1211 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ 1212 \ 1213 /* Set the wait time */ \ 1214 Thread->WaitTime = KeTickCount.LowPart; 1215 1216 #define KxQueueThreadWait() \ 1217 /* Setup the Wait Block */ \ 1218 Thread->WaitBlockList = WaitBlock; \ 1219 WaitBlock->WaitKey = STATUS_SUCCESS; \ 1220 WaitBlock->Object = Queue; \ 1221 WaitBlock->WaitType = WaitAny; \ 1222 WaitBlock->Thread = Thread; \ 1223 \ 1224 /* Clear wait status */ \ 1225 Thread->WaitStatus = STATUS_SUCCESS; \ 1226 \ 1227 /* Check if we have a timer */ \ 1228 if (Timeout) \ 1229 { \ 1230 /* Setup the timer */ \ 1231 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \ 1232 \ 1233 /* Save the due time for the caller */ \ 1234 DueTime.QuadPart = Timer->DueTime.QuadPart; \ 1235 \ 1236 /* Pointer to timer block */ \ 1237 WaitBlock->NextWaitBlock = TimerBlock; \ 1238 TimerBlock->NextWaitBlock = WaitBlock; \ 1239 \ 1240 /* Link the timer to this Wait Block */ \ 1241 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \ 1242 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \ 1243 } \ 1244 else \ 1245 { \ 1246 /* No timer block, just ourselves */ \ 1247 WaitBlock->NextWaitBlock = WaitBlock; \ 1248 } \ 1249 \ 1250 /* Set wait settings */ \ 1251 Thread->Alertable = FALSE; \ 1252 Thread->WaitMode = WaitMode; \ 1253 Thread->WaitReason = WrQueue; \ 1254 \ 1255 /* Check if we can swap the thread's stack */ \ 1256 Thread->WaitListEntry.Flink = NULL; \ 1257 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ 1258 \ 1259 /* Set the wait time */ \ 1260 Thread->WaitTime = KeTickCount.LowPart; 1261 1262 // 1263 // Unwaits a Thread 1264 // 1265 FORCEINLINE 1266 VOID 1267 KxUnwaitThread(IN DISPATCHER_HEADER *Object, 1268 IN KPRIORITY Increment) 1269 { 1270 PLIST_ENTRY WaitEntry, WaitList; 1271 PKWAIT_BLOCK WaitBlock; 1272 PKTHREAD WaitThread; 1273 ULONG WaitKey; 1274 1275 /* Loop the Wait Entries */ 1276 WaitList = &Object->WaitListHead; 1277 ASSERT(IsListEmpty(&Object->WaitListHead) == FALSE); 1278 WaitEntry = WaitList->Flink; 1279 do 1280 { 1281 /* Get the current wait block */ 1282 WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry); 1283 1284 /* Get the waiting thread */ 1285 WaitThread = WaitBlock->Thread; 1286 1287 /* Check the current Wait Mode */ 1288 if (WaitBlock->WaitType == WaitAny) 1289 { 1290 /* Use the actual wait key */ 1291 WaitKey = WaitBlock->WaitKey; 1292 } 1293 else 1294 { 1295 /* Otherwise, use STATUS_KERNEL_APC */ 1296 WaitKey = STATUS_KERNEL_APC; 1297 } 1298 1299 /* Unwait the thread */ 1300 KiUnwaitThread(WaitThread, WaitKey, Increment); 1301 1302 /* Next entry */ 1303 WaitEntry = WaitList->Flink; 1304 } while (WaitEntry != WaitList); 1305 } 1306 1307 // 1308 // Unwaits a Thread waiting on an event 1309 // 1310 FORCEINLINE 1311 VOID 1312 KxUnwaitThreadForEvent(IN PKEVENT Event, 1313 IN KPRIORITY Increment) 1314 { 1315 PLIST_ENTRY WaitEntry, WaitList; 1316 PKWAIT_BLOCK WaitBlock; 1317 PKTHREAD WaitThread; 1318 1319 /* Loop the Wait Entries */ 1320 WaitList = &Event->Header.WaitListHead; 1321 ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE); 1322 WaitEntry = WaitList->Flink; 1323 do 1324 { 1325 /* Get the current wait block */ 1326 WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry); 1327 1328 /* Get the waiting thread */ 1329 WaitThread = WaitBlock->Thread; 1330 1331 /* Check the current Wait Mode */ 1332 if (WaitBlock->WaitType == WaitAny) 1333 { 1334 /* Un-signal it */ 1335 Event->Header.SignalState = 0; 1336 1337 /* Un-signal the event and unwait the thread */ 1338 KiUnwaitThread(WaitThread, WaitBlock->WaitKey, Increment); 1339 break; 1340 } 1341 1342 /* Unwait the thread with STATUS_KERNEL_APC */ 1343 KiUnwaitThread(WaitThread, STATUS_KERNEL_APC, Increment); 1344 1345 /* Next entry */ 1346 WaitEntry = WaitList->Flink; 1347 } while (WaitEntry != WaitList); 1348 } 1349 1350 // 1351 // This routine queues a thread that is ready on the PRCB's ready lists. 1352 // If this thread cannot currently run on this CPU, then the thread is 1353 // added to the deferred ready list instead. 1354 // 1355 // This routine must be entered with the PRCB lock held and it will exit 1356 // with the PRCB lock released! 1357 // 1358 FORCEINLINE 1359 VOID 1360 KxQueueReadyThread(IN PKTHREAD Thread, 1361 IN PKPRCB Prcb) 1362 { 1363 BOOLEAN Preempted; 1364 KPRIORITY Priority; 1365 1366 /* Sanity checks */ 1367 ASSERT(Prcb == KeGetCurrentPrcb()); 1368 ASSERT(Thread->State == Running); 1369 ASSERT(Thread->NextProcessor == Prcb->Number); 1370 1371 /* Check if this thread is allowed to run in this CPU */ 1372 #ifdef CONFIG_SMP 1373 if ((Thread->Affinity) & (Prcb->SetMember)) 1374 #else 1375 if (TRUE) 1376 #endif 1377 { 1378 /* Set thread ready for execution */ 1379 Thread->State = Ready; 1380 1381 /* Save current priority and if someone had pre-empted it */ 1382 Priority = Thread->Priority; 1383 Preempted = Thread->Preempted; 1384 1385 /* We're not pre-empting now, and set the wait time */ 1386 Thread->Preempted = FALSE; 1387 Thread->WaitTime = KeTickCount.LowPart; 1388 1389 /* Sanity check */ 1390 ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY)); 1391 1392 /* Insert this thread in the appropriate order */ 1393 Preempted ? InsertHeadList(&Prcb->DispatcherReadyListHead[Priority], 1394 &Thread->WaitListEntry) : 1395 InsertTailList(&Prcb->DispatcherReadyListHead[Priority], 1396 &Thread->WaitListEntry); 1397 1398 /* Update the ready summary */ 1399 Prcb->ReadySummary |= PRIORITY_MASK(Priority); 1400 1401 /* Sanity check */ 1402 ASSERT(Priority == Thread->Priority); 1403 1404 /* Release the PRCB lock */ 1405 KiReleasePrcbLock(Prcb); 1406 } 1407 else 1408 { 1409 /* Otherwise, prepare this thread to be deferred */ 1410 Thread->State = DeferredReady; 1411 Thread->DeferredProcessor = Prcb->Number; 1412 1413 /* Release the lock and defer scheduling */ 1414 KiReleasePrcbLock(Prcb); 1415 KiDeferredReadyThread(Thread); 1416 } 1417 } 1418 1419 // 1420 // This routine scans for an appropriate ready thread to select at the 1421 // given priority and for the given CPU. 1422 // 1423 FORCEINLINE 1424 PKTHREAD 1425 KiSelectReadyThread(IN KPRIORITY Priority, 1426 IN PKPRCB Prcb) 1427 { 1428 ULONG PrioritySet; 1429 LONG HighPriority; 1430 PLIST_ENTRY ListEntry; 1431 PKTHREAD Thread = NULL; 1432 1433 /* Save the current mask and get the priority set for the CPU */ 1434 PrioritySet = Prcb->ReadySummary >> Priority; 1435 if (!PrioritySet) goto Quickie; 1436 1437 /* Get the highest priority possible */ 1438 BitScanReverse((PULONG)&HighPriority, PrioritySet); 1439 ASSERT((PrioritySet & PRIORITY_MASK(HighPriority)) != 0); 1440 HighPriority += Priority; 1441 1442 /* Make sure the list isn't empty at the highest priority */ 1443 ASSERT(IsListEmpty(&Prcb->DispatcherReadyListHead[HighPriority]) == FALSE); 1444 1445 /* Get the first thread on the list */ 1446 ListEntry = Prcb->DispatcherReadyListHead[HighPriority].Flink; 1447 Thread = CONTAINING_RECORD(ListEntry, KTHREAD, WaitListEntry); 1448 1449 /* Make sure this thread is here for a reason */ 1450 ASSERT(HighPriority == Thread->Priority); 1451 ASSERT(Thread->Affinity & AFFINITY_MASK(Prcb->Number)); 1452 ASSERT(Thread->NextProcessor == Prcb->Number); 1453 1454 /* Remove it from the list */ 1455 if (RemoveEntryList(&Thread->WaitListEntry)) 1456 { 1457 /* The list is empty now, reset the ready summary */ 1458 Prcb->ReadySummary ^= PRIORITY_MASK(HighPriority); 1459 } 1460 1461 /* Sanity check and return the thread */ 1462 Quickie: 1463 ASSERT((Thread == NULL) || 1464 (Thread->BasePriority == 0) || 1465 (Thread->Priority != 0)); 1466 return Thread; 1467 } 1468 1469 // 1470 // This routine computes the new priority for a thread. It is only valid for 1471 // threads with priorities in the dynamic priority range. 1472 // 1473 FORCEINLINE 1474 SCHAR 1475 KiComputeNewPriority(IN PKTHREAD Thread, 1476 IN SCHAR Adjustment) 1477 { 1478 SCHAR Priority; 1479 1480 /* Priority sanity checks */ 1481 ASSERT((Thread->PriorityDecrement >= 0) && 1482 (Thread->PriorityDecrement <= Thread->Priority)); 1483 ASSERT((Thread->Priority < LOW_REALTIME_PRIORITY) ? 1484 TRUE : (Thread->PriorityDecrement == 0)); 1485 1486 /* Get the current priority */ 1487 Priority = Thread->Priority; 1488 if (Priority < LOW_REALTIME_PRIORITY) 1489 { 1490 /* Decrease priority by the priority decrement */ 1491 Priority -= (Thread->PriorityDecrement + Adjustment); 1492 1493 /* Don't go out of bounds */ 1494 if (Priority < Thread->BasePriority) Priority = Thread->BasePriority; 1495 1496 /* Reset the priority decrement */ 1497 Thread->PriorityDecrement = 0; 1498 } 1499 1500 /* Sanity check */ 1501 ASSERT((Thread->BasePriority == 0) || (Priority != 0)); 1502 1503 /* Return the new priority */ 1504 return Priority; 1505 } 1506 1507 // 1508 // Guarded Mutex Routines 1509 // 1510 FORCEINLINE 1511 VOID 1512 _KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex) 1513 { 1514 /* Setup the Initial Data */ 1515 GuardedMutex->Count = GM_LOCK_BIT; 1516 GuardedMutex->Owner = NULL; 1517 GuardedMutex->Contention = 0; 1518 1519 /* Initialize the Wait Gate */ 1520 KeInitializeGate(&GuardedMutex->Gate); 1521 } 1522 1523 FORCEINLINE 1524 VOID 1525 _KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) 1526 { 1527 PKTHREAD Thread = KeGetCurrentThread(); 1528 1529 /* Sanity checks */ 1530 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1531 (Thread->SpecialApcDisable < 0) || 1532 (Thread->Teb == NULL) || 1533 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1534 ASSERT(GuardedMutex->Owner != Thread); 1535 1536 /* Remove the lock */ 1537 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V)) 1538 { 1539 /* The Guarded Mutex was already locked, enter contented case */ 1540 KiAcquireGuardedMutex(GuardedMutex); 1541 } 1542 1543 /* Set the Owner */ 1544 GuardedMutex->Owner = Thread; 1545 } 1546 1547 FORCEINLINE 1548 VOID 1549 _KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) 1550 { 1551 LONG OldValue, NewValue; 1552 1553 /* Sanity checks */ 1554 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1555 (KeGetCurrentThread()->SpecialApcDisable < 0) || 1556 (KeGetCurrentThread()->Teb == NULL) || 1557 (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1558 ASSERT(GuardedMutex->Owner == KeGetCurrentThread()); 1559 1560 /* Destroy the Owner */ 1561 GuardedMutex->Owner = NULL; 1562 1563 /* Add the Lock Bit */ 1564 OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT); 1565 ASSERT((OldValue & GM_LOCK_BIT) == 0); 1566 1567 /* Check if it was already locked, but not woken */ 1568 if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN)) 1569 { 1570 /* Update the Oldvalue to what it should be now */ 1571 OldValue += GM_LOCK_BIT; 1572 1573 /* The mutex will be woken, minus one waiter */ 1574 NewValue = OldValue + GM_LOCK_WAITER_WOKEN - 1575 GM_LOCK_WAITER_INC; 1576 1577 /* Remove the Woken bit */ 1578 if (InterlockedCompareExchange(&GuardedMutex->Count, 1579 NewValue, 1580 OldValue) == OldValue) 1581 { 1582 /* Signal the Gate */ 1583 KeSignalGateBoostPriority(&GuardedMutex->Gate); 1584 } 1585 } 1586 } 1587 1588 FORCEINLINE 1589 VOID 1590 _KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex) 1591 { 1592 PKTHREAD Thread = KeGetCurrentThread(); 1593 1594 /* Sanity checks */ 1595 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1596 ASSERT(GuardedMutex->Owner != Thread); 1597 1598 /* Disable Special APCs */ 1599 KeEnterGuardedRegion(); 1600 1601 /* Remove the lock */ 1602 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V)) 1603 { 1604 /* The Guarded Mutex was already locked, enter contented case */ 1605 KiAcquireGuardedMutex(GuardedMutex); 1606 } 1607 1608 /* Set the Owner and Special APC Disable state */ 1609 GuardedMutex->Owner = Thread; 1610 GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable; 1611 } 1612 1613 FORCEINLINE 1614 VOID 1615 _KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) 1616 { 1617 LONG OldValue, NewValue; 1618 1619 /* Sanity checks */ 1620 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1621 ASSERT(GuardedMutex->Owner == KeGetCurrentThread()); 1622 ASSERT(KeGetCurrentThread()->SpecialApcDisable == 1623 GuardedMutex->SpecialApcDisable); 1624 1625 /* Destroy the Owner */ 1626 GuardedMutex->Owner = NULL; 1627 1628 /* Add the Lock Bit */ 1629 OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT); 1630 ASSERT((OldValue & GM_LOCK_BIT) == 0); 1631 1632 /* Check if it was already locked, but not woken */ 1633 if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN)) 1634 { 1635 /* Update the Oldvalue to what it should be now */ 1636 OldValue += GM_LOCK_BIT; 1637 1638 /* The mutex will be woken, minus one waiter */ 1639 NewValue = OldValue + GM_LOCK_WAITER_WOKEN - 1640 GM_LOCK_WAITER_INC; 1641 1642 /* Remove the Woken bit */ 1643 if (InterlockedCompareExchange(&GuardedMutex->Count, 1644 NewValue, 1645 OldValue) == OldValue) 1646 { 1647 /* Signal the Gate */ 1648 KeSignalGateBoostPriority(&GuardedMutex->Gate); 1649 } 1650 } 1651 1652 /* Re-enable APCs */ 1653 KeLeaveGuardedRegion(); 1654 } 1655 1656 FORCEINLINE 1657 BOOLEAN 1658 _KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) 1659 { 1660 PKTHREAD Thread = KeGetCurrentThread(); 1661 1662 /* Block APCs */ 1663 KeEnterGuardedRegion(); 1664 1665 /* Remove the lock */ 1666 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V)) 1667 { 1668 /* Re-enable APCs */ 1669 KeLeaveGuardedRegion(); 1670 YieldProcessor(); 1671 1672 /* Return failure */ 1673 return FALSE; 1674 } 1675 1676 /* Set the Owner and APC State */ 1677 GuardedMutex->Owner = Thread; 1678 GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable; 1679 return TRUE; 1680 } 1681 1682 1683 FORCEINLINE 1684 VOID 1685 KiAcquireNmiListLock(OUT PKIRQL OldIrql) 1686 { 1687 KeAcquireSpinLock(&KiNmiCallbackListLock, OldIrql); 1688 } 1689 1690 FORCEINLINE 1691 VOID 1692 KiReleaseNmiListLock(IN KIRQL OldIrql) 1693 { 1694 KeReleaseSpinLock(&KiNmiCallbackListLock, OldIrql); 1695 } 1696