1 /* 2 * PROJECT: ReactOS Kernel 3 * LICENSE: GPL - See COPYING in the top level directory 4 * FILE: ntoskrnl/include/ke_x.h 5 * PURPOSE: Internal Inlined Functions for the Kernel 6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org) 7 */ 8 9 #ifndef _M_ARM 10 FORCEINLINE 11 KPROCESSOR_MODE 12 KeGetPreviousMode(VOID) 13 { 14 /* Return the current mode */ 15 return KeGetCurrentThread()->PreviousMode; 16 } 17 #endif 18 19 // 20 // Enters a Guarded Region 21 // 22 #define KeEnterGuardedRegion() \ 23 { \ 24 PKTHREAD _Thread = KeGetCurrentThread(); \ 25 \ 26 /* Sanity checks */ \ 27 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \ 28 ASSERT(_Thread == KeGetCurrentThread()); \ 29 ASSERT((_Thread->SpecialApcDisable <= 0) && \ 30 (_Thread->SpecialApcDisable != -32768)); \ 31 \ 32 /* Disable Special APCs */ \ 33 _Thread->SpecialApcDisable--; \ 34 } 35 36 // 37 // Leaves a Guarded Region 38 // 39 #define KeLeaveGuardedRegion() \ 40 { \ 41 PKTHREAD _Thread = KeGetCurrentThread(); \ 42 \ 43 /* Sanity checks */ \ 44 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \ 45 ASSERT(_Thread == KeGetCurrentThread()); \ 46 ASSERT(_Thread->SpecialApcDisable < 0); \ 47 \ 48 /* Leave region and check if APCs are OK now */ \ 49 if (!(++_Thread->SpecialApcDisable)) \ 50 { \ 51 /* Check for Kernel APCs on the list */ \ 52 if (!IsListEmpty(&_Thread->ApcState. \ 53 ApcListHead[KernelMode])) \ 54 { \ 55 /* Check for APC Delivery */ \ 56 KiCheckForKernelApcDelivery(); \ 57 } \ 58 } \ 59 } 60 61 // 62 // Enters a Critical Region 63 // 64 #define KeEnterCriticalRegion() \ 65 { \ 66 PKTHREAD _Thread = KeGetCurrentThread(); \ 67 \ 68 /* Sanity checks */ \ 69 ASSERT(_Thread == KeGetCurrentThread()); \ 70 ASSERT((_Thread->KernelApcDisable <= 0) && \ 71 (_Thread->KernelApcDisable != -32768)); \ 72 \ 73 /* Disable Kernel APCs */ \ 74 _Thread->KernelApcDisable--; \ 75 } 76 77 // 78 // Leaves a Critical Region 79 // 80 #define KeLeaveCriticalRegion() \ 81 { \ 82 PKTHREAD _Thread = KeGetCurrentThread(); \ 83 \ 84 /* Sanity checks */ \ 85 ASSERT(_Thread == KeGetCurrentThread()); \ 86 ASSERT(_Thread->KernelApcDisable < 0); \ 87 \ 88 /* Enable Kernel APCs */ \ 89 _Thread->KernelApcDisable++; \ 90 \ 91 /* Check if Kernel APCs are now enabled */ \ 92 if (!(_Thread->KernelApcDisable)) \ 93 { \ 94 /* Check if we need to request an APC Delivery */ \ 95 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \ 96 !(_Thread->SpecialApcDisable)) \ 97 { \ 98 /* Check for the right environment */ \ 99 KiCheckForKernelApcDelivery(); \ 100 } \ 101 } \ 102 } 103 104 #ifndef CONFIG_SMP 105 106 // 107 // This routine protects against multiple CPU acquires, it's meaningless on UP. 108 // 109 FORCEINLINE 110 VOID 111 KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object) 112 { 113 UNREFERENCED_PARAMETER(Object); 114 } 115 116 // 117 // This routine protects against multiple CPU acquires, it's meaningless on UP. 118 // 119 FORCEINLINE 120 VOID 121 KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object) 122 { 123 UNREFERENCED_PARAMETER(Object); 124 } 125 126 FORCEINLINE 127 KIRQL 128 KiAcquireDispatcherLock(VOID) 129 { 130 /* Raise to synch level */ 131 return KfRaiseIrql(SYNCH_LEVEL); 132 } 133 134 FORCEINLINE 135 VOID 136 KiReleaseDispatcherLock(IN KIRQL OldIrql) 137 { 138 /* Just exit the dispatcher */ 139 KiExitDispatcher(OldIrql); 140 } 141 142 FORCEINLINE 143 VOID 144 KiAcquireDispatcherLockAtDpcLevel(VOID) 145 { 146 /* This is a no-op at DPC Level for UP systems */ 147 return; 148 } 149 150 FORCEINLINE 151 VOID 152 KiReleaseDispatcherLockFromDpcLevel(VOID) 153 { 154 /* This is a no-op at DPC Level for UP systems */ 155 return; 156 } 157 158 // 159 // This routine makes the thread deferred ready on the boot CPU. 160 // 161 FORCEINLINE 162 VOID 163 KiInsertDeferredReadyList(IN PKTHREAD Thread) 164 { 165 /* Set the thread to deferred state and boot CPU */ 166 Thread->State = DeferredReady; 167 Thread->DeferredProcessor = 0; 168 169 /* Make the thread ready immediately */ 170 KiDeferredReadyThread(Thread); 171 } 172 173 FORCEINLINE 174 VOID 175 KiRescheduleThread(IN BOOLEAN NewThread, 176 IN ULONG Cpu) 177 { 178 /* This is meaningless on UP systems */ 179 UNREFERENCED_PARAMETER(NewThread); 180 UNREFERENCED_PARAMETER(Cpu); 181 } 182 183 // 184 // This routine protects against multiple CPU acquires, it's meaningless on UP. 185 // 186 FORCEINLINE 187 VOID 188 KiSetThreadSwapBusy(IN PKTHREAD Thread) 189 { 190 UNREFERENCED_PARAMETER(Thread); 191 } 192 193 // 194 // This routine protects against multiple CPU acquires, it's meaningless on UP. 195 // 196 FORCEINLINE 197 VOID 198 KiAcquirePrcbLock(IN PKPRCB Prcb) 199 { 200 UNREFERENCED_PARAMETER(Prcb); 201 } 202 203 // 204 // This routine protects against multiple CPU acquires, it's meaningless on UP. 205 // 206 FORCEINLINE 207 VOID 208 KiReleasePrcbLock(IN PKPRCB Prcb) 209 { 210 UNREFERENCED_PARAMETER(Prcb); 211 } 212 213 // 214 // This routine protects against multiple CPU acquires, it's meaningless on UP. 215 // 216 FORCEINLINE 217 VOID 218 KiAcquireThreadLock(IN PKTHREAD Thread) 219 { 220 UNREFERENCED_PARAMETER(Thread); 221 } 222 223 // 224 // This routine protects against multiple CPU acquires, it's meaningless on UP. 225 // 226 FORCEINLINE 227 VOID 228 KiReleaseThreadLock(IN PKTHREAD Thread) 229 { 230 UNREFERENCED_PARAMETER(Thread); 231 } 232 233 // 234 // This routine protects against multiple CPU acquires, it's meaningless on UP. 235 // 236 FORCEINLINE 237 BOOLEAN 238 KiTryThreadLock(IN PKTHREAD Thread) 239 { 240 UNREFERENCED_PARAMETER(Thread); 241 return FALSE; 242 } 243 244 FORCEINLINE 245 VOID 246 KiCheckDeferredReadyList(IN PKPRCB Prcb) 247 { 248 /* There are no deferred ready lists on UP systems */ 249 UNREFERENCED_PARAMETER(Prcb); 250 } 251 252 FORCEINLINE 253 VOID 254 KiRequestApcInterrupt(IN BOOLEAN NeedApc, 255 IN UCHAR Processor) 256 { 257 /* We deliver instantly on UP */ 258 UNREFERENCED_PARAMETER(NeedApc); 259 UNREFERENCED_PARAMETER(Processor); 260 } 261 262 FORCEINLINE 263 PKSPIN_LOCK_QUEUE 264 KiAcquireTimerLock(IN ULONG Hand) 265 { 266 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 267 268 /* Nothing to do on UP */ 269 UNREFERENCED_PARAMETER(Hand); 270 return NULL; 271 } 272 273 FORCEINLINE 274 VOID 275 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue) 276 { 277 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 278 279 /* Nothing to do on UP */ 280 UNREFERENCED_PARAMETER(LockQueue); 281 } 282 283 #else 284 285 FORCEINLINE 286 VOID 287 KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object) 288 { 289 LONG OldValue; 290 291 /* Make sure we're at a safe level to touch the lock */ 292 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 293 294 /* Start acquire loop */ 295 do 296 { 297 /* Loop until the other CPU releases it */ 298 while (TRUE) 299 { 300 /* Check if it got released */ 301 OldValue = Object->Lock; 302 if ((OldValue & KOBJECT_LOCK_BIT) == 0) break; 303 304 /* Let the CPU know that this is a loop */ 305 YieldProcessor(); 306 } 307 308 /* Try acquiring the lock now */ 309 } while (InterlockedCompareExchange(&Object->Lock, 310 OldValue | KOBJECT_LOCK_BIT, 311 OldValue) != OldValue); 312 } 313 314 FORCEINLINE 315 VOID 316 KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object) 317 { 318 /* Make sure we're at a safe level to touch the lock */ 319 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 320 321 /* Release it */ 322 InterlockedAnd(&Object->Lock, ~KOBJECT_LOCK_BIT); 323 } 324 325 FORCEINLINE 326 KIRQL 327 KiAcquireDispatcherLock(VOID) 328 { 329 /* Raise to synchronization level and acquire the dispatcher lock */ 330 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock); 331 } 332 333 FORCEINLINE 334 VOID 335 KiReleaseDispatcherLock(IN KIRQL OldIrql) 336 { 337 /* First release the lock */ 338 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()-> 339 LockQueue[LockQueueDispatcherLock]); 340 341 /* Then exit the dispatcher */ 342 KiExitDispatcher(OldIrql); 343 } 344 345 FORCEINLINE 346 VOID 347 KiAcquireDispatcherLockAtDpcLevel(VOID) 348 { 349 /* Acquire the dispatcher lock */ 350 KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()-> 351 LockQueue[LockQueueDispatcherLock]); 352 } 353 354 FORCEINLINE 355 VOID 356 KiReleaseDispatcherLockFromDpcLevel(VOID) 357 { 358 /* Release the dispatcher lock */ 359 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()-> 360 LockQueue[LockQueueDispatcherLock]); 361 } 362 363 // 364 // This routine inserts a thread into the deferred ready list of the current CPU 365 // 366 FORCEINLINE 367 VOID 368 KiInsertDeferredReadyList(IN PKTHREAD Thread) 369 { 370 PKPRCB Prcb = KeGetCurrentPrcb(); 371 372 /* Set the thread to deferred state and CPU */ 373 Thread->State = DeferredReady; 374 Thread->DeferredProcessor = Prcb->Number; 375 376 /* Add it on the list */ 377 PushEntryList(&Prcb->DeferredReadyListHead, &Thread->SwapListEntry); 378 } 379 380 FORCEINLINE 381 VOID 382 KiRescheduleThread(IN BOOLEAN NewThread, 383 IN ULONG Cpu) 384 { 385 /* Check if a new thread needs to be scheduled on a different CPU */ 386 if ((NewThread) && !(KeGetPcr()->Number == Cpu)) 387 { 388 /* Send an IPI to request delivery */ 389 KiIpiSend(AFFINITY_MASK(Cpu), IPI_DPC); 390 } 391 } 392 393 // 394 // This routine sets the current thread in a swap busy state, which ensure that 395 // nobody else tries to swap it concurrently. 396 // 397 FORCEINLINE 398 VOID 399 KiSetThreadSwapBusy(IN PKTHREAD Thread) 400 { 401 /* Make sure nobody already set it */ 402 ASSERT(Thread->SwapBusy == FALSE); 403 404 /* Set it ourselves */ 405 Thread->SwapBusy = TRUE; 406 } 407 408 // 409 // This routine acquires the PRCB lock so that only one caller can touch 410 // volatile PRCB data. 411 // 412 // Since this is a simple optimized spin-lock, it must only be acquired 413 // at dispatcher level or higher! 414 // 415 FORCEINLINE 416 VOID 417 KiAcquirePrcbLock(IN PKPRCB Prcb) 418 { 419 /* Make sure we're at a safe level to touch the PRCB lock */ 420 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 421 422 /* Start acquire loop */ 423 for (;;) 424 { 425 /* Acquire the lock and break out if we acquired it first */ 426 if (!InterlockedExchange((PLONG)&Prcb->PrcbLock, 1)) break; 427 428 /* Loop until the other CPU releases it */ 429 do 430 { 431 /* Let the CPU know that this is a loop */ 432 YieldProcessor(); 433 } while (Prcb->PrcbLock); 434 } 435 } 436 437 // 438 // This routine releases the PRCB lock so that other callers can touch 439 // volatile PRCB data. 440 // 441 // Since this is a simple optimized spin-lock, it must be be only acquired 442 // at dispatcher level or higher! 443 // 444 FORCEINLINE 445 VOID 446 KiReleasePrcbLock(IN PKPRCB Prcb) 447 { 448 /* Make sure we are above dispatch and the lock is acquired! */ 449 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 450 ASSERT(Prcb->PrcbLock != 0); 451 452 /* Release it */ 453 InterlockedAnd((PLONG)&Prcb->PrcbLock, 0); 454 } 455 456 // 457 // This routine acquires the thread lock so that only one caller can touch 458 // volatile thread data. 459 // 460 // Since this is a simple optimized spin-lock, it must be be only acquired 461 // at dispatcher level or higher! 462 // 463 FORCEINLINE 464 VOID 465 KiAcquireThreadLock(IN PKTHREAD Thread) 466 { 467 /* Make sure we're at a safe level to touch the thread lock */ 468 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 469 470 /* Start acquire loop */ 471 for (;;) 472 { 473 /* Acquire the lock and break out if we acquired it first */ 474 if (!InterlockedExchange((PLONG)&Thread->ThreadLock, 1)) break; 475 476 /* Loop until the other CPU releases it */ 477 do 478 { 479 /* Let the CPU know that this is a loop */ 480 YieldProcessor(); 481 } while (Thread->ThreadLock); 482 } 483 } 484 485 // 486 // This routine releases the thread lock so that other callers can touch 487 // volatile thread data. 488 // 489 // Since this is a simple optimized spin-lock, it must be be only acquired 490 // at dispatcher level or higher! 491 // 492 FORCEINLINE 493 VOID 494 KiReleaseThreadLock(IN PKTHREAD Thread) 495 { 496 /* Make sure we are still above dispatch */ 497 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 498 499 /* Release it */ 500 InterlockedAnd((PLONG)&Thread->ThreadLock, 0); 501 } 502 503 FORCEINLINE 504 BOOLEAN 505 KiTryThreadLock(IN PKTHREAD Thread) 506 { 507 LONG Value; 508 509 /* If the lock isn't acquired, return false */ 510 if (!Thread->ThreadLock) return FALSE; 511 512 /* Otherwise, try to acquire it and check the result */ 513 Value = 1; 514 Value = InterlockedExchange((PLONG)&Thread->ThreadLock, Value); 515 516 /* Return the lock state */ 517 return (Value == TRUE); 518 } 519 520 FORCEINLINE 521 VOID 522 KiCheckDeferredReadyList(IN PKPRCB Prcb) 523 { 524 /* Scan the deferred ready lists if required */ 525 if (Prcb->DeferredReadyListHead.Next) KiProcessDeferredReadyList(Prcb); 526 } 527 528 FORCEINLINE 529 VOID 530 KiRequestApcInterrupt(IN BOOLEAN NeedApc, 531 IN UCHAR Processor) 532 { 533 /* Check if we need to request APC delivery */ 534 if (NeedApc) 535 { 536 /* Check if it's on another CPU */ 537 if (KeGetPcr()->Number != Processor) 538 { 539 /* Send an IPI to request delivery */ 540 KiIpiSend(AFFINITY_MASK(Processor), IPI_APC); 541 } 542 else 543 { 544 /* Request a software interrupt */ 545 HalRequestSoftwareInterrupt(APC_LEVEL); 546 } 547 } 548 } 549 550 FORCEINLINE 551 PKSPIN_LOCK_QUEUE 552 KiAcquireTimerLock(IN ULONG Hand) 553 { 554 PKSPIN_LOCK_QUEUE LockQueue; 555 ULONG LockIndex; 556 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 557 558 /* Get the lock index */ 559 LockIndex = Hand >> LOCK_QUEUE_TIMER_LOCK_SHIFT; 560 LockIndex &= (LOCK_QUEUE_TIMER_TABLE_LOCKS - 1); 561 562 /* Now get the lock */ 563 LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueueTimerTableLock + LockIndex]; 564 565 /* Acquire it and return */ 566 KeAcquireQueuedSpinLockAtDpcLevel(LockQueue); 567 return LockQueue; 568 } 569 570 FORCEINLINE 571 VOID 572 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue) 573 { 574 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); 575 576 /* Release the lock */ 577 KeReleaseQueuedSpinLockFromDpcLevel(LockQueue); 578 } 579 580 #endif 581 582 FORCEINLINE 583 VOID 584 KiAcquireApcLock(IN PKTHREAD Thread, 585 IN PKLOCK_QUEUE_HANDLE Handle) 586 { 587 /* Acquire the lock and raise to synchronization level */ 588 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread->ApcQueueLock, Handle); 589 } 590 591 FORCEINLINE 592 VOID 593 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread, 594 IN PKLOCK_QUEUE_HANDLE Handle) 595 { 596 /* Acquire the lock */ 597 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread->ApcQueueLock, Handle); 598 } 599 600 FORCEINLINE 601 VOID 602 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread, 603 IN PKLOCK_QUEUE_HANDLE Handle) 604 { 605 /* Acquire the lock */ 606 KeAcquireInStackQueuedSpinLock(&Thread->ApcQueueLock, Handle); 607 } 608 609 FORCEINLINE 610 VOID 611 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle) 612 { 613 /* Release the lock */ 614 KeReleaseInStackQueuedSpinLock(Handle); 615 } 616 617 FORCEINLINE 618 VOID 619 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle) 620 { 621 /* Release the lock */ 622 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle); 623 } 624 625 FORCEINLINE 626 VOID 627 KiAcquireProcessLock(IN PKPROCESS Process, 628 IN PKLOCK_QUEUE_HANDLE Handle) 629 { 630 /* Acquire the lock and raise to synchronization level */ 631 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process->ProcessLock, Handle); 632 } 633 634 FORCEINLINE 635 VOID 636 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle) 637 { 638 /* Release the lock */ 639 KeReleaseInStackQueuedSpinLock(Handle); 640 } 641 642 FORCEINLINE 643 VOID 644 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle) 645 { 646 /* Release the lock */ 647 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle); 648 } 649 650 FORCEINLINE 651 VOID 652 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue, 653 IN PKLOCK_QUEUE_HANDLE DeviceLock) 654 { 655 /* Check if we were called from a threaded DPC */ 656 if (KeGetCurrentPrcb()->DpcThreadActive) 657 { 658 /* Lock the Queue, we're not at DPC level */ 659 KeAcquireInStackQueuedSpinLock(&DeviceQueue->Lock, DeviceLock); 660 } 661 else 662 { 663 /* We must be at DPC level, acquire the lock safely */ 664 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); 665 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue->Lock, 666 DeviceLock); 667 } 668 } 669 670 FORCEINLINE 671 VOID 672 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock) 673 { 674 /* Check if we were called from a threaded DPC */ 675 if (KeGetCurrentPrcb()->DpcThreadActive) 676 { 677 /* Unlock the Queue, we're not at DPC level */ 678 KeReleaseInStackQueuedSpinLock(DeviceLock); 679 } 680 else 681 { 682 /* We must be at DPC level, release the lock safely */ 683 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); 684 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock); 685 } 686 } 687 688 // 689 // Satisfies the wait of a mutant dispatcher object 690 // 691 #define KiSatisfyMutantWait(Object, Thread) \ 692 { \ 693 /* Decrease the Signal State */ \ 694 (Object)->Header.SignalState--; \ 695 \ 696 /* Check if it's now non-signaled */ \ 697 if (!(Object)->Header.SignalState) \ 698 { \ 699 /* Set the Owner Thread */ \ 700 (Object)->OwnerThread = Thread; \ 701 \ 702 /* Disable APCs if needed */ \ 703 Thread->KernelApcDisable = Thread->KernelApcDisable - \ 704 (Object)->ApcDisable; \ 705 \ 706 /* Check if it's abandoned */ \ 707 if ((Object)->Abandoned) \ 708 { \ 709 /* Unabandon it */ \ 710 (Object)->Abandoned = FALSE; \ 711 \ 712 /* Return Status */ \ 713 Thread->WaitStatus = STATUS_ABANDONED; \ 714 } \ 715 \ 716 /* Insert it into the Mutant List */ \ 717 InsertHeadList(Thread->MutantListHead.Blink, \ 718 &(Object)->MutantListEntry); \ 719 } \ 720 } 721 722 // 723 // Satisfies the wait of any nonmutant dispatcher object 724 // 725 #define KiSatisfyNonMutantWait(Object) \ 726 { \ 727 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \ 728 EventSynchronizationObject) \ 729 { \ 730 /* Synchronization Timers and Events just get un-signaled */ \ 731 (Object)->Header.SignalState = 0; \ 732 } \ 733 else if ((Object)->Header.Type == SemaphoreObject) \ 734 { \ 735 /* These ones can have multiple states, so we only decrease it */ \ 736 (Object)->Header.SignalState--; \ 737 } \ 738 } 739 740 // 741 // Satisfies the wait of any dispatcher object 742 // 743 #define KiSatisfyObjectWait(Object, Thread) \ 744 { \ 745 /* Special case for Mutants */ \ 746 if ((Object)->Header.Type == MutantObject) \ 747 { \ 748 KiSatisfyMutantWait((Object), (Thread)); \ 749 } \ 750 else \ 751 { \ 752 KiSatisfyNonMutantWait(Object); \ 753 } \ 754 } 755 756 // 757 // Recalculates the due time 758 // 759 FORCEINLINE 760 PLARGE_INTEGER 761 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime, 762 IN PLARGE_INTEGER DueTime, 763 IN OUT PLARGE_INTEGER NewDueTime) 764 { 765 /* Don't do anything for absolute waits */ 766 if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime; 767 768 /* Otherwise, query the interrupt time and recalculate */ 769 NewDueTime->QuadPart = KeQueryInterruptTime(); 770 NewDueTime->QuadPart -= DueTime->QuadPart; 771 return NewDueTime; 772 } 773 774 // 775 // Determines whether a thread should be added to the wait list 776 // 777 FORCEINLINE 778 BOOLEAN 779 KiCheckThreadStackSwap(IN PKTHREAD Thread, 780 IN KPROCESSOR_MODE WaitMode) 781 { 782 /* Check the required conditions */ 783 if ((WaitMode != KernelMode) && 784 (Thread->EnableStackSwap) && 785 (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9))) 786 { 787 /* We are go for swap */ 788 return TRUE; 789 } 790 else 791 { 792 /* Don't swap the thread */ 793 return FALSE; 794 } 795 } 796 797 // 798 // Adds a thread to the wait list 799 // 800 #define KiAddThreadToWaitList(Thread, Swappable) \ 801 { \ 802 /* Make sure it's swappable */ \ 803 if (Swappable) \ 804 { \ 805 /* Insert it into the PRCB's List */ \ 806 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \ 807 &Thread->WaitListEntry); \ 808 } \ 809 } 810 811 // 812 // Checks if a wait in progress should be interrupted by APCs or an alertable 813 // state. 814 // 815 FORCEINLINE 816 NTSTATUS 817 KiCheckAlertability(IN PKTHREAD Thread, 818 IN BOOLEAN Alertable, 819 IN KPROCESSOR_MODE WaitMode) 820 { 821 /* Check if the wait is alertable */ 822 if (Alertable) 823 { 824 /* It is, first check if the thread is alerted in this mode */ 825 if (Thread->Alerted[WaitMode]) 826 { 827 /* It is, so bail out of the wait */ 828 Thread->Alerted[WaitMode] = FALSE; 829 return STATUS_ALERTED; 830 } 831 else if ((WaitMode != KernelMode) && 832 (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode]))) 833 { 834 /* It's isn't, but this is a user wait with queued user APCs */ 835 Thread->ApcState.UserApcPending = TRUE; 836 return STATUS_USER_APC; 837 } 838 else if (Thread->Alerted[KernelMode]) 839 { 840 /* It isn't that either, but we're alered in kernel mode */ 841 Thread->Alerted[KernelMode] = FALSE; 842 return STATUS_ALERTED; 843 } 844 } 845 else if ((WaitMode != KernelMode) && (Thread->ApcState.UserApcPending)) 846 { 847 /* Not alertable, but this is a user wait with pending user APCs */ 848 return STATUS_USER_APC; 849 } 850 851 /* Otherwise, we're fine */ 852 return STATUS_WAIT_0; 853 } 854 855 ULONG 856 FORCEINLINE 857 KiComputeTimerTableIndex(IN ULONGLONG DueTime) 858 { 859 return (DueTime / KeMaximumIncrement) & (TIMER_TABLE_SIZE - 1); 860 } 861 862 // 863 // Called from KiCompleteTimer, KiInsertTreeTimer, KeSetSystemTime 864 // to remove timer entries 865 // See Windows HPI blog for more information. 866 FORCEINLINE 867 VOID 868 KiRemoveEntryTimer(IN PKTIMER Timer) 869 { 870 ULONG Hand; 871 PKTIMER_TABLE_ENTRY TableEntry; 872 873 /* Remove the timer from the timer list and check if it's empty */ 874 Hand = Timer->Header.Hand; 875 if (RemoveEntryList(&Timer->TimerListEntry)) 876 { 877 /* Get the respective timer table entry */ 878 TableEntry = &KiTimerTableListHead[Hand]; 879 if (&TableEntry->Entry == TableEntry->Entry.Flink) 880 { 881 /* Set the entry to an infinite absolute time */ 882 TableEntry->Time.HighPart = 0xFFFFFFFF; 883 } 884 } 885 886 /* Clear the list entries on dbg builds so we can tell the timer is gone */ 887 #if DBG 888 Timer->TimerListEntry.Flink = NULL; 889 Timer->TimerListEntry.Blink = NULL; 890 #endif 891 } 892 893 // 894 // Called by Wait and Queue code to insert a timer for dispatching. 895 // Also called by KeSetTimerEx to insert a timer from the caller. 896 // 897 FORCEINLINE 898 VOID 899 KxInsertTimer(IN PKTIMER Timer, 900 IN ULONG Hand) 901 { 902 PKSPIN_LOCK_QUEUE LockQueue; 903 904 /* Acquire the lock and release the dispatcher lock */ 905 LockQueue = KiAcquireTimerLock(Hand); 906 KiReleaseDispatcherLockFromDpcLevel(); 907 908 /* Try to insert the timer */ 909 if (KiInsertTimerTable(Timer, Hand)) 910 { 911 /* Complete it */ 912 KiCompleteTimer(Timer, LockQueue); 913 } 914 else 915 { 916 /* Do nothing, just release the lock */ 917 KiReleaseTimerLock(LockQueue); 918 } 919 } 920 921 // 922 // Called by KeSetTimerEx and KiInsertTreeTimer to calculate Due Time 923 // See the Windows HPI Blog for more information 924 // 925 FORCEINLINE 926 BOOLEAN 927 KiComputeDueTime(IN PKTIMER Timer, 928 IN LARGE_INTEGER DueTime, 929 OUT PULONG Hand) 930 { 931 LARGE_INTEGER InterruptTime, SystemTime, DifferenceTime; 932 933 /* Convert to relative time if needed */ 934 Timer->Header.Absolute = FALSE; 935 if (DueTime.HighPart >= 0) 936 { 937 /* Get System Time */ 938 KeQuerySystemTime(&SystemTime); 939 940 /* Do the conversion */ 941 DifferenceTime.QuadPart = SystemTime.QuadPart - DueTime.QuadPart; 942 943 /* Make sure it hasn't already expired */ 944 Timer->Header.Absolute = TRUE; 945 if (DifferenceTime.HighPart >= 0) 946 { 947 /* Cancel everything */ 948 Timer->Header.SignalState = TRUE; 949 Timer->Header.Hand = 0; 950 Timer->DueTime.QuadPart = 0; 951 *Hand = 0; 952 return FALSE; 953 } 954 955 /* Set the time as Absolute */ 956 DueTime = DifferenceTime; 957 } 958 959 /* Get the Interrupt Time */ 960 InterruptTime.QuadPart = KeQueryInterruptTime(); 961 962 /* Recalculate due time */ 963 Timer->DueTime.QuadPart = InterruptTime.QuadPart - DueTime.QuadPart; 964 965 /* Get the handle */ 966 *Hand = KiComputeTimerTableIndex(Timer->DueTime.QuadPart); 967 Timer->Header.Hand = (UCHAR)*Hand; 968 Timer->Header.Inserted = TRUE; 969 return TRUE; 970 } 971 972 // 973 // Called from Unlink and Queue Insert Code. 974 // Also called by timer code when canceling an inserted timer. 975 // Removes a timer from it's tree. 976 // 977 FORCEINLINE 978 VOID 979 KxRemoveTreeTimer(IN PKTIMER Timer) 980 { 981 ULONG Hand = Timer->Header.Hand; 982 PKSPIN_LOCK_QUEUE LockQueue; 983 PKTIMER_TABLE_ENTRY TimerEntry; 984 985 /* Acquire timer lock */ 986 LockQueue = KiAcquireTimerLock(Hand); 987 988 /* Set the timer as non-inserted */ 989 Timer->Header.Inserted = FALSE; 990 991 /* Remove it from the timer list */ 992 if (RemoveEntryList(&Timer->TimerListEntry)) 993 { 994 /* Get the entry and check if it's empty */ 995 TimerEntry = &KiTimerTableListHead[Hand]; 996 if (IsListEmpty(&TimerEntry->Entry)) 997 { 998 /* Clear the time then */ 999 TimerEntry->Time.HighPart = 0xFFFFFFFF; 1000 } 1001 } 1002 1003 /* Release the timer lock */ 1004 KiReleaseTimerLock(LockQueue); 1005 } 1006 1007 FORCEINLINE 1008 VOID 1009 KxSetTimerForThreadWait(IN PKTIMER Timer, 1010 IN LARGE_INTEGER Interval, 1011 OUT PULONG Hand) 1012 { 1013 ULONGLONG DueTime; 1014 LARGE_INTEGER InterruptTime, SystemTime, TimeDifference; 1015 1016 /* Check the timer's interval to see if it's absolute */ 1017 Timer->Header.Absolute = FALSE; 1018 if (Interval.HighPart >= 0) 1019 { 1020 /* Get the system time and calculate the relative time */ 1021 KeQuerySystemTime(&SystemTime); 1022 TimeDifference.QuadPart = SystemTime.QuadPart - Interval.QuadPart; 1023 Timer->Header.Absolute = TRUE; 1024 1025 /* Check if we've already expired */ 1026 if (TimeDifference.HighPart >= 0) 1027 { 1028 /* Reset everything */ 1029 Timer->DueTime.QuadPart = 0; 1030 *Hand = 0; 1031 Timer->Header.Hand = 0; 1032 return; 1033 } 1034 else 1035 { 1036 /* Update the interval */ 1037 Interval = TimeDifference; 1038 } 1039 } 1040 1041 /* Calculate the due time */ 1042 InterruptTime.QuadPart = KeQueryInterruptTime(); 1043 DueTime = InterruptTime.QuadPart - Interval.QuadPart; 1044 Timer->DueTime.QuadPart = DueTime; 1045 1046 /* Calculate the timer handle */ 1047 *Hand = KiComputeTimerTableIndex(DueTime); 1048 Timer->Header.Hand = (UCHAR)*Hand; 1049 } 1050 1051 #define KxDelayThreadWait() \ 1052 \ 1053 /* Setup the Wait Block */ \ 1054 Thread->WaitBlockList = TimerBlock; \ 1055 \ 1056 /* Setup the timer */ \ 1057 KxSetTimerForThreadWait(Timer, *Interval, &Hand); \ 1058 \ 1059 /* Save the due time for the caller */ \ 1060 DueTime.QuadPart = Timer->DueTime.QuadPart; \ 1061 \ 1062 /* Link the timer to this Wait Block */ \ 1063 TimerBlock->NextWaitBlock = TimerBlock; \ 1064 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \ 1065 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \ 1066 \ 1067 /* Clear wait status */ \ 1068 Thread->WaitStatus = STATUS_SUCCESS; \ 1069 \ 1070 /* Setup wait fields */ \ 1071 Thread->Alertable = Alertable; \ 1072 Thread->WaitReason = DelayExecution; \ 1073 Thread->WaitMode = WaitMode; \ 1074 \ 1075 /* Check if we can swap the thread's stack */ \ 1076 Thread->WaitListEntry.Flink = NULL; \ 1077 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ 1078 \ 1079 /* Set the wait time */ \ 1080 Thread->WaitTime = KeTickCount.LowPart; 1081 1082 #define KxMultiThreadWait() \ 1083 /* Link wait block array to the thread */ \ 1084 Thread->WaitBlockList = WaitBlockArray; \ 1085 \ 1086 /* Reset the index */ \ 1087 Index = 0; \ 1088 \ 1089 /* Loop wait blocks */ \ 1090 do \ 1091 { \ 1092 /* Fill out the wait block */ \ 1093 WaitBlock = &WaitBlockArray[Index]; \ 1094 WaitBlock->Object = Object[Index]; \ 1095 WaitBlock->WaitKey = (USHORT)Index; \ 1096 WaitBlock->WaitType = WaitType; \ 1097 WaitBlock->Thread = Thread; \ 1098 \ 1099 /* Link to next block */ \ 1100 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \ 1101 Index++; \ 1102 } while (Index < Count); \ 1103 \ 1104 /* Link the last block */ \ 1105 WaitBlock->NextWaitBlock = WaitBlockArray; \ 1106 \ 1107 /* Set default wait status */ \ 1108 Thread->WaitStatus = STATUS_WAIT_0; \ 1109 \ 1110 /* Check if we have a timer */ \ 1111 if (Timeout) \ 1112 { \ 1113 /* Link to the block */ \ 1114 TimerBlock->NextWaitBlock = WaitBlockArray; \ 1115 \ 1116 /* Setup the timer */ \ 1117 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \ 1118 \ 1119 /* Save the due time for the caller */ \ 1120 DueTime.QuadPart = Timer->DueTime.QuadPart; \ 1121 \ 1122 /* Initialize the list */ \ 1123 InitializeListHead(&Timer->Header.WaitListHead); \ 1124 } \ 1125 \ 1126 /* Set wait settings */ \ 1127 Thread->Alertable = Alertable; \ 1128 Thread->WaitMode = WaitMode; \ 1129 Thread->WaitReason = WaitReason; \ 1130 \ 1131 /* Check if we can swap the thread's stack */ \ 1132 Thread->WaitListEntry.Flink = NULL; \ 1133 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ 1134 \ 1135 /* Set the wait time */ \ 1136 Thread->WaitTime = KeTickCount.LowPart; 1137 1138 #define KxSingleThreadWait() \ 1139 /* Setup the Wait Block */ \ 1140 Thread->WaitBlockList = WaitBlock; \ 1141 WaitBlock->WaitKey = STATUS_SUCCESS; \ 1142 WaitBlock->Object = Object; \ 1143 WaitBlock->WaitType = WaitAny; \ 1144 \ 1145 /* Clear wait status */ \ 1146 Thread->WaitStatus = STATUS_SUCCESS; \ 1147 \ 1148 /* Check if we have a timer */ \ 1149 if (Timeout) \ 1150 { \ 1151 /* Setup the timer */ \ 1152 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \ 1153 \ 1154 /* Save the due time for the caller */ \ 1155 DueTime.QuadPart = Timer->DueTime.QuadPart; \ 1156 \ 1157 /* Pointer to timer block */ \ 1158 WaitBlock->NextWaitBlock = TimerBlock; \ 1159 TimerBlock->NextWaitBlock = WaitBlock; \ 1160 \ 1161 /* Link the timer to this Wait Block */ \ 1162 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \ 1163 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \ 1164 } \ 1165 else \ 1166 { \ 1167 /* No timer block, just ourselves */ \ 1168 WaitBlock->NextWaitBlock = WaitBlock; \ 1169 } \ 1170 \ 1171 /* Set wait settings */ \ 1172 Thread->Alertable = Alertable; \ 1173 Thread->WaitMode = WaitMode; \ 1174 Thread->WaitReason = WaitReason; \ 1175 \ 1176 /* Check if we can swap the thread's stack */ \ 1177 Thread->WaitListEntry.Flink = NULL; \ 1178 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ 1179 \ 1180 /* Set the wait time */ \ 1181 Thread->WaitTime = KeTickCount.LowPart; 1182 1183 #define KxQueueThreadWait() \ 1184 /* Setup the Wait Block */ \ 1185 Thread->WaitBlockList = WaitBlock; \ 1186 WaitBlock->WaitKey = STATUS_SUCCESS; \ 1187 WaitBlock->Object = Queue; \ 1188 WaitBlock->WaitType = WaitAny; \ 1189 WaitBlock->Thread = Thread; \ 1190 \ 1191 /* Clear wait status */ \ 1192 Thread->WaitStatus = STATUS_SUCCESS; \ 1193 \ 1194 /* Check if we have a timer */ \ 1195 if (Timeout) \ 1196 { \ 1197 /* Setup the timer */ \ 1198 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \ 1199 \ 1200 /* Save the due time for the caller */ \ 1201 DueTime.QuadPart = Timer->DueTime.QuadPart; \ 1202 \ 1203 /* Pointer to timer block */ \ 1204 WaitBlock->NextWaitBlock = TimerBlock; \ 1205 TimerBlock->NextWaitBlock = WaitBlock; \ 1206 \ 1207 /* Link the timer to this Wait Block */ \ 1208 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \ 1209 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \ 1210 } \ 1211 else \ 1212 { \ 1213 /* No timer block, just ourselves */ \ 1214 WaitBlock->NextWaitBlock = WaitBlock; \ 1215 } \ 1216 \ 1217 /* Set wait settings */ \ 1218 Thread->Alertable = FALSE; \ 1219 Thread->WaitMode = WaitMode; \ 1220 Thread->WaitReason = WrQueue; \ 1221 \ 1222 /* Check if we can swap the thread's stack */ \ 1223 Thread->WaitListEntry.Flink = NULL; \ 1224 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ 1225 \ 1226 /* Set the wait time */ \ 1227 Thread->WaitTime = KeTickCount.LowPart; 1228 1229 // 1230 // Unwaits a Thread 1231 // 1232 FORCEINLINE 1233 VOID 1234 KxUnwaitThread(IN DISPATCHER_HEADER *Object, 1235 IN KPRIORITY Increment) 1236 { 1237 PLIST_ENTRY WaitEntry, WaitList; 1238 PKWAIT_BLOCK WaitBlock; 1239 PKTHREAD WaitThread; 1240 ULONG WaitKey; 1241 1242 /* Loop the Wait Entries */ 1243 WaitList = &Object->WaitListHead; 1244 ASSERT(IsListEmpty(&Object->WaitListHead) == FALSE); 1245 WaitEntry = WaitList->Flink; 1246 do 1247 { 1248 /* Get the current wait block */ 1249 WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry); 1250 1251 /* Get the waiting thread */ 1252 WaitThread = WaitBlock->Thread; 1253 1254 /* Check the current Wait Mode */ 1255 if (WaitBlock->WaitType == WaitAny) 1256 { 1257 /* Use the actual wait key */ 1258 WaitKey = WaitBlock->WaitKey; 1259 } 1260 else 1261 { 1262 /* Otherwise, use STATUS_KERNEL_APC */ 1263 WaitKey = STATUS_KERNEL_APC; 1264 } 1265 1266 /* Unwait the thread */ 1267 KiUnwaitThread(WaitThread, WaitKey, Increment); 1268 1269 /* Next entry */ 1270 WaitEntry = WaitList->Flink; 1271 } while (WaitEntry != WaitList); 1272 } 1273 1274 // 1275 // Unwaits a Thread waiting on an event 1276 // 1277 FORCEINLINE 1278 VOID 1279 KxUnwaitThreadForEvent(IN PKEVENT Event, 1280 IN KPRIORITY Increment) 1281 { 1282 PLIST_ENTRY WaitEntry, WaitList; 1283 PKWAIT_BLOCK WaitBlock; 1284 PKTHREAD WaitThread; 1285 1286 /* Loop the Wait Entries */ 1287 WaitList = &Event->Header.WaitListHead; 1288 ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE); 1289 WaitEntry = WaitList->Flink; 1290 do 1291 { 1292 /* Get the current wait block */ 1293 WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry); 1294 1295 /* Get the waiting thread */ 1296 WaitThread = WaitBlock->Thread; 1297 1298 /* Check the current Wait Mode */ 1299 if (WaitBlock->WaitType == WaitAny) 1300 { 1301 /* Un-signal it */ 1302 Event->Header.SignalState = 0; 1303 1304 /* Un-signal the event and unwait the thread */ 1305 KiUnwaitThread(WaitThread, WaitBlock->WaitKey, Increment); 1306 break; 1307 } 1308 1309 /* Unwait the thread with STATUS_KERNEL_APC */ 1310 KiUnwaitThread(WaitThread, STATUS_KERNEL_APC, Increment); 1311 1312 /* Next entry */ 1313 WaitEntry = WaitList->Flink; 1314 } while (WaitEntry != WaitList); 1315 } 1316 1317 // 1318 // This routine queues a thread that is ready on the PRCB's ready lists. 1319 // If this thread cannot currently run on this CPU, then the thread is 1320 // added to the deferred ready list instead. 1321 // 1322 // This routine must be entered with the PRCB lock held and it will exit 1323 // with the PRCB lock released! 1324 // 1325 FORCEINLINE 1326 VOID 1327 KxQueueReadyThread(IN PKTHREAD Thread, 1328 IN PKPRCB Prcb) 1329 { 1330 BOOLEAN Preempted; 1331 KPRIORITY Priority; 1332 1333 /* Sanity checks */ 1334 ASSERT(Prcb == KeGetCurrentPrcb()); 1335 ASSERT(Thread->State == Running); 1336 ASSERT(Thread->NextProcessor == Prcb->Number); 1337 1338 /* Check if this thread is allowed to run in this CPU */ 1339 #ifdef CONFIG_SMP 1340 if ((Thread->Affinity) & (Prcb->SetMember)) 1341 #else 1342 if (TRUE) 1343 #endif 1344 { 1345 /* Set thread ready for execution */ 1346 Thread->State = Ready; 1347 1348 /* Save current priority and if someone had pre-empted it */ 1349 Priority = Thread->Priority; 1350 Preempted = Thread->Preempted; 1351 1352 /* We're not pre-empting now, and set the wait time */ 1353 Thread->Preempted = FALSE; 1354 Thread->WaitTime = KeTickCount.LowPart; 1355 1356 /* Sanity check */ 1357 ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY)); 1358 1359 /* Insert this thread in the appropriate order */ 1360 Preempted ? InsertHeadList(&Prcb->DispatcherReadyListHead[Priority], 1361 &Thread->WaitListEntry) : 1362 InsertTailList(&Prcb->DispatcherReadyListHead[Priority], 1363 &Thread->WaitListEntry); 1364 1365 /* Update the ready summary */ 1366 Prcb->ReadySummary |= PRIORITY_MASK(Priority); 1367 1368 /* Sanity check */ 1369 ASSERT(Priority == Thread->Priority); 1370 1371 /* Release the PRCB lock */ 1372 KiReleasePrcbLock(Prcb); 1373 } 1374 else 1375 { 1376 /* Otherwise, prepare this thread to be deferred */ 1377 Thread->State = DeferredReady; 1378 Thread->DeferredProcessor = Prcb->Number; 1379 1380 /* Release the lock and defer scheduling */ 1381 KiReleasePrcbLock(Prcb); 1382 KiDeferredReadyThread(Thread); 1383 } 1384 } 1385 1386 // 1387 // This routine scans for an appropriate ready thread to select at the 1388 // given priority and for the given CPU. 1389 // 1390 FORCEINLINE 1391 PKTHREAD 1392 KiSelectReadyThread(IN KPRIORITY Priority, 1393 IN PKPRCB Prcb) 1394 { 1395 ULONG PrioritySet; 1396 LONG HighPriority; 1397 PLIST_ENTRY ListEntry; 1398 PKTHREAD Thread = NULL; 1399 1400 /* Save the current mask and get the priority set for the CPU */ 1401 PrioritySet = Prcb->ReadySummary >> Priority; 1402 if (!PrioritySet) goto Quickie; 1403 1404 /* Get the highest priority possible */ 1405 BitScanReverse((PULONG)&HighPriority, PrioritySet); 1406 ASSERT((PrioritySet & PRIORITY_MASK(HighPriority)) != 0); 1407 HighPriority += Priority; 1408 1409 /* Make sure the list isn't empty at the highest priority */ 1410 ASSERT(IsListEmpty(&Prcb->DispatcherReadyListHead[HighPriority]) == FALSE); 1411 1412 /* Get the first thread on the list */ 1413 ListEntry = Prcb->DispatcherReadyListHead[HighPriority].Flink; 1414 Thread = CONTAINING_RECORD(ListEntry, KTHREAD, WaitListEntry); 1415 1416 /* Make sure this thread is here for a reason */ 1417 ASSERT(HighPriority == Thread->Priority); 1418 ASSERT(Thread->Affinity & AFFINITY_MASK(Prcb->Number)); 1419 ASSERT(Thread->NextProcessor == Prcb->Number); 1420 1421 /* Remove it from the list */ 1422 if (RemoveEntryList(&Thread->WaitListEntry)) 1423 { 1424 /* The list is empty now, reset the ready summary */ 1425 Prcb->ReadySummary ^= PRIORITY_MASK(HighPriority); 1426 } 1427 1428 /* Sanity check and return the thread */ 1429 Quickie: 1430 ASSERT((Thread == NULL) || 1431 (Thread->BasePriority == 0) || 1432 (Thread->Priority != 0)); 1433 return Thread; 1434 } 1435 1436 // 1437 // This routine computes the new priority for a thread. It is only valid for 1438 // threads with priorities in the dynamic priority range. 1439 // 1440 FORCEINLINE 1441 SCHAR 1442 KiComputeNewPriority(IN PKTHREAD Thread, 1443 IN SCHAR Adjustment) 1444 { 1445 SCHAR Priority; 1446 1447 /* Priority sanity checks */ 1448 ASSERT((Thread->PriorityDecrement >= 0) && 1449 (Thread->PriorityDecrement <= Thread->Priority)); 1450 ASSERT((Thread->Priority < LOW_REALTIME_PRIORITY) ? 1451 TRUE : (Thread->PriorityDecrement == 0)); 1452 1453 /* Get the current priority */ 1454 Priority = Thread->Priority; 1455 if (Priority < LOW_REALTIME_PRIORITY) 1456 { 1457 /* Decrease priority by the priority decrement */ 1458 Priority -= (Thread->PriorityDecrement + Adjustment); 1459 1460 /* Don't go out of bounds */ 1461 if (Priority < Thread->BasePriority) Priority = Thread->BasePriority; 1462 1463 /* Reset the priority decrement */ 1464 Thread->PriorityDecrement = 0; 1465 } 1466 1467 /* Sanity check */ 1468 ASSERT((Thread->BasePriority == 0) || (Priority != 0)); 1469 1470 /* Return the new priority */ 1471 return Priority; 1472 } 1473 1474 // 1475 // Guarded Mutex Routines 1476 // 1477 FORCEINLINE 1478 VOID 1479 _KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex) 1480 { 1481 /* Setup the Initial Data */ 1482 GuardedMutex->Count = GM_LOCK_BIT; 1483 GuardedMutex->Owner = NULL; 1484 GuardedMutex->Contention = 0; 1485 1486 /* Initialize the Wait Gate */ 1487 KeInitializeGate(&GuardedMutex->Gate); 1488 } 1489 1490 FORCEINLINE 1491 VOID 1492 _KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) 1493 { 1494 PKTHREAD Thread = KeGetCurrentThread(); 1495 1496 /* Sanity checks */ 1497 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1498 (Thread->SpecialApcDisable < 0) || 1499 (Thread->Teb == NULL) || 1500 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1501 ASSERT(GuardedMutex->Owner != Thread); 1502 1503 /* Remove the lock */ 1504 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V)) 1505 { 1506 /* The Guarded Mutex was already locked, enter contented case */ 1507 KiAcquireGuardedMutex(GuardedMutex); 1508 } 1509 1510 /* Set the Owner */ 1511 GuardedMutex->Owner = Thread; 1512 } 1513 1514 FORCEINLINE 1515 VOID 1516 _KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) 1517 { 1518 LONG OldValue, NewValue; 1519 1520 /* Sanity checks */ 1521 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1522 (KeGetCurrentThread()->SpecialApcDisable < 0) || 1523 (KeGetCurrentThread()->Teb == NULL) || 1524 (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1525 ASSERT(GuardedMutex->Owner == KeGetCurrentThread()); 1526 1527 /* Destroy the Owner */ 1528 GuardedMutex->Owner = NULL; 1529 1530 /* Add the Lock Bit */ 1531 OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT); 1532 ASSERT((OldValue & GM_LOCK_BIT) == 0); 1533 1534 /* Check if it was already locked, but not woken */ 1535 if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN)) 1536 { 1537 /* Update the Oldvalue to what it should be now */ 1538 OldValue += GM_LOCK_BIT; 1539 1540 /* The mutex will be woken, minus one waiter */ 1541 NewValue = OldValue + GM_LOCK_WAITER_WOKEN - 1542 GM_LOCK_WAITER_INC; 1543 1544 /* Remove the Woken bit */ 1545 if (InterlockedCompareExchange(&GuardedMutex->Count, 1546 NewValue, 1547 OldValue) == OldValue) 1548 { 1549 /* Signal the Gate */ 1550 KeSignalGateBoostPriority(&GuardedMutex->Gate); 1551 } 1552 } 1553 } 1554 1555 FORCEINLINE 1556 VOID 1557 _KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex) 1558 { 1559 PKTHREAD Thread = KeGetCurrentThread(); 1560 1561 /* Sanity checks */ 1562 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1563 ASSERT(GuardedMutex->Owner != Thread); 1564 1565 /* Disable Special APCs */ 1566 KeEnterGuardedRegion(); 1567 1568 /* Remove the lock */ 1569 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V)) 1570 { 1571 /* The Guarded Mutex was already locked, enter contented case */ 1572 KiAcquireGuardedMutex(GuardedMutex); 1573 } 1574 1575 /* Set the Owner and Special APC Disable state */ 1576 GuardedMutex->Owner = Thread; 1577 GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable; 1578 } 1579 1580 FORCEINLINE 1581 VOID 1582 _KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) 1583 { 1584 LONG OldValue, NewValue; 1585 1586 /* Sanity checks */ 1587 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1588 ASSERT(GuardedMutex->Owner == KeGetCurrentThread()); 1589 ASSERT(KeGetCurrentThread()->SpecialApcDisable == 1590 GuardedMutex->SpecialApcDisable); 1591 1592 /* Destroy the Owner */ 1593 GuardedMutex->Owner = NULL; 1594 1595 /* Add the Lock Bit */ 1596 OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT); 1597 ASSERT((OldValue & GM_LOCK_BIT) == 0); 1598 1599 /* Check if it was already locked, but not woken */ 1600 if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN)) 1601 { 1602 /* Update the Oldvalue to what it should be now */ 1603 OldValue += GM_LOCK_BIT; 1604 1605 /* The mutex will be woken, minus one waiter */ 1606 NewValue = OldValue + GM_LOCK_WAITER_WOKEN - 1607 GM_LOCK_WAITER_INC; 1608 1609 /* Remove the Woken bit */ 1610 if (InterlockedCompareExchange(&GuardedMutex->Count, 1611 NewValue, 1612 OldValue) == OldValue) 1613 { 1614 /* Signal the Gate */ 1615 KeSignalGateBoostPriority(&GuardedMutex->Gate); 1616 } 1617 } 1618 1619 /* Re-enable APCs */ 1620 KeLeaveGuardedRegion(); 1621 } 1622 1623 FORCEINLINE 1624 BOOLEAN 1625 _KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) 1626 { 1627 PKTHREAD Thread = KeGetCurrentThread(); 1628 1629 /* Block APCs */ 1630 KeEnterGuardedRegion(); 1631 1632 /* Remove the lock */ 1633 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V)) 1634 { 1635 /* Re-enable APCs */ 1636 KeLeaveGuardedRegion(); 1637 YieldProcessor(); 1638 1639 /* Return failure */ 1640 return FALSE; 1641 } 1642 1643 /* Set the Owner and APC State */ 1644 GuardedMutex->Owner = Thread; 1645 GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable; 1646 return TRUE; 1647 } 1648 1649 1650 FORCEINLINE 1651 VOID 1652 KiAcquireNmiListLock(OUT PKIRQL OldIrql) 1653 { 1654 KeAcquireSpinLock(&KiNmiCallbackListLock, OldIrql); 1655 } 1656 1657 FORCEINLINE 1658 VOID 1659 KiReleaseNmiListLock(IN KIRQL OldIrql) 1660 { 1661 KeReleaseSpinLock(&KiNmiCallbackListLock, OldIrql); 1662 } 1663