1 #pragma once 2 3 #ifdef __cplusplus 4 extern "C" 5 { 6 #endif 7 8 /* GLOBAL VARIABLES *********************************************************/ 9 10 extern RTL_TIME_ZONE_INFORMATION ExpTimeZoneInfo; 11 extern LARGE_INTEGER ExpTimeZoneBias; 12 extern ULONG ExpTimeZoneId; 13 extern ULONG ExpTickCountMultiplier; 14 extern ULONG ExpLastTimeZoneBias; 15 extern POBJECT_TYPE ExEventPairObjectType; 16 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType; 17 extern FAST_MUTEX ExpEnvironmentLock; 18 extern ERESOURCE ExpFirmwareTableResource; 19 extern ERESOURCE ExpTimeRefreshLock; 20 extern LIST_ENTRY ExpFirmwareTableProviderListHead; 21 extern BOOLEAN ExpIsWinPEMode; 22 extern ULONG ExpResourceTimeoutCount; 23 extern LIST_ENTRY ExpSystemResourcesList; 24 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset; 25 extern ULONG ExpUnicodeCaseTableDataOffset; 26 extern PVOID ExpNlsSectionPointer; 27 extern ULONG NtGlobalFlag; 28 extern UNICODE_STRING NtSystemRoot; 29 extern ULONG ExpInitializationPhase; 30 extern ULONG ExpAltTimeZoneBias; 31 extern LIST_ENTRY ExSystemLookasideListHead; 32 extern PCALLBACK_OBJECT PowerStateCallback; 33 extern LIST_ENTRY ExPoolLookasideListHead; 34 extern LIST_ENTRY ExpNonPagedLookasideListHead; 35 extern LIST_ENTRY ExpPagedLookasideListHead; 36 extern KSPIN_LOCK ExpNonPagedLookasideListLock; 37 extern KSPIN_LOCK ExpPagedLookasideListLock; 38 extern ULONG ExCriticalWorkerThreads; 39 extern ULONG ExDelayedWorkerThreads; 40 41 extern PVOID ExpDefaultErrorPort; 42 extern PEPROCESS ExpDefaultErrorPortProcess; 43 44 /* 45 * NT/Cm Version Info variables 46 */ 47 extern ULONG NtMajorVersion; 48 extern ULONG NtMinorVersion; 49 extern ULONG NtBuildNumber; 50 extern ULONG CmNtSpBuildNumber; 51 extern ULONG CmNtCSDVersion; 52 extern ULONG CmNtCSDReleaseType; 53 extern UNICODE_STRING CmVersionString; 54 extern UNICODE_STRING CmCSDVersionString; 55 extern CHAR NtBuildLab[]; 56 57 /* 58 * WinDBG Debugger Worker State Machine data (see dbgctrl.c) 59 */ 60 typedef enum _WINKD_WORKER_STATE 61 { 62 WinKdWorkerReady = 0, 63 WinKdWorkerStart, 64 WinKdWorkerInitialized 65 } WINKD_WORKER_STATE; 66 67 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem; 68 extern WINKD_WORKER_STATE ExpDebuggerWork; 69 extern PEPROCESS ExpDebuggerProcessAttach; 70 extern PEPROCESS ExpDebuggerProcessKill; 71 extern ULONG_PTR ExpDebuggerPageIn; 72 73 VOID 74 NTAPI 75 ExpDebuggerWorker( 76 _In_ PVOID Context); 77 78 #ifdef _WIN64 79 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4) 80 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3) 81 #else 82 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3) 83 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2) 84 #endif 85 #define HANDLE_TAG_BITS 2 86 #define HANDLE_INDEX_BITS (HANDLE_LOW_BITS + 2 * HANDLE_HIGH_BITS) 87 #define KERNEL_FLAG_BITS (sizeof(ULONG_PTR) * 8 - HANDLE_INDEX_BITS - HANDLE_TAG_BITS) 88 89 typedef union _EXHANDLE 90 { 91 struct 92 { 93 ULONG_PTR TagBits: HANDLE_TAG_BITS; 94 ULONG_PTR Index: HANDLE_INDEX_BITS; 95 ULONG_PTR KernelFlag: KERNEL_FLAG_BITS; 96 }; 97 struct 98 { 99 ULONG_PTR TagBits2: HANDLE_TAG_BITS; 100 ULONG_PTR LowIndex: HANDLE_LOW_BITS; 101 ULONG_PTR MidIndex: HANDLE_HIGH_BITS; 102 ULONG_PTR HighIndex: HANDLE_HIGH_BITS; 103 ULONG_PTR KernelFlag2: KERNEL_FLAG_BITS; 104 }; 105 HANDLE GenericHandleOverlay; 106 ULONG_PTR Value; 107 ULONG AsULONG; 108 } EXHANDLE, *PEXHANDLE; 109 110 typedef struct _ETIMER 111 { 112 KTIMER KeTimer; 113 KAPC TimerApc; 114 KDPC TimerDpc; 115 LIST_ENTRY ActiveTimerListEntry; 116 KSPIN_LOCK Lock; 117 LONG Period; 118 BOOLEAN ApcAssociated; 119 BOOLEAN WakeTimer; 120 LIST_ENTRY WakeTimerListEntry; 121 } ETIMER, *PETIMER; 122 123 typedef struct 124 { 125 PCALLBACK_OBJECT *CallbackObject; 126 PWSTR Name; 127 } SYSTEM_CALLBACKS; 128 129 typedef struct _HARDERROR_USER_PARAMETERS 130 { 131 ULONG_PTR Parameters[MAXIMUM_HARDERROR_PARAMETERS]; 132 UNICODE_STRING Strings[MAXIMUM_HARDERROR_PARAMETERS]; 133 WCHAR Buffer[ANYSIZE_ARRAY]; 134 } HARDERROR_USER_PARAMETERS, *PHARDERROR_USER_PARAMETERS; 135 136 #define MAX_FAST_REFS 7 137 138 #define ExAcquireRundownProtection _ExAcquireRundownProtection 139 #define ExReleaseRundownProtection _ExReleaseRundownProtection 140 #define ExInitializeRundownProtection _ExInitializeRundownProtection 141 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease 142 #define ExRundownCompleted _ExRundownCompleted 143 #define ExGetPreviousMode KeGetPreviousMode 144 145 146 // 147 // Various bits tagged on the handle or handle table 148 // 149 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1 150 #define FREE_HANDLE_MASK -1 151 152 // 153 // Number of entries in each table level 154 // 155 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY)) 156 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY)) 157 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES)) 158 159 // 160 // Maximum index in each table level before we need another table 161 // 162 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES 163 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES) 164 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES) 165 166 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&(x)->Ptr, (PVOID)(y), (PVOID)(z)) 167 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)(x), (PVOID)(y), (PVOID)(z)) 168 #define ExpSetRundown(x, y) InterlockedExchangePointer(&(x)->Ptr, (PVOID)(y)) 169 170 NTSTATUS 171 NTAPI 172 ExGetPoolTagInfo( 173 IN PSYSTEM_POOLTAG_INFORMATION SystemInformation, 174 IN ULONG SystemInformationLength, 175 IN OUT PULONG ReturnLength OPTIONAL 176 ); 177 178 typedef struct _UUID_CACHED_VALUES_STRUCT 179 { 180 ULONGLONG Time; 181 LONG AllocatedCount; 182 union 183 { 184 struct 185 { 186 UCHAR ClockSeqHiAndReserved; 187 UCHAR ClockSeqLow; 188 UCHAR NodeId[6 /*SEED_BUFFER_SIZE*/]; 189 }; 190 UCHAR GuidInit[8]; /* Match GUID.Data4 */ 191 }; 192 } UUID_CACHED_VALUES_STRUCT, *PUUID_CACHED_VALUES_STRUCT; 193 194 C_ASSERT(RTL_FIELD_SIZE(UUID_CACHED_VALUES_STRUCT, GuidInit) == RTL_FIELD_SIZE(UUID, Data4)); 195 196 /* INITIALIZATION FUNCTIONS *************************************************/ 197 198 CODE_SEG("INIT") 199 BOOLEAN 200 NTAPI 201 ExpWin32kInit(VOID); 202 203 VOID 204 NTAPI 205 ExInit2(VOID); 206 207 VOID 208 NTAPI 209 Phase1Initialization( 210 IN PVOID Context 211 ); 212 213 CODE_SEG("INIT") 214 VOID 215 NTAPI 216 ExpInitializePushLocks(VOID); 217 218 BOOLEAN 219 NTAPI 220 ExRefreshTimeZoneInformation( 221 IN PLARGE_INTEGER SystemBootTime 222 ); 223 224 CODE_SEG("INIT") 225 VOID 226 NTAPI 227 ExpInitializeWorkerThreads(VOID); 228 229 VOID 230 NTAPI 231 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap); 232 233 CODE_SEG("INIT") 234 VOID 235 NTAPI 236 ExpInitLookasideLists(VOID); 237 238 CODE_SEG("INIT") 239 VOID 240 NTAPI 241 ExInitializeSystemLookasideList( 242 IN PGENERAL_LOOKASIDE List, 243 IN POOL_TYPE Type, 244 IN ULONG Size, 245 IN ULONG Tag, 246 IN USHORT MaximumDepth, 247 IN PLIST_ENTRY ListHead 248 ); 249 250 CODE_SEG("INIT") 251 BOOLEAN 252 NTAPI 253 ExpInitializeCallbacks(VOID); 254 255 CODE_SEG("INIT") 256 BOOLEAN 257 NTAPI 258 ExpUuidInitialization(VOID); 259 260 CODE_SEG("INIT") 261 BOOLEAN 262 NTAPI 263 ExLuidInitialization(VOID); 264 265 CODE_SEG("INIT") 266 VOID 267 NTAPI 268 ExpInitializeExecutive( 269 IN ULONG Cpu, 270 IN PLOADER_PARAMETER_BLOCK LoaderBlock 271 ); 272 273 VOID 274 NTAPI 275 ExShutdownSystem(VOID); 276 277 CODE_SEG("INIT") 278 BOOLEAN 279 NTAPI 280 ExpInitializeEventImplementation(VOID); 281 282 CODE_SEG("INIT") 283 BOOLEAN 284 NTAPI 285 ExpInitializeKeyedEventImplementation(VOID); 286 287 CODE_SEG("INIT") 288 BOOLEAN 289 NTAPI 290 ExpInitializeEventPairImplementation(VOID); 291 292 CODE_SEG("INIT") 293 BOOLEAN 294 NTAPI 295 ExpInitializeSemaphoreImplementation(VOID); 296 297 CODE_SEG("INIT") 298 BOOLEAN 299 NTAPI 300 ExpInitializeMutantImplementation(VOID); 301 302 CODE_SEG("INIT") 303 BOOLEAN 304 NTAPI 305 ExpInitializeTimerImplementation(VOID); 306 307 CODE_SEG("INIT") 308 BOOLEAN 309 NTAPI 310 ExpInitializeProfileImplementation(VOID); 311 312 CODE_SEG("INIT") 313 VOID 314 NTAPI 315 ExpResourceInitialization(VOID); 316 317 CODE_SEG("INIT") 318 VOID 319 NTAPI 320 ExInitPoolLookasidePointers(VOID); 321 322 /* Callback Functions ********************************************************/ 323 324 VOID 325 NTAPI 326 ExInitializeCallBack( 327 IN OUT PEX_CALLBACK Callback 328 ); 329 330 PEX_CALLBACK_ROUTINE_BLOCK 331 NTAPI 332 ExAllocateCallBack( 333 IN PEX_CALLBACK_FUNCTION Function, 334 IN PVOID Context 335 ); 336 337 VOID 338 NTAPI 339 ExFreeCallBack( 340 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 341 ); 342 343 BOOLEAN 344 NTAPI 345 ExCompareExchangeCallBack ( 346 IN OUT PEX_CALLBACK CallBack, 347 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock, 348 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock 349 ); 350 351 PEX_CALLBACK_ROUTINE_BLOCK 352 NTAPI 353 ExReferenceCallBackBlock( 354 IN OUT PEX_CALLBACK CallBack 355 ); 356 357 VOID 358 NTAPI 359 ExDereferenceCallBackBlock( 360 IN OUT PEX_CALLBACK CallBack, 361 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 362 ); 363 364 PEX_CALLBACK_FUNCTION 365 NTAPI 366 ExGetCallBackBlockRoutine( 367 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 368 ); 369 370 PVOID 371 NTAPI 372 ExGetCallBackBlockContext( 373 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 374 ); 375 376 VOID 377 NTAPI 378 ExWaitForCallBacks( 379 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 380 ); 381 382 /* Rundown Functions ********************************************************/ 383 384 VOID 385 FASTCALL 386 ExfInitializeRundownProtection( 387 OUT PEX_RUNDOWN_REF RunRef 388 ); 389 390 VOID 391 FASTCALL 392 ExfReInitializeRundownProtection( 393 OUT PEX_RUNDOWN_REF RunRef 394 ); 395 396 BOOLEAN 397 FASTCALL 398 ExfAcquireRundownProtection( 399 IN OUT PEX_RUNDOWN_REF RunRef 400 ); 401 402 BOOLEAN 403 FASTCALL 404 ExfAcquireRundownProtectionEx( 405 IN OUT PEX_RUNDOWN_REF RunRef, 406 IN ULONG Count 407 ); 408 409 VOID 410 FASTCALL 411 ExfReleaseRundownProtection( 412 IN OUT PEX_RUNDOWN_REF RunRef 413 ); 414 415 VOID 416 FASTCALL 417 ExfReleaseRundownProtectionEx( 418 IN OUT PEX_RUNDOWN_REF RunRef, 419 IN ULONG Count 420 ); 421 422 VOID 423 FASTCALL 424 ExfRundownCompleted( 425 OUT PEX_RUNDOWN_REF RunRef 426 ); 427 428 VOID 429 FASTCALL 430 ExfWaitForRundownProtectionRelease( 431 IN OUT PEX_RUNDOWN_REF RunRef 432 ); 433 434 /* HANDLE TABLE FUNCTIONS ***************************************************/ 435 436 typedef BOOLEAN 437 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)( 438 PHANDLE_TABLE_ENTRY HandleTableEntry, 439 HANDLE Handle, 440 PVOID Context 441 ); 442 443 typedef BOOLEAN 444 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)( 445 IN PEPROCESS Process, 446 IN PHANDLE_TABLE HandleTable, 447 IN PHANDLE_TABLE_ENTRY HandleTableEntry, 448 IN PHANDLE_TABLE_ENTRY NewEntry 449 ); 450 451 typedef BOOLEAN 452 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)( 453 PHANDLE_TABLE_ENTRY HandleTableEntry, 454 ULONG_PTR Context 455 ); 456 457 CODE_SEG("INIT") 458 VOID 459 NTAPI 460 ExpInitializeHandleTables( 461 VOID 462 ); 463 464 PHANDLE_TABLE 465 NTAPI 466 ExCreateHandleTable( 467 IN PEPROCESS Process OPTIONAL 468 ); 469 470 VOID 471 NTAPI 472 ExUnlockHandleTableEntry( 473 IN PHANDLE_TABLE HandleTable, 474 IN PHANDLE_TABLE_ENTRY HandleTableEntry 475 ); 476 477 HANDLE 478 NTAPI 479 ExCreateHandle( 480 IN PHANDLE_TABLE HandleTable, 481 IN PHANDLE_TABLE_ENTRY HandleTableEntry 482 ); 483 484 VOID 485 NTAPI 486 ExDestroyHandleTable( 487 IN PHANDLE_TABLE HandleTable, 488 IN PVOID DestroyHandleProcedure OPTIONAL 489 ); 490 491 BOOLEAN 492 NTAPI 493 ExDestroyHandle( 494 IN PHANDLE_TABLE HandleTable, 495 IN HANDLE Handle, 496 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL 497 ); 498 499 PHANDLE_TABLE_ENTRY 500 NTAPI 501 ExMapHandleToPointer( 502 IN PHANDLE_TABLE HandleTable, 503 IN HANDLE Handle 504 ); 505 506 PHANDLE_TABLE 507 NTAPI 508 ExDupHandleTable( 509 IN PEPROCESS Process, 510 IN PHANDLE_TABLE HandleTable, 511 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure, 512 IN ULONG_PTR Mask 513 ); 514 515 BOOLEAN 516 NTAPI 517 ExChangeHandle( 518 IN PHANDLE_TABLE HandleTable, 519 IN HANDLE Handle, 520 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine, 521 IN ULONG_PTR Context 522 ); 523 524 VOID 525 NTAPI 526 ExSweepHandleTable( 527 IN PHANDLE_TABLE HandleTable, 528 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure, 529 IN PVOID Context 530 ); 531 532 PHANDLE_TABLE_ENTRY 533 NTAPI 534 ExpLookupHandleTableEntry( 535 IN PHANDLE_TABLE HandleTable, 536 IN EXHANDLE Handle 537 ); 538 539 BOOLEAN 540 NTAPI 541 ExpLockHandleTableEntry( 542 IN PHANDLE_TABLE HandleTable, 543 IN PHANDLE_TABLE_ENTRY HandleTableEntry 544 ); 545 546 /* PSEH EXCEPTION HANDLING **************************************************/ 547 548 LONG 549 NTAPI 550 ExSystemExceptionFilter(VOID); 551 552 /* CALLBACKS *****************************************************************/ 553 554 FORCEINLINE 555 VOID 556 ExDoCallBack(IN OUT PEX_CALLBACK Callback, 557 IN PVOID Context, 558 IN PVOID Argument1, 559 IN PVOID Argument2) 560 { 561 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock; 562 PEX_CALLBACK_FUNCTION Function; 563 564 /* Reference the block */ 565 CallbackBlock = ExReferenceCallBackBlock(Callback); 566 if (CallbackBlock) 567 { 568 /* Get the function */ 569 Function = ExGetCallBackBlockRoutine(CallbackBlock); 570 571 /* Do the callback */ 572 Function(Context, Argument1, Argument2); 573 574 /* Now dereference it */ 575 ExDereferenceCallBackBlock(Callback, CallbackBlock); 576 } 577 } 578 579 /* FAST REFS ******************************************************************/ 580 581 FORCEINLINE 582 PVOID 583 ExGetObjectFastReference(IN EX_FAST_REF FastRef) 584 { 585 /* Return the unbiased pointer */ 586 return (PVOID)(FastRef.Value & ~MAX_FAST_REFS); 587 } 588 589 FORCEINLINE 590 ULONG 591 ExGetCountFastReference(IN EX_FAST_REF FastRef) 592 { 593 /* Return the reference count */ 594 return (ULONG)FastRef.RefCnt; 595 } 596 597 FORCEINLINE 598 VOID 599 ExInitializeFastReference(OUT PEX_FAST_REF FastRef, 600 IN OPTIONAL PVOID Object) 601 { 602 /* Sanity check */ 603 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0); 604 605 /* Check if an object is being set */ 606 if (!Object) 607 { 608 /* Clear the field */ 609 FastRef->Object = NULL; 610 } 611 else 612 { 613 /* Otherwise, we assume the object was referenced and is ready */ 614 FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS; 615 } 616 } 617 618 FORCEINLINE 619 EX_FAST_REF 620 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef) 621 { 622 EX_FAST_REF OldValue, NewValue; 623 624 /* Start reference loop */ 625 for (;;) 626 { 627 /* Get the current reference count */ 628 OldValue = *FastRef; 629 if (OldValue.RefCnt) 630 { 631 /* Increase the reference count */ 632 NewValue.Value = OldValue.Value - 1; 633 NewValue.Object = ExpChangePushlock(&FastRef->Object, 634 NewValue.Object, 635 OldValue.Object); 636 if (NewValue.Object != OldValue.Object) continue; 637 } 638 639 /* We are done */ 640 break; 641 } 642 643 /* Return the old value */ 644 return OldValue; 645 } 646 647 FORCEINLINE 648 BOOLEAN 649 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef, 650 IN PVOID Object) 651 { 652 EX_FAST_REF OldValue, NewValue; 653 654 /* Sanity checks */ 655 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 656 657 /* Start update loop */ 658 for (;;) 659 { 660 /* Get the current reference count */ 661 OldValue = *FastRef; 662 663 /* Check if the current count is too high or if the pointer changed */ 664 if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) || 665 ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object)) 666 { 667 /* Fail */ 668 return FALSE; 669 } 670 671 /* Update the reference count */ 672 NewValue.Value = OldValue.Value + MAX_FAST_REFS; 673 NewValue.Object = ExpChangePushlock(&FastRef->Object, 674 NewValue.Object, 675 OldValue.Object); 676 if (NewValue.Object != OldValue.Object) continue; 677 678 /* We are done */ 679 break; 680 } 681 682 /* Return success */ 683 return TRUE; 684 } 685 686 FORCEINLINE 687 BOOLEAN 688 ExReleaseFastReference(IN PEX_FAST_REF FastRef, 689 IN PVOID Object) 690 { 691 EX_FAST_REF OldValue, NewValue; 692 693 /* Sanity checks */ 694 ASSERT(Object != NULL); 695 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 696 697 /* Start reference loop */ 698 for (;;) 699 { 700 /* Get the current reference count */ 701 OldValue = *FastRef; 702 703 /* Check if we're full if if the pointer changed */ 704 if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE; 705 706 /* Decrease the reference count */ 707 NewValue.Value = OldValue.Value + 1; 708 NewValue.Object = ExpChangePushlock(&FastRef->Object, 709 NewValue.Object, 710 OldValue.Object); 711 if (NewValue.Object != OldValue.Object) continue; 712 713 /* We are done */ 714 break; 715 } 716 717 /* Return success */ 718 return TRUE; 719 } 720 721 FORCEINLINE 722 EX_FAST_REF 723 ExSwapFastReference(IN PEX_FAST_REF FastRef, 724 IN PVOID Object) 725 { 726 EX_FAST_REF NewValue, OldValue; 727 728 /* Sanity check */ 729 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0); 730 731 /* Check if an object is being set */ 732 if (!Object) 733 { 734 /* Clear the field */ 735 NewValue.Object = NULL; 736 } 737 else 738 { 739 /* Otherwise, we assume the object was referenced and is ready */ 740 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS; 741 } 742 743 /* Update the object */ 744 OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object); 745 return OldValue; 746 } 747 748 FORCEINLINE 749 EX_FAST_REF 750 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef, 751 IN PVOID Object, 752 IN PVOID OldObject) 753 { 754 EX_FAST_REF OldValue, NewValue; 755 756 /* Sanity check and start swap loop */ 757 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 758 for (;;) 759 { 760 /* Get the current value */ 761 OldValue = *FastRef; 762 763 /* Make sure there's enough references to swap */ 764 if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break; 765 766 /* Check if we have an object to swap */ 767 if (Object) 768 { 769 /* Set up the value with maximum fast references */ 770 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS; 771 } 772 else 773 { 774 /* Write the object address itself (which is empty) */ 775 NewValue.Value = (ULONG_PTR)Object; 776 } 777 778 /* Do the actual compare exchange */ 779 NewValue.Object = ExpChangePushlock(&FastRef->Object, 780 NewValue.Object, 781 OldValue.Object); 782 if (NewValue.Object != OldValue.Object) continue; 783 784 /* All done */ 785 break; 786 } 787 788 /* Return the old value */ 789 return OldValue; 790 } 791 792 /* RUNDOWN *******************************************************************/ 793 794 FORCEINLINE 795 PEX_RUNDOWN_REF 796 ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, 797 IN ULONG ProcNumber) 798 { 799 return (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs + 800 RunRefCacheAware->RunRefSize * 801 (ProcNumber % RunRefCacheAware->Number)); 802 } 803 804 /*++ 805 * @name ExfAcquireRundownProtection 806 * INTERNAL MACRO 807 * 808 * The ExfAcquireRundownProtection routine acquires rundown protection for 809 * the specified descriptor. 810 * 811 * @param RunRef 812 * Pointer to a rundown reference descriptor. 813 * 814 * @return TRUE if access to the protected structure was granted, FALSE otherwise. 815 * 816 * @remarks This is the internal macro for system use only.In case the rundown 817 * was active, then the slow-path will be called through the exported 818 * function. 819 * 820 *--*/ 821 FORCEINLINE 822 BOOLEAN 823 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef) 824 { 825 ULONG_PTR Value, NewValue; 826 827 /* Get the current value and mask the active bit */ 828 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE; 829 830 /* Add a reference */ 831 NewValue = Value + EX_RUNDOWN_COUNT_INC; 832 833 /* Change the value */ 834 NewValue = ExpChangeRundown(RunRef, NewValue, Value); 835 if (NewValue != Value) 836 { 837 /* Rundown was active, use long path */ 838 return ExfAcquireRundownProtection(RunRef); 839 } 840 841 /* Success */ 842 return TRUE; 843 } 844 845 /*++ 846 * @name ExReleaseRundownProtection 847 * INTERNAL MACRO 848 * 849 * The ExReleaseRundownProtection routine releases rundown protection for 850 * the specified descriptor. 851 * 852 * @param RunRef 853 * Pointer to a rundown reference descriptor. 854 * 855 * @return TRUE if access to the protected structure was granted, FALSE otherwise. 856 * 857 * @remarks This is the internal macro for system use only.In case the rundown 858 * was active, then the slow-path will be called through the exported 859 * function. 860 * 861 *--*/ 862 FORCEINLINE 863 VOID 864 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef) 865 { 866 ULONG_PTR Value, NewValue; 867 868 /* Get the current value and mask the active bit */ 869 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE; 870 871 /* Remove a reference */ 872 NewValue = Value - EX_RUNDOWN_COUNT_INC; 873 874 /* Change the value */ 875 NewValue = ExpChangeRundown(RunRef, NewValue, Value); 876 877 /* Check if the rundown was active */ 878 if (NewValue != Value) 879 { 880 /* Rundown was active, use long path */ 881 ExfReleaseRundownProtection(RunRef); 882 } 883 else 884 { 885 /* Sanity check */ 886 ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1)); 887 } 888 } 889 890 /*++ 891 * @name ExInitializeRundownProtection 892 * INTERNAL MACRO 893 * 894 * The ExInitializeRundownProtection routine initializes a rundown 895 * protection descriptor. 896 * 897 * @param RunRef 898 * Pointer to a rundown reference descriptor. 899 * 900 * @return None. 901 * 902 * @remarks This is the internal macro for system use only. 903 * 904 *--*/ 905 FORCEINLINE 906 VOID 907 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef) 908 { 909 /* Set the count to zero */ 910 RunRef->Count = 0; 911 } 912 913 /*++ 914 * @name ExWaitForRundownProtectionRelease 915 * INTERNAL MACRO 916 * 917 * The ExWaitForRundownProtectionRelease routine waits until the specified 918 * rundown descriptor has been released. 919 * 920 * @param RunRef 921 * Pointer to a rundown reference descriptor. 922 * 923 * @return None. 924 * 925 * @remarks This is the internal macro for system use only. If a wait is actually 926 * necessary, then the slow path is taken through the exported function. 927 * 928 *--*/ 929 FORCEINLINE 930 VOID 931 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef) 932 { 933 ULONG_PTR Value; 934 935 /* Set the active bit */ 936 Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0); 937 if ((Value) && (Value != EX_RUNDOWN_ACTIVE)) 938 { 939 /* If the the rundown wasn't already active, then take the long path */ 940 ExfWaitForRundownProtectionRelease(RunRef); 941 } 942 } 943 944 /*++ 945 * @name ExRundownCompleted 946 * INTERNAL MACRO 947 * 948 * The ExRundownCompleted routine completes the rundown of the specified 949 * descriptor by setting the active bit. 950 * 951 * @param RunRef 952 * Pointer to a rundown reference descriptor. 953 * 954 * @return None. 955 * 956 * @remarks This is the internal macro for system use only. 957 * 958 *--*/ 959 FORCEINLINE 960 VOID 961 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef) 962 { 963 /* Sanity check */ 964 ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0); 965 966 /* Mark the counter as active */ 967 ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE); 968 } 969 970 /* PUSHLOCKS *****************************************************************/ 971 972 /* FIXME: VERIFY THESE! */ 973 974 VOID 975 FASTCALL 976 ExBlockPushLock( 977 IN PEX_PUSH_LOCK PushLock, 978 IN PVOID WaitBlock 979 ); 980 981 VOID 982 FASTCALL 983 ExfUnblockPushLock( 984 IN PEX_PUSH_LOCK PushLock, 985 IN PVOID CurrentWaitBlock 986 ); 987 988 VOID 989 FASTCALL 990 ExWaitForUnblockPushLock( 991 IN PEX_PUSH_LOCK PushLock, 992 IN PVOID WaitBlock 993 ); 994 995 /*++ 996 * @name _ExInitializePushLock 997 * INTERNAL MACRO 998 * 999 * The _ExInitializePushLock macro initializes a PushLock. 1000 * 1001 * @params PushLock 1002 * Pointer to the pushlock which is to be initialized. 1003 * 1004 * @return None. 1005 * 1006 * @remarks None. 1007 * 1008 *--*/ 1009 FORCEINLINE 1010 VOID 1011 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock) 1012 { 1013 /* Set the value to 0 */ 1014 PushLock->Ptr = 0; 1015 } 1016 #define ExInitializePushLock _ExInitializePushLock 1017 1018 /*++ 1019 * @name ExAcquirePushLockExclusive 1020 * INTERNAL MACRO 1021 * 1022 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock. 1023 * 1024 * @params PushLock 1025 * Pointer to the pushlock which is to be acquired. 1026 * 1027 * @return None. 1028 * 1029 * @remarks The function attempts the quickest route to acquire the lock, which is 1030 * to simply set the lock bit. 1031 * However, if the pushlock is already shared, the slower path is taken. 1032 * 1033 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1034 * This macro should usually be paired up with KeAcquireCriticalRegion. 1035 * 1036 *--*/ 1037 FORCEINLINE 1038 VOID 1039 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock) 1040 { 1041 /* Try acquiring the lock */ 1042 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V)) 1043 { 1044 /* Someone changed it, use the slow path */ 1045 ExfAcquirePushLockExclusive(PushLock); 1046 } 1047 1048 /* Sanity check */ 1049 ASSERT(PushLock->Locked); 1050 } 1051 1052 /*++ 1053 * @name ExTryToAcquirePushLockExclusive 1054 * INTERNAL MACRO 1055 * 1056 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock. 1057 * 1058 * @params PushLock 1059 * Pointer to the pushlock which is to be acquired. 1060 * 1061 * @return None. 1062 * 1063 * @remarks The function attempts the quickest route to acquire the lock, which is 1064 * to simply set the lock bit. 1065 * However, if the pushlock is already shared, the slower path is taken. 1066 * 1067 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1068 * This macro should usually be paired up with KeAcquireCriticalRegion. 1069 * 1070 *--*/ 1071 FORCEINLINE 1072 BOOLEAN 1073 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock) 1074 { 1075 /* Try acquiring the lock */ 1076 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V)) 1077 { 1078 /* Can't acquire */ 1079 return FALSE; 1080 } 1081 1082 /* Got acquired */ 1083 ASSERT (PushLock->Locked); 1084 return TRUE; 1085 } 1086 1087 /*++ 1088 * @name ExAcquirePushLockShared 1089 * INTERNAL MACRO 1090 * 1091 * The ExAcquirePushLockShared macro acquires a shared PushLock. 1092 * 1093 * @params PushLock 1094 * Pointer to the pushlock which is to be acquired. 1095 * 1096 * @return None. 1097 * 1098 * @remarks The function attempts the quickest route to acquire the lock, which is 1099 * to simply set the lock bit and set the share count to one. 1100 * However, if the pushlock is already shared, the slower path is taken. 1101 * 1102 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1103 * This macro should usually be paired up with KeAcquireCriticalRegion. 1104 * 1105 *--*/ 1106 FORCEINLINE 1107 VOID 1108 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock) 1109 { 1110 EX_PUSH_LOCK NewValue; 1111 1112 /* Try acquiring the lock */ 1113 NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1114 if (ExpChangePushlock(PushLock, NewValue.Ptr, 0)) 1115 { 1116 /* Someone changed it, use the slow path */ 1117 ExfAcquirePushLockShared(PushLock); 1118 } 1119 1120 /* Sanity checks */ 1121 ASSERT(PushLock->Locked); 1122 } 1123 1124 /*++ 1125 * @name ExConvertPushLockSharedToExclusive 1126 * INTERNAL MACRO 1127 * 1128 * The ExConvertPushLockSharedToExclusive macro converts an exclusive 1129 * pushlock to a shared pushlock. 1130 * 1131 * @params PushLock 1132 * Pointer to the pushlock which is to be converted. 1133 * 1134 * @return FALSE if conversion failed, TRUE otherwise. 1135 * 1136 * @remarks The function attempts the quickest route to convert the lock, which is 1137 * to simply set the lock bit and remove any other bits. 1138 * 1139 *--*/ 1140 FORCEINLINE 1141 BOOLEAN 1142 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock) 1143 { 1144 EX_PUSH_LOCK OldValue; 1145 1146 /* Set the expected old value */ 1147 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1148 1149 /* Try converting the lock */ 1150 if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) != 1151 OldValue.Ptr) 1152 { 1153 /* Conversion failed */ 1154 return FALSE; 1155 } 1156 1157 /* Sanity check */ 1158 ASSERT(PushLock->Locked); 1159 return TRUE; 1160 } 1161 1162 /*++ 1163 * @name ExWaitOnPushLock 1164 * INTERNAL MACRO 1165 * 1166 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock. 1167 * 1168 * @params PushLock 1169 * Pointer to a pushlock. 1170 * 1171 * @return None. 1172 * 1173 * @remarks The function attempts to get any exclusive waiters out of their slow 1174 * path by forcing an instant acquire/release operation. 1175 * 1176 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL. 1177 * 1178 *--*/ 1179 FORCEINLINE 1180 VOID 1181 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock) 1182 { 1183 /* Check if we're locked */ 1184 if (PushLock->Locked) 1185 { 1186 /* Acquire the lock */ 1187 ExfAcquirePushLockExclusive(PushLock); 1188 ASSERT(PushLock->Locked); 1189 1190 /* Release it */ 1191 ExfReleasePushLockExclusive(PushLock); 1192 } 1193 } 1194 1195 /*++ 1196 * @name ExReleasePushLockShared 1197 * INTERNAL MACRO 1198 * 1199 * The ExReleasePushLockShared macro releases a previously acquired PushLock. 1200 * 1201 * @params PushLock 1202 * Pointer to a previously acquired pushlock. 1203 * 1204 * @return None. 1205 * 1206 * @remarks The function attempts the quickest route to release the lock, which is 1207 * to simply decrease the share count and remove the lock bit. 1208 * However, if the pushlock is being waited on then the long path is taken. 1209 * 1210 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL. 1211 * This macro should usually be paired up with KeLeaveCriticalRegion. 1212 * 1213 *--*/ 1214 FORCEINLINE 1215 VOID 1216 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock) 1217 { 1218 EX_PUSH_LOCK OldValue; 1219 1220 /* Sanity checks */ 1221 ASSERT(PushLock->Locked); 1222 1223 /* Try to clear the pushlock */ 1224 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1225 if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr) 1226 { 1227 /* There are still other people waiting on it */ 1228 ExfReleasePushLockShared(PushLock); 1229 } 1230 } 1231 1232 /*++ 1233 * @name ExReleasePushLockExclusive 1234 * INTERNAL MACRO 1235 * 1236 * The ExReleasePushLockExclusive macro releases a previously 1237 * exclusively acquired PushLock. 1238 * 1239 * @params PushLock 1240 * Pointer to a previously acquired pushlock. 1241 * 1242 * @return None. 1243 * 1244 * @remarks The function attempts the quickest route to release the lock, which is 1245 * to simply clear the locked bit. 1246 * However, if the pushlock is being waited on, the slow path is taken 1247 * in an attempt to wake up the lock. 1248 * 1249 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL. 1250 * This macro should usually be paired up with KeLeaveCriticalRegion. 1251 * 1252 *--*/ 1253 FORCEINLINE 1254 VOID 1255 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock) 1256 { 1257 EX_PUSH_LOCK OldValue; 1258 1259 /* Sanity checks */ 1260 ASSERT(PushLock->Locked); 1261 1262 /* Unlock the pushlock */ 1263 OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock, 1264 -(SSIZE_T)EX_PUSH_LOCK_LOCK); 1265 1266 /* Sanity checks */ 1267 ASSERT(OldValue.Locked); 1268 ASSERT(OldValue.Waiting || OldValue.Shared == 0); 1269 1270 /* Check if anyone is waiting on it and it's not already waking*/ 1271 if ((OldValue.Waiting) && !(OldValue.Waking)) 1272 { 1273 /* Wake it up */ 1274 ExfTryToWakePushLock(PushLock); 1275 } 1276 } 1277 1278 /*++ 1279 * @name ExReleasePushLock 1280 * INTERNAL MACRO 1281 * 1282 * The ExReleasePushLock macro releases a previously acquired PushLock. 1283 * 1284 * @params PushLock 1285 * Pointer to a previously acquired pushlock. 1286 * 1287 * @return None. 1288 * 1289 * @remarks The function attempts the quickest route to release the lock, which is 1290 * to simply clear all the fields and decrease the share count if required. 1291 * However, if the pushlock is being waited on then the long path is taken. 1292 * 1293 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL. 1294 * This macro should usually be paired up with KeLeaveCriticalRegion. 1295 * 1296 *--*/ 1297 FORCEINLINE 1298 VOID 1299 ExReleasePushLock(PEX_PUSH_LOCK PushLock) 1300 { 1301 EX_PUSH_LOCK OldValue = *PushLock; 1302 EX_PUSH_LOCK NewValue; 1303 1304 /* Sanity checks */ 1305 ASSERT(OldValue.Locked); 1306 1307 /* Check if the pushlock is shared */ 1308 if (OldValue.Shared > 1) 1309 { 1310 /* Decrease the share count */ 1311 NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC; 1312 } 1313 else 1314 { 1315 /* Clear the pushlock entirely */ 1316 NewValue.Value = 0; 1317 } 1318 1319 /* Check if nobody is waiting on us and try clearing the lock here */ 1320 if ((OldValue.Waiting) || 1321 (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) != 1322 OldValue.Ptr)) 1323 { 1324 /* We have waiters, use the long path */ 1325 ExfReleasePushLock(PushLock); 1326 } 1327 } 1328 1329 /* FAST MUTEX INLINES *********************************************************/ 1330 1331 FORCEINLINE 1332 VOID 1333 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex) 1334 { 1335 PKTHREAD Thread = KeGetCurrentThread(); 1336 1337 /* Sanity check */ 1338 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1339 (Thread->CombinedApcDisable != 0) || 1340 (Thread->Teb == NULL) || 1341 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1342 ASSERT(FastMutex->Owner != Thread); 1343 1344 /* Decrease the count */ 1345 if (InterlockedDecrement(&FastMutex->Count)) 1346 { 1347 /* Someone is still holding it, use slow path */ 1348 KiAcquireFastMutex(FastMutex); 1349 } 1350 1351 /* Set the owner */ 1352 FastMutex->Owner = Thread; 1353 } 1354 1355 FORCEINLINE 1356 VOID 1357 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) 1358 { 1359 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1360 (KeGetCurrentThread()->CombinedApcDisable != 0) || 1361 (KeGetCurrentThread()->Teb == NULL) || 1362 (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1363 ASSERT(FastMutex->Owner == KeGetCurrentThread()); 1364 1365 /* Erase the owner */ 1366 FastMutex->Owner = NULL; 1367 1368 /* Increase the count */ 1369 if (InterlockedIncrement(&FastMutex->Count) <= 0) 1370 { 1371 /* Someone was waiting for it, signal the waiter */ 1372 KeSetEventBoostPriority(&FastMutex->Event, NULL); 1373 } 1374 } 1375 1376 FORCEINLINE 1377 VOID 1378 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex) 1379 { 1380 KIRQL OldIrql; 1381 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1382 1383 /* Raise IRQL to APC */ 1384 KeRaiseIrql(APC_LEVEL, &OldIrql); 1385 1386 /* Decrease the count */ 1387 if (InterlockedDecrement(&FastMutex->Count)) 1388 { 1389 /* Someone is still holding it, use slow path */ 1390 KiAcquireFastMutex(FastMutex); 1391 } 1392 1393 /* Set the owner and IRQL */ 1394 FastMutex->Owner = KeGetCurrentThread(); 1395 FastMutex->OldIrql = OldIrql; 1396 } 1397 1398 FORCEINLINE 1399 VOID 1400 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex) 1401 { 1402 KIRQL OldIrql; 1403 ASSERT(KeGetCurrentIrql() == APC_LEVEL); 1404 1405 /* Erase the owner */ 1406 FastMutex->Owner = NULL; 1407 OldIrql = (KIRQL)FastMutex->OldIrql; 1408 1409 /* Increase the count */ 1410 if (InterlockedIncrement(&FastMutex->Count) <= 0) 1411 { 1412 /* Someone was waiting for it, signal the waiter */ 1413 KeSetEventBoostPriority(&FastMutex->Event, NULL); 1414 } 1415 1416 /* Lower IRQL back */ 1417 KeLowerIrql(OldIrql); 1418 } 1419 1420 FORCEINLINE 1421 BOOLEAN 1422 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex) 1423 { 1424 KIRQL OldIrql; 1425 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1426 1427 /* Raise to APC_LEVEL */ 1428 KeRaiseIrql(APC_LEVEL, &OldIrql); 1429 1430 /* Check if we can quickly acquire it */ 1431 if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1) 1432 { 1433 /* We have, set us as owners */ 1434 FastMutex->Owner = KeGetCurrentThread(); 1435 FastMutex->OldIrql = OldIrql; 1436 return TRUE; 1437 } 1438 else 1439 { 1440 /* Acquire attempt failed */ 1441 KeLowerIrql(OldIrql); 1442 YieldProcessor(); 1443 return FALSE; 1444 } 1445 } 1446 1447 FORCEINLINE 1448 VOID 1449 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) 1450 { 1451 /* Enter the Critical Region */ 1452 KeEnterCriticalRegion(); 1453 1454 /* Acquire the mutex unsafely */ 1455 _ExAcquireFastMutexUnsafe(FastMutex); 1456 } 1457 1458 FORCEINLINE 1459 VOID 1460 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex) 1461 { 1462 /* Release the mutex unsafely */ 1463 _ExReleaseFastMutexUnsafe(FastMutex); 1464 1465 /* Leave the critical region */ 1466 KeLeaveCriticalRegion(); 1467 } 1468 1469 /* OTHER FUNCTIONS **********************************************************/ 1470 1471 BOOLEAN 1472 NTAPI 1473 ExTryToAcquireResourceExclusiveLite( 1474 IN PERESOURCE Resource 1475 ); 1476 1477 NTSTATUS 1478 ExpSetTimeZoneInformation( 1479 IN PRTL_TIME_ZONE_INFORMATION TimeZoneInformation 1480 ); 1481 1482 BOOLEAN 1483 NTAPI 1484 ExAcquireTimeRefreshLock( 1485 IN BOOLEAN Wait 1486 ); 1487 1488 VOID 1489 NTAPI 1490 ExReleaseTimeRefreshLock( 1491 VOID 1492 ); 1493 1494 VOID 1495 NTAPI 1496 ExUpdateSystemTimeFromCmos( 1497 IN BOOLEAN UpdateInterruptTime, 1498 IN ULONG MaxSepInSeconds 1499 ); 1500 1501 VOID 1502 NTAPI 1503 ExAllocateLocallyUniqueId( 1504 OUT LUID *LocallyUniqueId 1505 ); 1506 1507 VOID 1508 NTAPI 1509 ExTimerRundown( 1510 VOID 1511 ); 1512 1513 CODE_SEG("INIT") 1514 VOID 1515 NTAPI 1516 HeadlessInit( 1517 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1518 ); 1519 1520 CODE_SEG("INIT") 1521 VOID 1522 NTAPI 1523 XIPInit( 1524 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1525 ); 1526 1527 #define InterlockedDecrementUL(Addend) \ 1528 (ULONG)InterlockedDecrement((PLONG)(Addend)) 1529 1530 #define InterlockedIncrementUL(Addend) \ 1531 (ULONG)InterlockedIncrement((PLONG)(Addend)) 1532 1533 #define InterlockedExchangeUL(Target, Value) \ 1534 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value)) 1535 1536 #define InterlockedExchangeAddUL(Addend, Value) \ 1537 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value)) 1538 1539 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \ 1540 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand)) 1541 1542 #define InterlockedCompareExchangeSizeT(Destination, Exchange, Comperand) \ 1543 (SIZE_T)InterlockedCompareExchangePointer((PVOID*)(Destination), (PVOID)(SIZE_T)(Exchange), (PVOID)(SIZE_T)(Comperand)) 1544 1545 #ifdef _WIN64 1546 #define InterlockedExchangeSizeT(Target, Value) \ 1547 (SIZE_T)InterlockedExchange64((PLONG64)(Target), (LONG64)(Value)) 1548 #else 1549 #define InterlockedExchangeSizeT(Target, Value) \ 1550 (SIZE_T)InterlockedExchange((PLONG)(Target), (LONG)(Value)) 1551 #endif 1552 1553 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \ 1554 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand)) 1555 1556 #ifdef __cplusplus 1557 } // extern "C" 1558 #endif 1559