1 #pragma once 2 3 #ifdef __cplusplus 4 extern "C" 5 { 6 #endif 7 8 /* GLOBAL VARIABLES *********************************************************/ 9 10 extern RTL_TIME_ZONE_INFORMATION ExpTimeZoneInfo; 11 extern LARGE_INTEGER ExpTimeZoneBias; 12 extern ULONG ExpTimeZoneId; 13 extern ULONG ExpTickCountMultiplier; 14 extern ULONG ExpLastTimeZoneBias; 15 extern POBJECT_TYPE ExEventPairObjectType; 16 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType; 17 extern FAST_MUTEX ExpEnvironmentLock; 18 extern ERESOURCE ExpFirmwareTableResource; 19 extern ERESOURCE ExpTimeRefreshLock; 20 extern LIST_ENTRY ExpFirmwareTableProviderListHead; 21 extern BOOLEAN ExpIsWinPEMode; 22 extern ULONG ExpResourceTimeoutCount; 23 extern LIST_ENTRY ExpSystemResourcesList; 24 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset; 25 extern ULONG ExpUnicodeCaseTableDataOffset; 26 extern PVOID ExpNlsSectionPointer; 27 extern ULONG NtGlobalFlag; 28 extern UNICODE_STRING NtSystemRoot; 29 extern ULONG ExpInitializationPhase; 30 extern ULONG ExpAltTimeZoneBias; 31 extern LIST_ENTRY ExSystemLookasideListHead; 32 extern PCALLBACK_OBJECT PowerStateCallback; 33 extern LIST_ENTRY ExPoolLookasideListHead; 34 extern LIST_ENTRY ExpNonPagedLookasideListHead; 35 extern LIST_ENTRY ExpPagedLookasideListHead; 36 extern KSPIN_LOCK ExpNonPagedLookasideListLock; 37 extern KSPIN_LOCK ExpPagedLookasideListLock; 38 extern ULONG ExCriticalWorkerThreads; 39 extern ULONG ExDelayedWorkerThreads; 40 41 extern PVOID ExpDefaultErrorPort; 42 extern PEPROCESS ExpDefaultErrorPortProcess; 43 44 /* 45 * NT/Cm Version Info variables 46 */ 47 extern ULONG NtMajorVersion; 48 extern ULONG NtMinorVersion; 49 extern ULONG NtBuildNumber; 50 extern ULONG CmNtSpBuildNumber; 51 extern ULONG CmNtCSDVersion; 52 extern ULONG CmNtCSDReleaseType; 53 extern UNICODE_STRING CmVersionString; 54 extern UNICODE_STRING CmCSDVersionString; 55 extern CHAR NtBuildLab[]; 56 57 /* 58 * WinDBG Debugger Worker State Machine data (see dbgctrl.c) 59 */ 60 typedef enum _WINKD_WORKER_STATE 61 { 62 WinKdWorkerReady = 0, 63 WinKdWorkerStart, 64 WinKdWorkerInitialized 65 } WINKD_WORKER_STATE; 66 67 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem; 68 extern WINKD_WORKER_STATE ExpDebuggerWork; 69 extern PEPROCESS ExpDebuggerProcessAttach; 70 extern PEPROCESS ExpDebuggerProcessKill; 71 extern ULONG_PTR ExpDebuggerPageIn; 72 73 VOID NTAPI ExpDebuggerWorker(IN PVOID Context); 74 75 #ifdef _WIN64 76 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4) 77 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3) 78 #else 79 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3) 80 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2) 81 #endif 82 #define HANDLE_TAG_BITS 2 83 #define HANDLE_INDEX_BITS (HANDLE_LOW_BITS + 2 * HANDLE_HIGH_BITS) 84 #define KERNEL_FLAG_BITS (sizeof(ULONG_PTR) * 8 - HANDLE_INDEX_BITS - HANDLE_TAG_BITS) 85 86 typedef union _EXHANDLE 87 { 88 struct 89 { 90 ULONG_PTR TagBits: HANDLE_TAG_BITS; 91 ULONG_PTR Index: HANDLE_INDEX_BITS; 92 ULONG_PTR KernelFlag: KERNEL_FLAG_BITS; 93 }; 94 struct 95 { 96 ULONG_PTR TagBits2: HANDLE_TAG_BITS; 97 ULONG_PTR LowIndex: HANDLE_LOW_BITS; 98 ULONG_PTR MidIndex: HANDLE_HIGH_BITS; 99 ULONG_PTR HighIndex: HANDLE_HIGH_BITS; 100 ULONG_PTR KernelFlag2: KERNEL_FLAG_BITS; 101 }; 102 HANDLE GenericHandleOverlay; 103 ULONG_PTR Value; 104 ULONG AsULONG; 105 } EXHANDLE, *PEXHANDLE; 106 107 typedef struct _ETIMER 108 { 109 KTIMER KeTimer; 110 KAPC TimerApc; 111 KDPC TimerDpc; 112 LIST_ENTRY ActiveTimerListEntry; 113 KSPIN_LOCK Lock; 114 LONG Period; 115 BOOLEAN ApcAssociated; 116 BOOLEAN WakeTimer; 117 LIST_ENTRY WakeTimerListEntry; 118 } ETIMER, *PETIMER; 119 120 typedef struct 121 { 122 PCALLBACK_OBJECT *CallbackObject; 123 PWSTR Name; 124 } SYSTEM_CALLBACKS; 125 126 typedef struct _HARDERROR_USER_PARAMETERS 127 { 128 ULONG_PTR Parameters[MAXIMUM_HARDERROR_PARAMETERS]; 129 UNICODE_STRING Strings[MAXIMUM_HARDERROR_PARAMETERS]; 130 WCHAR Buffer[ANYSIZE_ARRAY]; 131 } HARDERROR_USER_PARAMETERS, *PHARDERROR_USER_PARAMETERS; 132 133 #define MAX_FAST_REFS 7 134 135 #define ExAcquireRundownProtection _ExAcquireRundownProtection 136 #define ExReleaseRundownProtection _ExReleaseRundownProtection 137 #define ExInitializeRundownProtection _ExInitializeRundownProtection 138 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease 139 #define ExRundownCompleted _ExRundownCompleted 140 #define ExGetPreviousMode KeGetPreviousMode 141 142 143 // 144 // Various bits tagged on the handle or handle table 145 // 146 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1 147 #define FREE_HANDLE_MASK -1 148 149 // 150 // Number of entries in each table level 151 // 152 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY)) 153 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY)) 154 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES)) 155 156 // 157 // Maximum index in each table level before we need another table 158 // 159 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES 160 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES) 161 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES) 162 163 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&(x)->Ptr, (PVOID)(y), (PVOID)(z)) 164 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)(x), (PVOID)(y), (PVOID)(z)) 165 #define ExpSetRundown(x, y) InterlockedExchangePointer(&(x)->Ptr, (PVOID)(y)) 166 167 NTSTATUS 168 NTAPI 169 ExGetPoolTagInfo( 170 IN PSYSTEM_POOLTAG_INFORMATION SystemInformation, 171 IN ULONG SystemInformationLength, 172 IN OUT PULONG ReturnLength OPTIONAL 173 ); 174 175 typedef struct _UUID_CACHED_VALUES_STRUCT 176 { 177 ULONGLONG Time; 178 LONG AllocatedCount; 179 union 180 { 181 struct 182 { 183 UCHAR ClockSeqHiAndReserved; 184 UCHAR ClockSeqLow; 185 UCHAR NodeId[6 /*SEED_BUFFER_SIZE*/]; 186 }; 187 UCHAR GuidInit[8]; /* Match GUID.Data4 */ 188 }; 189 } UUID_CACHED_VALUES_STRUCT, *PUUID_CACHED_VALUES_STRUCT; 190 191 C_ASSERT(RTL_FIELD_SIZE(UUID_CACHED_VALUES_STRUCT, GuidInit) == RTL_FIELD_SIZE(UUID, Data4)); 192 193 /* INITIALIZATION FUNCTIONS *************************************************/ 194 195 CODE_SEG("INIT") 196 BOOLEAN 197 NTAPI 198 ExpWin32kInit(VOID); 199 200 VOID 201 NTAPI 202 ExInit2(VOID); 203 204 VOID 205 NTAPI 206 Phase1Initialization( 207 IN PVOID Context 208 ); 209 210 CODE_SEG("INIT") 211 VOID 212 NTAPI 213 ExpInitializePushLocks(VOID); 214 215 BOOLEAN 216 NTAPI 217 ExRefreshTimeZoneInformation( 218 IN PLARGE_INTEGER SystemBootTime 219 ); 220 221 CODE_SEG("INIT") 222 VOID 223 NTAPI 224 ExpInitializeWorkerThreads(VOID); 225 226 VOID 227 NTAPI 228 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap); 229 230 CODE_SEG("INIT") 231 VOID 232 NTAPI 233 ExpInitLookasideLists(VOID); 234 235 CODE_SEG("INIT") 236 VOID 237 NTAPI 238 ExInitializeSystemLookasideList( 239 IN PGENERAL_LOOKASIDE List, 240 IN POOL_TYPE Type, 241 IN ULONG Size, 242 IN ULONG Tag, 243 IN USHORT MaximumDepth, 244 IN PLIST_ENTRY ListHead 245 ); 246 247 CODE_SEG("INIT") 248 BOOLEAN 249 NTAPI 250 ExpInitializeCallbacks(VOID); 251 252 CODE_SEG("INIT") 253 BOOLEAN 254 NTAPI 255 ExpUuidInitialization(VOID); 256 257 CODE_SEG("INIT") 258 BOOLEAN 259 NTAPI 260 ExLuidInitialization(VOID); 261 262 CODE_SEG("INIT") 263 VOID 264 NTAPI 265 ExpInitializeExecutive( 266 IN ULONG Cpu, 267 IN PLOADER_PARAMETER_BLOCK LoaderBlock 268 ); 269 270 VOID 271 NTAPI 272 ExShutdownSystem(VOID); 273 274 CODE_SEG("INIT") 275 BOOLEAN 276 NTAPI 277 ExpInitializeEventImplementation(VOID); 278 279 CODE_SEG("INIT") 280 BOOLEAN 281 NTAPI 282 ExpInitializeKeyedEventImplementation(VOID); 283 284 CODE_SEG("INIT") 285 BOOLEAN 286 NTAPI 287 ExpInitializeEventPairImplementation(VOID); 288 289 CODE_SEG("INIT") 290 BOOLEAN 291 NTAPI 292 ExpInitializeSemaphoreImplementation(VOID); 293 294 CODE_SEG("INIT") 295 BOOLEAN 296 NTAPI 297 ExpInitializeMutantImplementation(VOID); 298 299 CODE_SEG("INIT") 300 BOOLEAN 301 NTAPI 302 ExpInitializeTimerImplementation(VOID); 303 304 CODE_SEG("INIT") 305 BOOLEAN 306 NTAPI 307 ExpInitializeProfileImplementation(VOID); 308 309 CODE_SEG("INIT") 310 VOID 311 NTAPI 312 ExpResourceInitialization(VOID); 313 314 CODE_SEG("INIT") 315 VOID 316 NTAPI 317 ExInitPoolLookasidePointers(VOID); 318 319 /* Callback Functions ********************************************************/ 320 321 VOID 322 NTAPI 323 ExInitializeCallBack( 324 IN OUT PEX_CALLBACK Callback 325 ); 326 327 PEX_CALLBACK_ROUTINE_BLOCK 328 NTAPI 329 ExAllocateCallBack( 330 IN PEX_CALLBACK_FUNCTION Function, 331 IN PVOID Context 332 ); 333 334 VOID 335 NTAPI 336 ExFreeCallBack( 337 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 338 ); 339 340 BOOLEAN 341 NTAPI 342 ExCompareExchangeCallBack ( 343 IN OUT PEX_CALLBACK CallBack, 344 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock, 345 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock 346 ); 347 348 PEX_CALLBACK_ROUTINE_BLOCK 349 NTAPI 350 ExReferenceCallBackBlock( 351 IN OUT PEX_CALLBACK CallBack 352 ); 353 354 VOID 355 NTAPI 356 ExDereferenceCallBackBlock( 357 IN OUT PEX_CALLBACK CallBack, 358 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 359 ); 360 361 PEX_CALLBACK_FUNCTION 362 NTAPI 363 ExGetCallBackBlockRoutine( 364 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 365 ); 366 367 PVOID 368 NTAPI 369 ExGetCallBackBlockContext( 370 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 371 ); 372 373 VOID 374 NTAPI 375 ExWaitForCallBacks( 376 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 377 ); 378 379 /* Rundown Functions ********************************************************/ 380 381 VOID 382 FASTCALL 383 ExfInitializeRundownProtection( 384 OUT PEX_RUNDOWN_REF RunRef 385 ); 386 387 VOID 388 FASTCALL 389 ExfReInitializeRundownProtection( 390 OUT PEX_RUNDOWN_REF RunRef 391 ); 392 393 BOOLEAN 394 FASTCALL 395 ExfAcquireRundownProtection( 396 IN OUT PEX_RUNDOWN_REF RunRef 397 ); 398 399 BOOLEAN 400 FASTCALL 401 ExfAcquireRundownProtectionEx( 402 IN OUT PEX_RUNDOWN_REF RunRef, 403 IN ULONG Count 404 ); 405 406 VOID 407 FASTCALL 408 ExfReleaseRundownProtection( 409 IN OUT PEX_RUNDOWN_REF RunRef 410 ); 411 412 VOID 413 FASTCALL 414 ExfReleaseRundownProtectionEx( 415 IN OUT PEX_RUNDOWN_REF RunRef, 416 IN ULONG Count 417 ); 418 419 VOID 420 FASTCALL 421 ExfRundownCompleted( 422 OUT PEX_RUNDOWN_REF RunRef 423 ); 424 425 VOID 426 FASTCALL 427 ExfWaitForRundownProtectionRelease( 428 IN OUT PEX_RUNDOWN_REF RunRef 429 ); 430 431 /* HANDLE TABLE FUNCTIONS ***************************************************/ 432 433 typedef BOOLEAN 434 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)( 435 PHANDLE_TABLE_ENTRY HandleTableEntry, 436 HANDLE Handle, 437 PVOID Context 438 ); 439 440 typedef BOOLEAN 441 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)( 442 IN PEPROCESS Process, 443 IN PHANDLE_TABLE HandleTable, 444 IN PHANDLE_TABLE_ENTRY HandleTableEntry, 445 IN PHANDLE_TABLE_ENTRY NewEntry 446 ); 447 448 typedef BOOLEAN 449 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)( 450 PHANDLE_TABLE_ENTRY HandleTableEntry, 451 ULONG_PTR Context 452 ); 453 454 CODE_SEG("INIT") 455 VOID 456 NTAPI 457 ExpInitializeHandleTables( 458 VOID 459 ); 460 461 PHANDLE_TABLE 462 NTAPI 463 ExCreateHandleTable( 464 IN PEPROCESS Process OPTIONAL 465 ); 466 467 VOID 468 NTAPI 469 ExUnlockHandleTableEntry( 470 IN PHANDLE_TABLE HandleTable, 471 IN PHANDLE_TABLE_ENTRY HandleTableEntry 472 ); 473 474 HANDLE 475 NTAPI 476 ExCreateHandle( 477 IN PHANDLE_TABLE HandleTable, 478 IN PHANDLE_TABLE_ENTRY HandleTableEntry 479 ); 480 481 VOID 482 NTAPI 483 ExDestroyHandleTable( 484 IN PHANDLE_TABLE HandleTable, 485 IN PVOID DestroyHandleProcedure OPTIONAL 486 ); 487 488 BOOLEAN 489 NTAPI 490 ExDestroyHandle( 491 IN PHANDLE_TABLE HandleTable, 492 IN HANDLE Handle, 493 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL 494 ); 495 496 PHANDLE_TABLE_ENTRY 497 NTAPI 498 ExMapHandleToPointer( 499 IN PHANDLE_TABLE HandleTable, 500 IN HANDLE Handle 501 ); 502 503 PHANDLE_TABLE 504 NTAPI 505 ExDupHandleTable( 506 IN PEPROCESS Process, 507 IN PHANDLE_TABLE HandleTable, 508 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure, 509 IN ULONG_PTR Mask 510 ); 511 512 BOOLEAN 513 NTAPI 514 ExChangeHandle( 515 IN PHANDLE_TABLE HandleTable, 516 IN HANDLE Handle, 517 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine, 518 IN ULONG_PTR Context 519 ); 520 521 VOID 522 NTAPI 523 ExSweepHandleTable( 524 IN PHANDLE_TABLE HandleTable, 525 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure, 526 IN PVOID Context 527 ); 528 529 PHANDLE_TABLE_ENTRY 530 NTAPI 531 ExpLookupHandleTableEntry( 532 IN PHANDLE_TABLE HandleTable, 533 IN EXHANDLE Handle 534 ); 535 536 BOOLEAN 537 NTAPI 538 ExpLockHandleTableEntry( 539 IN PHANDLE_TABLE HandleTable, 540 IN PHANDLE_TABLE_ENTRY HandleTableEntry 541 ); 542 543 /* PSEH EXCEPTION HANDLING **************************************************/ 544 545 LONG 546 NTAPI 547 ExSystemExceptionFilter(VOID); 548 549 /* CALLBACKS *****************************************************************/ 550 551 FORCEINLINE 552 VOID 553 ExDoCallBack(IN OUT PEX_CALLBACK Callback, 554 IN PVOID Context, 555 IN PVOID Argument1, 556 IN PVOID Argument2) 557 { 558 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock; 559 PEX_CALLBACK_FUNCTION Function; 560 561 /* Reference the block */ 562 CallbackBlock = ExReferenceCallBackBlock(Callback); 563 if (CallbackBlock) 564 { 565 /* Get the function */ 566 Function = ExGetCallBackBlockRoutine(CallbackBlock); 567 568 /* Do the callback */ 569 Function(Context, Argument1, Argument2); 570 571 /* Now dereference it */ 572 ExDereferenceCallBackBlock(Callback, CallbackBlock); 573 } 574 } 575 576 /* FAST REFS ******************************************************************/ 577 578 FORCEINLINE 579 PVOID 580 ExGetObjectFastReference(IN EX_FAST_REF FastRef) 581 { 582 /* Return the unbiased pointer */ 583 return (PVOID)(FastRef.Value & ~MAX_FAST_REFS); 584 } 585 586 FORCEINLINE 587 ULONG 588 ExGetCountFastReference(IN EX_FAST_REF FastRef) 589 { 590 /* Return the reference count */ 591 return (ULONG)FastRef.RefCnt; 592 } 593 594 FORCEINLINE 595 VOID 596 ExInitializeFastReference(OUT PEX_FAST_REF FastRef, 597 IN OPTIONAL PVOID Object) 598 { 599 /* Sanity check */ 600 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0); 601 602 /* Check if an object is being set */ 603 if (!Object) 604 { 605 /* Clear the field */ 606 FastRef->Object = NULL; 607 } 608 else 609 { 610 /* Otherwise, we assume the object was referenced and is ready */ 611 FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS; 612 } 613 } 614 615 FORCEINLINE 616 EX_FAST_REF 617 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef) 618 { 619 EX_FAST_REF OldValue, NewValue; 620 621 /* Start reference loop */ 622 for (;;) 623 { 624 /* Get the current reference count */ 625 OldValue = *FastRef; 626 if (OldValue.RefCnt) 627 { 628 /* Increase the reference count */ 629 NewValue.Value = OldValue.Value - 1; 630 NewValue.Object = ExpChangePushlock(&FastRef->Object, 631 NewValue.Object, 632 OldValue.Object); 633 if (NewValue.Object != OldValue.Object) continue; 634 } 635 636 /* We are done */ 637 break; 638 } 639 640 /* Return the old value */ 641 return OldValue; 642 } 643 644 FORCEINLINE 645 BOOLEAN 646 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef, 647 IN PVOID Object) 648 { 649 EX_FAST_REF OldValue, NewValue; 650 651 /* Sanity checks */ 652 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 653 654 /* Start update loop */ 655 for (;;) 656 { 657 /* Get the current reference count */ 658 OldValue = *FastRef; 659 660 /* Check if the current count is too high or if the pointer changed */ 661 if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) || 662 ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object)) 663 { 664 /* Fail */ 665 return FALSE; 666 } 667 668 /* Update the reference count */ 669 NewValue.Value = OldValue.Value + MAX_FAST_REFS; 670 NewValue.Object = ExpChangePushlock(&FastRef->Object, 671 NewValue.Object, 672 OldValue.Object); 673 if (NewValue.Object != OldValue.Object) continue; 674 675 /* We are done */ 676 break; 677 } 678 679 /* Return success */ 680 return TRUE; 681 } 682 683 FORCEINLINE 684 BOOLEAN 685 ExReleaseFastReference(IN PEX_FAST_REF FastRef, 686 IN PVOID Object) 687 { 688 EX_FAST_REF OldValue, NewValue; 689 690 /* Sanity checks */ 691 ASSERT(Object != NULL); 692 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 693 694 /* Start reference loop */ 695 for (;;) 696 { 697 /* Get the current reference count */ 698 OldValue = *FastRef; 699 700 /* Check if we're full if if the pointer changed */ 701 if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE; 702 703 /* Decrease the reference count */ 704 NewValue.Value = OldValue.Value + 1; 705 NewValue.Object = ExpChangePushlock(&FastRef->Object, 706 NewValue.Object, 707 OldValue.Object); 708 if (NewValue.Object != OldValue.Object) continue; 709 710 /* We are done */ 711 break; 712 } 713 714 /* Return success */ 715 return TRUE; 716 } 717 718 FORCEINLINE 719 EX_FAST_REF 720 ExSwapFastReference(IN PEX_FAST_REF FastRef, 721 IN PVOID Object) 722 { 723 EX_FAST_REF NewValue, OldValue; 724 725 /* Sanity check */ 726 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0); 727 728 /* Check if an object is being set */ 729 if (!Object) 730 { 731 /* Clear the field */ 732 NewValue.Object = NULL; 733 } 734 else 735 { 736 /* Otherwise, we assume the object was referenced and is ready */ 737 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS; 738 } 739 740 /* Update the object */ 741 OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object); 742 return OldValue; 743 } 744 745 FORCEINLINE 746 EX_FAST_REF 747 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef, 748 IN PVOID Object, 749 IN PVOID OldObject) 750 { 751 EX_FAST_REF OldValue, NewValue; 752 753 /* Sanity check and start swap loop */ 754 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 755 for (;;) 756 { 757 /* Get the current value */ 758 OldValue = *FastRef; 759 760 /* Make sure there's enough references to swap */ 761 if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break; 762 763 /* Check if we have an object to swap */ 764 if (Object) 765 { 766 /* Set up the value with maximum fast references */ 767 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS; 768 } 769 else 770 { 771 /* Write the object address itself (which is empty) */ 772 NewValue.Value = (ULONG_PTR)Object; 773 } 774 775 /* Do the actual compare exchange */ 776 NewValue.Object = ExpChangePushlock(&FastRef->Object, 777 NewValue.Object, 778 OldValue.Object); 779 if (NewValue.Object != OldValue.Object) continue; 780 781 /* All done */ 782 break; 783 } 784 785 /* Return the old value */ 786 return OldValue; 787 } 788 789 /* RUNDOWN *******************************************************************/ 790 791 FORCEINLINE 792 PEX_RUNDOWN_REF 793 ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, 794 IN ULONG ProcNumber) 795 { 796 return (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs + 797 RunRefCacheAware->RunRefSize * 798 (ProcNumber % RunRefCacheAware->Number)); 799 } 800 801 /*++ 802 * @name ExfAcquireRundownProtection 803 * INTERNAL MACRO 804 * 805 * The ExfAcquireRundownProtection routine acquires rundown protection for 806 * the specified descriptor. 807 * 808 * @param RunRef 809 * Pointer to a rundown reference descriptor. 810 * 811 * @return TRUE if access to the protected structure was granted, FALSE otherwise. 812 * 813 * @remarks This is the internal macro for system use only.In case the rundown 814 * was active, then the slow-path will be called through the exported 815 * function. 816 * 817 *--*/ 818 FORCEINLINE 819 BOOLEAN 820 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef) 821 { 822 ULONG_PTR Value, NewValue; 823 824 /* Get the current value and mask the active bit */ 825 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE; 826 827 /* Add a reference */ 828 NewValue = Value + EX_RUNDOWN_COUNT_INC; 829 830 /* Change the value */ 831 NewValue = ExpChangeRundown(RunRef, NewValue, Value); 832 if (NewValue != Value) 833 { 834 /* Rundown was active, use long path */ 835 return ExfAcquireRundownProtection(RunRef); 836 } 837 838 /* Success */ 839 return TRUE; 840 } 841 842 /*++ 843 * @name ExReleaseRundownProtection 844 * INTERNAL MACRO 845 * 846 * The ExReleaseRundownProtection routine releases rundown protection for 847 * the specified descriptor. 848 * 849 * @param RunRef 850 * Pointer to a rundown reference descriptor. 851 * 852 * @return TRUE if access to the protected structure was granted, FALSE otherwise. 853 * 854 * @remarks This is the internal macro for system use only.In case the rundown 855 * was active, then the slow-path will be called through the exported 856 * function. 857 * 858 *--*/ 859 FORCEINLINE 860 VOID 861 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef) 862 { 863 ULONG_PTR Value, NewValue; 864 865 /* Get the current value and mask the active bit */ 866 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE; 867 868 /* Remove a reference */ 869 NewValue = Value - EX_RUNDOWN_COUNT_INC; 870 871 /* Change the value */ 872 NewValue = ExpChangeRundown(RunRef, NewValue, Value); 873 874 /* Check if the rundown was active */ 875 if (NewValue != Value) 876 { 877 /* Rundown was active, use long path */ 878 ExfReleaseRundownProtection(RunRef); 879 } 880 else 881 { 882 /* Sanity check */ 883 ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1)); 884 } 885 } 886 887 /*++ 888 * @name ExInitializeRundownProtection 889 * INTERNAL MACRO 890 * 891 * The ExInitializeRundownProtection routine initializes a rundown 892 * protection descriptor. 893 * 894 * @param RunRef 895 * Pointer to a rundown reference descriptor. 896 * 897 * @return None. 898 * 899 * @remarks This is the internal macro for system use only. 900 * 901 *--*/ 902 FORCEINLINE 903 VOID 904 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef) 905 { 906 /* Set the count to zero */ 907 RunRef->Count = 0; 908 } 909 910 /*++ 911 * @name ExWaitForRundownProtectionRelease 912 * INTERNAL MACRO 913 * 914 * The ExWaitForRundownProtectionRelease routine waits until the specified 915 * rundown descriptor has been released. 916 * 917 * @param RunRef 918 * Pointer to a rundown reference descriptor. 919 * 920 * @return None. 921 * 922 * @remarks This is the internal macro for system use only. If a wait is actually 923 * necessary, then the slow path is taken through the exported function. 924 * 925 *--*/ 926 FORCEINLINE 927 VOID 928 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef) 929 { 930 ULONG_PTR Value; 931 932 /* Set the active bit */ 933 Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0); 934 if ((Value) && (Value != EX_RUNDOWN_ACTIVE)) 935 { 936 /* If the the rundown wasn't already active, then take the long path */ 937 ExfWaitForRundownProtectionRelease(RunRef); 938 } 939 } 940 941 /*++ 942 * @name ExRundownCompleted 943 * INTERNAL MACRO 944 * 945 * The ExRundownCompleted routine completes the rundown of the specified 946 * descriptor by setting the active bit. 947 * 948 * @param RunRef 949 * Pointer to a rundown reference descriptor. 950 * 951 * @return None. 952 * 953 * @remarks This is the internal macro for system use only. 954 * 955 *--*/ 956 FORCEINLINE 957 VOID 958 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef) 959 { 960 /* Sanity check */ 961 ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0); 962 963 /* Mark the counter as active */ 964 ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE); 965 } 966 967 /* PUSHLOCKS *****************************************************************/ 968 969 /* FIXME: VERIFY THESE! */ 970 971 VOID 972 FASTCALL 973 ExBlockPushLock( 974 IN PEX_PUSH_LOCK PushLock, 975 IN PVOID WaitBlock 976 ); 977 978 VOID 979 FASTCALL 980 ExfUnblockPushLock( 981 IN PEX_PUSH_LOCK PushLock, 982 IN PVOID CurrentWaitBlock 983 ); 984 985 VOID 986 FASTCALL 987 ExWaitForUnblockPushLock( 988 IN PEX_PUSH_LOCK PushLock, 989 IN PVOID WaitBlock 990 ); 991 992 /*++ 993 * @name _ExInitializePushLock 994 * INTERNAL MACRO 995 * 996 * The _ExInitializePushLock macro initializes a PushLock. 997 * 998 * @params PushLock 999 * Pointer to the pushlock which is to be initialized. 1000 * 1001 * @return None. 1002 * 1003 * @remarks None. 1004 * 1005 *--*/ 1006 FORCEINLINE 1007 VOID 1008 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock) 1009 { 1010 /* Set the value to 0 */ 1011 PushLock->Ptr = 0; 1012 } 1013 #define ExInitializePushLock _ExInitializePushLock 1014 1015 /*++ 1016 * @name ExAcquirePushLockExclusive 1017 * INTERNAL MACRO 1018 * 1019 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock. 1020 * 1021 * @params PushLock 1022 * Pointer to the pushlock which is to be acquired. 1023 * 1024 * @return None. 1025 * 1026 * @remarks The function attempts the quickest route to acquire the lock, which is 1027 * to simply set the lock bit. 1028 * However, if the pushlock is already shared, the slower path is taken. 1029 * 1030 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1031 * This macro should usually be paired up with KeAcquireCriticalRegion. 1032 * 1033 *--*/ 1034 FORCEINLINE 1035 VOID 1036 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock) 1037 { 1038 /* Try acquiring the lock */ 1039 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V)) 1040 { 1041 /* Someone changed it, use the slow path */ 1042 ExfAcquirePushLockExclusive(PushLock); 1043 } 1044 1045 /* Sanity check */ 1046 ASSERT(PushLock->Locked); 1047 } 1048 1049 /*++ 1050 * @name ExTryToAcquirePushLockExclusive 1051 * INTERNAL MACRO 1052 * 1053 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock. 1054 * 1055 * @params PushLock 1056 * Pointer to the pushlock which is to be acquired. 1057 * 1058 * @return None. 1059 * 1060 * @remarks The function attempts the quickest route to acquire the lock, which is 1061 * to simply set the lock bit. 1062 * However, if the pushlock is already shared, the slower path is taken. 1063 * 1064 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1065 * This macro should usually be paired up with KeAcquireCriticalRegion. 1066 * 1067 *--*/ 1068 FORCEINLINE 1069 BOOLEAN 1070 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock) 1071 { 1072 /* Try acquiring the lock */ 1073 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V)) 1074 { 1075 /* Can't acquire */ 1076 return FALSE; 1077 } 1078 1079 /* Got acquired */ 1080 ASSERT (PushLock->Locked); 1081 return TRUE; 1082 } 1083 1084 /*++ 1085 * @name ExAcquirePushLockShared 1086 * INTERNAL MACRO 1087 * 1088 * The ExAcquirePushLockShared macro acquires a shared PushLock. 1089 * 1090 * @params PushLock 1091 * Pointer to the pushlock which is to be acquired. 1092 * 1093 * @return None. 1094 * 1095 * @remarks The function attempts the quickest route to acquire the lock, which is 1096 * to simply set the lock bit and set the share count to one. 1097 * However, if the pushlock is already shared, the slower path is taken. 1098 * 1099 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1100 * This macro should usually be paired up with KeAcquireCriticalRegion. 1101 * 1102 *--*/ 1103 FORCEINLINE 1104 VOID 1105 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock) 1106 { 1107 EX_PUSH_LOCK NewValue; 1108 1109 /* Try acquiring the lock */ 1110 NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1111 if (ExpChangePushlock(PushLock, NewValue.Ptr, 0)) 1112 { 1113 /* Someone changed it, use the slow path */ 1114 ExfAcquirePushLockShared(PushLock); 1115 } 1116 1117 /* Sanity checks */ 1118 ASSERT(PushLock->Locked); 1119 } 1120 1121 /*++ 1122 * @name ExConvertPushLockSharedToExclusive 1123 * INTERNAL MACRO 1124 * 1125 * The ExConvertPushLockSharedToExclusive macro converts an exclusive 1126 * pushlock to a shared pushlock. 1127 * 1128 * @params PushLock 1129 * Pointer to the pushlock which is to be converted. 1130 * 1131 * @return FALSE if conversion failed, TRUE otherwise. 1132 * 1133 * @remarks The function attempts the quickest route to convert the lock, which is 1134 * to simply set the lock bit and remove any other bits. 1135 * 1136 *--*/ 1137 FORCEINLINE 1138 BOOLEAN 1139 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock) 1140 { 1141 EX_PUSH_LOCK OldValue; 1142 1143 /* Set the expected old value */ 1144 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1145 1146 /* Try converting the lock */ 1147 if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) != 1148 OldValue.Ptr) 1149 { 1150 /* Conversion failed */ 1151 return FALSE; 1152 } 1153 1154 /* Sanity check */ 1155 ASSERT(PushLock->Locked); 1156 return TRUE; 1157 } 1158 1159 /*++ 1160 * @name ExWaitOnPushLock 1161 * INTERNAL MACRO 1162 * 1163 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock. 1164 * 1165 * @params PushLock 1166 * Pointer to a pushlock. 1167 * 1168 * @return None. 1169 * 1170 * @remarks The function attempts to get any exclusive waiters out of their slow 1171 * path by forcing an instant acquire/release operation. 1172 * 1173 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL. 1174 * 1175 *--*/ 1176 FORCEINLINE 1177 VOID 1178 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock) 1179 { 1180 /* Check if we're locked */ 1181 if (PushLock->Locked) 1182 { 1183 /* Acquire the lock */ 1184 ExfAcquirePushLockExclusive(PushLock); 1185 ASSERT(PushLock->Locked); 1186 1187 /* Release it */ 1188 ExfReleasePushLockExclusive(PushLock); 1189 } 1190 } 1191 1192 /*++ 1193 * @name ExReleasePushLockShared 1194 * INTERNAL MACRO 1195 * 1196 * The ExReleasePushLockShared macro releases a previously acquired PushLock. 1197 * 1198 * @params PushLock 1199 * Pointer to a previously acquired pushlock. 1200 * 1201 * @return None. 1202 * 1203 * @remarks The function attempts the quickest route to release the lock, which is 1204 * to simply decrease the share count and remove the lock bit. 1205 * However, if the pushlock is being waited on then the long path is taken. 1206 * 1207 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL. 1208 * This macro should usually be paired up with KeLeaveCriticalRegion. 1209 * 1210 *--*/ 1211 FORCEINLINE 1212 VOID 1213 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock) 1214 { 1215 EX_PUSH_LOCK OldValue; 1216 1217 /* Sanity checks */ 1218 ASSERT(PushLock->Locked); 1219 1220 /* Try to clear the pushlock */ 1221 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1222 if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr) 1223 { 1224 /* There are still other people waiting on it */ 1225 ExfReleasePushLockShared(PushLock); 1226 } 1227 } 1228 1229 /*++ 1230 * @name ExReleasePushLockExclusive 1231 * INTERNAL MACRO 1232 * 1233 * The ExReleasePushLockExclusive macro releases a previously 1234 * exclusively acquired PushLock. 1235 * 1236 * @params PushLock 1237 * Pointer to a previously acquired pushlock. 1238 * 1239 * @return None. 1240 * 1241 * @remarks The function attempts the quickest route to release the lock, which is 1242 * to simply clear the locked bit. 1243 * However, if the pushlock is being waited on, the slow path is taken 1244 * in an attempt to wake up the lock. 1245 * 1246 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL. 1247 * This macro should usually be paired up with KeLeaveCriticalRegion. 1248 * 1249 *--*/ 1250 FORCEINLINE 1251 VOID 1252 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock) 1253 { 1254 EX_PUSH_LOCK OldValue; 1255 1256 /* Sanity checks */ 1257 ASSERT(PushLock->Locked); 1258 1259 /* Unlock the pushlock */ 1260 OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock, 1261 -(SSIZE_T)EX_PUSH_LOCK_LOCK); 1262 1263 /* Sanity checks */ 1264 ASSERT(OldValue.Locked); 1265 ASSERT(OldValue.Waiting || OldValue.Shared == 0); 1266 1267 /* Check if anyone is waiting on it and it's not already waking*/ 1268 if ((OldValue.Waiting) && !(OldValue.Waking)) 1269 { 1270 /* Wake it up */ 1271 ExfTryToWakePushLock(PushLock); 1272 } 1273 } 1274 1275 /*++ 1276 * @name ExReleasePushLock 1277 * INTERNAL MACRO 1278 * 1279 * The ExReleasePushLock macro releases a previously acquired PushLock. 1280 * 1281 * @params PushLock 1282 * Pointer to a previously acquired pushlock. 1283 * 1284 * @return None. 1285 * 1286 * @remarks The function attempts the quickest route to release the lock, which is 1287 * to simply clear all the fields and decrease the share count if required. 1288 * However, if the pushlock is being waited on then the long path is taken. 1289 * 1290 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL. 1291 * This macro should usually be paired up with KeLeaveCriticalRegion. 1292 * 1293 *--*/ 1294 FORCEINLINE 1295 VOID 1296 ExReleasePushLock(PEX_PUSH_LOCK PushLock) 1297 { 1298 EX_PUSH_LOCK OldValue = *PushLock; 1299 EX_PUSH_LOCK NewValue; 1300 1301 /* Sanity checks */ 1302 ASSERT(OldValue.Locked); 1303 1304 /* Check if the pushlock is shared */ 1305 if (OldValue.Shared > 1) 1306 { 1307 /* Decrease the share count */ 1308 NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC; 1309 } 1310 else 1311 { 1312 /* Clear the pushlock entirely */ 1313 NewValue.Value = 0; 1314 } 1315 1316 /* Check if nobody is waiting on us and try clearing the lock here */ 1317 if ((OldValue.Waiting) || 1318 (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) != 1319 OldValue.Ptr)) 1320 { 1321 /* We have waiters, use the long path */ 1322 ExfReleasePushLock(PushLock); 1323 } 1324 } 1325 1326 /* FAST MUTEX INLINES *********************************************************/ 1327 1328 FORCEINLINE 1329 VOID 1330 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex) 1331 { 1332 PKTHREAD Thread = KeGetCurrentThread(); 1333 1334 /* Sanity check */ 1335 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1336 (Thread->CombinedApcDisable != 0) || 1337 (Thread->Teb == NULL) || 1338 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1339 ASSERT(FastMutex->Owner != Thread); 1340 1341 /* Decrease the count */ 1342 if (InterlockedDecrement(&FastMutex->Count)) 1343 { 1344 /* Someone is still holding it, use slow path */ 1345 KiAcquireFastMutex(FastMutex); 1346 } 1347 1348 /* Set the owner */ 1349 FastMutex->Owner = Thread; 1350 } 1351 1352 FORCEINLINE 1353 VOID 1354 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) 1355 { 1356 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1357 (KeGetCurrentThread()->CombinedApcDisable != 0) || 1358 (KeGetCurrentThread()->Teb == NULL) || 1359 (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1360 ASSERT(FastMutex->Owner == KeGetCurrentThread()); 1361 1362 /* Erase the owner */ 1363 FastMutex->Owner = NULL; 1364 1365 /* Increase the count */ 1366 if (InterlockedIncrement(&FastMutex->Count) <= 0) 1367 { 1368 /* Someone was waiting for it, signal the waiter */ 1369 KeSetEventBoostPriority(&FastMutex->Event, NULL); 1370 } 1371 } 1372 1373 FORCEINLINE 1374 VOID 1375 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex) 1376 { 1377 KIRQL OldIrql; 1378 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1379 1380 /* Raise IRQL to APC */ 1381 KeRaiseIrql(APC_LEVEL, &OldIrql); 1382 1383 /* Decrease the count */ 1384 if (InterlockedDecrement(&FastMutex->Count)) 1385 { 1386 /* Someone is still holding it, use slow path */ 1387 KiAcquireFastMutex(FastMutex); 1388 } 1389 1390 /* Set the owner and IRQL */ 1391 FastMutex->Owner = KeGetCurrentThread(); 1392 FastMutex->OldIrql = OldIrql; 1393 } 1394 1395 FORCEINLINE 1396 VOID 1397 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex) 1398 { 1399 KIRQL OldIrql; 1400 ASSERT(KeGetCurrentIrql() == APC_LEVEL); 1401 1402 /* Erase the owner */ 1403 FastMutex->Owner = NULL; 1404 OldIrql = (KIRQL)FastMutex->OldIrql; 1405 1406 /* Increase the count */ 1407 if (InterlockedIncrement(&FastMutex->Count) <= 0) 1408 { 1409 /* Someone was waiting for it, signal the waiter */ 1410 KeSetEventBoostPriority(&FastMutex->Event, NULL); 1411 } 1412 1413 /* Lower IRQL back */ 1414 KeLowerIrql(OldIrql); 1415 } 1416 1417 FORCEINLINE 1418 BOOLEAN 1419 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex) 1420 { 1421 KIRQL OldIrql; 1422 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1423 1424 /* Raise to APC_LEVEL */ 1425 KeRaiseIrql(APC_LEVEL, &OldIrql); 1426 1427 /* Check if we can quickly acquire it */ 1428 if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1) 1429 { 1430 /* We have, set us as owners */ 1431 FastMutex->Owner = KeGetCurrentThread(); 1432 FastMutex->OldIrql = OldIrql; 1433 return TRUE; 1434 } 1435 else 1436 { 1437 /* Acquire attempt failed */ 1438 KeLowerIrql(OldIrql); 1439 YieldProcessor(); 1440 return FALSE; 1441 } 1442 } 1443 1444 FORCEINLINE 1445 VOID 1446 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) 1447 { 1448 /* Enter the Critical Region */ 1449 KeEnterCriticalRegion(); 1450 1451 /* Acquire the mutex unsafely */ 1452 _ExAcquireFastMutexUnsafe(FastMutex); 1453 } 1454 1455 FORCEINLINE 1456 VOID 1457 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex) 1458 { 1459 /* Release the mutex unsafely */ 1460 _ExReleaseFastMutexUnsafe(FastMutex); 1461 1462 /* Leave the critical region */ 1463 KeLeaveCriticalRegion(); 1464 } 1465 1466 /* OTHER FUNCTIONS **********************************************************/ 1467 1468 BOOLEAN 1469 NTAPI 1470 ExTryToAcquireResourceExclusiveLite( 1471 IN PERESOURCE Resource 1472 ); 1473 1474 NTSTATUS 1475 ExpSetTimeZoneInformation( 1476 IN PRTL_TIME_ZONE_INFORMATION TimeZoneInformation 1477 ); 1478 1479 BOOLEAN 1480 NTAPI 1481 ExAcquireTimeRefreshLock( 1482 IN BOOLEAN Wait 1483 ); 1484 1485 VOID 1486 NTAPI 1487 ExReleaseTimeRefreshLock( 1488 VOID 1489 ); 1490 1491 VOID 1492 NTAPI 1493 ExUpdateSystemTimeFromCmos( 1494 IN BOOLEAN UpdateInterruptTime, 1495 IN ULONG MaxSepInSeconds 1496 ); 1497 1498 VOID 1499 NTAPI 1500 ExAllocateLocallyUniqueId( 1501 OUT LUID *LocallyUniqueId 1502 ); 1503 1504 VOID 1505 NTAPI 1506 ExTimerRundown( 1507 VOID 1508 ); 1509 1510 CODE_SEG("INIT") 1511 VOID 1512 NTAPI 1513 HeadlessInit( 1514 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1515 ); 1516 1517 CODE_SEG("INIT") 1518 VOID 1519 NTAPI 1520 XIPInit( 1521 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1522 ); 1523 1524 #define InterlockedDecrementUL(Addend) \ 1525 (ULONG)InterlockedDecrement((PLONG)(Addend)) 1526 1527 #define InterlockedIncrementUL(Addend) \ 1528 (ULONG)InterlockedIncrement((PLONG)(Addend)) 1529 1530 #define InterlockedExchangeUL(Target, Value) \ 1531 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value)) 1532 1533 #define InterlockedExchangeAddUL(Addend, Value) \ 1534 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value)) 1535 1536 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \ 1537 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand)) 1538 1539 #define InterlockedCompareExchangeSizeT(Destination, Exchange, Comperand) \ 1540 (SIZE_T)InterlockedCompareExchangePointer((PVOID*)(Destination), (PVOID)(SIZE_T)(Exchange), (PVOID)(SIZE_T)(Comperand)) 1541 1542 #ifdef _WIN64 1543 #define InterlockedExchangeSizeT(Target, Value) \ 1544 (SIZE_T)InterlockedExchange64((PLONG64)(Target), (LONG64)(Value)) 1545 #else 1546 #define InterlockedExchangeSizeT(Target, Value) \ 1547 (SIZE_T)InterlockedExchange((PLONG)(Target), (LONG)(Value)) 1548 #endif 1549 1550 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \ 1551 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand)) 1552 1553 #ifdef __cplusplus 1554 } // extern "C" 1555 #endif 1556