1 #pragma once 2 3 #ifdef __cplusplus 4 extern "C" 5 { 6 #endif 7 8 /* GLOBAL VARIABLES *********************************************************/ 9 10 extern RTL_TIME_ZONE_INFORMATION ExpTimeZoneInfo; 11 extern LARGE_INTEGER ExpTimeZoneBias; 12 extern ULONG ExpTimeZoneId; 13 extern ULONG ExpTickCountMultiplier; 14 extern ULONG ExpLastTimeZoneBias; 15 extern POBJECT_TYPE ExEventPairObjectType; 16 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType; 17 extern FAST_MUTEX ExpEnvironmentLock; 18 extern ERESOURCE ExpFirmwareTableResource; 19 extern ERESOURCE ExpTimeRefreshLock; 20 extern LIST_ENTRY ExpFirmwareTableProviderListHead; 21 extern BOOLEAN ExpIsWinPEMode; 22 extern LIST_ENTRY ExpSystemResourcesList; 23 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset; 24 extern ULONG ExpUnicodeCaseTableDataOffset; 25 extern PVOID ExpNlsSectionPointer; 26 extern ULONG NtGlobalFlag; 27 extern UNICODE_STRING NtSystemRoot; 28 extern ULONG ExpInitializationPhase; 29 extern ULONG ExpAltTimeZoneBias; 30 extern LIST_ENTRY ExSystemLookasideListHead; 31 extern PCALLBACK_OBJECT PowerStateCallback; 32 extern LIST_ENTRY ExPoolLookasideListHead; 33 extern LIST_ENTRY ExpNonPagedLookasideListHead; 34 extern LIST_ENTRY ExpPagedLookasideListHead; 35 extern KSPIN_LOCK ExpNonPagedLookasideListLock; 36 extern KSPIN_LOCK ExpPagedLookasideListLock; 37 extern ULONG ExCriticalWorkerThreads; 38 extern ULONG ExDelayedWorkerThreads; 39 40 extern PVOID ExpDefaultErrorPort; 41 extern PEPROCESS ExpDefaultErrorPortProcess; 42 43 /* 44 * NT/Cm Version Info variables 45 */ 46 extern ULONG NtMajorVersion; 47 extern ULONG NtMinorVersion; 48 extern ULONG NtBuildNumber; 49 extern ULONG CmNtSpBuildNumber; 50 extern ULONG CmNtCSDVersion; 51 extern ULONG CmNtCSDReleaseType; 52 extern UNICODE_STRING CmVersionString; 53 extern UNICODE_STRING CmCSDVersionString; 54 extern CHAR NtBuildLab[]; 55 56 /* 57 * WinDBG Debugger Worker State Machine data (see dbgctrl.c) 58 */ 59 typedef enum _WINKD_WORKER_STATE 60 { 61 WinKdWorkerReady = 0, 62 WinKdWorkerStart, 63 WinKdWorkerInitialized 64 } WINKD_WORKER_STATE; 65 66 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem; 67 extern WINKD_WORKER_STATE ExpDebuggerWork; 68 extern PEPROCESS ExpDebuggerProcessAttach; 69 extern PEPROCESS ExpDebuggerProcessKill; 70 extern ULONG_PTR ExpDebuggerPageIn; 71 72 VOID NTAPI ExpDebuggerWorker(IN PVOID Context); 73 74 #ifdef _WIN64 75 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4) 76 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3) 77 #else 78 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3) 79 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2) 80 #endif 81 #define HANDLE_TAG_BITS (2) 82 #define HANDLE_INDEX_BITS (HANDLE_LOW_BITS + 2*HANDLE_HIGH_BITS) 83 #define KERNEL_FLAG_BITS (sizeof(PVOID)*8 - HANDLE_INDEX_BITS - HANDLE_TAG_BITS) 84 85 typedef union _EXHANDLE 86 { 87 struct 88 { 89 ULONG_PTR TagBits: HANDLE_TAG_BITS; 90 ULONG_PTR Index: HANDLE_INDEX_BITS; 91 ULONG_PTR KernelFlag : KERNEL_FLAG_BITS; 92 }; 93 struct 94 { 95 ULONG_PTR TagBits2: HANDLE_TAG_BITS; 96 ULONG_PTR LowIndex: HANDLE_LOW_BITS; 97 ULONG_PTR MidIndex: HANDLE_HIGH_BITS; 98 ULONG_PTR HighIndex: HANDLE_HIGH_BITS; 99 ULONG_PTR KernelFlag2: KERNEL_FLAG_BITS; 100 }; 101 HANDLE GenericHandleOverlay; 102 ULONG_PTR Value; 103 ULONG AsULONG; 104 } EXHANDLE, *PEXHANDLE; 105 106 typedef struct _ETIMER 107 { 108 KTIMER KeTimer; 109 KAPC TimerApc; 110 KDPC TimerDpc; 111 LIST_ENTRY ActiveTimerListEntry; 112 KSPIN_LOCK Lock; 113 LONG Period; 114 BOOLEAN ApcAssociated; 115 BOOLEAN WakeTimer; 116 LIST_ENTRY WakeTimerListEntry; 117 } ETIMER, *PETIMER; 118 119 typedef struct 120 { 121 PCALLBACK_OBJECT *CallbackObject; 122 PWSTR Name; 123 } SYSTEM_CALLBACKS; 124 125 typedef struct _HARDERROR_USER_PARAMETERS 126 { 127 ULONG_PTR Parameters[MAXIMUM_HARDERROR_PARAMETERS]; 128 UNICODE_STRING Strings[MAXIMUM_HARDERROR_PARAMETERS]; 129 WCHAR Buffer[ANYSIZE_ARRAY]; 130 } HARDERROR_USER_PARAMETERS, *PHARDERROR_USER_PARAMETERS; 131 132 #define MAX_FAST_REFS 7 133 134 #define ExAcquireRundownProtection _ExAcquireRundownProtection 135 #define ExReleaseRundownProtection _ExReleaseRundownProtection 136 #define ExInitializeRundownProtection _ExInitializeRundownProtection 137 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease 138 #define ExRundownCompleted _ExRundownCompleted 139 #define ExGetPreviousMode KeGetPreviousMode 140 141 142 // 143 // Various bits tagged on the handle or handle table 144 // 145 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1 146 #define FREE_HANDLE_MASK -1 147 148 // 149 // Number of entries in each table level 150 // 151 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY)) 152 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY)) 153 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES)) 154 155 // 156 // Maximum index in each table level before we need another table 157 // 158 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES 159 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES) 160 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES) 161 162 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&x->Ptr, (PVOID)y, (PVOID)z) 163 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z) 164 #define ExpSetRundown(x, y) InterlockedExchangePointer(&x->Ptr, (PVOID)y) 165 166 NTSTATUS 167 NTAPI 168 ExGetPoolTagInfo( 169 IN PSYSTEM_POOLTAG_INFORMATION SystemInformation, 170 IN ULONG SystemInformationLength, 171 IN OUT PULONG ReturnLength OPTIONAL 172 ); 173 174 typedef struct _UUID_CACHED_VALUES_STRUCT 175 { 176 ULONGLONG Time; 177 LONG AllocatedCount; 178 union 179 { 180 struct 181 { 182 UCHAR ClockSeqHiAndReserved; 183 UCHAR ClockSeqLow; 184 UCHAR NodeId[6 /*SEED_BUFFER_SIZE*/]; 185 }; 186 UCHAR GuidInit[8]; /* Match GUID.Data4 */ 187 }; 188 } UUID_CACHED_VALUES_STRUCT, *PUUID_CACHED_VALUES_STRUCT; 189 190 C_ASSERT(RTL_FIELD_SIZE(UUID_CACHED_VALUES_STRUCT, GuidInit) == RTL_FIELD_SIZE(UUID, Data4)); 191 192 /* INITIALIZATION FUNCTIONS *************************************************/ 193 194 CODE_SEG("INIT") 195 BOOLEAN 196 NTAPI 197 ExpWin32kInit(VOID); 198 199 VOID 200 NTAPI 201 ExInit2(VOID); 202 203 VOID 204 NTAPI 205 Phase1Initialization( 206 IN PVOID Context 207 ); 208 209 CODE_SEG("INIT") 210 VOID 211 NTAPI 212 ExpInitializePushLocks(VOID); 213 214 BOOLEAN 215 NTAPI 216 ExRefreshTimeZoneInformation( 217 IN PLARGE_INTEGER SystemBootTime 218 ); 219 220 CODE_SEG("INIT") 221 VOID 222 NTAPI 223 ExpInitializeWorkerThreads(VOID); 224 225 VOID 226 NTAPI 227 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap); 228 229 CODE_SEG("INIT") 230 VOID 231 NTAPI 232 ExpInitLookasideLists(VOID); 233 234 CODE_SEG("INIT") 235 VOID 236 NTAPI 237 ExInitializeSystemLookasideList( 238 IN PGENERAL_LOOKASIDE List, 239 IN POOL_TYPE Type, 240 IN ULONG Size, 241 IN ULONG Tag, 242 IN USHORT MaximumDepth, 243 IN PLIST_ENTRY ListHead 244 ); 245 246 CODE_SEG("INIT") 247 BOOLEAN 248 NTAPI 249 ExpInitializeCallbacks(VOID); 250 251 CODE_SEG("INIT") 252 BOOLEAN 253 NTAPI 254 ExpUuidInitialization(VOID); 255 256 CODE_SEG("INIT") 257 BOOLEAN 258 NTAPI 259 ExLuidInitialization(VOID); 260 261 CODE_SEG("INIT") 262 VOID 263 NTAPI 264 ExpInitializeExecutive( 265 IN ULONG Cpu, 266 IN PLOADER_PARAMETER_BLOCK LoaderBlock 267 ); 268 269 VOID 270 NTAPI 271 ExShutdownSystem(VOID); 272 273 CODE_SEG("INIT") 274 BOOLEAN 275 NTAPI 276 ExpInitializeEventImplementation(VOID); 277 278 CODE_SEG("INIT") 279 BOOLEAN 280 NTAPI 281 ExpInitializeKeyedEventImplementation(VOID); 282 283 CODE_SEG("INIT") 284 BOOLEAN 285 NTAPI 286 ExpInitializeEventPairImplementation(VOID); 287 288 CODE_SEG("INIT") 289 BOOLEAN 290 NTAPI 291 ExpInitializeSemaphoreImplementation(VOID); 292 293 CODE_SEG("INIT") 294 BOOLEAN 295 NTAPI 296 ExpInitializeMutantImplementation(VOID); 297 298 CODE_SEG("INIT") 299 BOOLEAN 300 NTAPI 301 ExpInitializeTimerImplementation(VOID); 302 303 CODE_SEG("INIT") 304 BOOLEAN 305 NTAPI 306 ExpInitializeProfileImplementation(VOID); 307 308 CODE_SEG("INIT") 309 VOID 310 NTAPI 311 ExpResourceInitialization(VOID); 312 313 CODE_SEG("INIT") 314 VOID 315 NTAPI 316 ExInitPoolLookasidePointers(VOID); 317 318 /* Callback Functions ********************************************************/ 319 320 VOID 321 NTAPI 322 ExInitializeCallBack( 323 IN OUT PEX_CALLBACK Callback 324 ); 325 326 PEX_CALLBACK_ROUTINE_BLOCK 327 NTAPI 328 ExAllocateCallBack( 329 IN PEX_CALLBACK_FUNCTION Function, 330 IN PVOID Context 331 ); 332 333 VOID 334 NTAPI 335 ExFreeCallBack( 336 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 337 ); 338 339 BOOLEAN 340 NTAPI 341 ExCompareExchangeCallBack ( 342 IN OUT PEX_CALLBACK CallBack, 343 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock, 344 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock 345 ); 346 347 PEX_CALLBACK_ROUTINE_BLOCK 348 NTAPI 349 ExReferenceCallBackBlock( 350 IN OUT PEX_CALLBACK CallBack 351 ); 352 353 VOID 354 NTAPI 355 ExDereferenceCallBackBlock( 356 IN OUT PEX_CALLBACK CallBack, 357 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 358 ); 359 360 PEX_CALLBACK_FUNCTION 361 NTAPI 362 ExGetCallBackBlockRoutine( 363 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 364 ); 365 366 PVOID 367 NTAPI 368 ExGetCallBackBlockContext( 369 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 370 ); 371 372 VOID 373 NTAPI 374 ExWaitForCallBacks( 375 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 376 ); 377 378 /* Rundown Functions ********************************************************/ 379 380 VOID 381 FASTCALL 382 ExfInitializeRundownProtection( 383 OUT PEX_RUNDOWN_REF RunRef 384 ); 385 386 VOID 387 FASTCALL 388 ExfReInitializeRundownProtection( 389 OUT PEX_RUNDOWN_REF RunRef 390 ); 391 392 BOOLEAN 393 FASTCALL 394 ExfAcquireRundownProtection( 395 IN OUT PEX_RUNDOWN_REF RunRef 396 ); 397 398 BOOLEAN 399 FASTCALL 400 ExfAcquireRundownProtectionEx( 401 IN OUT PEX_RUNDOWN_REF RunRef, 402 IN ULONG Count 403 ); 404 405 VOID 406 FASTCALL 407 ExfReleaseRundownProtection( 408 IN OUT PEX_RUNDOWN_REF RunRef 409 ); 410 411 VOID 412 FASTCALL 413 ExfReleaseRundownProtectionEx( 414 IN OUT PEX_RUNDOWN_REF RunRef, 415 IN ULONG Count 416 ); 417 418 VOID 419 FASTCALL 420 ExfRundownCompleted( 421 OUT PEX_RUNDOWN_REF RunRef 422 ); 423 424 VOID 425 FASTCALL 426 ExfWaitForRundownProtectionRelease( 427 IN OUT PEX_RUNDOWN_REF RunRef 428 ); 429 430 /* HANDLE TABLE FUNCTIONS ***************************************************/ 431 432 typedef BOOLEAN 433 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)( 434 PHANDLE_TABLE_ENTRY HandleTableEntry, 435 HANDLE Handle, 436 PVOID Context 437 ); 438 439 typedef BOOLEAN 440 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)( 441 IN PEPROCESS Process, 442 IN PHANDLE_TABLE HandleTable, 443 IN PHANDLE_TABLE_ENTRY HandleTableEntry, 444 IN PHANDLE_TABLE_ENTRY NewEntry 445 ); 446 447 typedef BOOLEAN 448 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)( 449 PHANDLE_TABLE_ENTRY HandleTableEntry, 450 ULONG_PTR Context 451 ); 452 453 CODE_SEG("INIT") 454 VOID 455 NTAPI 456 ExpInitializeHandleTables( 457 VOID 458 ); 459 460 PHANDLE_TABLE 461 NTAPI 462 ExCreateHandleTable( 463 IN PEPROCESS Process OPTIONAL 464 ); 465 466 VOID 467 NTAPI 468 ExUnlockHandleTableEntry( 469 IN PHANDLE_TABLE HandleTable, 470 IN PHANDLE_TABLE_ENTRY HandleTableEntry 471 ); 472 473 HANDLE 474 NTAPI 475 ExCreateHandle( 476 IN PHANDLE_TABLE HandleTable, 477 IN PHANDLE_TABLE_ENTRY HandleTableEntry 478 ); 479 480 VOID 481 NTAPI 482 ExDestroyHandleTable( 483 IN PHANDLE_TABLE HandleTable, 484 IN PVOID DestroyHandleProcedure OPTIONAL 485 ); 486 487 BOOLEAN 488 NTAPI 489 ExDestroyHandle( 490 IN PHANDLE_TABLE HandleTable, 491 IN HANDLE Handle, 492 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL 493 ); 494 495 PHANDLE_TABLE_ENTRY 496 NTAPI 497 ExMapHandleToPointer( 498 IN PHANDLE_TABLE HandleTable, 499 IN HANDLE Handle 500 ); 501 502 PHANDLE_TABLE 503 NTAPI 504 ExDupHandleTable( 505 IN PEPROCESS Process, 506 IN PHANDLE_TABLE HandleTable, 507 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure, 508 IN ULONG_PTR Mask 509 ); 510 511 BOOLEAN 512 NTAPI 513 ExChangeHandle( 514 IN PHANDLE_TABLE HandleTable, 515 IN HANDLE Handle, 516 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine, 517 IN ULONG_PTR Context 518 ); 519 520 VOID 521 NTAPI 522 ExSweepHandleTable( 523 IN PHANDLE_TABLE HandleTable, 524 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure, 525 IN PVOID Context 526 ); 527 528 PHANDLE_TABLE_ENTRY 529 NTAPI 530 ExpLookupHandleTableEntry( 531 IN PHANDLE_TABLE HandleTable, 532 IN EXHANDLE Handle 533 ); 534 535 BOOLEAN 536 NTAPI 537 ExpLockHandleTableEntry( 538 IN PHANDLE_TABLE HandleTable, 539 IN PHANDLE_TABLE_ENTRY HandleTableEntry 540 ); 541 542 /* PSEH EXCEPTION HANDLING **************************************************/ 543 544 LONG 545 NTAPI 546 ExSystemExceptionFilter(VOID); 547 548 /* CALLBACKS *****************************************************************/ 549 550 FORCEINLINE 551 VOID 552 ExDoCallBack(IN OUT PEX_CALLBACK Callback, 553 IN PVOID Context, 554 IN PVOID Argument1, 555 IN PVOID Argument2) 556 { 557 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock; 558 PEX_CALLBACK_FUNCTION Function; 559 560 /* Reference the block */ 561 CallbackBlock = ExReferenceCallBackBlock(Callback); 562 if (CallbackBlock) 563 { 564 /* Get the function */ 565 Function = ExGetCallBackBlockRoutine(CallbackBlock); 566 567 /* Do the callback */ 568 Function(Context, Argument1, Argument2); 569 570 /* Now dereference it */ 571 ExDereferenceCallBackBlock(Callback, CallbackBlock); 572 } 573 } 574 575 /* FAST REFS ******************************************************************/ 576 577 FORCEINLINE 578 PVOID 579 ExGetObjectFastReference(IN EX_FAST_REF FastRef) 580 { 581 /* Return the unbiased pointer */ 582 return (PVOID)(FastRef.Value & ~MAX_FAST_REFS); 583 } 584 585 FORCEINLINE 586 ULONG 587 ExGetCountFastReference(IN EX_FAST_REF FastRef) 588 { 589 /* Return the reference count */ 590 return (ULONG)FastRef.RefCnt; 591 } 592 593 FORCEINLINE 594 VOID 595 ExInitializeFastReference(OUT PEX_FAST_REF FastRef, 596 IN OPTIONAL PVOID Object) 597 { 598 /* Sanity check */ 599 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0); 600 601 /* Check if an object is being set */ 602 if (!Object) 603 { 604 /* Clear the field */ 605 FastRef->Object = NULL; 606 } 607 else 608 { 609 /* Otherwise, we assume the object was referenced and is ready */ 610 FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS; 611 } 612 } 613 614 FORCEINLINE 615 EX_FAST_REF 616 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef) 617 { 618 EX_FAST_REF OldValue, NewValue; 619 620 /* Start reference loop */ 621 for (;;) 622 { 623 /* Get the current reference count */ 624 OldValue = *FastRef; 625 if (OldValue.RefCnt) 626 { 627 /* Increase the reference count */ 628 NewValue.Value = OldValue.Value - 1; 629 NewValue.Object = ExpChangePushlock(&FastRef->Object, 630 NewValue.Object, 631 OldValue.Object); 632 if (NewValue.Object != OldValue.Object) continue; 633 } 634 635 /* We are done */ 636 break; 637 } 638 639 /* Return the old value */ 640 return OldValue; 641 } 642 643 FORCEINLINE 644 BOOLEAN 645 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef, 646 IN PVOID Object) 647 { 648 EX_FAST_REF OldValue, NewValue; 649 650 /* Sanity checks */ 651 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 652 653 /* Start update loop */ 654 for (;;) 655 { 656 /* Get the current reference count */ 657 OldValue = *FastRef; 658 659 /* Check if the current count is too high or if the pointer changed */ 660 if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) || 661 ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object)) 662 { 663 /* Fail */ 664 return FALSE; 665 } 666 667 /* Update the reference count */ 668 NewValue.Value = OldValue.Value + MAX_FAST_REFS; 669 NewValue.Object = ExpChangePushlock(&FastRef->Object, 670 NewValue.Object, 671 OldValue.Object); 672 if (NewValue.Object != OldValue.Object) continue; 673 674 /* We are done */ 675 break; 676 } 677 678 /* Return success */ 679 return TRUE; 680 } 681 682 FORCEINLINE 683 BOOLEAN 684 ExReleaseFastReference(IN PEX_FAST_REF FastRef, 685 IN PVOID Object) 686 { 687 EX_FAST_REF OldValue, NewValue; 688 689 /* Sanity checks */ 690 ASSERT(Object != NULL); 691 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 692 693 /* Start reference loop */ 694 for (;;) 695 { 696 /* Get the current reference count */ 697 OldValue = *FastRef; 698 699 /* Check if we're full if if the pointer changed */ 700 if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE; 701 702 /* Decrease the reference count */ 703 NewValue.Value = OldValue.Value + 1; 704 NewValue.Object = ExpChangePushlock(&FastRef->Object, 705 NewValue.Object, 706 OldValue.Object); 707 if (NewValue.Object != OldValue.Object) continue; 708 709 /* We are done */ 710 break; 711 } 712 713 /* Return success */ 714 return TRUE; 715 } 716 717 FORCEINLINE 718 EX_FAST_REF 719 ExSwapFastReference(IN PEX_FAST_REF FastRef, 720 IN PVOID Object) 721 { 722 EX_FAST_REF NewValue, OldValue; 723 724 /* Sanity check */ 725 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0); 726 727 /* Check if an object is being set */ 728 if (!Object) 729 { 730 /* Clear the field */ 731 NewValue.Object = NULL; 732 } 733 else 734 { 735 /* Otherwise, we assume the object was referenced and is ready */ 736 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS; 737 } 738 739 /* Update the object */ 740 OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object); 741 return OldValue; 742 } 743 744 FORCEINLINE 745 EX_FAST_REF 746 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef, 747 IN PVOID Object, 748 IN PVOID OldObject) 749 { 750 EX_FAST_REF OldValue, NewValue; 751 752 /* Sanity check and start swap loop */ 753 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 754 for (;;) 755 { 756 /* Get the current value */ 757 OldValue = *FastRef; 758 759 /* Make sure there's enough references to swap */ 760 if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break; 761 762 /* Check if we have an object to swap */ 763 if (Object) 764 { 765 /* Set up the value with maximum fast references */ 766 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS; 767 } 768 else 769 { 770 /* Write the object address itself (which is empty) */ 771 NewValue.Value = (ULONG_PTR)Object; 772 } 773 774 /* Do the actual compare exchange */ 775 NewValue.Object = ExpChangePushlock(&FastRef->Object, 776 NewValue.Object, 777 OldValue.Object); 778 if (NewValue.Object != OldValue.Object) continue; 779 780 /* All done */ 781 break; 782 } 783 784 /* Return the old value */ 785 return OldValue; 786 } 787 788 /* RUNDOWN *******************************************************************/ 789 790 FORCEINLINE 791 PEX_RUNDOWN_REF 792 ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, 793 IN ULONG ProcNumber) 794 { 795 return (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs + 796 RunRefCacheAware->RunRefSize * 797 (ProcNumber % RunRefCacheAware->Number)); 798 } 799 800 /*++ 801 * @name ExfAcquireRundownProtection 802 * INTERNAL MACRO 803 * 804 * The ExfAcquireRundownProtection routine acquires rundown protection for 805 * the specified descriptor. 806 * 807 * @param RunRef 808 * Pointer to a rundown reference descriptor. 809 * 810 * @return TRUE if access to the protected structure was granted, FALSE otherwise. 811 * 812 * @remarks This is the internal macro for system use only.In case the rundown 813 * was active, then the slow-path will be called through the exported 814 * function. 815 * 816 *--*/ 817 FORCEINLINE 818 BOOLEAN 819 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef) 820 { 821 ULONG_PTR Value, NewValue; 822 823 /* Get the current value and mask the active bit */ 824 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE; 825 826 /* Add a reference */ 827 NewValue = Value + EX_RUNDOWN_COUNT_INC; 828 829 /* Change the value */ 830 NewValue = ExpChangeRundown(RunRef, NewValue, Value); 831 if (NewValue != Value) 832 { 833 /* Rundown was active, use long path */ 834 return ExfAcquireRundownProtection(RunRef); 835 } 836 837 /* Success */ 838 return TRUE; 839 } 840 841 /*++ 842 * @name ExReleaseRundownProtection 843 * INTERNAL MACRO 844 * 845 * The ExReleaseRundownProtection routine releases rundown protection for 846 * the specified descriptor. 847 * 848 * @param RunRef 849 * Pointer to a rundown reference descriptor. 850 * 851 * @return TRUE if access to the protected structure was granted, FALSE otherwise. 852 * 853 * @remarks This is the internal macro for system use only.In case the rundown 854 * was active, then the slow-path will be called through the exported 855 * function. 856 * 857 *--*/ 858 FORCEINLINE 859 VOID 860 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef) 861 { 862 ULONG_PTR Value, NewValue; 863 864 /* Get the current value and mask the active bit */ 865 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE; 866 867 /* Remove a reference */ 868 NewValue = Value - EX_RUNDOWN_COUNT_INC; 869 870 /* Change the value */ 871 NewValue = ExpChangeRundown(RunRef, NewValue, Value); 872 873 /* Check if the rundown was active */ 874 if (NewValue != Value) 875 { 876 /* Rundown was active, use long path */ 877 ExfReleaseRundownProtection(RunRef); 878 } 879 else 880 { 881 /* Sanity check */ 882 ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1)); 883 } 884 } 885 886 /*++ 887 * @name ExInitializeRundownProtection 888 * INTERNAL MACRO 889 * 890 * The ExInitializeRundownProtection routine initializes a rundown 891 * protection descriptor. 892 * 893 * @param RunRef 894 * Pointer to a rundown reference descriptor. 895 * 896 * @return None. 897 * 898 * @remarks This is the internal macro for system use only. 899 * 900 *--*/ 901 FORCEINLINE 902 VOID 903 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef) 904 { 905 /* Set the count to zero */ 906 RunRef->Count = 0; 907 } 908 909 /*++ 910 * @name ExWaitForRundownProtectionRelease 911 * INTERNAL MACRO 912 * 913 * The ExWaitForRundownProtectionRelease routine waits until the specified 914 * rundown descriptor has been released. 915 * 916 * @param RunRef 917 * Pointer to a rundown reference descriptor. 918 * 919 * @return None. 920 * 921 * @remarks This is the internal macro for system use only. If a wait is actually 922 * necessary, then the slow path is taken through the exported function. 923 * 924 *--*/ 925 FORCEINLINE 926 VOID 927 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef) 928 { 929 ULONG_PTR Value; 930 931 /* Set the active bit */ 932 Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0); 933 if ((Value) && (Value != EX_RUNDOWN_ACTIVE)) 934 { 935 /* If the the rundown wasn't already active, then take the long path */ 936 ExfWaitForRundownProtectionRelease(RunRef); 937 } 938 } 939 940 /*++ 941 * @name ExRundownCompleted 942 * INTERNAL MACRO 943 * 944 * The ExRundownCompleted routine completes the rundown of the specified 945 * descriptor by setting the active bit. 946 * 947 * @param RunRef 948 * Pointer to a rundown reference descriptor. 949 * 950 * @return None. 951 * 952 * @remarks This is the internal macro for system use only. 953 * 954 *--*/ 955 FORCEINLINE 956 VOID 957 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef) 958 { 959 /* Sanity check */ 960 ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0); 961 962 /* Mark the counter as active */ 963 ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE); 964 } 965 966 /* PUSHLOCKS *****************************************************************/ 967 968 /* FIXME: VERIFY THESE! */ 969 970 VOID 971 FASTCALL 972 ExBlockPushLock( 973 IN PEX_PUSH_LOCK PushLock, 974 IN PVOID WaitBlock 975 ); 976 977 VOID 978 FASTCALL 979 ExfUnblockPushLock( 980 IN PEX_PUSH_LOCK PushLock, 981 IN PVOID CurrentWaitBlock 982 ); 983 984 VOID 985 FASTCALL 986 ExWaitForUnblockPushLock( 987 IN PEX_PUSH_LOCK PushLock, 988 IN PVOID WaitBlock 989 ); 990 991 /*++ 992 * @name _ExInitializePushLock 993 * INTERNAL MACRO 994 * 995 * The _ExInitializePushLock macro initializes a PushLock. 996 * 997 * @params PushLock 998 * Pointer to the pushlock which is to be initialized. 999 * 1000 * @return None. 1001 * 1002 * @remarks None. 1003 * 1004 *--*/ 1005 FORCEINLINE 1006 VOID 1007 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock) 1008 { 1009 /* Set the value to 0 */ 1010 PushLock->Ptr = 0; 1011 } 1012 #define ExInitializePushLock _ExInitializePushLock 1013 1014 /*++ 1015 * @name ExAcquirePushLockExclusive 1016 * INTERNAL MACRO 1017 * 1018 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock. 1019 * 1020 * @params PushLock 1021 * Pointer to the pushlock which is to be acquired. 1022 * 1023 * @return None. 1024 * 1025 * @remarks The function attempts the quickest route to acquire the lock, which is 1026 * to simply set the lock bit. 1027 * However, if the pushlock is already shared, the slower path is taken. 1028 * 1029 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1030 * This macro should usually be paired up with KeAcquireCriticalRegion. 1031 * 1032 *--*/ 1033 FORCEINLINE 1034 VOID 1035 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock) 1036 { 1037 /* Try acquiring the lock */ 1038 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V)) 1039 { 1040 /* Someone changed it, use the slow path */ 1041 ExfAcquirePushLockExclusive(PushLock); 1042 } 1043 1044 /* Sanity check */ 1045 ASSERT(PushLock->Locked); 1046 } 1047 1048 /*++ 1049 * @name ExTryToAcquirePushLockExclusive 1050 * INTERNAL MACRO 1051 * 1052 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock. 1053 * 1054 * @params PushLock 1055 * Pointer to the pushlock which is to be acquired. 1056 * 1057 * @return None. 1058 * 1059 * @remarks The function attempts the quickest route to acquire the lock, which is 1060 * to simply set the lock bit. 1061 * However, if the pushlock is already shared, the slower path is taken. 1062 * 1063 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1064 * This macro should usually be paired up with KeAcquireCriticalRegion. 1065 * 1066 *--*/ 1067 FORCEINLINE 1068 BOOLEAN 1069 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock) 1070 { 1071 /* Try acquiring the lock */ 1072 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V)) 1073 { 1074 /* Can't acquire */ 1075 return FALSE; 1076 } 1077 1078 /* Got acquired */ 1079 ASSERT (PushLock->Locked); 1080 return TRUE; 1081 } 1082 1083 /*++ 1084 * @name ExAcquirePushLockShared 1085 * INTERNAL MACRO 1086 * 1087 * The ExAcquirePushLockShared macro acquires a shared PushLock. 1088 * 1089 * @params PushLock 1090 * Pointer to the pushlock which is to be acquired. 1091 * 1092 * @return None. 1093 * 1094 * @remarks The function attempts the quickest route to acquire the lock, which is 1095 * to simply set the lock bit and set the share count to one. 1096 * However, if the pushlock is already shared, the slower path is taken. 1097 * 1098 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1099 * This macro should usually be paired up with KeAcquireCriticalRegion. 1100 * 1101 *--*/ 1102 FORCEINLINE 1103 VOID 1104 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock) 1105 { 1106 EX_PUSH_LOCK NewValue; 1107 1108 /* Try acquiring the lock */ 1109 NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1110 if (ExpChangePushlock(PushLock, NewValue.Ptr, 0)) 1111 { 1112 /* Someone changed it, use the slow path */ 1113 ExfAcquirePushLockShared(PushLock); 1114 } 1115 1116 /* Sanity checks */ 1117 ASSERT(PushLock->Locked); 1118 } 1119 1120 /*++ 1121 * @name ExConvertPushLockSharedToExclusive 1122 * INTERNAL MACRO 1123 * 1124 * The ExConvertPushLockSharedToExclusive macro converts an exclusive 1125 * pushlock to a shared pushlock. 1126 * 1127 * @params PushLock 1128 * Pointer to the pushlock which is to be converted. 1129 * 1130 * @return FALSE if conversion failed, TRUE otherwise. 1131 * 1132 * @remarks The function attempts the quickest route to convert the lock, which is 1133 * to simply set the lock bit and remove any other bits. 1134 * 1135 *--*/ 1136 FORCEINLINE 1137 BOOLEAN 1138 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock) 1139 { 1140 EX_PUSH_LOCK OldValue; 1141 1142 /* Set the expected old value */ 1143 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1144 1145 /* Try converting the lock */ 1146 if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) != 1147 OldValue.Ptr) 1148 { 1149 /* Conversion failed */ 1150 return FALSE; 1151 } 1152 1153 /* Sanity check */ 1154 ASSERT(PushLock->Locked); 1155 return TRUE; 1156 } 1157 1158 /*++ 1159 * @name ExWaitOnPushLock 1160 * INTERNAL MACRO 1161 * 1162 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock. 1163 * 1164 * @params PushLock 1165 * Pointer to a pushlock. 1166 * 1167 * @return None. 1168 * 1169 * @remarks The function attempts to get any exclusive waiters out of their slow 1170 * path by forcing an instant acquire/release operation. 1171 * 1172 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL. 1173 * 1174 *--*/ 1175 FORCEINLINE 1176 VOID 1177 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock) 1178 { 1179 /* Check if we're locked */ 1180 if (PushLock->Locked) 1181 { 1182 /* Acquire the lock */ 1183 ExfAcquirePushLockExclusive(PushLock); 1184 ASSERT(PushLock->Locked); 1185 1186 /* Release it */ 1187 ExfReleasePushLockExclusive(PushLock); 1188 } 1189 } 1190 1191 /*++ 1192 * @name ExReleasePushLockShared 1193 * INTERNAL MACRO 1194 * 1195 * The ExReleasePushLockShared macro releases a previously acquired PushLock. 1196 * 1197 * @params PushLock 1198 * Pointer to a previously acquired pushlock. 1199 * 1200 * @return None. 1201 * 1202 * @remarks The function attempts the quickest route to release the lock, which is 1203 * to simply decrease the share count and remove the lock bit. 1204 * However, if the pushlock is being waited on then the long path is taken. 1205 * 1206 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL. 1207 * This macro should usually be paired up with KeLeaveCriticalRegion. 1208 * 1209 *--*/ 1210 FORCEINLINE 1211 VOID 1212 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock) 1213 { 1214 EX_PUSH_LOCK OldValue; 1215 1216 /* Sanity checks */ 1217 ASSERT(PushLock->Locked); 1218 1219 /* Try to clear the pushlock */ 1220 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1221 if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr) 1222 { 1223 /* There are still other people waiting on it */ 1224 ExfReleasePushLockShared(PushLock); 1225 } 1226 } 1227 1228 /*++ 1229 * @name ExReleasePushLockExclusive 1230 * INTERNAL MACRO 1231 * 1232 * The ExReleasePushLockExclusive macro releases a previously 1233 * exclusively acquired PushLock. 1234 * 1235 * @params PushLock 1236 * Pointer to a previously acquired pushlock. 1237 * 1238 * @return None. 1239 * 1240 * @remarks The function attempts the quickest route to release the lock, which is 1241 * to simply clear the locked bit. 1242 * However, if the pushlock is being waited on, the slow path is taken 1243 * in an attempt to wake up the lock. 1244 * 1245 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL. 1246 * This macro should usually be paired up with KeLeaveCriticalRegion. 1247 * 1248 *--*/ 1249 FORCEINLINE 1250 VOID 1251 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock) 1252 { 1253 EX_PUSH_LOCK OldValue; 1254 1255 /* Sanity checks */ 1256 ASSERT(PushLock->Locked); 1257 1258 /* Unlock the pushlock */ 1259 OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock, 1260 -(SSIZE_T)EX_PUSH_LOCK_LOCK); 1261 1262 /* Sanity checks */ 1263 ASSERT(OldValue.Locked); 1264 ASSERT(OldValue.Waiting || OldValue.Shared == 0); 1265 1266 /* Check if anyone is waiting on it and it's not already waking*/ 1267 if ((OldValue.Waiting) && !(OldValue.Waking)) 1268 { 1269 /* Wake it up */ 1270 ExfTryToWakePushLock(PushLock); 1271 } 1272 } 1273 1274 /*++ 1275 * @name ExReleasePushLock 1276 * INTERNAL MACRO 1277 * 1278 * The ExReleasePushLock macro releases a previously acquired PushLock. 1279 * 1280 * @params PushLock 1281 * Pointer to a previously acquired pushlock. 1282 * 1283 * @return None. 1284 * 1285 * @remarks The function attempts the quickest route to release the lock, which is 1286 * to simply clear all the fields and decrease the share count if required. 1287 * However, if the pushlock is being waited on then the long path is taken. 1288 * 1289 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL. 1290 * This macro should usually be paired up with KeLeaveCriticalRegion. 1291 * 1292 *--*/ 1293 FORCEINLINE 1294 VOID 1295 ExReleasePushLock(PEX_PUSH_LOCK PushLock) 1296 { 1297 EX_PUSH_LOCK OldValue = *PushLock; 1298 EX_PUSH_LOCK NewValue; 1299 1300 /* Sanity checks */ 1301 ASSERT(OldValue.Locked); 1302 1303 /* Check if the pushlock is shared */ 1304 if (OldValue.Shared > 1) 1305 { 1306 /* Decrease the share count */ 1307 NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC; 1308 } 1309 else 1310 { 1311 /* Clear the pushlock entirely */ 1312 NewValue.Value = 0; 1313 } 1314 1315 /* Check if nobody is waiting on us and try clearing the lock here */ 1316 if ((OldValue.Waiting) || 1317 (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) != 1318 OldValue.Ptr)) 1319 { 1320 /* We have waiters, use the long path */ 1321 ExfReleasePushLock(PushLock); 1322 } 1323 } 1324 1325 /* FAST MUTEX INLINES *********************************************************/ 1326 1327 FORCEINLINE 1328 VOID 1329 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex) 1330 { 1331 PKTHREAD Thread = KeGetCurrentThread(); 1332 1333 /* Sanity check */ 1334 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1335 (Thread->CombinedApcDisable != 0) || 1336 (Thread->Teb == NULL) || 1337 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1338 ASSERT(FastMutex->Owner != Thread); 1339 1340 /* Decrease the count */ 1341 if (InterlockedDecrement(&FastMutex->Count)) 1342 { 1343 /* Someone is still holding it, use slow path */ 1344 KiAcquireFastMutex(FastMutex); 1345 } 1346 1347 /* Set the owner */ 1348 FastMutex->Owner = Thread; 1349 } 1350 1351 FORCEINLINE 1352 VOID 1353 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) 1354 { 1355 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1356 (KeGetCurrentThread()->CombinedApcDisable != 0) || 1357 (KeGetCurrentThread()->Teb == NULL) || 1358 (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1359 ASSERT(FastMutex->Owner == KeGetCurrentThread()); 1360 1361 /* Erase the owner */ 1362 FastMutex->Owner = NULL; 1363 1364 /* Increase the count */ 1365 if (InterlockedIncrement(&FastMutex->Count) <= 0) 1366 { 1367 /* Someone was waiting for it, signal the waiter */ 1368 KeSetEventBoostPriority(&FastMutex->Event, NULL); 1369 } 1370 } 1371 1372 FORCEINLINE 1373 VOID 1374 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex) 1375 { 1376 KIRQL OldIrql; 1377 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1378 1379 /* Raise IRQL to APC */ 1380 KeRaiseIrql(APC_LEVEL, &OldIrql); 1381 1382 /* Decrease the count */ 1383 if (InterlockedDecrement(&FastMutex->Count)) 1384 { 1385 /* Someone is still holding it, use slow path */ 1386 KiAcquireFastMutex(FastMutex); 1387 } 1388 1389 /* Set the owner and IRQL */ 1390 FastMutex->Owner = KeGetCurrentThread(); 1391 FastMutex->OldIrql = OldIrql; 1392 } 1393 1394 FORCEINLINE 1395 VOID 1396 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex) 1397 { 1398 KIRQL OldIrql; 1399 ASSERT(KeGetCurrentIrql() == APC_LEVEL); 1400 1401 /* Erase the owner */ 1402 FastMutex->Owner = NULL; 1403 OldIrql = (KIRQL)FastMutex->OldIrql; 1404 1405 /* Increase the count */ 1406 if (InterlockedIncrement(&FastMutex->Count) <= 0) 1407 { 1408 /* Someone was waiting for it, signal the waiter */ 1409 KeSetEventBoostPriority(&FastMutex->Event, NULL); 1410 } 1411 1412 /* Lower IRQL back */ 1413 KeLowerIrql(OldIrql); 1414 } 1415 1416 FORCEINLINE 1417 BOOLEAN 1418 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex) 1419 { 1420 KIRQL OldIrql; 1421 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1422 1423 /* Raise to APC_LEVEL */ 1424 KeRaiseIrql(APC_LEVEL, &OldIrql); 1425 1426 /* Check if we can quickly acquire it */ 1427 if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1) 1428 { 1429 /* We have, set us as owners */ 1430 FastMutex->Owner = KeGetCurrentThread(); 1431 FastMutex->OldIrql = OldIrql; 1432 return TRUE; 1433 } 1434 else 1435 { 1436 /* Acquire attempt failed */ 1437 KeLowerIrql(OldIrql); 1438 YieldProcessor(); 1439 return FALSE; 1440 } 1441 } 1442 1443 FORCEINLINE 1444 VOID 1445 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) 1446 { 1447 /* Enter the Critical Region */ 1448 KeEnterCriticalRegion(); 1449 1450 /* Acquire the mutex unsafely */ 1451 _ExAcquireFastMutexUnsafe(FastMutex); 1452 } 1453 1454 FORCEINLINE 1455 VOID 1456 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex) 1457 { 1458 /* Release the mutex unsafely */ 1459 _ExReleaseFastMutexUnsafe(FastMutex); 1460 1461 /* Leave the critical region */ 1462 KeLeaveCriticalRegion(); 1463 } 1464 1465 /* OTHER FUNCTIONS **********************************************************/ 1466 1467 BOOLEAN 1468 NTAPI 1469 ExTryToAcquireResourceExclusiveLite( 1470 IN PERESOURCE Resource 1471 ); 1472 1473 NTSTATUS 1474 ExpSetTimeZoneInformation( 1475 IN PRTL_TIME_ZONE_INFORMATION TimeZoneInformation 1476 ); 1477 1478 BOOLEAN 1479 NTAPI 1480 ExAcquireTimeRefreshLock( 1481 IN BOOLEAN Wait 1482 ); 1483 1484 VOID 1485 NTAPI 1486 ExReleaseTimeRefreshLock( 1487 VOID 1488 ); 1489 1490 VOID 1491 NTAPI 1492 ExUpdateSystemTimeFromCmos( 1493 IN BOOLEAN UpdateInterruptTime, 1494 IN ULONG MaxSepInSeconds 1495 ); 1496 1497 VOID 1498 NTAPI 1499 ExAllocateLocallyUniqueId( 1500 OUT LUID *LocallyUniqueId 1501 ); 1502 1503 VOID 1504 NTAPI 1505 ExTimerRundown( 1506 VOID 1507 ); 1508 1509 CODE_SEG("INIT") 1510 VOID 1511 NTAPI 1512 HeadlessInit( 1513 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1514 ); 1515 1516 CODE_SEG("INIT") 1517 VOID 1518 NTAPI 1519 XIPInit( 1520 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1521 ); 1522 1523 #define InterlockedDecrementUL(Addend) \ 1524 (ULONG)InterlockedDecrement((PLONG)(Addend)) 1525 1526 #define InterlockedIncrementUL(Addend) \ 1527 (ULONG)InterlockedIncrement((PLONG)(Addend)) 1528 1529 #define InterlockedExchangeUL(Target, Value) \ 1530 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value)) 1531 1532 #define InterlockedExchangeAddUL(Addend, Value) \ 1533 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value)) 1534 1535 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \ 1536 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand)) 1537 1538 #define InterlockedCompareExchangeSizeT(Destination, Exchange, Comperand) \ 1539 (SIZE_T)InterlockedCompareExchangePointer((PVOID*)(Destination), (PVOID)(SIZE_T)(Exchange), (PVOID)(SIZE_T)(Comperand)) 1540 1541 #ifdef _WIN64 1542 #define InterlockedExchangeSizeT(Target, Value) \ 1543 (SIZE_T)InterlockedExchange64((PLONG64)Target, (LONG64)Value) 1544 #else 1545 #define InterlockedExchangeSizeT(Target, Value) \ 1546 (SIZE_T)InterlockedExchange((PLONG)Target, (LONG)Value) 1547 #endif 1548 1549 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \ 1550 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand)) 1551 1552 #ifdef __cplusplus 1553 } // extern "C" 1554 #endif 1555