1 #pragma once 2 3 /* GLOBAL VARIABLES *********************************************************/ 4 5 extern RTL_TIME_ZONE_INFORMATION ExpTimeZoneInfo; 6 extern LARGE_INTEGER ExpTimeZoneBias; 7 extern ULONG ExpTimeZoneId; 8 extern ULONG ExpTickCountMultiplier; 9 extern ULONG ExpLastTimeZoneBias; 10 extern POBJECT_TYPE ExEventPairObjectType; 11 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType; 12 extern FAST_MUTEX ExpEnvironmentLock; 13 extern ERESOURCE ExpFirmwareTableResource; 14 extern LIST_ENTRY ExpFirmwareTableProviderListHead; 15 extern BOOLEAN ExpIsWinPEMode; 16 extern LIST_ENTRY ExpSystemResourcesList; 17 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset; 18 extern ULONG ExpUnicodeCaseTableDataOffset; 19 extern PVOID ExpNlsSectionPointer; 20 extern ULONG NtGlobalFlag; 21 extern UNICODE_STRING NtSystemRoot; 22 extern ULONG ExpInitializationPhase; 23 extern ULONG ExpAltTimeZoneBias; 24 extern LIST_ENTRY ExSystemLookasideListHead; 25 extern PCALLBACK_OBJECT PowerStateCallback; 26 extern LIST_ENTRY ExPoolLookasideListHead; 27 extern LIST_ENTRY ExpNonPagedLookasideListHead; 28 extern LIST_ENTRY ExpPagedLookasideListHead; 29 extern KSPIN_LOCK ExpNonPagedLookasideListLock; 30 extern KSPIN_LOCK ExpPagedLookasideListLock; 31 extern ULONG ExCriticalWorkerThreads; 32 extern ULONG ExDelayedWorkerThreads; 33 34 extern PVOID ExpDefaultErrorPort; 35 extern PEPROCESS ExpDefaultErrorPortProcess; 36 37 /* 38 * NT/Cm Version Info variables 39 */ 40 extern ULONG NtMajorVersion; 41 extern ULONG NtMinorVersion; 42 extern ULONG NtBuildNumber; 43 extern ULONG CmNtSpBuildNumber; 44 extern ULONG CmNtCSDVersion; 45 extern ULONG CmNtCSDReleaseType; 46 extern UNICODE_STRING CmVersionString; 47 extern UNICODE_STRING CmCSDVersionString; 48 extern CHAR NtBuildLab[]; 49 50 // #ifdef _WINKD_ 51 /* 52 * WinDBG Debugger Worker State Machine data (see dbgctrl.c) 53 */ 54 typedef enum _WINKD_WORKER_STATE 55 { 56 WinKdWorkerReady = 0, 57 WinKdWorkerStart, 58 WinKdWorkerInitialized 59 } WINKD_WORKER_STATE; 60 61 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem; 62 extern WINKD_WORKER_STATE ExpDebuggerWork; 63 extern PEPROCESS ExpDebuggerProcessAttach; 64 extern PEPROCESS ExpDebuggerProcessKill; 65 extern ULONG_PTR ExpDebuggerPageIn; 66 67 VOID NTAPI ExpDebuggerWorker(IN PVOID Context); 68 // #endif /* _WINKD_ */ 69 70 #ifdef _WIN64 71 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4) 72 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3) 73 #else 74 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3) 75 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2) 76 #endif 77 #define HANDLE_TAG_BITS (2) 78 #define HANDLE_INDEX_BITS (HANDLE_LOW_BITS + 2*HANDLE_HIGH_BITS) 79 #define KERNEL_FLAG_BITS (sizeof(PVOID)*8 - HANDLE_INDEX_BITS - HANDLE_TAG_BITS) 80 81 typedef union _EXHANDLE 82 { 83 struct 84 { 85 ULONG_PTR TagBits: HANDLE_TAG_BITS; 86 ULONG_PTR Index: HANDLE_INDEX_BITS; 87 ULONG_PTR KernelFlag : KERNEL_FLAG_BITS; 88 }; 89 struct 90 { 91 ULONG_PTR TagBits2: HANDLE_TAG_BITS; 92 ULONG_PTR LowIndex: HANDLE_LOW_BITS; 93 ULONG_PTR MidIndex: HANDLE_HIGH_BITS; 94 ULONG_PTR HighIndex: HANDLE_HIGH_BITS; 95 ULONG_PTR KernelFlag2: KERNEL_FLAG_BITS; 96 }; 97 HANDLE GenericHandleOverlay; 98 ULONG_PTR Value; 99 ULONG AsULONG; 100 } EXHANDLE, *PEXHANDLE; 101 102 typedef struct _ETIMER 103 { 104 KTIMER KeTimer; 105 KAPC TimerApc; 106 KDPC TimerDpc; 107 LIST_ENTRY ActiveTimerListEntry; 108 KSPIN_LOCK Lock; 109 LONG Period; 110 BOOLEAN ApcAssociated; 111 BOOLEAN WakeTimer; 112 LIST_ENTRY WakeTimerListEntry; 113 } ETIMER, *PETIMER; 114 115 typedef struct 116 { 117 PCALLBACK_OBJECT *CallbackObject; 118 PWSTR Name; 119 } SYSTEM_CALLBACKS; 120 121 typedef struct _HARDERROR_USER_PARAMETERS 122 { 123 ULONG_PTR Parameters[MAXIMUM_HARDERROR_PARAMETERS]; 124 UNICODE_STRING Strings[MAXIMUM_HARDERROR_PARAMETERS]; 125 WCHAR Buffer[ANYSIZE_ARRAY]; 126 } HARDERROR_USER_PARAMETERS, *PHARDERROR_USER_PARAMETERS; 127 128 #define MAX_FAST_REFS 7 129 130 #define ExAcquireRundownProtection _ExAcquireRundownProtection 131 #define ExReleaseRundownProtection _ExReleaseRundownProtection 132 #define ExInitializeRundownProtection _ExInitializeRundownProtection 133 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease 134 #define ExRundownCompleted _ExRundownCompleted 135 #define ExGetPreviousMode KeGetPreviousMode 136 137 138 // 139 // Various bits tagged on the handle or handle table 140 // 141 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1 142 #define FREE_HANDLE_MASK -1 143 144 // 145 // Number of entries in each table level 146 // 147 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY)) 148 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY)) 149 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES)) 150 151 // 152 // Maximum index in each table level before we need another table 153 // 154 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES 155 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES) 156 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES) 157 158 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&x->Ptr, (PVOID)y, (PVOID)z) 159 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z) 160 #define ExpSetRundown(x, y) InterlockedExchangePointer(&x->Ptr, (PVOID)y) 161 162 NTSTATUS 163 NTAPI 164 ExGetPoolTagInfo( 165 IN PSYSTEM_POOLTAG_INFORMATION SystemInformation, 166 IN ULONG SystemInformationLength, 167 IN OUT PULONG ReturnLength OPTIONAL 168 ); 169 170 typedef struct _UUID_CACHED_VALUES_STRUCT 171 { 172 ULONGLONG Time; 173 LONG AllocatedCount; 174 UCHAR ClockSeqHiAndReserved; 175 UCHAR ClockSeqLow; 176 UCHAR NodeId [ 6 ] ; 177 } UUID_CACHED_VALUES_STRUCT, *PUUID_CACHED_VALUES_STRUCT; 178 179 /* INITIALIZATION FUNCTIONS *************************************************/ 180 181 INIT_FUNCTION 182 BOOLEAN 183 NTAPI 184 ExpWin32kInit(VOID); 185 186 VOID 187 NTAPI 188 ExInit2(VOID); 189 190 VOID 191 NTAPI 192 Phase1Initialization( 193 IN PVOID Context 194 ); 195 196 INIT_FUNCTION 197 VOID 198 NTAPI 199 ExpInitializePushLocks(VOID); 200 201 BOOLEAN 202 NTAPI 203 ExRefreshTimeZoneInformation( 204 IN PLARGE_INTEGER SystemBootTime 205 ); 206 207 INIT_FUNCTION 208 VOID 209 NTAPI 210 ExpInitializeWorkerThreads(VOID); 211 212 VOID 213 NTAPI 214 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap); 215 216 INIT_FUNCTION 217 VOID 218 NTAPI 219 ExpInitLookasideLists(VOID); 220 221 INIT_FUNCTION 222 VOID 223 NTAPI 224 ExInitializeSystemLookasideList( 225 IN PGENERAL_LOOKASIDE List, 226 IN POOL_TYPE Type, 227 IN ULONG Size, 228 IN ULONG Tag, 229 IN USHORT MaximumDepth, 230 IN PLIST_ENTRY ListHead 231 ); 232 233 INIT_FUNCTION 234 BOOLEAN 235 NTAPI 236 ExpInitializeCallbacks(VOID); 237 238 INIT_FUNCTION 239 BOOLEAN 240 NTAPI 241 ExpUuidInitialization(VOID); 242 243 INIT_FUNCTION 244 BOOLEAN 245 NTAPI 246 ExLuidInitialization(VOID); 247 248 INIT_FUNCTION 249 VOID 250 NTAPI 251 ExpInitializeExecutive( 252 IN ULONG Cpu, 253 IN PLOADER_PARAMETER_BLOCK LoaderBlock 254 ); 255 256 VOID 257 NTAPI 258 ExShutdownSystem(VOID); 259 260 INIT_FUNCTION 261 BOOLEAN 262 NTAPI 263 ExpInitializeEventImplementation(VOID); 264 265 INIT_FUNCTION 266 BOOLEAN 267 NTAPI 268 ExpInitializeKeyedEventImplementation(VOID); 269 270 INIT_FUNCTION 271 BOOLEAN 272 NTAPI 273 ExpInitializeEventPairImplementation(VOID); 274 275 INIT_FUNCTION 276 BOOLEAN 277 NTAPI 278 ExpInitializeSemaphoreImplementation(VOID); 279 280 INIT_FUNCTION 281 BOOLEAN 282 NTAPI 283 ExpInitializeMutantImplementation(VOID); 284 285 INIT_FUNCTION 286 BOOLEAN 287 NTAPI 288 ExpInitializeTimerImplementation(VOID); 289 290 INIT_FUNCTION 291 BOOLEAN 292 NTAPI 293 ExpInitializeProfileImplementation(VOID); 294 295 INIT_FUNCTION 296 VOID 297 NTAPI 298 ExpResourceInitialization(VOID); 299 300 INIT_FUNCTION 301 VOID 302 NTAPI 303 ExInitPoolLookasidePointers(VOID); 304 305 /* Callback Functions ********************************************************/ 306 307 VOID 308 NTAPI 309 ExInitializeCallBack( 310 IN OUT PEX_CALLBACK Callback 311 ); 312 313 PEX_CALLBACK_ROUTINE_BLOCK 314 NTAPI 315 ExAllocateCallBack( 316 IN PEX_CALLBACK_FUNCTION Function, 317 IN PVOID Context 318 ); 319 320 VOID 321 NTAPI 322 ExFreeCallBack( 323 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 324 ); 325 326 BOOLEAN 327 NTAPI 328 ExCompareExchangeCallBack ( 329 IN OUT PEX_CALLBACK CallBack, 330 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock, 331 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock 332 ); 333 334 PEX_CALLBACK_ROUTINE_BLOCK 335 NTAPI 336 ExReferenceCallBackBlock( 337 IN OUT PEX_CALLBACK CallBack 338 ); 339 340 VOID 341 NTAPI 342 ExDereferenceCallBackBlock( 343 IN OUT PEX_CALLBACK CallBack, 344 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 345 ); 346 347 PEX_CALLBACK_FUNCTION 348 NTAPI 349 ExGetCallBackBlockRoutine( 350 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 351 ); 352 353 PVOID 354 NTAPI 355 ExGetCallBackBlockContext( 356 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 357 ); 358 359 VOID 360 NTAPI 361 ExWaitForCallBacks( 362 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 363 ); 364 365 /* Rundown Functions ********************************************************/ 366 367 VOID 368 FASTCALL 369 ExfInitializeRundownProtection( 370 OUT PEX_RUNDOWN_REF RunRef 371 ); 372 373 VOID 374 FASTCALL 375 ExfReInitializeRundownProtection( 376 OUT PEX_RUNDOWN_REF RunRef 377 ); 378 379 BOOLEAN 380 FASTCALL 381 ExfAcquireRundownProtection( 382 IN OUT PEX_RUNDOWN_REF RunRef 383 ); 384 385 BOOLEAN 386 FASTCALL 387 ExfAcquireRundownProtectionEx( 388 IN OUT PEX_RUNDOWN_REF RunRef, 389 IN ULONG Count 390 ); 391 392 VOID 393 FASTCALL 394 ExfReleaseRundownProtection( 395 IN OUT PEX_RUNDOWN_REF RunRef 396 ); 397 398 VOID 399 FASTCALL 400 ExfReleaseRundownProtectionEx( 401 IN OUT PEX_RUNDOWN_REF RunRef, 402 IN ULONG Count 403 ); 404 405 VOID 406 FASTCALL 407 ExfRundownCompleted( 408 OUT PEX_RUNDOWN_REF RunRef 409 ); 410 411 VOID 412 FASTCALL 413 ExfWaitForRundownProtectionRelease( 414 IN OUT PEX_RUNDOWN_REF RunRef 415 ); 416 417 /* HANDLE TABLE FUNCTIONS ***************************************************/ 418 419 typedef BOOLEAN 420 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)( 421 PHANDLE_TABLE_ENTRY HandleTableEntry, 422 HANDLE Handle, 423 PVOID Context 424 ); 425 426 typedef BOOLEAN 427 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)( 428 IN PEPROCESS Process, 429 IN PHANDLE_TABLE HandleTable, 430 IN PHANDLE_TABLE_ENTRY HandleTableEntry, 431 IN PHANDLE_TABLE_ENTRY NewEntry 432 ); 433 434 typedef BOOLEAN 435 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)( 436 PHANDLE_TABLE_ENTRY HandleTableEntry, 437 ULONG_PTR Context 438 ); 439 440 INIT_FUNCTION 441 VOID 442 NTAPI 443 ExpInitializeHandleTables( 444 VOID 445 ); 446 447 PHANDLE_TABLE 448 NTAPI 449 ExCreateHandleTable( 450 IN PEPROCESS Process OPTIONAL 451 ); 452 453 VOID 454 NTAPI 455 ExUnlockHandleTableEntry( 456 IN PHANDLE_TABLE HandleTable, 457 IN PHANDLE_TABLE_ENTRY HandleTableEntry 458 ); 459 460 HANDLE 461 NTAPI 462 ExCreateHandle( 463 IN PHANDLE_TABLE HandleTable, 464 IN PHANDLE_TABLE_ENTRY HandleTableEntry 465 ); 466 467 VOID 468 NTAPI 469 ExDestroyHandleTable( 470 IN PHANDLE_TABLE HandleTable, 471 IN PVOID DestroyHandleProcedure OPTIONAL 472 ); 473 474 BOOLEAN 475 NTAPI 476 ExDestroyHandle( 477 IN PHANDLE_TABLE HandleTable, 478 IN HANDLE Handle, 479 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL 480 ); 481 482 PHANDLE_TABLE_ENTRY 483 NTAPI 484 ExMapHandleToPointer( 485 IN PHANDLE_TABLE HandleTable, 486 IN HANDLE Handle 487 ); 488 489 PHANDLE_TABLE 490 NTAPI 491 ExDupHandleTable( 492 IN PEPROCESS Process, 493 IN PHANDLE_TABLE HandleTable, 494 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure, 495 IN ULONG_PTR Mask 496 ); 497 498 BOOLEAN 499 NTAPI 500 ExChangeHandle( 501 IN PHANDLE_TABLE HandleTable, 502 IN HANDLE Handle, 503 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine, 504 IN ULONG_PTR Context 505 ); 506 507 VOID 508 NTAPI 509 ExSweepHandleTable( 510 IN PHANDLE_TABLE HandleTable, 511 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure, 512 IN PVOID Context 513 ); 514 515 PHANDLE_TABLE_ENTRY 516 NTAPI 517 ExpLookupHandleTableEntry( 518 IN PHANDLE_TABLE HandleTable, 519 IN EXHANDLE Handle 520 ); 521 522 BOOLEAN 523 NTAPI 524 ExpLockHandleTableEntry( 525 IN PHANDLE_TABLE HandleTable, 526 IN PHANDLE_TABLE_ENTRY HandleTableEntry 527 ); 528 529 /* PSEH EXCEPTION HANDLING **************************************************/ 530 531 LONG 532 NTAPI 533 ExSystemExceptionFilter(VOID); 534 535 /* CALLBACKS *****************************************************************/ 536 537 FORCEINLINE 538 VOID 539 ExDoCallBack(IN OUT PEX_CALLBACK Callback, 540 IN PVOID Context, 541 IN PVOID Argument1, 542 IN PVOID Argument2) 543 { 544 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock; 545 PEX_CALLBACK_FUNCTION Function; 546 547 /* Reference the block */ 548 CallbackBlock = ExReferenceCallBackBlock(Callback); 549 if (CallbackBlock) 550 { 551 /* Get the function */ 552 Function = ExGetCallBackBlockRoutine(CallbackBlock); 553 554 /* Do the callback */ 555 Function(Context, Argument1, Argument2); 556 557 /* Now dereference it */ 558 ExDereferenceCallBackBlock(Callback, CallbackBlock); 559 } 560 } 561 562 /* FAST REFS ******************************************************************/ 563 564 FORCEINLINE 565 PVOID 566 ExGetObjectFastReference(IN EX_FAST_REF FastRef) 567 { 568 /* Return the unbiased pointer */ 569 return (PVOID)(FastRef.Value & ~MAX_FAST_REFS); 570 } 571 572 FORCEINLINE 573 ULONG 574 ExGetCountFastReference(IN EX_FAST_REF FastRef) 575 { 576 /* Return the reference count */ 577 return (ULONG)FastRef.RefCnt; 578 } 579 580 FORCEINLINE 581 VOID 582 ExInitializeFastReference(OUT PEX_FAST_REF FastRef, 583 IN OPTIONAL PVOID Object) 584 { 585 /* Sanity check */ 586 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0); 587 588 /* Check if an object is being set */ 589 if (!Object) 590 { 591 /* Clear the field */ 592 FastRef->Object = NULL; 593 } 594 else 595 { 596 /* Otherwise, we assume the object was referenced and is ready */ 597 FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS; 598 } 599 } 600 601 FORCEINLINE 602 EX_FAST_REF 603 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef) 604 { 605 EX_FAST_REF OldValue, NewValue; 606 607 /* Start reference loop */ 608 for (;;) 609 { 610 /* Get the current reference count */ 611 OldValue = *FastRef; 612 if (OldValue.RefCnt) 613 { 614 /* Increase the reference count */ 615 NewValue.Value = OldValue.Value - 1; 616 NewValue.Object = ExpChangePushlock(&FastRef->Object, 617 NewValue.Object, 618 OldValue.Object); 619 if (NewValue.Object != OldValue.Object) continue; 620 } 621 622 /* We are done */ 623 break; 624 } 625 626 /* Return the old value */ 627 return OldValue; 628 } 629 630 FORCEINLINE 631 BOOLEAN 632 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef, 633 IN PVOID Object) 634 { 635 EX_FAST_REF OldValue, NewValue; 636 637 /* Sanity checks */ 638 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 639 640 /* Start update loop */ 641 for (;;) 642 { 643 /* Get the current reference count */ 644 OldValue = *FastRef; 645 646 /* Check if the current count is too high or if the pointer changed */ 647 if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) || 648 ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object)) 649 { 650 /* Fail */ 651 return FALSE; 652 } 653 654 /* Update the reference count */ 655 NewValue.Value = OldValue.Value + MAX_FAST_REFS; 656 NewValue.Object = ExpChangePushlock(&FastRef->Object, 657 NewValue.Object, 658 OldValue.Object); 659 if (NewValue.Object != OldValue.Object) continue; 660 661 /* We are done */ 662 break; 663 } 664 665 /* Return success */ 666 return TRUE; 667 } 668 669 FORCEINLINE 670 BOOLEAN 671 ExReleaseFastReference(IN PEX_FAST_REF FastRef, 672 IN PVOID Object) 673 { 674 EX_FAST_REF OldValue, NewValue; 675 676 /* Sanity checks */ 677 ASSERT(Object != NULL); 678 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 679 680 /* Start reference loop */ 681 for (;;) 682 { 683 /* Get the current reference count */ 684 OldValue = *FastRef; 685 686 /* Check if we're full if if the pointer changed */ 687 if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE; 688 689 /* Decrease the reference count */ 690 NewValue.Value = OldValue.Value + 1; 691 NewValue.Object = ExpChangePushlock(&FastRef->Object, 692 NewValue.Object, 693 OldValue.Object); 694 if (NewValue.Object != OldValue.Object) continue; 695 696 /* We are done */ 697 break; 698 } 699 700 /* Return success */ 701 return TRUE; 702 } 703 704 FORCEINLINE 705 EX_FAST_REF 706 ExSwapFastReference(IN PEX_FAST_REF FastRef, 707 IN PVOID Object) 708 { 709 EX_FAST_REF NewValue, OldValue; 710 711 /* Sanity check */ 712 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0); 713 714 /* Check if an object is being set */ 715 if (!Object) 716 { 717 /* Clear the field */ 718 NewValue.Object = NULL; 719 } 720 else 721 { 722 /* Otherwise, we assume the object was referenced and is ready */ 723 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS; 724 } 725 726 /* Update the object */ 727 OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object); 728 return OldValue; 729 } 730 731 FORCEINLINE 732 EX_FAST_REF 733 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef, 734 IN PVOID Object, 735 IN PVOID OldObject) 736 { 737 EX_FAST_REF OldValue, NewValue; 738 739 /* Sanity check and start swap loop */ 740 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 741 for (;;) 742 { 743 /* Get the current value */ 744 OldValue = *FastRef; 745 746 /* Make sure there's enough references to swap */ 747 if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break; 748 749 /* Check if we have an object to swap */ 750 if (Object) 751 { 752 /* Set up the value with maximum fast references */ 753 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS; 754 } 755 else 756 { 757 /* Write the object address itself (which is empty) */ 758 NewValue.Value = (ULONG_PTR)Object; 759 } 760 761 /* Do the actual compare exchange */ 762 NewValue.Object = ExpChangePushlock(&FastRef->Object, 763 NewValue.Object, 764 OldValue.Object); 765 if (NewValue.Object != OldValue.Object) continue; 766 767 /* All done */ 768 break; 769 } 770 771 /* Return the old value */ 772 return OldValue; 773 } 774 775 /* RUNDOWN *******************************************************************/ 776 777 FORCEINLINE 778 PEX_RUNDOWN_REF 779 ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, 780 IN ULONG ProcNumber) 781 { 782 return (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs + 783 RunRefCacheAware->RunRefSize * 784 (ProcNumber % RunRefCacheAware->Number)); 785 } 786 787 /*++ 788 * @name ExfAcquireRundownProtection 789 * INTERNAL MACRO 790 * 791 * The ExfAcquireRundownProtection routine acquires rundown protection for 792 * the specified descriptor. 793 * 794 * @param RunRef 795 * Pointer to a rundown reference descriptor. 796 * 797 * @return TRUE if access to the protected structure was granted, FALSE otherwise. 798 * 799 * @remarks This is the internal macro for system use only.In case the rundown 800 * was active, then the slow-path will be called through the exported 801 * function. 802 * 803 *--*/ 804 FORCEINLINE 805 BOOLEAN 806 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef) 807 { 808 ULONG_PTR Value, NewValue; 809 810 /* Get the current value and mask the active bit */ 811 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE; 812 813 /* Add a reference */ 814 NewValue = Value + EX_RUNDOWN_COUNT_INC; 815 816 /* Change the value */ 817 NewValue = ExpChangeRundown(RunRef, NewValue, Value); 818 if (NewValue != Value) 819 { 820 /* Rundown was active, use long path */ 821 return ExfAcquireRundownProtection(RunRef); 822 } 823 824 /* Success */ 825 return TRUE; 826 } 827 828 /*++ 829 * @name ExReleaseRundownProtection 830 * INTERNAL MACRO 831 * 832 * The ExReleaseRundownProtection routine releases rundown protection for 833 * the specified descriptor. 834 * 835 * @param RunRef 836 * Pointer to a rundown reference descriptor. 837 * 838 * @return TRUE if access to the protected structure was granted, FALSE otherwise. 839 * 840 * @remarks This is the internal macro for system use only.In case the rundown 841 * was active, then the slow-path will be called through the exported 842 * function. 843 * 844 *--*/ 845 FORCEINLINE 846 VOID 847 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef) 848 { 849 ULONG_PTR Value, NewValue; 850 851 /* Get the current value and mask the active bit */ 852 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE; 853 854 /* Remove a reference */ 855 NewValue = Value - EX_RUNDOWN_COUNT_INC; 856 857 /* Change the value */ 858 NewValue = ExpChangeRundown(RunRef, NewValue, Value); 859 860 /* Check if the rundown was active */ 861 if (NewValue != Value) 862 { 863 /* Rundown was active, use long path */ 864 ExfReleaseRundownProtection(RunRef); 865 } 866 else 867 { 868 /* Sanity check */ 869 ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1)); 870 } 871 } 872 873 /*++ 874 * @name ExInitializeRundownProtection 875 * INTERNAL MACRO 876 * 877 * The ExInitializeRundownProtection routine initializes a rundown 878 * protection descriptor. 879 * 880 * @param RunRef 881 * Pointer to a rundown reference descriptor. 882 * 883 * @return None. 884 * 885 * @remarks This is the internal macro for system use only. 886 * 887 *--*/ 888 FORCEINLINE 889 VOID 890 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef) 891 { 892 /* Set the count to zero */ 893 RunRef->Count = 0; 894 } 895 896 /*++ 897 * @name ExWaitForRundownProtectionRelease 898 * INTERNAL MACRO 899 * 900 * The ExWaitForRundownProtectionRelease routine waits until the specified 901 * rundown descriptor has been released. 902 * 903 * @param RunRef 904 * Pointer to a rundown reference descriptor. 905 * 906 * @return None. 907 * 908 * @remarks This is the internal macro for system use only. If a wait is actually 909 * necessary, then the slow path is taken through the exported function. 910 * 911 *--*/ 912 FORCEINLINE 913 VOID 914 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef) 915 { 916 ULONG_PTR Value; 917 918 /* Set the active bit */ 919 Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0); 920 if ((Value) && (Value != EX_RUNDOWN_ACTIVE)) 921 { 922 /* If the the rundown wasn't already active, then take the long path */ 923 ExfWaitForRundownProtectionRelease(RunRef); 924 } 925 } 926 927 /*++ 928 * @name ExRundownCompleted 929 * INTERNAL MACRO 930 * 931 * The ExRundownCompleted routine completes the rundown of the specified 932 * descriptor by setting the active bit. 933 * 934 * @param RunRef 935 * Pointer to a rundown reference descriptor. 936 * 937 * @return None. 938 * 939 * @remarks This is the internal macro for system use only. 940 * 941 *--*/ 942 FORCEINLINE 943 VOID 944 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef) 945 { 946 /* Sanity check */ 947 ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0); 948 949 /* Mark the counter as active */ 950 ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE); 951 } 952 953 /* PUSHLOCKS *****************************************************************/ 954 955 /* FIXME: VERIFY THESE! */ 956 957 VOID 958 FASTCALL 959 ExBlockPushLock( 960 IN PEX_PUSH_LOCK PushLock, 961 IN PVOID WaitBlock 962 ); 963 964 VOID 965 FASTCALL 966 ExfUnblockPushLock( 967 IN PEX_PUSH_LOCK PushLock, 968 IN PVOID CurrentWaitBlock 969 ); 970 971 VOID 972 FASTCALL 973 ExWaitForUnblockPushLock( 974 IN PEX_PUSH_LOCK PushLock, 975 IN PVOID WaitBlock 976 ); 977 978 /*++ 979 * @name _ExInitializePushLock 980 * INTERNAL MACRO 981 * 982 * The _ExInitializePushLock macro initializes a PushLock. 983 * 984 * @params PushLock 985 * Pointer to the pushlock which is to be initialized. 986 * 987 * @return None. 988 * 989 * @remarks None. 990 * 991 *--*/ 992 FORCEINLINE 993 VOID 994 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock) 995 { 996 /* Set the value to 0 */ 997 PushLock->Ptr = 0; 998 } 999 #define ExInitializePushLock _ExInitializePushLock 1000 1001 /*++ 1002 * @name ExAcquirePushLockExclusive 1003 * INTERNAL MACRO 1004 * 1005 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock. 1006 * 1007 * @params PushLock 1008 * Pointer to the pushlock which is to be acquired. 1009 * 1010 * @return None. 1011 * 1012 * @remarks The function attempts the quickest route to acquire the lock, which is 1013 * to simply set the lock bit. 1014 * However, if the pushlock is already shared, the slower path is taken. 1015 * 1016 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1017 * This macro should usually be paired up with KeAcquireCriticalRegion. 1018 * 1019 *--*/ 1020 FORCEINLINE 1021 VOID 1022 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock) 1023 { 1024 /* Try acquiring the lock */ 1025 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V)) 1026 { 1027 /* Someone changed it, use the slow path */ 1028 ExfAcquirePushLockExclusive(PushLock); 1029 } 1030 1031 /* Sanity check */ 1032 ASSERT(PushLock->Locked); 1033 } 1034 1035 /*++ 1036 * @name ExTryToAcquirePushLockExclusive 1037 * INTERNAL MACRO 1038 * 1039 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock. 1040 * 1041 * @params PushLock 1042 * Pointer to the pushlock which is to be acquired. 1043 * 1044 * @return None. 1045 * 1046 * @remarks The function attempts the quickest route to acquire the lock, which is 1047 * to simply set the lock bit. 1048 * However, if the pushlock is already shared, the slower path is taken. 1049 * 1050 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1051 * This macro should usually be paired up with KeAcquireCriticalRegion. 1052 * 1053 *--*/ 1054 FORCEINLINE 1055 BOOLEAN 1056 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock) 1057 { 1058 /* Try acquiring the lock */ 1059 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V)) 1060 { 1061 /* Can't acquire */ 1062 return FALSE; 1063 } 1064 1065 /* Got acquired */ 1066 ASSERT (PushLock->Locked); 1067 return TRUE; 1068 } 1069 1070 /*++ 1071 * @name ExAcquirePushLockShared 1072 * INTERNAL MACRO 1073 * 1074 * The ExAcquirePushLockShared macro acquires a shared PushLock. 1075 * 1076 * @params PushLock 1077 * Pointer to the pushlock which is to be acquired. 1078 * 1079 * @return None. 1080 * 1081 * @remarks The function attempts the quickest route to acquire the lock, which is 1082 * to simply set the lock bit and set the share count to one. 1083 * However, if the pushlock is already shared, the slower path is taken. 1084 * 1085 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1086 * This macro should usually be paired up with KeAcquireCriticalRegion. 1087 * 1088 *--*/ 1089 FORCEINLINE 1090 VOID 1091 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock) 1092 { 1093 EX_PUSH_LOCK NewValue; 1094 1095 /* Try acquiring the lock */ 1096 NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1097 if (ExpChangePushlock(PushLock, NewValue.Ptr, 0)) 1098 { 1099 /* Someone changed it, use the slow path */ 1100 ExfAcquirePushLockShared(PushLock); 1101 } 1102 1103 /* Sanity checks */ 1104 ASSERT(PushLock->Locked); 1105 } 1106 1107 /*++ 1108 * @name ExConvertPushLockSharedToExclusive 1109 * INTERNAL MACRO 1110 * 1111 * The ExConvertPushLockSharedToExclusive macro converts an exclusive 1112 * pushlock to a shared pushlock. 1113 * 1114 * @params PushLock 1115 * Pointer to the pushlock which is to be converted. 1116 * 1117 * @return FALSE if conversion failed, TRUE otherwise. 1118 * 1119 * @remarks The function attempts the quickest route to convert the lock, which is 1120 * to simply set the lock bit and remove any other bits. 1121 * 1122 *--*/ 1123 FORCEINLINE 1124 BOOLEAN 1125 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock) 1126 { 1127 EX_PUSH_LOCK OldValue; 1128 1129 /* Set the expected old value */ 1130 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1131 1132 /* Try converting the lock */ 1133 if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) != 1134 OldValue.Ptr) 1135 { 1136 /* Conversion failed */ 1137 return FALSE; 1138 } 1139 1140 /* Sanity check */ 1141 ASSERT(PushLock->Locked); 1142 return TRUE; 1143 } 1144 1145 /*++ 1146 * @name ExWaitOnPushLock 1147 * INTERNAL MACRO 1148 * 1149 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock. 1150 * 1151 * @params PushLock 1152 * Pointer to a pushlock. 1153 * 1154 * @return None. 1155 * 1156 * @remarks The function attempts to get any exclusive waiters out of their slow 1157 * path by forcing an instant acquire/release operation. 1158 * 1159 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL. 1160 * 1161 *--*/ 1162 FORCEINLINE 1163 VOID 1164 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock) 1165 { 1166 /* Check if we're locked */ 1167 if (PushLock->Locked) 1168 { 1169 /* Acquire the lock */ 1170 ExfAcquirePushLockExclusive(PushLock); 1171 ASSERT(PushLock->Locked); 1172 1173 /* Release it */ 1174 ExfReleasePushLockExclusive(PushLock); 1175 } 1176 } 1177 1178 /*++ 1179 * @name ExReleasePushLockShared 1180 * INTERNAL MACRO 1181 * 1182 * The ExReleasePushLockShared macro releases a previously acquired PushLock. 1183 * 1184 * @params PushLock 1185 * Pointer to a previously acquired pushlock. 1186 * 1187 * @return None. 1188 * 1189 * @remarks The function attempts the quickest route to release the lock, which is 1190 * to simply decrease the share count and remove the lock bit. 1191 * However, if the pushlock is being waited on then the long path is taken. 1192 * 1193 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL. 1194 * This macro should usually be paired up with KeLeaveCriticalRegion. 1195 * 1196 *--*/ 1197 FORCEINLINE 1198 VOID 1199 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock) 1200 { 1201 EX_PUSH_LOCK OldValue; 1202 1203 /* Sanity checks */ 1204 ASSERT(PushLock->Locked); 1205 1206 /* Try to clear the pushlock */ 1207 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1208 if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr) 1209 { 1210 /* There are still other people waiting on it */ 1211 ExfReleasePushLockShared(PushLock); 1212 } 1213 } 1214 1215 /*++ 1216 * @name ExReleasePushLockExclusive 1217 * INTERNAL MACRO 1218 * 1219 * The ExReleasePushLockExclusive macro releases a previously 1220 * exclusively acquired PushLock. 1221 * 1222 * @params PushLock 1223 * Pointer to a previously acquired pushlock. 1224 * 1225 * @return None. 1226 * 1227 * @remarks The function attempts the quickest route to release the lock, which is 1228 * to simply clear the locked bit. 1229 * However, if the pushlock is being waited on, the slow path is taken 1230 * in an attempt to wake up the lock. 1231 * 1232 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL. 1233 * This macro should usually be paired up with KeLeaveCriticalRegion. 1234 * 1235 *--*/ 1236 FORCEINLINE 1237 VOID 1238 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock) 1239 { 1240 EX_PUSH_LOCK OldValue; 1241 1242 /* Sanity checks */ 1243 ASSERT(PushLock->Locked); 1244 1245 /* Unlock the pushlock */ 1246 OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock, 1247 -(SSIZE_T)EX_PUSH_LOCK_LOCK); 1248 1249 /* Sanity checks */ 1250 ASSERT(OldValue.Locked); 1251 ASSERT(OldValue.Waiting || OldValue.Shared == 0); 1252 1253 /* Check if anyone is waiting on it and it's not already waking*/ 1254 if ((OldValue.Waiting) && !(OldValue.Waking)) 1255 { 1256 /* Wake it up */ 1257 ExfTryToWakePushLock(PushLock); 1258 } 1259 } 1260 1261 /*++ 1262 * @name ExReleasePushLock 1263 * INTERNAL MACRO 1264 * 1265 * The ExReleasePushLock macro releases a previously acquired PushLock. 1266 * 1267 * @params PushLock 1268 * Pointer to a previously acquired pushlock. 1269 * 1270 * @return None. 1271 * 1272 * @remarks The function attempts the quickest route to release the lock, which is 1273 * to simply clear all the fields and decrease the share count if required. 1274 * However, if the pushlock is being waited on then the long path is taken. 1275 * 1276 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL. 1277 * This macro should usually be paired up with KeLeaveCriticalRegion. 1278 * 1279 *--*/ 1280 FORCEINLINE 1281 VOID 1282 ExReleasePushLock(PEX_PUSH_LOCK PushLock) 1283 { 1284 EX_PUSH_LOCK OldValue = *PushLock; 1285 EX_PUSH_LOCK NewValue; 1286 1287 /* Sanity checks */ 1288 ASSERT(OldValue.Locked); 1289 1290 /* Check if the pushlock is shared */ 1291 if (OldValue.Shared > 1) 1292 { 1293 /* Decrease the share count */ 1294 NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC; 1295 } 1296 else 1297 { 1298 /* Clear the pushlock entirely */ 1299 NewValue.Value = 0; 1300 } 1301 1302 /* Check if nobody is waiting on us and try clearing the lock here */ 1303 if ((OldValue.Waiting) || 1304 (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) != 1305 OldValue.Ptr)) 1306 { 1307 /* We have waiters, use the long path */ 1308 ExfReleasePushLock(PushLock); 1309 } 1310 } 1311 1312 /* FAST MUTEX INLINES *********************************************************/ 1313 1314 FORCEINLINE 1315 VOID 1316 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex) 1317 { 1318 PKTHREAD Thread = KeGetCurrentThread(); 1319 1320 /* Sanity check */ 1321 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1322 (Thread->CombinedApcDisable != 0) || 1323 (Thread->Teb == NULL) || 1324 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1325 ASSERT(FastMutex->Owner != Thread); 1326 1327 /* Decrease the count */ 1328 if (InterlockedDecrement(&FastMutex->Count)) 1329 { 1330 /* Someone is still holding it, use slow path */ 1331 KiAcquireFastMutex(FastMutex); 1332 } 1333 1334 /* Set the owner */ 1335 FastMutex->Owner = Thread; 1336 } 1337 1338 FORCEINLINE 1339 VOID 1340 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) 1341 { 1342 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1343 (KeGetCurrentThread()->CombinedApcDisable != 0) || 1344 (KeGetCurrentThread()->Teb == NULL) || 1345 (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1346 ASSERT(FastMutex->Owner == KeGetCurrentThread()); 1347 1348 /* Erase the owner */ 1349 FastMutex->Owner = NULL; 1350 1351 /* Increase the count */ 1352 if (InterlockedIncrement(&FastMutex->Count) <= 0) 1353 { 1354 /* Someone was waiting for it, signal the waiter */ 1355 KeSetEventBoostPriority(&FastMutex->Event, NULL); 1356 } 1357 } 1358 1359 FORCEINLINE 1360 VOID 1361 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex) 1362 { 1363 KIRQL OldIrql; 1364 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1365 1366 /* Raise IRQL to APC */ 1367 KeRaiseIrql(APC_LEVEL, &OldIrql); 1368 1369 /* Decrease the count */ 1370 if (InterlockedDecrement(&FastMutex->Count)) 1371 { 1372 /* Someone is still holding it, use slow path */ 1373 KiAcquireFastMutex(FastMutex); 1374 } 1375 1376 /* Set the owner and IRQL */ 1377 FastMutex->Owner = KeGetCurrentThread(); 1378 FastMutex->OldIrql = OldIrql; 1379 } 1380 1381 FORCEINLINE 1382 VOID 1383 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex) 1384 { 1385 KIRQL OldIrql; 1386 ASSERT(KeGetCurrentIrql() == APC_LEVEL); 1387 1388 /* Erase the owner */ 1389 FastMutex->Owner = NULL; 1390 OldIrql = (KIRQL)FastMutex->OldIrql; 1391 1392 /* Increase the count */ 1393 if (InterlockedIncrement(&FastMutex->Count) <= 0) 1394 { 1395 /* Someone was waiting for it, signal the waiter */ 1396 KeSetEventBoostPriority(&FastMutex->Event, NULL); 1397 } 1398 1399 /* Lower IRQL back */ 1400 KeLowerIrql(OldIrql); 1401 } 1402 1403 FORCEINLINE 1404 BOOLEAN 1405 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex) 1406 { 1407 KIRQL OldIrql; 1408 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1409 1410 /* Raise to APC_LEVEL */ 1411 KeRaiseIrql(APC_LEVEL, &OldIrql); 1412 1413 /* Check if we can quickly acquire it */ 1414 if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1) 1415 { 1416 /* We have, set us as owners */ 1417 FastMutex->Owner = KeGetCurrentThread(); 1418 FastMutex->OldIrql = OldIrql; 1419 return TRUE; 1420 } 1421 else 1422 { 1423 /* Acquire attempt failed */ 1424 KeLowerIrql(OldIrql); 1425 YieldProcessor(); 1426 return FALSE; 1427 } 1428 } 1429 1430 FORCEINLINE 1431 VOID 1432 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) 1433 { 1434 /* Enter the Critical Region */ 1435 KeEnterCriticalRegion(); 1436 1437 /* Acquire the mutex unsafely */ 1438 _ExAcquireFastMutexUnsafe(FastMutex); 1439 } 1440 1441 FORCEINLINE 1442 VOID 1443 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex) 1444 { 1445 /* Release the mutex unsafely */ 1446 _ExReleaseFastMutexUnsafe(FastMutex); 1447 1448 /* Leave the critical region */ 1449 KeLeaveCriticalRegion(); 1450 } 1451 1452 /* OTHER FUNCTIONS **********************************************************/ 1453 1454 BOOLEAN 1455 NTAPI 1456 ExTryToAcquireResourceExclusiveLite( 1457 IN PERESOURCE Resource 1458 ); 1459 1460 NTSTATUS 1461 ExpSetTimeZoneInformation( 1462 IN PRTL_TIME_ZONE_INFORMATION TimeZoneInformation 1463 ); 1464 1465 BOOLEAN 1466 NTAPI 1467 ExAcquireTimeRefreshLock( 1468 IN BOOLEAN Wait 1469 ); 1470 1471 VOID 1472 NTAPI 1473 ExReleaseTimeRefreshLock( 1474 VOID 1475 ); 1476 1477 VOID 1478 NTAPI 1479 ExUpdateSystemTimeFromCmos( 1480 IN BOOLEAN UpdateInterruptTime, 1481 IN ULONG MaxSepInSeconds 1482 ); 1483 1484 VOID 1485 NTAPI 1486 ExAllocateLocallyUniqueId( 1487 OUT LUID *LocallyUniqueId 1488 ); 1489 1490 VOID 1491 NTAPI 1492 ExTimerRundown( 1493 VOID 1494 ); 1495 1496 INIT_FUNCTION 1497 VOID 1498 NTAPI 1499 HeadlessInit( 1500 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1501 ); 1502 1503 INIT_FUNCTION 1504 VOID 1505 NTAPI 1506 XIPInit( 1507 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1508 ); 1509 1510 #define InterlockedDecrementUL(Addend) \ 1511 (ULONG)InterlockedDecrement((PLONG)(Addend)) 1512 1513 #define InterlockedIncrementUL(Addend) \ 1514 (ULONG)InterlockedIncrement((PLONG)(Addend)) 1515 1516 #define InterlockedExchangeUL(Target, Value) \ 1517 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value)) 1518 1519 #define InterlockedExchangeAddUL(Addend, Value) \ 1520 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value)) 1521 1522 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \ 1523 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand)) 1524 1525 #define InterlockedCompareExchangeSizeT(Destination, Exchange, Comperand) \ 1526 (SIZE_T)InterlockedCompareExchangePointer((PVOID*)(Destination), (PVOID)(SIZE_T)(Exchange), (PVOID)(SIZE_T)(Comperand)) 1527 1528 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \ 1529 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand)) 1530