1 #pragma once 2 3 #ifdef __cplusplus 4 extern "C" 5 { 6 #endif 7 8 /* GLOBAL VARIABLES *********************************************************/ 9 10 extern RTL_TIME_ZONE_INFORMATION ExpTimeZoneInfo; 11 extern LARGE_INTEGER ExpTimeZoneBias; 12 extern ULONG ExpTimeZoneId; 13 extern ULONG ExpTickCountMultiplier; 14 extern ULONG ExpLastTimeZoneBias; 15 extern POBJECT_TYPE ExEventPairObjectType; 16 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType; 17 extern FAST_MUTEX ExpEnvironmentLock; 18 extern ERESOURCE ExpFirmwareTableResource; 19 extern LIST_ENTRY ExpFirmwareTableProviderListHead; 20 extern BOOLEAN ExpIsWinPEMode; 21 extern LIST_ENTRY ExpSystemResourcesList; 22 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset; 23 extern ULONG ExpUnicodeCaseTableDataOffset; 24 extern PVOID ExpNlsSectionPointer; 25 extern ULONG NtGlobalFlag; 26 extern UNICODE_STRING NtSystemRoot; 27 extern ULONG ExpInitializationPhase; 28 extern ULONG ExpAltTimeZoneBias; 29 extern LIST_ENTRY ExSystemLookasideListHead; 30 extern PCALLBACK_OBJECT PowerStateCallback; 31 extern LIST_ENTRY ExPoolLookasideListHead; 32 extern LIST_ENTRY ExpNonPagedLookasideListHead; 33 extern LIST_ENTRY ExpPagedLookasideListHead; 34 extern KSPIN_LOCK ExpNonPagedLookasideListLock; 35 extern KSPIN_LOCK ExpPagedLookasideListLock; 36 extern ULONG ExCriticalWorkerThreads; 37 extern ULONG ExDelayedWorkerThreads; 38 39 extern PVOID ExpDefaultErrorPort; 40 extern PEPROCESS ExpDefaultErrorPortProcess; 41 42 /* 43 * NT/Cm Version Info variables 44 */ 45 extern ULONG NtMajorVersion; 46 extern ULONG NtMinorVersion; 47 extern ULONG NtBuildNumber; 48 extern ULONG CmNtSpBuildNumber; 49 extern ULONG CmNtCSDVersion; 50 extern ULONG CmNtCSDReleaseType; 51 extern UNICODE_STRING CmVersionString; 52 extern UNICODE_STRING CmCSDVersionString; 53 extern CHAR NtBuildLab[]; 54 55 /* 56 * WinDBG Debugger Worker State Machine data (see dbgctrl.c) 57 */ 58 typedef enum _WINKD_WORKER_STATE 59 { 60 WinKdWorkerReady = 0, 61 WinKdWorkerStart, 62 WinKdWorkerInitialized 63 } WINKD_WORKER_STATE; 64 65 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem; 66 extern WINKD_WORKER_STATE ExpDebuggerWork; 67 extern PEPROCESS ExpDebuggerProcessAttach; 68 extern PEPROCESS ExpDebuggerProcessKill; 69 extern ULONG_PTR ExpDebuggerPageIn; 70 71 VOID NTAPI ExpDebuggerWorker(IN PVOID Context); 72 73 #ifdef _WIN64 74 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4) 75 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3) 76 #else 77 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3) 78 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2) 79 #endif 80 #define HANDLE_TAG_BITS (2) 81 #define HANDLE_INDEX_BITS (HANDLE_LOW_BITS + 2*HANDLE_HIGH_BITS) 82 #define KERNEL_FLAG_BITS (sizeof(PVOID)*8 - HANDLE_INDEX_BITS - HANDLE_TAG_BITS) 83 84 typedef union _EXHANDLE 85 { 86 struct 87 { 88 ULONG_PTR TagBits: HANDLE_TAG_BITS; 89 ULONG_PTR Index: HANDLE_INDEX_BITS; 90 ULONG_PTR KernelFlag : KERNEL_FLAG_BITS; 91 }; 92 struct 93 { 94 ULONG_PTR TagBits2: HANDLE_TAG_BITS; 95 ULONG_PTR LowIndex: HANDLE_LOW_BITS; 96 ULONG_PTR MidIndex: HANDLE_HIGH_BITS; 97 ULONG_PTR HighIndex: HANDLE_HIGH_BITS; 98 ULONG_PTR KernelFlag2: KERNEL_FLAG_BITS; 99 }; 100 HANDLE GenericHandleOverlay; 101 ULONG_PTR Value; 102 ULONG AsULONG; 103 } EXHANDLE, *PEXHANDLE; 104 105 typedef struct _ETIMER 106 { 107 KTIMER KeTimer; 108 KAPC TimerApc; 109 KDPC TimerDpc; 110 LIST_ENTRY ActiveTimerListEntry; 111 KSPIN_LOCK Lock; 112 LONG Period; 113 BOOLEAN ApcAssociated; 114 BOOLEAN WakeTimer; 115 LIST_ENTRY WakeTimerListEntry; 116 } ETIMER, *PETIMER; 117 118 typedef struct 119 { 120 PCALLBACK_OBJECT *CallbackObject; 121 PWSTR Name; 122 } SYSTEM_CALLBACKS; 123 124 typedef struct _HARDERROR_USER_PARAMETERS 125 { 126 ULONG_PTR Parameters[MAXIMUM_HARDERROR_PARAMETERS]; 127 UNICODE_STRING Strings[MAXIMUM_HARDERROR_PARAMETERS]; 128 WCHAR Buffer[ANYSIZE_ARRAY]; 129 } HARDERROR_USER_PARAMETERS, *PHARDERROR_USER_PARAMETERS; 130 131 #define MAX_FAST_REFS 7 132 133 #define ExAcquireRundownProtection _ExAcquireRundownProtection 134 #define ExReleaseRundownProtection _ExReleaseRundownProtection 135 #define ExInitializeRundownProtection _ExInitializeRundownProtection 136 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease 137 #define ExRundownCompleted _ExRundownCompleted 138 #define ExGetPreviousMode KeGetPreviousMode 139 140 141 // 142 // Various bits tagged on the handle or handle table 143 // 144 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1 145 #define FREE_HANDLE_MASK -1 146 147 // 148 // Number of entries in each table level 149 // 150 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY)) 151 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY)) 152 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES)) 153 154 // 155 // Maximum index in each table level before we need another table 156 // 157 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES 158 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES) 159 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES) 160 161 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&x->Ptr, (PVOID)y, (PVOID)z) 162 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z) 163 #define ExpSetRundown(x, y) InterlockedExchangePointer(&x->Ptr, (PVOID)y) 164 165 NTSTATUS 166 NTAPI 167 ExGetPoolTagInfo( 168 IN PSYSTEM_POOLTAG_INFORMATION SystemInformation, 169 IN ULONG SystemInformationLength, 170 IN OUT PULONG ReturnLength OPTIONAL 171 ); 172 173 typedef struct _UUID_CACHED_VALUES_STRUCT 174 { 175 ULONGLONG Time; 176 LONG AllocatedCount; 177 union 178 { 179 struct 180 { 181 UCHAR ClockSeqHiAndReserved; 182 UCHAR ClockSeqLow; 183 UCHAR NodeId[6 /*SEED_BUFFER_SIZE*/]; 184 }; 185 UCHAR GuidInit[8]; /* Match GUID.Data4 */ 186 }; 187 } UUID_CACHED_VALUES_STRUCT, *PUUID_CACHED_VALUES_STRUCT; 188 189 C_ASSERT(RTL_FIELD_SIZE(UUID_CACHED_VALUES_STRUCT, GuidInit) == RTL_FIELD_SIZE(UUID, Data4)); 190 191 /* INITIALIZATION FUNCTIONS *************************************************/ 192 193 CODE_SEG("INIT") 194 BOOLEAN 195 NTAPI 196 ExpWin32kInit(VOID); 197 198 VOID 199 NTAPI 200 ExInit2(VOID); 201 202 VOID 203 NTAPI 204 Phase1Initialization( 205 IN PVOID Context 206 ); 207 208 CODE_SEG("INIT") 209 VOID 210 NTAPI 211 ExpInitializePushLocks(VOID); 212 213 BOOLEAN 214 NTAPI 215 ExRefreshTimeZoneInformation( 216 IN PLARGE_INTEGER SystemBootTime 217 ); 218 219 CODE_SEG("INIT") 220 VOID 221 NTAPI 222 ExpInitializeWorkerThreads(VOID); 223 224 VOID 225 NTAPI 226 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap); 227 228 CODE_SEG("INIT") 229 VOID 230 NTAPI 231 ExpInitLookasideLists(VOID); 232 233 CODE_SEG("INIT") 234 VOID 235 NTAPI 236 ExInitializeSystemLookasideList( 237 IN PGENERAL_LOOKASIDE List, 238 IN POOL_TYPE Type, 239 IN ULONG Size, 240 IN ULONG Tag, 241 IN USHORT MaximumDepth, 242 IN PLIST_ENTRY ListHead 243 ); 244 245 CODE_SEG("INIT") 246 BOOLEAN 247 NTAPI 248 ExpInitializeCallbacks(VOID); 249 250 CODE_SEG("INIT") 251 BOOLEAN 252 NTAPI 253 ExpUuidInitialization(VOID); 254 255 CODE_SEG("INIT") 256 BOOLEAN 257 NTAPI 258 ExLuidInitialization(VOID); 259 260 CODE_SEG("INIT") 261 VOID 262 NTAPI 263 ExpInitializeExecutive( 264 IN ULONG Cpu, 265 IN PLOADER_PARAMETER_BLOCK LoaderBlock 266 ); 267 268 VOID 269 NTAPI 270 ExShutdownSystem(VOID); 271 272 CODE_SEG("INIT") 273 BOOLEAN 274 NTAPI 275 ExpInitializeEventImplementation(VOID); 276 277 CODE_SEG("INIT") 278 BOOLEAN 279 NTAPI 280 ExpInitializeKeyedEventImplementation(VOID); 281 282 CODE_SEG("INIT") 283 BOOLEAN 284 NTAPI 285 ExpInitializeEventPairImplementation(VOID); 286 287 CODE_SEG("INIT") 288 BOOLEAN 289 NTAPI 290 ExpInitializeSemaphoreImplementation(VOID); 291 292 CODE_SEG("INIT") 293 BOOLEAN 294 NTAPI 295 ExpInitializeMutantImplementation(VOID); 296 297 CODE_SEG("INIT") 298 BOOLEAN 299 NTAPI 300 ExpInitializeTimerImplementation(VOID); 301 302 CODE_SEG("INIT") 303 BOOLEAN 304 NTAPI 305 ExpInitializeProfileImplementation(VOID); 306 307 CODE_SEG("INIT") 308 VOID 309 NTAPI 310 ExpResourceInitialization(VOID); 311 312 CODE_SEG("INIT") 313 VOID 314 NTAPI 315 ExInitPoolLookasidePointers(VOID); 316 317 /* Callback Functions ********************************************************/ 318 319 VOID 320 NTAPI 321 ExInitializeCallBack( 322 IN OUT PEX_CALLBACK Callback 323 ); 324 325 PEX_CALLBACK_ROUTINE_BLOCK 326 NTAPI 327 ExAllocateCallBack( 328 IN PEX_CALLBACK_FUNCTION Function, 329 IN PVOID Context 330 ); 331 332 VOID 333 NTAPI 334 ExFreeCallBack( 335 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 336 ); 337 338 BOOLEAN 339 NTAPI 340 ExCompareExchangeCallBack ( 341 IN OUT PEX_CALLBACK CallBack, 342 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock, 343 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock 344 ); 345 346 PEX_CALLBACK_ROUTINE_BLOCK 347 NTAPI 348 ExReferenceCallBackBlock( 349 IN OUT PEX_CALLBACK CallBack 350 ); 351 352 VOID 353 NTAPI 354 ExDereferenceCallBackBlock( 355 IN OUT PEX_CALLBACK CallBack, 356 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 357 ); 358 359 PEX_CALLBACK_FUNCTION 360 NTAPI 361 ExGetCallBackBlockRoutine( 362 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 363 ); 364 365 PVOID 366 NTAPI 367 ExGetCallBackBlockContext( 368 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 369 ); 370 371 VOID 372 NTAPI 373 ExWaitForCallBacks( 374 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock 375 ); 376 377 /* Rundown Functions ********************************************************/ 378 379 VOID 380 FASTCALL 381 ExfInitializeRundownProtection( 382 OUT PEX_RUNDOWN_REF RunRef 383 ); 384 385 VOID 386 FASTCALL 387 ExfReInitializeRundownProtection( 388 OUT PEX_RUNDOWN_REF RunRef 389 ); 390 391 BOOLEAN 392 FASTCALL 393 ExfAcquireRundownProtection( 394 IN OUT PEX_RUNDOWN_REF RunRef 395 ); 396 397 BOOLEAN 398 FASTCALL 399 ExfAcquireRundownProtectionEx( 400 IN OUT PEX_RUNDOWN_REF RunRef, 401 IN ULONG Count 402 ); 403 404 VOID 405 FASTCALL 406 ExfReleaseRundownProtection( 407 IN OUT PEX_RUNDOWN_REF RunRef 408 ); 409 410 VOID 411 FASTCALL 412 ExfReleaseRundownProtectionEx( 413 IN OUT PEX_RUNDOWN_REF RunRef, 414 IN ULONG Count 415 ); 416 417 VOID 418 FASTCALL 419 ExfRundownCompleted( 420 OUT PEX_RUNDOWN_REF RunRef 421 ); 422 423 VOID 424 FASTCALL 425 ExfWaitForRundownProtectionRelease( 426 IN OUT PEX_RUNDOWN_REF RunRef 427 ); 428 429 /* HANDLE TABLE FUNCTIONS ***************************************************/ 430 431 typedef BOOLEAN 432 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)( 433 PHANDLE_TABLE_ENTRY HandleTableEntry, 434 HANDLE Handle, 435 PVOID Context 436 ); 437 438 typedef BOOLEAN 439 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)( 440 IN PEPROCESS Process, 441 IN PHANDLE_TABLE HandleTable, 442 IN PHANDLE_TABLE_ENTRY HandleTableEntry, 443 IN PHANDLE_TABLE_ENTRY NewEntry 444 ); 445 446 typedef BOOLEAN 447 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)( 448 PHANDLE_TABLE_ENTRY HandleTableEntry, 449 ULONG_PTR Context 450 ); 451 452 CODE_SEG("INIT") 453 VOID 454 NTAPI 455 ExpInitializeHandleTables( 456 VOID 457 ); 458 459 PHANDLE_TABLE 460 NTAPI 461 ExCreateHandleTable( 462 IN PEPROCESS Process OPTIONAL 463 ); 464 465 VOID 466 NTAPI 467 ExUnlockHandleTableEntry( 468 IN PHANDLE_TABLE HandleTable, 469 IN PHANDLE_TABLE_ENTRY HandleTableEntry 470 ); 471 472 HANDLE 473 NTAPI 474 ExCreateHandle( 475 IN PHANDLE_TABLE HandleTable, 476 IN PHANDLE_TABLE_ENTRY HandleTableEntry 477 ); 478 479 VOID 480 NTAPI 481 ExDestroyHandleTable( 482 IN PHANDLE_TABLE HandleTable, 483 IN PVOID DestroyHandleProcedure OPTIONAL 484 ); 485 486 BOOLEAN 487 NTAPI 488 ExDestroyHandle( 489 IN PHANDLE_TABLE HandleTable, 490 IN HANDLE Handle, 491 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL 492 ); 493 494 PHANDLE_TABLE_ENTRY 495 NTAPI 496 ExMapHandleToPointer( 497 IN PHANDLE_TABLE HandleTable, 498 IN HANDLE Handle 499 ); 500 501 PHANDLE_TABLE 502 NTAPI 503 ExDupHandleTable( 504 IN PEPROCESS Process, 505 IN PHANDLE_TABLE HandleTable, 506 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure, 507 IN ULONG_PTR Mask 508 ); 509 510 BOOLEAN 511 NTAPI 512 ExChangeHandle( 513 IN PHANDLE_TABLE HandleTable, 514 IN HANDLE Handle, 515 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine, 516 IN ULONG_PTR Context 517 ); 518 519 VOID 520 NTAPI 521 ExSweepHandleTable( 522 IN PHANDLE_TABLE HandleTable, 523 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure, 524 IN PVOID Context 525 ); 526 527 PHANDLE_TABLE_ENTRY 528 NTAPI 529 ExpLookupHandleTableEntry( 530 IN PHANDLE_TABLE HandleTable, 531 IN EXHANDLE Handle 532 ); 533 534 BOOLEAN 535 NTAPI 536 ExpLockHandleTableEntry( 537 IN PHANDLE_TABLE HandleTable, 538 IN PHANDLE_TABLE_ENTRY HandleTableEntry 539 ); 540 541 /* PSEH EXCEPTION HANDLING **************************************************/ 542 543 LONG 544 NTAPI 545 ExSystemExceptionFilter(VOID); 546 547 /* CALLBACKS *****************************************************************/ 548 549 FORCEINLINE 550 VOID 551 ExDoCallBack(IN OUT PEX_CALLBACK Callback, 552 IN PVOID Context, 553 IN PVOID Argument1, 554 IN PVOID Argument2) 555 { 556 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock; 557 PEX_CALLBACK_FUNCTION Function; 558 559 /* Reference the block */ 560 CallbackBlock = ExReferenceCallBackBlock(Callback); 561 if (CallbackBlock) 562 { 563 /* Get the function */ 564 Function = ExGetCallBackBlockRoutine(CallbackBlock); 565 566 /* Do the callback */ 567 Function(Context, Argument1, Argument2); 568 569 /* Now dereference it */ 570 ExDereferenceCallBackBlock(Callback, CallbackBlock); 571 } 572 } 573 574 /* FAST REFS ******************************************************************/ 575 576 FORCEINLINE 577 PVOID 578 ExGetObjectFastReference(IN EX_FAST_REF FastRef) 579 { 580 /* Return the unbiased pointer */ 581 return (PVOID)(FastRef.Value & ~MAX_FAST_REFS); 582 } 583 584 FORCEINLINE 585 ULONG 586 ExGetCountFastReference(IN EX_FAST_REF FastRef) 587 { 588 /* Return the reference count */ 589 return (ULONG)FastRef.RefCnt; 590 } 591 592 FORCEINLINE 593 VOID 594 ExInitializeFastReference(OUT PEX_FAST_REF FastRef, 595 IN OPTIONAL PVOID Object) 596 { 597 /* Sanity check */ 598 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0); 599 600 /* Check if an object is being set */ 601 if (!Object) 602 { 603 /* Clear the field */ 604 FastRef->Object = NULL; 605 } 606 else 607 { 608 /* Otherwise, we assume the object was referenced and is ready */ 609 FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS; 610 } 611 } 612 613 FORCEINLINE 614 EX_FAST_REF 615 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef) 616 { 617 EX_FAST_REF OldValue, NewValue; 618 619 /* Start reference loop */ 620 for (;;) 621 { 622 /* Get the current reference count */ 623 OldValue = *FastRef; 624 if (OldValue.RefCnt) 625 { 626 /* Increase the reference count */ 627 NewValue.Value = OldValue.Value - 1; 628 NewValue.Object = ExpChangePushlock(&FastRef->Object, 629 NewValue.Object, 630 OldValue.Object); 631 if (NewValue.Object != OldValue.Object) continue; 632 } 633 634 /* We are done */ 635 break; 636 } 637 638 /* Return the old value */ 639 return OldValue; 640 } 641 642 FORCEINLINE 643 BOOLEAN 644 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef, 645 IN PVOID Object) 646 { 647 EX_FAST_REF OldValue, NewValue; 648 649 /* Sanity checks */ 650 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 651 652 /* Start update loop */ 653 for (;;) 654 { 655 /* Get the current reference count */ 656 OldValue = *FastRef; 657 658 /* Check if the current count is too high or if the pointer changed */ 659 if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) || 660 ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object)) 661 { 662 /* Fail */ 663 return FALSE; 664 } 665 666 /* Update the reference count */ 667 NewValue.Value = OldValue.Value + MAX_FAST_REFS; 668 NewValue.Object = ExpChangePushlock(&FastRef->Object, 669 NewValue.Object, 670 OldValue.Object); 671 if (NewValue.Object != OldValue.Object) continue; 672 673 /* We are done */ 674 break; 675 } 676 677 /* Return success */ 678 return TRUE; 679 } 680 681 FORCEINLINE 682 BOOLEAN 683 ExReleaseFastReference(IN PEX_FAST_REF FastRef, 684 IN PVOID Object) 685 { 686 EX_FAST_REF OldValue, NewValue; 687 688 /* Sanity checks */ 689 ASSERT(Object != NULL); 690 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 691 692 /* Start reference loop */ 693 for (;;) 694 { 695 /* Get the current reference count */ 696 OldValue = *FastRef; 697 698 /* Check if we're full if if the pointer changed */ 699 if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE; 700 701 /* Decrease the reference count */ 702 NewValue.Value = OldValue.Value + 1; 703 NewValue.Object = ExpChangePushlock(&FastRef->Object, 704 NewValue.Object, 705 OldValue.Object); 706 if (NewValue.Object != OldValue.Object) continue; 707 708 /* We are done */ 709 break; 710 } 711 712 /* Return success */ 713 return TRUE; 714 } 715 716 FORCEINLINE 717 EX_FAST_REF 718 ExSwapFastReference(IN PEX_FAST_REF FastRef, 719 IN PVOID Object) 720 { 721 EX_FAST_REF NewValue, OldValue; 722 723 /* Sanity check */ 724 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0); 725 726 /* Check if an object is being set */ 727 if (!Object) 728 { 729 /* Clear the field */ 730 NewValue.Object = NULL; 731 } 732 else 733 { 734 /* Otherwise, we assume the object was referenced and is ready */ 735 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS; 736 } 737 738 /* Update the object */ 739 OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object); 740 return OldValue; 741 } 742 743 FORCEINLINE 744 EX_FAST_REF 745 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef, 746 IN PVOID Object, 747 IN PVOID OldObject) 748 { 749 EX_FAST_REF OldValue, NewValue; 750 751 /* Sanity check and start swap loop */ 752 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS)); 753 for (;;) 754 { 755 /* Get the current value */ 756 OldValue = *FastRef; 757 758 /* Make sure there's enough references to swap */ 759 if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break; 760 761 /* Check if we have an object to swap */ 762 if (Object) 763 { 764 /* Set up the value with maximum fast references */ 765 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS; 766 } 767 else 768 { 769 /* Write the object address itself (which is empty) */ 770 NewValue.Value = (ULONG_PTR)Object; 771 } 772 773 /* Do the actual compare exchange */ 774 NewValue.Object = ExpChangePushlock(&FastRef->Object, 775 NewValue.Object, 776 OldValue.Object); 777 if (NewValue.Object != OldValue.Object) continue; 778 779 /* All done */ 780 break; 781 } 782 783 /* Return the old value */ 784 return OldValue; 785 } 786 787 /* RUNDOWN *******************************************************************/ 788 789 FORCEINLINE 790 PEX_RUNDOWN_REF 791 ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware, 792 IN ULONG ProcNumber) 793 { 794 return (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs + 795 RunRefCacheAware->RunRefSize * 796 (ProcNumber % RunRefCacheAware->Number)); 797 } 798 799 /*++ 800 * @name ExfAcquireRundownProtection 801 * INTERNAL MACRO 802 * 803 * The ExfAcquireRundownProtection routine acquires rundown protection for 804 * the specified descriptor. 805 * 806 * @param RunRef 807 * Pointer to a rundown reference descriptor. 808 * 809 * @return TRUE if access to the protected structure was granted, FALSE otherwise. 810 * 811 * @remarks This is the internal macro for system use only.In case the rundown 812 * was active, then the slow-path will be called through the exported 813 * function. 814 * 815 *--*/ 816 FORCEINLINE 817 BOOLEAN 818 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef) 819 { 820 ULONG_PTR Value, NewValue; 821 822 /* Get the current value and mask the active bit */ 823 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE; 824 825 /* Add a reference */ 826 NewValue = Value + EX_RUNDOWN_COUNT_INC; 827 828 /* Change the value */ 829 NewValue = ExpChangeRundown(RunRef, NewValue, Value); 830 if (NewValue != Value) 831 { 832 /* Rundown was active, use long path */ 833 return ExfAcquireRundownProtection(RunRef); 834 } 835 836 /* Success */ 837 return TRUE; 838 } 839 840 /*++ 841 * @name ExReleaseRundownProtection 842 * INTERNAL MACRO 843 * 844 * The ExReleaseRundownProtection routine releases rundown protection for 845 * the specified descriptor. 846 * 847 * @param RunRef 848 * Pointer to a rundown reference descriptor. 849 * 850 * @return TRUE if access to the protected structure was granted, FALSE otherwise. 851 * 852 * @remarks This is the internal macro for system use only.In case the rundown 853 * was active, then the slow-path will be called through the exported 854 * function. 855 * 856 *--*/ 857 FORCEINLINE 858 VOID 859 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef) 860 { 861 ULONG_PTR Value, NewValue; 862 863 /* Get the current value and mask the active bit */ 864 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE; 865 866 /* Remove a reference */ 867 NewValue = Value - EX_RUNDOWN_COUNT_INC; 868 869 /* Change the value */ 870 NewValue = ExpChangeRundown(RunRef, NewValue, Value); 871 872 /* Check if the rundown was active */ 873 if (NewValue != Value) 874 { 875 /* Rundown was active, use long path */ 876 ExfReleaseRundownProtection(RunRef); 877 } 878 else 879 { 880 /* Sanity check */ 881 ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1)); 882 } 883 } 884 885 /*++ 886 * @name ExInitializeRundownProtection 887 * INTERNAL MACRO 888 * 889 * The ExInitializeRundownProtection routine initializes a rundown 890 * protection descriptor. 891 * 892 * @param RunRef 893 * Pointer to a rundown reference descriptor. 894 * 895 * @return None. 896 * 897 * @remarks This is the internal macro for system use only. 898 * 899 *--*/ 900 FORCEINLINE 901 VOID 902 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef) 903 { 904 /* Set the count to zero */ 905 RunRef->Count = 0; 906 } 907 908 /*++ 909 * @name ExWaitForRundownProtectionRelease 910 * INTERNAL MACRO 911 * 912 * The ExWaitForRundownProtectionRelease routine waits until the specified 913 * rundown descriptor has been released. 914 * 915 * @param RunRef 916 * Pointer to a rundown reference descriptor. 917 * 918 * @return None. 919 * 920 * @remarks This is the internal macro for system use only. If a wait is actually 921 * necessary, then the slow path is taken through the exported function. 922 * 923 *--*/ 924 FORCEINLINE 925 VOID 926 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef) 927 { 928 ULONG_PTR Value; 929 930 /* Set the active bit */ 931 Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0); 932 if ((Value) && (Value != EX_RUNDOWN_ACTIVE)) 933 { 934 /* If the the rundown wasn't already active, then take the long path */ 935 ExfWaitForRundownProtectionRelease(RunRef); 936 } 937 } 938 939 /*++ 940 * @name ExRundownCompleted 941 * INTERNAL MACRO 942 * 943 * The ExRundownCompleted routine completes the rundown of the specified 944 * descriptor by setting the active bit. 945 * 946 * @param RunRef 947 * Pointer to a rundown reference descriptor. 948 * 949 * @return None. 950 * 951 * @remarks This is the internal macro for system use only. 952 * 953 *--*/ 954 FORCEINLINE 955 VOID 956 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef) 957 { 958 /* Sanity check */ 959 ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0); 960 961 /* Mark the counter as active */ 962 ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE); 963 } 964 965 /* PUSHLOCKS *****************************************************************/ 966 967 /* FIXME: VERIFY THESE! */ 968 969 VOID 970 FASTCALL 971 ExBlockPushLock( 972 IN PEX_PUSH_LOCK PushLock, 973 IN PVOID WaitBlock 974 ); 975 976 VOID 977 FASTCALL 978 ExfUnblockPushLock( 979 IN PEX_PUSH_LOCK PushLock, 980 IN PVOID CurrentWaitBlock 981 ); 982 983 VOID 984 FASTCALL 985 ExWaitForUnblockPushLock( 986 IN PEX_PUSH_LOCK PushLock, 987 IN PVOID WaitBlock 988 ); 989 990 /*++ 991 * @name _ExInitializePushLock 992 * INTERNAL MACRO 993 * 994 * The _ExInitializePushLock macro initializes a PushLock. 995 * 996 * @params PushLock 997 * Pointer to the pushlock which is to be initialized. 998 * 999 * @return None. 1000 * 1001 * @remarks None. 1002 * 1003 *--*/ 1004 FORCEINLINE 1005 VOID 1006 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock) 1007 { 1008 /* Set the value to 0 */ 1009 PushLock->Ptr = 0; 1010 } 1011 #define ExInitializePushLock _ExInitializePushLock 1012 1013 /*++ 1014 * @name ExAcquirePushLockExclusive 1015 * INTERNAL MACRO 1016 * 1017 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock. 1018 * 1019 * @params PushLock 1020 * Pointer to the pushlock which is to be acquired. 1021 * 1022 * @return None. 1023 * 1024 * @remarks The function attempts the quickest route to acquire the lock, which is 1025 * to simply set the lock bit. 1026 * However, if the pushlock is already shared, the slower path is taken. 1027 * 1028 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1029 * This macro should usually be paired up with KeAcquireCriticalRegion. 1030 * 1031 *--*/ 1032 FORCEINLINE 1033 VOID 1034 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock) 1035 { 1036 /* Try acquiring the lock */ 1037 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V)) 1038 { 1039 /* Someone changed it, use the slow path */ 1040 ExfAcquirePushLockExclusive(PushLock); 1041 } 1042 1043 /* Sanity check */ 1044 ASSERT(PushLock->Locked); 1045 } 1046 1047 /*++ 1048 * @name ExTryToAcquirePushLockExclusive 1049 * INTERNAL MACRO 1050 * 1051 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock. 1052 * 1053 * @params PushLock 1054 * Pointer to the pushlock which is to be acquired. 1055 * 1056 * @return None. 1057 * 1058 * @remarks The function attempts the quickest route to acquire the lock, which is 1059 * to simply set the lock bit. 1060 * However, if the pushlock is already shared, the slower path is taken. 1061 * 1062 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1063 * This macro should usually be paired up with KeAcquireCriticalRegion. 1064 * 1065 *--*/ 1066 FORCEINLINE 1067 BOOLEAN 1068 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock) 1069 { 1070 /* Try acquiring the lock */ 1071 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V)) 1072 { 1073 /* Can't acquire */ 1074 return FALSE; 1075 } 1076 1077 /* Got acquired */ 1078 ASSERT (PushLock->Locked); 1079 return TRUE; 1080 } 1081 1082 /*++ 1083 * @name ExAcquirePushLockShared 1084 * INTERNAL MACRO 1085 * 1086 * The ExAcquirePushLockShared macro acquires a shared PushLock. 1087 * 1088 * @params PushLock 1089 * Pointer to the pushlock which is to be acquired. 1090 * 1091 * @return None. 1092 * 1093 * @remarks The function attempts the quickest route to acquire the lock, which is 1094 * to simply set the lock bit and set the share count to one. 1095 * However, if the pushlock is already shared, the slower path is taken. 1096 * 1097 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL. 1098 * This macro should usually be paired up with KeAcquireCriticalRegion. 1099 * 1100 *--*/ 1101 FORCEINLINE 1102 VOID 1103 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock) 1104 { 1105 EX_PUSH_LOCK NewValue; 1106 1107 /* Try acquiring the lock */ 1108 NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1109 if (ExpChangePushlock(PushLock, NewValue.Ptr, 0)) 1110 { 1111 /* Someone changed it, use the slow path */ 1112 ExfAcquirePushLockShared(PushLock); 1113 } 1114 1115 /* Sanity checks */ 1116 ASSERT(PushLock->Locked); 1117 } 1118 1119 /*++ 1120 * @name ExConvertPushLockSharedToExclusive 1121 * INTERNAL MACRO 1122 * 1123 * The ExConvertPushLockSharedToExclusive macro converts an exclusive 1124 * pushlock to a shared pushlock. 1125 * 1126 * @params PushLock 1127 * Pointer to the pushlock which is to be converted. 1128 * 1129 * @return FALSE if conversion failed, TRUE otherwise. 1130 * 1131 * @remarks The function attempts the quickest route to convert the lock, which is 1132 * to simply set the lock bit and remove any other bits. 1133 * 1134 *--*/ 1135 FORCEINLINE 1136 BOOLEAN 1137 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock) 1138 { 1139 EX_PUSH_LOCK OldValue; 1140 1141 /* Set the expected old value */ 1142 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1143 1144 /* Try converting the lock */ 1145 if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) != 1146 OldValue.Ptr) 1147 { 1148 /* Conversion failed */ 1149 return FALSE; 1150 } 1151 1152 /* Sanity check */ 1153 ASSERT(PushLock->Locked); 1154 return TRUE; 1155 } 1156 1157 /*++ 1158 * @name ExWaitOnPushLock 1159 * INTERNAL MACRO 1160 * 1161 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock. 1162 * 1163 * @params PushLock 1164 * Pointer to a pushlock. 1165 * 1166 * @return None. 1167 * 1168 * @remarks The function attempts to get any exclusive waiters out of their slow 1169 * path by forcing an instant acquire/release operation. 1170 * 1171 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL. 1172 * 1173 *--*/ 1174 FORCEINLINE 1175 VOID 1176 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock) 1177 { 1178 /* Check if we're locked */ 1179 if (PushLock->Locked) 1180 { 1181 /* Acquire the lock */ 1182 ExfAcquirePushLockExclusive(PushLock); 1183 ASSERT(PushLock->Locked); 1184 1185 /* Release it */ 1186 ExfReleasePushLockExclusive(PushLock); 1187 } 1188 } 1189 1190 /*++ 1191 * @name ExReleasePushLockShared 1192 * INTERNAL MACRO 1193 * 1194 * The ExReleasePushLockShared macro releases a previously acquired PushLock. 1195 * 1196 * @params PushLock 1197 * Pointer to a previously acquired pushlock. 1198 * 1199 * @return None. 1200 * 1201 * @remarks The function attempts the quickest route to release the lock, which is 1202 * to simply decrease the share count and remove the lock bit. 1203 * However, if the pushlock is being waited on then the long path is taken. 1204 * 1205 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL. 1206 * This macro should usually be paired up with KeLeaveCriticalRegion. 1207 * 1208 *--*/ 1209 FORCEINLINE 1210 VOID 1211 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock) 1212 { 1213 EX_PUSH_LOCK OldValue; 1214 1215 /* Sanity checks */ 1216 ASSERT(PushLock->Locked); 1217 1218 /* Try to clear the pushlock */ 1219 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC; 1220 if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr) 1221 { 1222 /* There are still other people waiting on it */ 1223 ExfReleasePushLockShared(PushLock); 1224 } 1225 } 1226 1227 /*++ 1228 * @name ExReleasePushLockExclusive 1229 * INTERNAL MACRO 1230 * 1231 * The ExReleasePushLockExclusive macro releases a previously 1232 * exclusively acquired PushLock. 1233 * 1234 * @params PushLock 1235 * Pointer to a previously acquired pushlock. 1236 * 1237 * @return None. 1238 * 1239 * @remarks The function attempts the quickest route to release the lock, which is 1240 * to simply clear the locked bit. 1241 * However, if the pushlock is being waited on, the slow path is taken 1242 * in an attempt to wake up the lock. 1243 * 1244 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL. 1245 * This macro should usually be paired up with KeLeaveCriticalRegion. 1246 * 1247 *--*/ 1248 FORCEINLINE 1249 VOID 1250 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock) 1251 { 1252 EX_PUSH_LOCK OldValue; 1253 1254 /* Sanity checks */ 1255 ASSERT(PushLock->Locked); 1256 1257 /* Unlock the pushlock */ 1258 OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock, 1259 -(SSIZE_T)EX_PUSH_LOCK_LOCK); 1260 1261 /* Sanity checks */ 1262 ASSERT(OldValue.Locked); 1263 ASSERT(OldValue.Waiting || OldValue.Shared == 0); 1264 1265 /* Check if anyone is waiting on it and it's not already waking*/ 1266 if ((OldValue.Waiting) && !(OldValue.Waking)) 1267 { 1268 /* Wake it up */ 1269 ExfTryToWakePushLock(PushLock); 1270 } 1271 } 1272 1273 /*++ 1274 * @name ExReleasePushLock 1275 * INTERNAL MACRO 1276 * 1277 * The ExReleasePushLock macro releases a previously acquired PushLock. 1278 * 1279 * @params PushLock 1280 * Pointer to a previously acquired pushlock. 1281 * 1282 * @return None. 1283 * 1284 * @remarks The function attempts the quickest route to release the lock, which is 1285 * to simply clear all the fields and decrease the share count if required. 1286 * However, if the pushlock is being waited on then the long path is taken. 1287 * 1288 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL. 1289 * This macro should usually be paired up with KeLeaveCriticalRegion. 1290 * 1291 *--*/ 1292 FORCEINLINE 1293 VOID 1294 ExReleasePushLock(PEX_PUSH_LOCK PushLock) 1295 { 1296 EX_PUSH_LOCK OldValue = *PushLock; 1297 EX_PUSH_LOCK NewValue; 1298 1299 /* Sanity checks */ 1300 ASSERT(OldValue.Locked); 1301 1302 /* Check if the pushlock is shared */ 1303 if (OldValue.Shared > 1) 1304 { 1305 /* Decrease the share count */ 1306 NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC; 1307 } 1308 else 1309 { 1310 /* Clear the pushlock entirely */ 1311 NewValue.Value = 0; 1312 } 1313 1314 /* Check if nobody is waiting on us and try clearing the lock here */ 1315 if ((OldValue.Waiting) || 1316 (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) != 1317 OldValue.Ptr)) 1318 { 1319 /* We have waiters, use the long path */ 1320 ExfReleasePushLock(PushLock); 1321 } 1322 } 1323 1324 /* FAST MUTEX INLINES *********************************************************/ 1325 1326 FORCEINLINE 1327 VOID 1328 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex) 1329 { 1330 PKTHREAD Thread = KeGetCurrentThread(); 1331 1332 /* Sanity check */ 1333 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1334 (Thread->CombinedApcDisable != 0) || 1335 (Thread->Teb == NULL) || 1336 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1337 ASSERT(FastMutex->Owner != Thread); 1338 1339 /* Decrease the count */ 1340 if (InterlockedDecrement(&FastMutex->Count)) 1341 { 1342 /* Someone is still holding it, use slow path */ 1343 KiAcquireFastMutex(FastMutex); 1344 } 1345 1346 /* Set the owner */ 1347 FastMutex->Owner = Thread; 1348 } 1349 1350 FORCEINLINE 1351 VOID 1352 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) 1353 { 1354 ASSERT((KeGetCurrentIrql() == APC_LEVEL) || 1355 (KeGetCurrentThread()->CombinedApcDisable != 0) || 1356 (KeGetCurrentThread()->Teb == NULL) || 1357 (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); 1358 ASSERT(FastMutex->Owner == KeGetCurrentThread()); 1359 1360 /* Erase the owner */ 1361 FastMutex->Owner = NULL; 1362 1363 /* Increase the count */ 1364 if (InterlockedIncrement(&FastMutex->Count) <= 0) 1365 { 1366 /* Someone was waiting for it, signal the waiter */ 1367 KeSetEventBoostPriority(&FastMutex->Event, NULL); 1368 } 1369 } 1370 1371 FORCEINLINE 1372 VOID 1373 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex) 1374 { 1375 KIRQL OldIrql; 1376 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1377 1378 /* Raise IRQL to APC */ 1379 KeRaiseIrql(APC_LEVEL, &OldIrql); 1380 1381 /* Decrease the count */ 1382 if (InterlockedDecrement(&FastMutex->Count)) 1383 { 1384 /* Someone is still holding it, use slow path */ 1385 KiAcquireFastMutex(FastMutex); 1386 } 1387 1388 /* Set the owner and IRQL */ 1389 FastMutex->Owner = KeGetCurrentThread(); 1390 FastMutex->OldIrql = OldIrql; 1391 } 1392 1393 FORCEINLINE 1394 VOID 1395 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex) 1396 { 1397 KIRQL OldIrql; 1398 ASSERT(KeGetCurrentIrql() == APC_LEVEL); 1399 1400 /* Erase the owner */ 1401 FastMutex->Owner = NULL; 1402 OldIrql = (KIRQL)FastMutex->OldIrql; 1403 1404 /* Increase the count */ 1405 if (InterlockedIncrement(&FastMutex->Count) <= 0) 1406 { 1407 /* Someone was waiting for it, signal the waiter */ 1408 KeSetEventBoostPriority(&FastMutex->Event, NULL); 1409 } 1410 1411 /* Lower IRQL back */ 1412 KeLowerIrql(OldIrql); 1413 } 1414 1415 FORCEINLINE 1416 BOOLEAN 1417 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex) 1418 { 1419 KIRQL OldIrql; 1420 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1421 1422 /* Raise to APC_LEVEL */ 1423 KeRaiseIrql(APC_LEVEL, &OldIrql); 1424 1425 /* Check if we can quickly acquire it */ 1426 if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1) 1427 { 1428 /* We have, set us as owners */ 1429 FastMutex->Owner = KeGetCurrentThread(); 1430 FastMutex->OldIrql = OldIrql; 1431 return TRUE; 1432 } 1433 else 1434 { 1435 /* Acquire attempt failed */ 1436 KeLowerIrql(OldIrql); 1437 YieldProcessor(); 1438 return FALSE; 1439 } 1440 } 1441 1442 FORCEINLINE 1443 VOID 1444 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) 1445 { 1446 /* Enter the Critical Region */ 1447 KeEnterCriticalRegion(); 1448 1449 /* Acquire the mutex unsafely */ 1450 _ExAcquireFastMutexUnsafe(FastMutex); 1451 } 1452 1453 FORCEINLINE 1454 VOID 1455 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex) 1456 { 1457 /* Release the mutex unsafely */ 1458 _ExReleaseFastMutexUnsafe(FastMutex); 1459 1460 /* Leave the critical region */ 1461 KeLeaveCriticalRegion(); 1462 } 1463 1464 /* OTHER FUNCTIONS **********************************************************/ 1465 1466 BOOLEAN 1467 NTAPI 1468 ExTryToAcquireResourceExclusiveLite( 1469 IN PERESOURCE Resource 1470 ); 1471 1472 NTSTATUS 1473 ExpSetTimeZoneInformation( 1474 IN PRTL_TIME_ZONE_INFORMATION TimeZoneInformation 1475 ); 1476 1477 BOOLEAN 1478 NTAPI 1479 ExAcquireTimeRefreshLock( 1480 IN BOOLEAN Wait 1481 ); 1482 1483 VOID 1484 NTAPI 1485 ExReleaseTimeRefreshLock( 1486 VOID 1487 ); 1488 1489 VOID 1490 NTAPI 1491 ExUpdateSystemTimeFromCmos( 1492 IN BOOLEAN UpdateInterruptTime, 1493 IN ULONG MaxSepInSeconds 1494 ); 1495 1496 VOID 1497 NTAPI 1498 ExAllocateLocallyUniqueId( 1499 OUT LUID *LocallyUniqueId 1500 ); 1501 1502 VOID 1503 NTAPI 1504 ExTimerRundown( 1505 VOID 1506 ); 1507 1508 CODE_SEG("INIT") 1509 VOID 1510 NTAPI 1511 HeadlessInit( 1512 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1513 ); 1514 1515 CODE_SEG("INIT") 1516 VOID 1517 NTAPI 1518 XIPInit( 1519 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1520 ); 1521 1522 #define InterlockedDecrementUL(Addend) \ 1523 (ULONG)InterlockedDecrement((PLONG)(Addend)) 1524 1525 #define InterlockedIncrementUL(Addend) \ 1526 (ULONG)InterlockedIncrement((PLONG)(Addend)) 1527 1528 #define InterlockedExchangeUL(Target, Value) \ 1529 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value)) 1530 1531 #define InterlockedExchangeAddUL(Addend, Value) \ 1532 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value)) 1533 1534 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \ 1535 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand)) 1536 1537 #define InterlockedCompareExchangeSizeT(Destination, Exchange, Comperand) \ 1538 (SIZE_T)InterlockedCompareExchangePointer((PVOID*)(Destination), (PVOID)(SIZE_T)(Exchange), (PVOID)(SIZE_T)(Comperand)) 1539 1540 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \ 1541 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand)) 1542 1543 #ifdef __cplusplus 1544 } // extern "C" 1545 #endif 1546