1 #pragma once
2
3 #ifdef __cplusplus
4 extern "C"
5 {
6 #endif
7
8 /* GLOBAL VARIABLES *********************************************************/
9
10 extern RTL_TIME_ZONE_INFORMATION ExpTimeZoneInfo;
11 extern LARGE_INTEGER ExpTimeZoneBias;
12 extern ULONG ExpTimeZoneId;
13 extern ULONG ExpTickCountMultiplier;
14 extern ULONG ExpLastTimeZoneBias;
15 extern POBJECT_TYPE ExEventPairObjectType;
16 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType;
17 extern FAST_MUTEX ExpEnvironmentLock;
18 extern ERESOURCE ExpFirmwareTableResource;
19 extern ERESOURCE ExpTimeRefreshLock;
20 extern LIST_ENTRY ExpFirmwareTableProviderListHead;
21 extern BOOLEAN ExpIsWinPEMode;
22 extern ULONG ExpResourceTimeoutCount;
23 extern LIST_ENTRY ExpSystemResourcesList;
24 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
25 extern ULONG ExpUnicodeCaseTableDataOffset;
26 extern PVOID ExpNlsSectionPointer;
27 extern ULONG NtGlobalFlag;
28 extern UNICODE_STRING NtSystemRoot;
29 extern ULONG ExpInitializationPhase;
30 extern ULONG ExpAltTimeZoneBias;
31 extern LIST_ENTRY ExSystemLookasideListHead;
32 extern PCALLBACK_OBJECT PowerStateCallback;
33 extern LIST_ENTRY ExPoolLookasideListHead;
34 extern LIST_ENTRY ExpNonPagedLookasideListHead;
35 extern LIST_ENTRY ExpPagedLookasideListHead;
36 extern KSPIN_LOCK ExpNonPagedLookasideListLock;
37 extern KSPIN_LOCK ExpPagedLookasideListLock;
38 extern ULONG ExCriticalWorkerThreads;
39 extern ULONG ExDelayedWorkerThreads;
40
41 extern PVOID ExpDefaultErrorPort;
42 extern PEPROCESS ExpDefaultErrorPortProcess;
43
44 /*
45 * NT/Cm Version Info variables
46 */
47 extern ULONG NtMajorVersion;
48 extern ULONG NtMinorVersion;
49 extern ULONG NtBuildNumber;
50 extern ULONG CmNtSpBuildNumber;
51 extern ULONG CmNtCSDVersion;
52 extern ULONG CmNtCSDReleaseType;
53 extern UNICODE_STRING CmVersionString;
54 extern UNICODE_STRING CmCSDVersionString;
55 extern CHAR NtBuildLab[];
56
57 /*
58 * WinDBG Debugger Worker State Machine data (see dbgctrl.c)
59 */
60 typedef enum _WINKD_WORKER_STATE
61 {
62 WinKdWorkerReady = 0,
63 WinKdWorkerStart,
64 WinKdWorkerInitialized
65 } WINKD_WORKER_STATE;
66
67 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem;
68 extern WINKD_WORKER_STATE ExpDebuggerWork;
69 extern PEPROCESS ExpDebuggerProcessAttach;
70 extern PEPROCESS ExpDebuggerProcessKill;
71 extern ULONG_PTR ExpDebuggerPageIn;
72
73 VOID
74 NTAPI
75 ExpDebuggerWorker(
76 _In_ PVOID Context);
77
78 #ifdef _WIN64
79 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4)
80 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3)
81 #else
82 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3)
83 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2)
84 #endif
85 #define HANDLE_TAG_BITS 2
86 #define HANDLE_INDEX_BITS (HANDLE_LOW_BITS + 2 * HANDLE_HIGH_BITS)
87 #define KERNEL_FLAG_BITS (sizeof(ULONG_PTR) * 8 - HANDLE_INDEX_BITS - HANDLE_TAG_BITS)
88
89 typedef union _EXHANDLE
90 {
91 struct
92 {
93 ULONG_PTR TagBits: HANDLE_TAG_BITS;
94 ULONG_PTR Index: HANDLE_INDEX_BITS;
95 ULONG_PTR KernelFlag: KERNEL_FLAG_BITS;
96 };
97 struct
98 {
99 ULONG_PTR TagBits2: HANDLE_TAG_BITS;
100 ULONG_PTR LowIndex: HANDLE_LOW_BITS;
101 ULONG_PTR MidIndex: HANDLE_HIGH_BITS;
102 ULONG_PTR HighIndex: HANDLE_HIGH_BITS;
103 ULONG_PTR KernelFlag2: KERNEL_FLAG_BITS;
104 };
105 HANDLE GenericHandleOverlay;
106 ULONG_PTR Value;
107 ULONG AsULONG;
108 } EXHANDLE, *PEXHANDLE;
109
110 typedef struct _ETIMER
111 {
112 KTIMER KeTimer;
113 KAPC TimerApc;
114 KDPC TimerDpc;
115 LIST_ENTRY ActiveTimerListEntry;
116 KSPIN_LOCK Lock;
117 LONG Period;
118 BOOLEAN ApcAssociated;
119 BOOLEAN WakeTimer;
120 LIST_ENTRY WakeTimerListEntry;
121 } ETIMER, *PETIMER;
122
123 typedef struct
124 {
125 PCALLBACK_OBJECT *CallbackObject;
126 PWSTR Name;
127 } SYSTEM_CALLBACKS;
128
129 typedef struct _HARDERROR_USER_PARAMETERS
130 {
131 ULONG_PTR Parameters[MAXIMUM_HARDERROR_PARAMETERS];
132 UNICODE_STRING Strings[MAXIMUM_HARDERROR_PARAMETERS];
133 WCHAR Buffer[ANYSIZE_ARRAY];
134 } HARDERROR_USER_PARAMETERS, *PHARDERROR_USER_PARAMETERS;
135
136 #define MAX_FAST_REFS 7
137
138 #define ExAcquireRundownProtection _ExAcquireRundownProtection
139 #define ExReleaseRundownProtection _ExReleaseRundownProtection
140 #define ExInitializeRundownProtection _ExInitializeRundownProtection
141 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease
142 #define ExRundownCompleted _ExRundownCompleted
143 #define ExGetPreviousMode KeGetPreviousMode
144
145
146 //
147 // Various bits tagged on the handle or handle table
148 //
149 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1
150 #define FREE_HANDLE_MASK -1
151
152 //
153 // Number of entries in each table level
154 //
155 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
156 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
157 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
158
159 //
160 // Maximum index in each table level before we need another table
161 //
162 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES
163 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
164 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
165
166 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&(x)->Ptr, (PVOID)(y), (PVOID)(z))
167 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)(x), (PVOID)(y), (PVOID)(z))
168 #define ExpSetRundown(x, y) InterlockedExchangePointer(&(x)->Ptr, (PVOID)(y))
169
170 NTSTATUS
171 NTAPI
172 ExGetPoolTagInfo(
173 IN PSYSTEM_POOLTAG_INFORMATION SystemInformation,
174 IN ULONG SystemInformationLength,
175 IN OUT PULONG ReturnLength OPTIONAL
176 );
177
178 typedef struct _UUID_CACHED_VALUES_STRUCT
179 {
180 ULONGLONG Time;
181 LONG AllocatedCount;
182 union
183 {
184 struct
185 {
186 UCHAR ClockSeqHiAndReserved;
187 UCHAR ClockSeqLow;
188 UCHAR NodeId[6 /*SEED_BUFFER_SIZE*/];
189 };
190 UCHAR GuidInit[8]; /* Match GUID.Data4 */
191 };
192 } UUID_CACHED_VALUES_STRUCT, *PUUID_CACHED_VALUES_STRUCT;
193
194 C_ASSERT(RTL_FIELD_SIZE(UUID_CACHED_VALUES_STRUCT, GuidInit) == RTL_FIELD_SIZE(UUID, Data4));
195
196 /* INITIALIZATION FUNCTIONS *************************************************/
197
198 CODE_SEG("INIT")
199 BOOLEAN
200 NTAPI
201 ExpWin32kInit(VOID);
202
203 VOID
204 NTAPI
205 ExInit2(VOID);
206
207 VOID
208 NTAPI
209 Phase1Initialization(
210 IN PVOID Context
211 );
212
213 CODE_SEG("INIT")
214 VOID
215 NTAPI
216 ExpInitializePushLocks(VOID);
217
218 BOOLEAN
219 NTAPI
220 ExRefreshTimeZoneInformation(
221 IN PLARGE_INTEGER SystemBootTime
222 );
223
224 CODE_SEG("INIT")
225 VOID
226 NTAPI
227 ExpInitializeWorkerThreads(VOID);
228
229 VOID
230 NTAPI
231 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap);
232
233 CODE_SEG("INIT")
234 VOID
235 NTAPI
236 ExpInitLookasideLists(VOID);
237
238 CODE_SEG("INIT")
239 VOID
240 NTAPI
241 ExInitializeSystemLookasideList(
242 IN PGENERAL_LOOKASIDE List,
243 IN POOL_TYPE Type,
244 IN ULONG Size,
245 IN ULONG Tag,
246 IN USHORT MaximumDepth,
247 IN PLIST_ENTRY ListHead
248 );
249
250 CODE_SEG("INIT")
251 BOOLEAN
252 NTAPI
253 ExpInitializeCallbacks(VOID);
254
255 CODE_SEG("INIT")
256 BOOLEAN
257 NTAPI
258 ExpUuidInitialization(VOID);
259
260 CODE_SEG("INIT")
261 BOOLEAN
262 NTAPI
263 ExLuidInitialization(VOID);
264
265 CODE_SEG("INIT")
266 VOID
267 NTAPI
268 ExpInitializeExecutive(
269 IN ULONG Cpu,
270 IN PLOADER_PARAMETER_BLOCK LoaderBlock
271 );
272
273 VOID
274 NTAPI
275 ExShutdownSystem(VOID);
276
277 CODE_SEG("INIT")
278 BOOLEAN
279 NTAPI
280 ExpInitializeEventImplementation(VOID);
281
282 CODE_SEG("INIT")
283 BOOLEAN
284 NTAPI
285 ExpInitializeKeyedEventImplementation(VOID);
286
287 CODE_SEG("INIT")
288 BOOLEAN
289 NTAPI
290 ExpInitializeEventPairImplementation(VOID);
291
292 CODE_SEG("INIT")
293 BOOLEAN
294 NTAPI
295 ExpInitializeSemaphoreImplementation(VOID);
296
297 CODE_SEG("INIT")
298 BOOLEAN
299 NTAPI
300 ExpInitializeMutantImplementation(VOID);
301
302 CODE_SEG("INIT")
303 BOOLEAN
304 NTAPI
305 ExpInitializeTimerImplementation(VOID);
306
307 CODE_SEG("INIT")
308 BOOLEAN
309 NTAPI
310 ExpInitializeProfileImplementation(VOID);
311
312 CODE_SEG("INIT")
313 VOID
314 NTAPI
315 ExpResourceInitialization(VOID);
316
317 CODE_SEG("INIT")
318 VOID
319 NTAPI
320 ExInitPoolLookasidePointers(VOID);
321
322 /* Callback Functions ********************************************************/
323
324 VOID
325 NTAPI
326 ExInitializeCallBack(
327 IN OUT PEX_CALLBACK Callback
328 );
329
330 PEX_CALLBACK_ROUTINE_BLOCK
331 NTAPI
332 ExAllocateCallBack(
333 IN PEX_CALLBACK_FUNCTION Function,
334 IN PVOID Context
335 );
336
337 VOID
338 NTAPI
339 ExFreeCallBack(
340 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
341 );
342
343 BOOLEAN
344 NTAPI
345 ExCompareExchangeCallBack (
346 IN OUT PEX_CALLBACK CallBack,
347 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock,
348 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
349 );
350
351 PEX_CALLBACK_ROUTINE_BLOCK
352 NTAPI
353 ExReferenceCallBackBlock(
354 IN OUT PEX_CALLBACK CallBack
355 );
356
357 VOID
358 NTAPI
359 ExDereferenceCallBackBlock(
360 IN OUT PEX_CALLBACK CallBack,
361 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
362 );
363
364 PEX_CALLBACK_FUNCTION
365 NTAPI
366 ExGetCallBackBlockRoutine(
367 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
368 );
369
370 PVOID
371 NTAPI
372 ExGetCallBackBlockContext(
373 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
374 );
375
376 VOID
377 NTAPI
378 ExWaitForCallBacks(
379 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
380 );
381
382 /* Rundown Functions ********************************************************/
383
384 VOID
385 FASTCALL
386 ExfInitializeRundownProtection(
387 OUT PEX_RUNDOWN_REF RunRef
388 );
389
390 VOID
391 FASTCALL
392 ExfReInitializeRundownProtection(
393 OUT PEX_RUNDOWN_REF RunRef
394 );
395
396 BOOLEAN
397 FASTCALL
398 ExfAcquireRundownProtection(
399 IN OUT PEX_RUNDOWN_REF RunRef
400 );
401
402 BOOLEAN
403 FASTCALL
404 ExfAcquireRundownProtectionEx(
405 IN OUT PEX_RUNDOWN_REF RunRef,
406 IN ULONG Count
407 );
408
409 VOID
410 FASTCALL
411 ExfReleaseRundownProtection(
412 IN OUT PEX_RUNDOWN_REF RunRef
413 );
414
415 VOID
416 FASTCALL
417 ExfReleaseRundownProtectionEx(
418 IN OUT PEX_RUNDOWN_REF RunRef,
419 IN ULONG Count
420 );
421
422 VOID
423 FASTCALL
424 ExfRundownCompleted(
425 OUT PEX_RUNDOWN_REF RunRef
426 );
427
428 VOID
429 FASTCALL
430 ExfWaitForRundownProtectionRelease(
431 IN OUT PEX_RUNDOWN_REF RunRef
432 );
433
434 /* HANDLE TABLE FUNCTIONS ***************************************************/
435
436 typedef BOOLEAN
437 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)(
438 PHANDLE_TABLE_ENTRY HandleTableEntry,
439 HANDLE Handle,
440 PVOID Context
441 );
442
443 typedef BOOLEAN
444 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)(
445 IN PEPROCESS Process,
446 IN PHANDLE_TABLE HandleTable,
447 IN PHANDLE_TABLE_ENTRY HandleTableEntry,
448 IN PHANDLE_TABLE_ENTRY NewEntry
449 );
450
451 typedef BOOLEAN
452 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)(
453 PHANDLE_TABLE_ENTRY HandleTableEntry,
454 ULONG_PTR Context
455 );
456
457 CODE_SEG("INIT")
458 VOID
459 NTAPI
460 ExpInitializeHandleTables(
461 VOID
462 );
463
464 PHANDLE_TABLE
465 NTAPI
466 ExCreateHandleTable(
467 IN PEPROCESS Process OPTIONAL
468 );
469
470 VOID
471 NTAPI
472 ExUnlockHandleTableEntry(
473 IN PHANDLE_TABLE HandleTable,
474 IN PHANDLE_TABLE_ENTRY HandleTableEntry
475 );
476
477 HANDLE
478 NTAPI
479 ExCreateHandle(
480 IN PHANDLE_TABLE HandleTable,
481 IN PHANDLE_TABLE_ENTRY HandleTableEntry
482 );
483
484 VOID
485 NTAPI
486 ExDestroyHandleTable(
487 IN PHANDLE_TABLE HandleTable,
488 IN PVOID DestroyHandleProcedure OPTIONAL
489 );
490
491 BOOLEAN
492 NTAPI
493 ExDestroyHandle(
494 IN PHANDLE_TABLE HandleTable,
495 IN HANDLE Handle,
496 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
497 );
498
499 PHANDLE_TABLE_ENTRY
500 NTAPI
501 ExMapHandleToPointer(
502 IN PHANDLE_TABLE HandleTable,
503 IN HANDLE Handle
504 );
505
506 PHANDLE_TABLE
507 NTAPI
508 ExDupHandleTable(
509 IN PEPROCESS Process,
510 IN PHANDLE_TABLE HandleTable,
511 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure,
512 IN ULONG_PTR Mask
513 );
514
515 BOOLEAN
516 NTAPI
517 ExChangeHandle(
518 IN PHANDLE_TABLE HandleTable,
519 IN HANDLE Handle,
520 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine,
521 IN ULONG_PTR Context
522 );
523
524 VOID
525 NTAPI
526 ExSweepHandleTable(
527 IN PHANDLE_TABLE HandleTable,
528 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure,
529 IN PVOID Context
530 );
531
532 PHANDLE_TABLE_ENTRY
533 NTAPI
534 ExpLookupHandleTableEntry(
535 IN PHANDLE_TABLE HandleTable,
536 IN EXHANDLE Handle
537 );
538
539 BOOLEAN
540 NTAPI
541 ExpLockHandleTableEntry(
542 IN PHANDLE_TABLE HandleTable,
543 IN PHANDLE_TABLE_ENTRY HandleTableEntry
544 );
545
546 /* PSEH EXCEPTION HANDLING **************************************************/
547
548 LONG
549 NTAPI
550 ExSystemExceptionFilter(VOID);
551
552 /* CALLBACKS *****************************************************************/
553
554 FORCEINLINE
555 VOID
ExDoCallBack(IN OUT PEX_CALLBACK Callback,IN PVOID Context,IN PVOID Argument1,IN PVOID Argument2)556 ExDoCallBack(IN OUT PEX_CALLBACK Callback,
557 IN PVOID Context,
558 IN PVOID Argument1,
559 IN PVOID Argument2)
560 {
561 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
562 PEX_CALLBACK_FUNCTION Function;
563
564 /* Reference the block */
565 CallbackBlock = ExReferenceCallBackBlock(Callback);
566 if (CallbackBlock)
567 {
568 /* Get the function */
569 Function = ExGetCallBackBlockRoutine(CallbackBlock);
570
571 /* Do the callback */
572 Function(Context, Argument1, Argument2);
573
574 /* Now dereference it */
575 ExDereferenceCallBackBlock(Callback, CallbackBlock);
576 }
577 }
578
579 /* FAST REFS ******************************************************************/
580
581 FORCEINLINE
582 PVOID
ExGetObjectFastReference(IN EX_FAST_REF FastRef)583 ExGetObjectFastReference(IN EX_FAST_REF FastRef)
584 {
585 /* Return the unbiased pointer */
586 return (PVOID)(FastRef.Value & ~MAX_FAST_REFS);
587 }
588
589 FORCEINLINE
590 ULONG
ExGetCountFastReference(IN EX_FAST_REF FastRef)591 ExGetCountFastReference(IN EX_FAST_REF FastRef)
592 {
593 /* Return the reference count */
594 return (ULONG)FastRef.RefCnt;
595 }
596
597 FORCEINLINE
598 VOID
ExInitializeFastReference(OUT PEX_FAST_REF FastRef,IN OPTIONAL PVOID Object)599 ExInitializeFastReference(OUT PEX_FAST_REF FastRef,
600 IN OPTIONAL PVOID Object)
601 {
602 /* Sanity check */
603 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
604
605 /* Check if an object is being set */
606 if (!Object)
607 {
608 /* Clear the field */
609 FastRef->Object = NULL;
610 }
611 else
612 {
613 /* Otherwise, we assume the object was referenced and is ready */
614 FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS;
615 }
616 }
617
618 FORCEINLINE
619 EX_FAST_REF
ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)620 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
621 {
622 EX_FAST_REF OldValue, NewValue;
623
624 /* Start reference loop */
625 for (;;)
626 {
627 /* Get the current reference count */
628 OldValue = *FastRef;
629 if (OldValue.RefCnt)
630 {
631 /* Increase the reference count */
632 NewValue.Value = OldValue.Value - 1;
633 NewValue.Object = ExpChangePushlock(&FastRef->Object,
634 NewValue.Object,
635 OldValue.Object);
636 if (NewValue.Object != OldValue.Object) continue;
637 }
638
639 /* We are done */
640 break;
641 }
642
643 /* Return the old value */
644 return OldValue;
645 }
646
647 FORCEINLINE
648 BOOLEAN
ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,IN PVOID Object)649 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,
650 IN PVOID Object)
651 {
652 EX_FAST_REF OldValue, NewValue;
653
654 /* Sanity checks */
655 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
656
657 /* Start update loop */
658 for (;;)
659 {
660 /* Get the current reference count */
661 OldValue = *FastRef;
662
663 /* Check if the current count is too high or if the pointer changed */
664 if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
665 ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
666 {
667 /* Fail */
668 return FALSE;
669 }
670
671 /* Update the reference count */
672 NewValue.Value = OldValue.Value + MAX_FAST_REFS;
673 NewValue.Object = ExpChangePushlock(&FastRef->Object,
674 NewValue.Object,
675 OldValue.Object);
676 if (NewValue.Object != OldValue.Object) continue;
677
678 /* We are done */
679 break;
680 }
681
682 /* Return success */
683 return TRUE;
684 }
685
686 FORCEINLINE
687 BOOLEAN
ExReleaseFastReference(IN PEX_FAST_REF FastRef,IN PVOID Object)688 ExReleaseFastReference(IN PEX_FAST_REF FastRef,
689 IN PVOID Object)
690 {
691 EX_FAST_REF OldValue, NewValue;
692
693 /* Sanity checks */
694 ASSERT(Object != NULL);
695 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
696
697 /* Start reference loop */
698 for (;;)
699 {
700 /* Get the current reference count */
701 OldValue = *FastRef;
702
703 /* Check if we're full if if the pointer changed */
704 if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
705
706 /* Decrease the reference count */
707 NewValue.Value = OldValue.Value + 1;
708 NewValue.Object = ExpChangePushlock(&FastRef->Object,
709 NewValue.Object,
710 OldValue.Object);
711 if (NewValue.Object != OldValue.Object) continue;
712
713 /* We are done */
714 break;
715 }
716
717 /* Return success */
718 return TRUE;
719 }
720
721 FORCEINLINE
722 EX_FAST_REF
ExSwapFastReference(IN PEX_FAST_REF FastRef,IN PVOID Object)723 ExSwapFastReference(IN PEX_FAST_REF FastRef,
724 IN PVOID Object)
725 {
726 EX_FAST_REF NewValue, OldValue;
727
728 /* Sanity check */
729 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
730
731 /* Check if an object is being set */
732 if (!Object)
733 {
734 /* Clear the field */
735 NewValue.Object = NULL;
736 }
737 else
738 {
739 /* Otherwise, we assume the object was referenced and is ready */
740 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
741 }
742
743 /* Update the object */
744 OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
745 return OldValue;
746 }
747
748 FORCEINLINE
749 EX_FAST_REF
ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,IN PVOID Object,IN PVOID OldObject)750 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,
751 IN PVOID Object,
752 IN PVOID OldObject)
753 {
754 EX_FAST_REF OldValue, NewValue;
755
756 /* Sanity check and start swap loop */
757 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
758 for (;;)
759 {
760 /* Get the current value */
761 OldValue = *FastRef;
762
763 /* Make sure there's enough references to swap */
764 if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
765
766 /* Check if we have an object to swap */
767 if (Object)
768 {
769 /* Set up the value with maximum fast references */
770 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
771 }
772 else
773 {
774 /* Write the object address itself (which is empty) */
775 NewValue.Value = (ULONG_PTR)Object;
776 }
777
778 /* Do the actual compare exchange */
779 NewValue.Object = ExpChangePushlock(&FastRef->Object,
780 NewValue.Object,
781 OldValue.Object);
782 if (NewValue.Object != OldValue.Object) continue;
783
784 /* All done */
785 break;
786 }
787
788 /* Return the old value */
789 return OldValue;
790 }
791
792 /* RUNDOWN *******************************************************************/
793
794 FORCEINLINE
795 PEX_RUNDOWN_REF
ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,IN ULONG ProcNumber)796 ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
797 IN ULONG ProcNumber)
798 {
799 return (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs +
800 RunRefCacheAware->RunRefSize *
801 (ProcNumber % RunRefCacheAware->Number));
802 }
803
804 /*++
805 * @name ExfAcquireRundownProtection
806 * INTERNAL MACRO
807 *
808 * The ExfAcquireRundownProtection routine acquires rundown protection for
809 * the specified descriptor.
810 *
811 * @param RunRef
812 * Pointer to a rundown reference descriptor.
813 *
814 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
815 *
816 * @remarks This is the internal macro for system use only.In case the rundown
817 * was active, then the slow-path will be called through the exported
818 * function.
819 *
820 *--*/
821 FORCEINLINE
822 BOOLEAN
_ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)823 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
824 {
825 ULONG_PTR Value, NewValue;
826
827 /* Get the current value and mask the active bit */
828 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
829
830 /* Add a reference */
831 NewValue = Value + EX_RUNDOWN_COUNT_INC;
832
833 /* Change the value */
834 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
835 if (NewValue != Value)
836 {
837 /* Rundown was active, use long path */
838 return ExfAcquireRundownProtection(RunRef);
839 }
840
841 /* Success */
842 return TRUE;
843 }
844
845 /*++
846 * @name ExReleaseRundownProtection
847 * INTERNAL MACRO
848 *
849 * The ExReleaseRundownProtection routine releases rundown protection for
850 * the specified descriptor.
851 *
852 * @param RunRef
853 * Pointer to a rundown reference descriptor.
854 *
855 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
856 *
857 * @remarks This is the internal macro for system use only.In case the rundown
858 * was active, then the slow-path will be called through the exported
859 * function.
860 *
861 *--*/
862 FORCEINLINE
863 VOID
_ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)864 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
865 {
866 ULONG_PTR Value, NewValue;
867
868 /* Get the current value and mask the active bit */
869 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
870
871 /* Remove a reference */
872 NewValue = Value - EX_RUNDOWN_COUNT_INC;
873
874 /* Change the value */
875 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
876
877 /* Check if the rundown was active */
878 if (NewValue != Value)
879 {
880 /* Rundown was active, use long path */
881 ExfReleaseRundownProtection(RunRef);
882 }
883 else
884 {
885 /* Sanity check */
886 ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
887 }
888 }
889
890 /*++
891 * @name ExInitializeRundownProtection
892 * INTERNAL MACRO
893 *
894 * The ExInitializeRundownProtection routine initializes a rundown
895 * protection descriptor.
896 *
897 * @param RunRef
898 * Pointer to a rundown reference descriptor.
899 *
900 * @return None.
901 *
902 * @remarks This is the internal macro for system use only.
903 *
904 *--*/
905 FORCEINLINE
906 VOID
_ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)907 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
908 {
909 /* Set the count to zero */
910 RunRef->Count = 0;
911 }
912
913 /*++
914 * @name ExWaitForRundownProtectionRelease
915 * INTERNAL MACRO
916 *
917 * The ExWaitForRundownProtectionRelease routine waits until the specified
918 * rundown descriptor has been released.
919 *
920 * @param RunRef
921 * Pointer to a rundown reference descriptor.
922 *
923 * @return None.
924 *
925 * @remarks This is the internal macro for system use only. If a wait is actually
926 * necessary, then the slow path is taken through the exported function.
927 *
928 *--*/
929 FORCEINLINE
930 VOID
_ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)931 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
932 {
933 ULONG_PTR Value;
934
935 /* Set the active bit */
936 Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
937 if ((Value) && (Value != EX_RUNDOWN_ACTIVE))
938 {
939 /* If the the rundown wasn't already active, then take the long path */
940 ExfWaitForRundownProtectionRelease(RunRef);
941 }
942 }
943
944 /*++
945 * @name ExRundownCompleted
946 * INTERNAL MACRO
947 *
948 * The ExRundownCompleted routine completes the rundown of the specified
949 * descriptor by setting the active bit.
950 *
951 * @param RunRef
952 * Pointer to a rundown reference descriptor.
953 *
954 * @return None.
955 *
956 * @remarks This is the internal macro for system use only.
957 *
958 *--*/
959 FORCEINLINE
960 VOID
_ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)961 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
962 {
963 /* Sanity check */
964 ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
965
966 /* Mark the counter as active */
967 ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
968 }
969
970 /* PUSHLOCKS *****************************************************************/
971
972 /* FIXME: VERIFY THESE! */
973
974 VOID
975 FASTCALL
976 ExBlockPushLock(
977 IN PEX_PUSH_LOCK PushLock,
978 IN PVOID WaitBlock
979 );
980
981 VOID
982 FASTCALL
983 ExfUnblockPushLock(
984 IN PEX_PUSH_LOCK PushLock,
985 IN PVOID CurrentWaitBlock
986 );
987
988 VOID
989 FASTCALL
990 ExWaitForUnblockPushLock(
991 IN PEX_PUSH_LOCK PushLock,
992 IN PVOID WaitBlock
993 );
994
995 /*++
996 * @name _ExInitializePushLock
997 * INTERNAL MACRO
998 *
999 * The _ExInitializePushLock macro initializes a PushLock.
1000 *
1001 * @params PushLock
1002 * Pointer to the pushlock which is to be initialized.
1003 *
1004 * @return None.
1005 *
1006 * @remarks None.
1007 *
1008 *--*/
1009 FORCEINLINE
1010 VOID
_ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock)1011 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock)
1012 {
1013 /* Set the value to 0 */
1014 PushLock->Ptr = 0;
1015 }
1016 #define ExInitializePushLock _ExInitializePushLock
1017
1018 /*++
1019 * @name ExAcquirePushLockExclusive
1020 * INTERNAL MACRO
1021 *
1022 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
1023 *
1024 * @params PushLock
1025 * Pointer to the pushlock which is to be acquired.
1026 *
1027 * @return None.
1028 *
1029 * @remarks The function attempts the quickest route to acquire the lock, which is
1030 * to simply set the lock bit.
1031 * However, if the pushlock is already shared, the slower path is taken.
1032 *
1033 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1034 * This macro should usually be paired up with KeAcquireCriticalRegion.
1035 *
1036 *--*/
1037 FORCEINLINE
1038 VOID
ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)1039 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
1040 {
1041 /* Try acquiring the lock */
1042 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
1043 {
1044 /* Someone changed it, use the slow path */
1045 ExfAcquirePushLockExclusive(PushLock);
1046 }
1047
1048 /* Sanity check */
1049 ASSERT(PushLock->Locked);
1050 }
1051
1052 /*++
1053 * @name ExTryToAcquirePushLockExclusive
1054 * INTERNAL MACRO
1055 *
1056 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
1057 *
1058 * @params PushLock
1059 * Pointer to the pushlock which is to be acquired.
1060 *
1061 * @return None.
1062 *
1063 * @remarks The function attempts the quickest route to acquire the lock, which is
1064 * to simply set the lock bit.
1065 * However, if the pushlock is already shared, the slower path is taken.
1066 *
1067 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1068 * This macro should usually be paired up with KeAcquireCriticalRegion.
1069 *
1070 *--*/
1071 FORCEINLINE
1072 BOOLEAN
ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)1073 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
1074 {
1075 /* Try acquiring the lock */
1076 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
1077 {
1078 /* Can't acquire */
1079 return FALSE;
1080 }
1081
1082 /* Got acquired */
1083 ASSERT (PushLock->Locked);
1084 return TRUE;
1085 }
1086
1087 /*++
1088 * @name ExAcquirePushLockShared
1089 * INTERNAL MACRO
1090 *
1091 * The ExAcquirePushLockShared macro acquires a shared PushLock.
1092 *
1093 * @params PushLock
1094 * Pointer to the pushlock which is to be acquired.
1095 *
1096 * @return None.
1097 *
1098 * @remarks The function attempts the quickest route to acquire the lock, which is
1099 * to simply set the lock bit and set the share count to one.
1100 * However, if the pushlock is already shared, the slower path is taken.
1101 *
1102 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1103 * This macro should usually be paired up with KeAcquireCriticalRegion.
1104 *
1105 *--*/
1106 FORCEINLINE
1107 VOID
ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)1108 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
1109 {
1110 EX_PUSH_LOCK NewValue;
1111
1112 /* Try acquiring the lock */
1113 NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1114 if (ExpChangePushlock(PushLock, NewValue.Ptr, 0))
1115 {
1116 /* Someone changed it, use the slow path */
1117 ExfAcquirePushLockShared(PushLock);
1118 }
1119
1120 /* Sanity checks */
1121 ASSERT(PushLock->Locked);
1122 }
1123
1124 /*++
1125 * @name ExConvertPushLockSharedToExclusive
1126 * INTERNAL MACRO
1127 *
1128 * The ExConvertPushLockSharedToExclusive macro converts an exclusive
1129 * pushlock to a shared pushlock.
1130 *
1131 * @params PushLock
1132 * Pointer to the pushlock which is to be converted.
1133 *
1134 * @return FALSE if conversion failed, TRUE otherwise.
1135 *
1136 * @remarks The function attempts the quickest route to convert the lock, which is
1137 * to simply set the lock bit and remove any other bits.
1138 *
1139 *--*/
1140 FORCEINLINE
1141 BOOLEAN
ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)1142 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)
1143 {
1144 EX_PUSH_LOCK OldValue;
1145
1146 /* Set the expected old value */
1147 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1148
1149 /* Try converting the lock */
1150 if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) !=
1151 OldValue.Ptr)
1152 {
1153 /* Conversion failed */
1154 return FALSE;
1155 }
1156
1157 /* Sanity check */
1158 ASSERT(PushLock->Locked);
1159 return TRUE;
1160 }
1161
1162 /*++
1163 * @name ExWaitOnPushLock
1164 * INTERNAL MACRO
1165 *
1166 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1167 *
1168 * @params PushLock
1169 * Pointer to a pushlock.
1170 *
1171 * @return None.
1172 *
1173 * @remarks The function attempts to get any exclusive waiters out of their slow
1174 * path by forcing an instant acquire/release operation.
1175 *
1176 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1177 *
1178 *--*/
1179 FORCEINLINE
1180 VOID
ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)1181 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)
1182 {
1183 /* Check if we're locked */
1184 if (PushLock->Locked)
1185 {
1186 /* Acquire the lock */
1187 ExfAcquirePushLockExclusive(PushLock);
1188 ASSERT(PushLock->Locked);
1189
1190 /* Release it */
1191 ExfReleasePushLockExclusive(PushLock);
1192 }
1193 }
1194
1195 /*++
1196 * @name ExReleasePushLockShared
1197 * INTERNAL MACRO
1198 *
1199 * The ExReleasePushLockShared macro releases a previously acquired PushLock.
1200 *
1201 * @params PushLock
1202 * Pointer to a previously acquired pushlock.
1203 *
1204 * @return None.
1205 *
1206 * @remarks The function attempts the quickest route to release the lock, which is
1207 * to simply decrease the share count and remove the lock bit.
1208 * However, if the pushlock is being waited on then the long path is taken.
1209 *
1210 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1211 * This macro should usually be paired up with KeLeaveCriticalRegion.
1212 *
1213 *--*/
1214 FORCEINLINE
1215 VOID
ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)1216 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)
1217 {
1218 EX_PUSH_LOCK OldValue;
1219
1220 /* Sanity checks */
1221 ASSERT(PushLock->Locked);
1222
1223 /* Try to clear the pushlock */
1224 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1225 if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr)
1226 {
1227 /* There are still other people waiting on it */
1228 ExfReleasePushLockShared(PushLock);
1229 }
1230 }
1231
1232 /*++
1233 * @name ExReleasePushLockExclusive
1234 * INTERNAL MACRO
1235 *
1236 * The ExReleasePushLockExclusive macro releases a previously
1237 * exclusively acquired PushLock.
1238 *
1239 * @params PushLock
1240 * Pointer to a previously acquired pushlock.
1241 *
1242 * @return None.
1243 *
1244 * @remarks The function attempts the quickest route to release the lock, which is
1245 * to simply clear the locked bit.
1246 * However, if the pushlock is being waited on, the slow path is taken
1247 * in an attempt to wake up the lock.
1248 *
1249 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1250 * This macro should usually be paired up with KeLeaveCriticalRegion.
1251 *
1252 *--*/
1253 FORCEINLINE
1254 VOID
ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)1255 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
1256 {
1257 EX_PUSH_LOCK OldValue;
1258
1259 /* Sanity checks */
1260 ASSERT(PushLock->Locked);
1261
1262 /* Unlock the pushlock */
1263 OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock,
1264 -(SSIZE_T)EX_PUSH_LOCK_LOCK);
1265
1266 /* Sanity checks */
1267 ASSERT(OldValue.Locked);
1268 ASSERT(OldValue.Waiting || OldValue.Shared == 0);
1269
1270 /* Check if anyone is waiting on it and it's not already waking*/
1271 if ((OldValue.Waiting) && !(OldValue.Waking))
1272 {
1273 /* Wake it up */
1274 ExfTryToWakePushLock(PushLock);
1275 }
1276 }
1277
1278 /*++
1279 * @name ExReleasePushLock
1280 * INTERNAL MACRO
1281 *
1282 * The ExReleasePushLock macro releases a previously acquired PushLock.
1283 *
1284 * @params PushLock
1285 * Pointer to a previously acquired pushlock.
1286 *
1287 * @return None.
1288 *
1289 * @remarks The function attempts the quickest route to release the lock, which is
1290 * to simply clear all the fields and decrease the share count if required.
1291 * However, if the pushlock is being waited on then the long path is taken.
1292 *
1293 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1294 * This macro should usually be paired up with KeLeaveCriticalRegion.
1295 *
1296 *--*/
1297 FORCEINLINE
1298 VOID
ExReleasePushLock(PEX_PUSH_LOCK PushLock)1299 ExReleasePushLock(PEX_PUSH_LOCK PushLock)
1300 {
1301 EX_PUSH_LOCK OldValue = *PushLock;
1302 EX_PUSH_LOCK NewValue;
1303
1304 /* Sanity checks */
1305 ASSERT(OldValue.Locked);
1306
1307 /* Check if the pushlock is shared */
1308 if (OldValue.Shared > 1)
1309 {
1310 /* Decrease the share count */
1311 NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC;
1312 }
1313 else
1314 {
1315 /* Clear the pushlock entirely */
1316 NewValue.Value = 0;
1317 }
1318
1319 /* Check if nobody is waiting on us and try clearing the lock here */
1320 if ((OldValue.Waiting) ||
1321 (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) !=
1322 OldValue.Ptr))
1323 {
1324 /* We have waiters, use the long path */
1325 ExfReleasePushLock(PushLock);
1326 }
1327 }
1328
1329 /* FAST MUTEX INLINES *********************************************************/
1330
1331 FORCEINLINE
1332 VOID
_ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)1333 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
1334 {
1335 PKTHREAD Thread = KeGetCurrentThread();
1336
1337 /* Sanity check */
1338 ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1339 (Thread->CombinedApcDisable != 0) ||
1340 (Thread->Teb == NULL) ||
1341 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1342 ASSERT(FastMutex->Owner != Thread);
1343
1344 /* Decrease the count */
1345 if (InterlockedDecrement(&FastMutex->Count))
1346 {
1347 /* Someone is still holding it, use slow path */
1348 KiAcquireFastMutex(FastMutex);
1349 }
1350
1351 /* Set the owner */
1352 FastMutex->Owner = Thread;
1353 }
1354
1355 FORCEINLINE
1356 VOID
_ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)1357 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1358 {
1359 ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1360 (KeGetCurrentThread()->CombinedApcDisable != 0) ||
1361 (KeGetCurrentThread()->Teb == NULL) ||
1362 (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1363 ASSERT(FastMutex->Owner == KeGetCurrentThread());
1364
1365 /* Erase the owner */
1366 FastMutex->Owner = NULL;
1367
1368 /* Increase the count */
1369 if (InterlockedIncrement(&FastMutex->Count) <= 0)
1370 {
1371 /* Someone was waiting for it, signal the waiter */
1372 KeSetEventBoostPriority(&FastMutex->Event, NULL);
1373 }
1374 }
1375
1376 FORCEINLINE
1377 VOID
_ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)1378 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)
1379 {
1380 KIRQL OldIrql;
1381 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1382
1383 /* Raise IRQL to APC */
1384 KeRaiseIrql(APC_LEVEL, &OldIrql);
1385
1386 /* Decrease the count */
1387 if (InterlockedDecrement(&FastMutex->Count))
1388 {
1389 /* Someone is still holding it, use slow path */
1390 KiAcquireFastMutex(FastMutex);
1391 }
1392
1393 /* Set the owner and IRQL */
1394 FastMutex->Owner = KeGetCurrentThread();
1395 FastMutex->OldIrql = OldIrql;
1396 }
1397
1398 FORCEINLINE
1399 VOID
_ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)1400 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
1401 {
1402 KIRQL OldIrql;
1403 ASSERT(KeGetCurrentIrql() == APC_LEVEL);
1404
1405 /* Erase the owner */
1406 FastMutex->Owner = NULL;
1407 OldIrql = (KIRQL)FastMutex->OldIrql;
1408
1409 /* Increase the count */
1410 if (InterlockedIncrement(&FastMutex->Count) <= 0)
1411 {
1412 /* Someone was waiting for it, signal the waiter */
1413 KeSetEventBoostPriority(&FastMutex->Event, NULL);
1414 }
1415
1416 /* Lower IRQL back */
1417 KeLowerIrql(OldIrql);
1418 }
1419
1420 FORCEINLINE
1421 BOOLEAN
_ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)1422 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
1423 {
1424 KIRQL OldIrql;
1425 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1426
1427 /* Raise to APC_LEVEL */
1428 KeRaiseIrql(APC_LEVEL, &OldIrql);
1429
1430 /* Check if we can quickly acquire it */
1431 if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
1432 {
1433 /* We have, set us as owners */
1434 FastMutex->Owner = KeGetCurrentThread();
1435 FastMutex->OldIrql = OldIrql;
1436 return TRUE;
1437 }
1438 else
1439 {
1440 /* Acquire attempt failed */
1441 KeLowerIrql(OldIrql);
1442 YieldProcessor();
1443 return FALSE;
1444 }
1445 }
1446
1447 FORCEINLINE
1448 VOID
_ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)1449 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1450 {
1451 /* Enter the Critical Region */
1452 KeEnterCriticalRegion();
1453
1454 /* Acquire the mutex unsafely */
1455 _ExAcquireFastMutexUnsafe(FastMutex);
1456 }
1457
1458 FORCEINLINE
1459 VOID
_ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)1460 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
1461 {
1462 /* Release the mutex unsafely */
1463 _ExReleaseFastMutexUnsafe(FastMutex);
1464
1465 /* Leave the critical region */
1466 KeLeaveCriticalRegion();
1467 }
1468
1469 /* OTHER FUNCTIONS **********************************************************/
1470
1471 BOOLEAN
1472 NTAPI
1473 ExTryToAcquireResourceExclusiveLite(
1474 IN PERESOURCE Resource
1475 );
1476
1477 NTSTATUS
1478 ExpSetTimeZoneInformation(
1479 IN PRTL_TIME_ZONE_INFORMATION TimeZoneInformation
1480 );
1481
1482 BOOLEAN
1483 NTAPI
1484 ExAcquireTimeRefreshLock(
1485 IN BOOLEAN Wait
1486 );
1487
1488 VOID
1489 NTAPI
1490 ExReleaseTimeRefreshLock(
1491 VOID
1492 );
1493
1494 VOID
1495 NTAPI
1496 ExUpdateSystemTimeFromCmos(
1497 IN BOOLEAN UpdateInterruptTime,
1498 IN ULONG MaxSepInSeconds
1499 );
1500
1501 VOID
1502 NTAPI
1503 ExAllocateLocallyUniqueId(
1504 OUT LUID *LocallyUniqueId
1505 );
1506
1507 VOID
1508 NTAPI
1509 ExTimerRundown(
1510 VOID
1511 );
1512
1513 CODE_SEG("INIT")
1514 VOID
1515 NTAPI
1516 HeadlessInit(
1517 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1518 );
1519
1520 CODE_SEG("INIT")
1521 VOID
1522 NTAPI
1523 XIPInit(
1524 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1525 );
1526
1527 #define InterlockedDecrementUL(Addend) \
1528 (ULONG)InterlockedDecrement((PLONG)(Addend))
1529
1530 #define InterlockedIncrementUL(Addend) \
1531 (ULONG)InterlockedIncrement((PLONG)(Addend))
1532
1533 #define InterlockedExchangeUL(Target, Value) \
1534 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1535
1536 #define InterlockedExchangeAddUL(Addend, Value) \
1537 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1538
1539 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1540 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1541
1542 #define InterlockedCompareExchangeSizeT(Destination, Exchange, Comperand) \
1543 (SIZE_T)InterlockedCompareExchangePointer((PVOID*)(Destination), (PVOID)(SIZE_T)(Exchange), (PVOID)(SIZE_T)(Comperand))
1544
1545 #ifdef _WIN64
1546 #define InterlockedExchangeSizeT(Target, Value) \
1547 (SIZE_T)InterlockedExchange64((PLONG64)(Target), (LONG64)(Value))
1548 #else
1549 #define InterlockedExchangeSizeT(Target, Value) \
1550 (SIZE_T)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1551 #endif
1552
1553 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1554 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))
1555
1556 #ifdef __cplusplus
1557 } // extern "C"
1558 #endif
1559