1 #pragma once
2
3 #ifdef __cplusplus
4 extern "C"
5 {
6 #endif
7
8 /* GLOBAL VARIABLES *********************************************************/
9
10 extern RTL_TIME_ZONE_INFORMATION ExpTimeZoneInfo;
11 extern LARGE_INTEGER ExpTimeZoneBias;
12 extern ULONG ExpTimeZoneId;
13 extern ULONG ExpTickCountMultiplier;
14 extern ULONG ExpLastTimeZoneBias;
15 extern POBJECT_TYPE ExEventPairObjectType;
16 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType;
17 extern FAST_MUTEX ExpEnvironmentLock;
18 extern ERESOURCE ExpFirmwareTableResource;
19 extern ERESOURCE ExpTimeRefreshLock;
20 extern LIST_ENTRY ExpFirmwareTableProviderListHead;
21 extern BOOLEAN ExpIsWinPEMode;
22 extern ULONG ExpResourceTimeoutCount;
23 extern LIST_ENTRY ExpSystemResourcesList;
24 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
25 extern ULONG ExpUnicodeCaseTableDataOffset;
26 extern PVOID ExpNlsSectionPointer;
27 extern ULONG NtGlobalFlag;
28 extern UNICODE_STRING NtSystemRoot;
29 extern ULONG ExpInitializationPhase;
30 extern ULONG ExpAltTimeZoneBias;
31 extern LIST_ENTRY ExSystemLookasideListHead;
32 extern PCALLBACK_OBJECT PowerStateCallback;
33 extern LIST_ENTRY ExPoolLookasideListHead;
34 extern LIST_ENTRY ExpNonPagedLookasideListHead;
35 extern LIST_ENTRY ExpPagedLookasideListHead;
36 extern KSPIN_LOCK ExpNonPagedLookasideListLock;
37 extern KSPIN_LOCK ExpPagedLookasideListLock;
38 extern ULONG ExCriticalWorkerThreads;
39 extern ULONG ExDelayedWorkerThreads;
40
41 extern PVOID ExpDefaultErrorPort;
42 extern PEPROCESS ExpDefaultErrorPortProcess;
43
44 /*
45 * NT/Cm Version Info variables
46 */
47 extern ULONG NtMajorVersion;
48 extern ULONG NtMinorVersion;
49 extern ULONG NtBuildNumber;
50 extern ULONG CmNtSpBuildNumber;
51 extern ULONG CmNtCSDVersion;
52 extern ULONG CmNtCSDReleaseType;
53 extern UNICODE_STRING CmVersionString;
54 extern UNICODE_STRING CmCSDVersionString;
55 extern CHAR NtBuildLab[];
56
57 /*
58 * WinDBG Debugger Worker State Machine data (see dbgctrl.c)
59 */
60 typedef enum _WINKD_WORKER_STATE
61 {
62 WinKdWorkerReady = 0,
63 WinKdWorkerStart,
64 WinKdWorkerInitialized
65 } WINKD_WORKER_STATE;
66
67 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem;
68 extern WINKD_WORKER_STATE ExpDebuggerWork;
69 extern PEPROCESS ExpDebuggerProcessAttach;
70 extern PEPROCESS ExpDebuggerProcessKill;
71 extern ULONG_PTR ExpDebuggerPageIn;
72
73 VOID NTAPI ExpDebuggerWorker(IN PVOID Context);
74
75 #ifdef _WIN64
76 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4)
77 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3)
78 #else
79 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3)
80 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2)
81 #endif
82 #define HANDLE_TAG_BITS 2
83 #define HANDLE_INDEX_BITS (HANDLE_LOW_BITS + 2 * HANDLE_HIGH_BITS)
84 #define KERNEL_FLAG_BITS (sizeof(ULONG_PTR) * 8 - HANDLE_INDEX_BITS - HANDLE_TAG_BITS)
85
86 typedef union _EXHANDLE
87 {
88 struct
89 {
90 ULONG_PTR TagBits: HANDLE_TAG_BITS;
91 ULONG_PTR Index: HANDLE_INDEX_BITS;
92 ULONG_PTR KernelFlag: KERNEL_FLAG_BITS;
93 };
94 struct
95 {
96 ULONG_PTR TagBits2: HANDLE_TAG_BITS;
97 ULONG_PTR LowIndex: HANDLE_LOW_BITS;
98 ULONG_PTR MidIndex: HANDLE_HIGH_BITS;
99 ULONG_PTR HighIndex: HANDLE_HIGH_BITS;
100 ULONG_PTR KernelFlag2: KERNEL_FLAG_BITS;
101 };
102 HANDLE GenericHandleOverlay;
103 ULONG_PTR Value;
104 ULONG AsULONG;
105 } EXHANDLE, *PEXHANDLE;
106
107 typedef struct _ETIMER
108 {
109 KTIMER KeTimer;
110 KAPC TimerApc;
111 KDPC TimerDpc;
112 LIST_ENTRY ActiveTimerListEntry;
113 KSPIN_LOCK Lock;
114 LONG Period;
115 BOOLEAN ApcAssociated;
116 BOOLEAN WakeTimer;
117 LIST_ENTRY WakeTimerListEntry;
118 } ETIMER, *PETIMER;
119
120 typedef struct
121 {
122 PCALLBACK_OBJECT *CallbackObject;
123 PWSTR Name;
124 } SYSTEM_CALLBACKS;
125
126 typedef struct _HARDERROR_USER_PARAMETERS
127 {
128 ULONG_PTR Parameters[MAXIMUM_HARDERROR_PARAMETERS];
129 UNICODE_STRING Strings[MAXIMUM_HARDERROR_PARAMETERS];
130 WCHAR Buffer[ANYSIZE_ARRAY];
131 } HARDERROR_USER_PARAMETERS, *PHARDERROR_USER_PARAMETERS;
132
133 #define MAX_FAST_REFS 7
134
135 #define ExAcquireRundownProtection _ExAcquireRundownProtection
136 #define ExReleaseRundownProtection _ExReleaseRundownProtection
137 #define ExInitializeRundownProtection _ExInitializeRundownProtection
138 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease
139 #define ExRundownCompleted _ExRundownCompleted
140 #define ExGetPreviousMode KeGetPreviousMode
141
142
143 //
144 // Various bits tagged on the handle or handle table
145 //
146 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1
147 #define FREE_HANDLE_MASK -1
148
149 //
150 // Number of entries in each table level
151 //
152 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
153 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
154 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
155
156 //
157 // Maximum index in each table level before we need another table
158 //
159 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES
160 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
161 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
162
163 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&(x)->Ptr, (PVOID)(y), (PVOID)(z))
164 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)(x), (PVOID)(y), (PVOID)(z))
165 #define ExpSetRundown(x, y) InterlockedExchangePointer(&(x)->Ptr, (PVOID)(y))
166
167 NTSTATUS
168 NTAPI
169 ExGetPoolTagInfo(
170 IN PSYSTEM_POOLTAG_INFORMATION SystemInformation,
171 IN ULONG SystemInformationLength,
172 IN OUT PULONG ReturnLength OPTIONAL
173 );
174
175 typedef struct _UUID_CACHED_VALUES_STRUCT
176 {
177 ULONGLONG Time;
178 LONG AllocatedCount;
179 union
180 {
181 struct
182 {
183 UCHAR ClockSeqHiAndReserved;
184 UCHAR ClockSeqLow;
185 UCHAR NodeId[6 /*SEED_BUFFER_SIZE*/];
186 };
187 UCHAR GuidInit[8]; /* Match GUID.Data4 */
188 };
189 } UUID_CACHED_VALUES_STRUCT, *PUUID_CACHED_VALUES_STRUCT;
190
191 C_ASSERT(RTL_FIELD_SIZE(UUID_CACHED_VALUES_STRUCT, GuidInit) == RTL_FIELD_SIZE(UUID, Data4));
192
193 /* INITIALIZATION FUNCTIONS *************************************************/
194
195 CODE_SEG("INIT")
196 BOOLEAN
197 NTAPI
198 ExpWin32kInit(VOID);
199
200 VOID
201 NTAPI
202 ExInit2(VOID);
203
204 VOID
205 NTAPI
206 Phase1Initialization(
207 IN PVOID Context
208 );
209
210 CODE_SEG("INIT")
211 VOID
212 NTAPI
213 ExpInitializePushLocks(VOID);
214
215 BOOLEAN
216 NTAPI
217 ExRefreshTimeZoneInformation(
218 IN PLARGE_INTEGER SystemBootTime
219 );
220
221 CODE_SEG("INIT")
222 VOID
223 NTAPI
224 ExpInitializeWorkerThreads(VOID);
225
226 VOID
227 NTAPI
228 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap);
229
230 CODE_SEG("INIT")
231 VOID
232 NTAPI
233 ExpInitLookasideLists(VOID);
234
235 CODE_SEG("INIT")
236 VOID
237 NTAPI
238 ExInitializeSystemLookasideList(
239 IN PGENERAL_LOOKASIDE List,
240 IN POOL_TYPE Type,
241 IN ULONG Size,
242 IN ULONG Tag,
243 IN USHORT MaximumDepth,
244 IN PLIST_ENTRY ListHead
245 );
246
247 CODE_SEG("INIT")
248 BOOLEAN
249 NTAPI
250 ExpInitializeCallbacks(VOID);
251
252 CODE_SEG("INIT")
253 BOOLEAN
254 NTAPI
255 ExpUuidInitialization(VOID);
256
257 CODE_SEG("INIT")
258 BOOLEAN
259 NTAPI
260 ExLuidInitialization(VOID);
261
262 CODE_SEG("INIT")
263 VOID
264 NTAPI
265 ExpInitializeExecutive(
266 IN ULONG Cpu,
267 IN PLOADER_PARAMETER_BLOCK LoaderBlock
268 );
269
270 VOID
271 NTAPI
272 ExShutdownSystem(VOID);
273
274 CODE_SEG("INIT")
275 BOOLEAN
276 NTAPI
277 ExpInitializeEventImplementation(VOID);
278
279 CODE_SEG("INIT")
280 BOOLEAN
281 NTAPI
282 ExpInitializeKeyedEventImplementation(VOID);
283
284 CODE_SEG("INIT")
285 BOOLEAN
286 NTAPI
287 ExpInitializeEventPairImplementation(VOID);
288
289 CODE_SEG("INIT")
290 BOOLEAN
291 NTAPI
292 ExpInitializeSemaphoreImplementation(VOID);
293
294 CODE_SEG("INIT")
295 BOOLEAN
296 NTAPI
297 ExpInitializeMutantImplementation(VOID);
298
299 CODE_SEG("INIT")
300 BOOLEAN
301 NTAPI
302 ExpInitializeTimerImplementation(VOID);
303
304 CODE_SEG("INIT")
305 BOOLEAN
306 NTAPI
307 ExpInitializeProfileImplementation(VOID);
308
309 CODE_SEG("INIT")
310 VOID
311 NTAPI
312 ExpResourceInitialization(VOID);
313
314 CODE_SEG("INIT")
315 VOID
316 NTAPI
317 ExInitPoolLookasidePointers(VOID);
318
319 /* Callback Functions ********************************************************/
320
321 VOID
322 NTAPI
323 ExInitializeCallBack(
324 IN OUT PEX_CALLBACK Callback
325 );
326
327 PEX_CALLBACK_ROUTINE_BLOCK
328 NTAPI
329 ExAllocateCallBack(
330 IN PEX_CALLBACK_FUNCTION Function,
331 IN PVOID Context
332 );
333
334 VOID
335 NTAPI
336 ExFreeCallBack(
337 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
338 );
339
340 BOOLEAN
341 NTAPI
342 ExCompareExchangeCallBack (
343 IN OUT PEX_CALLBACK CallBack,
344 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock,
345 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
346 );
347
348 PEX_CALLBACK_ROUTINE_BLOCK
349 NTAPI
350 ExReferenceCallBackBlock(
351 IN OUT PEX_CALLBACK CallBack
352 );
353
354 VOID
355 NTAPI
356 ExDereferenceCallBackBlock(
357 IN OUT PEX_CALLBACK CallBack,
358 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
359 );
360
361 PEX_CALLBACK_FUNCTION
362 NTAPI
363 ExGetCallBackBlockRoutine(
364 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
365 );
366
367 PVOID
368 NTAPI
369 ExGetCallBackBlockContext(
370 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
371 );
372
373 VOID
374 NTAPI
375 ExWaitForCallBacks(
376 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
377 );
378
379 /* Rundown Functions ********************************************************/
380
381 VOID
382 FASTCALL
383 ExfInitializeRundownProtection(
384 OUT PEX_RUNDOWN_REF RunRef
385 );
386
387 VOID
388 FASTCALL
389 ExfReInitializeRundownProtection(
390 OUT PEX_RUNDOWN_REF RunRef
391 );
392
393 BOOLEAN
394 FASTCALL
395 ExfAcquireRundownProtection(
396 IN OUT PEX_RUNDOWN_REF RunRef
397 );
398
399 BOOLEAN
400 FASTCALL
401 ExfAcquireRundownProtectionEx(
402 IN OUT PEX_RUNDOWN_REF RunRef,
403 IN ULONG Count
404 );
405
406 VOID
407 FASTCALL
408 ExfReleaseRundownProtection(
409 IN OUT PEX_RUNDOWN_REF RunRef
410 );
411
412 VOID
413 FASTCALL
414 ExfReleaseRundownProtectionEx(
415 IN OUT PEX_RUNDOWN_REF RunRef,
416 IN ULONG Count
417 );
418
419 VOID
420 FASTCALL
421 ExfRundownCompleted(
422 OUT PEX_RUNDOWN_REF RunRef
423 );
424
425 VOID
426 FASTCALL
427 ExfWaitForRundownProtectionRelease(
428 IN OUT PEX_RUNDOWN_REF RunRef
429 );
430
431 /* HANDLE TABLE FUNCTIONS ***************************************************/
432
433 typedef BOOLEAN
434 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)(
435 PHANDLE_TABLE_ENTRY HandleTableEntry,
436 HANDLE Handle,
437 PVOID Context
438 );
439
440 typedef BOOLEAN
441 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)(
442 IN PEPROCESS Process,
443 IN PHANDLE_TABLE HandleTable,
444 IN PHANDLE_TABLE_ENTRY HandleTableEntry,
445 IN PHANDLE_TABLE_ENTRY NewEntry
446 );
447
448 typedef BOOLEAN
449 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)(
450 PHANDLE_TABLE_ENTRY HandleTableEntry,
451 ULONG_PTR Context
452 );
453
454 CODE_SEG("INIT")
455 VOID
456 NTAPI
457 ExpInitializeHandleTables(
458 VOID
459 );
460
461 PHANDLE_TABLE
462 NTAPI
463 ExCreateHandleTable(
464 IN PEPROCESS Process OPTIONAL
465 );
466
467 VOID
468 NTAPI
469 ExUnlockHandleTableEntry(
470 IN PHANDLE_TABLE HandleTable,
471 IN PHANDLE_TABLE_ENTRY HandleTableEntry
472 );
473
474 HANDLE
475 NTAPI
476 ExCreateHandle(
477 IN PHANDLE_TABLE HandleTable,
478 IN PHANDLE_TABLE_ENTRY HandleTableEntry
479 );
480
481 VOID
482 NTAPI
483 ExDestroyHandleTable(
484 IN PHANDLE_TABLE HandleTable,
485 IN PVOID DestroyHandleProcedure OPTIONAL
486 );
487
488 BOOLEAN
489 NTAPI
490 ExDestroyHandle(
491 IN PHANDLE_TABLE HandleTable,
492 IN HANDLE Handle,
493 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
494 );
495
496 PHANDLE_TABLE_ENTRY
497 NTAPI
498 ExMapHandleToPointer(
499 IN PHANDLE_TABLE HandleTable,
500 IN HANDLE Handle
501 );
502
503 PHANDLE_TABLE
504 NTAPI
505 ExDupHandleTable(
506 IN PEPROCESS Process,
507 IN PHANDLE_TABLE HandleTable,
508 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure,
509 IN ULONG_PTR Mask
510 );
511
512 BOOLEAN
513 NTAPI
514 ExChangeHandle(
515 IN PHANDLE_TABLE HandleTable,
516 IN HANDLE Handle,
517 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine,
518 IN ULONG_PTR Context
519 );
520
521 VOID
522 NTAPI
523 ExSweepHandleTable(
524 IN PHANDLE_TABLE HandleTable,
525 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure,
526 IN PVOID Context
527 );
528
529 PHANDLE_TABLE_ENTRY
530 NTAPI
531 ExpLookupHandleTableEntry(
532 IN PHANDLE_TABLE HandleTable,
533 IN EXHANDLE Handle
534 );
535
536 BOOLEAN
537 NTAPI
538 ExpLockHandleTableEntry(
539 IN PHANDLE_TABLE HandleTable,
540 IN PHANDLE_TABLE_ENTRY HandleTableEntry
541 );
542
543 /* PSEH EXCEPTION HANDLING **************************************************/
544
545 LONG
546 NTAPI
547 ExSystemExceptionFilter(VOID);
548
549 /* CALLBACKS *****************************************************************/
550
551 FORCEINLINE
552 VOID
ExDoCallBack(IN OUT PEX_CALLBACK Callback,IN PVOID Context,IN PVOID Argument1,IN PVOID Argument2)553 ExDoCallBack(IN OUT PEX_CALLBACK Callback,
554 IN PVOID Context,
555 IN PVOID Argument1,
556 IN PVOID Argument2)
557 {
558 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
559 PEX_CALLBACK_FUNCTION Function;
560
561 /* Reference the block */
562 CallbackBlock = ExReferenceCallBackBlock(Callback);
563 if (CallbackBlock)
564 {
565 /* Get the function */
566 Function = ExGetCallBackBlockRoutine(CallbackBlock);
567
568 /* Do the callback */
569 Function(Context, Argument1, Argument2);
570
571 /* Now dereference it */
572 ExDereferenceCallBackBlock(Callback, CallbackBlock);
573 }
574 }
575
576 /* FAST REFS ******************************************************************/
577
578 FORCEINLINE
579 PVOID
ExGetObjectFastReference(IN EX_FAST_REF FastRef)580 ExGetObjectFastReference(IN EX_FAST_REF FastRef)
581 {
582 /* Return the unbiased pointer */
583 return (PVOID)(FastRef.Value & ~MAX_FAST_REFS);
584 }
585
586 FORCEINLINE
587 ULONG
ExGetCountFastReference(IN EX_FAST_REF FastRef)588 ExGetCountFastReference(IN EX_FAST_REF FastRef)
589 {
590 /* Return the reference count */
591 return (ULONG)FastRef.RefCnt;
592 }
593
594 FORCEINLINE
595 VOID
ExInitializeFastReference(OUT PEX_FAST_REF FastRef,IN OPTIONAL PVOID Object)596 ExInitializeFastReference(OUT PEX_FAST_REF FastRef,
597 IN OPTIONAL PVOID Object)
598 {
599 /* Sanity check */
600 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
601
602 /* Check if an object is being set */
603 if (!Object)
604 {
605 /* Clear the field */
606 FastRef->Object = NULL;
607 }
608 else
609 {
610 /* Otherwise, we assume the object was referenced and is ready */
611 FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS;
612 }
613 }
614
615 FORCEINLINE
616 EX_FAST_REF
ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)617 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
618 {
619 EX_FAST_REF OldValue, NewValue;
620
621 /* Start reference loop */
622 for (;;)
623 {
624 /* Get the current reference count */
625 OldValue = *FastRef;
626 if (OldValue.RefCnt)
627 {
628 /* Increase the reference count */
629 NewValue.Value = OldValue.Value - 1;
630 NewValue.Object = ExpChangePushlock(&FastRef->Object,
631 NewValue.Object,
632 OldValue.Object);
633 if (NewValue.Object != OldValue.Object) continue;
634 }
635
636 /* We are done */
637 break;
638 }
639
640 /* Return the old value */
641 return OldValue;
642 }
643
644 FORCEINLINE
645 BOOLEAN
ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,IN PVOID Object)646 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,
647 IN PVOID Object)
648 {
649 EX_FAST_REF OldValue, NewValue;
650
651 /* Sanity checks */
652 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
653
654 /* Start update loop */
655 for (;;)
656 {
657 /* Get the current reference count */
658 OldValue = *FastRef;
659
660 /* Check if the current count is too high or if the pointer changed */
661 if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
662 ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
663 {
664 /* Fail */
665 return FALSE;
666 }
667
668 /* Update the reference count */
669 NewValue.Value = OldValue.Value + MAX_FAST_REFS;
670 NewValue.Object = ExpChangePushlock(&FastRef->Object,
671 NewValue.Object,
672 OldValue.Object);
673 if (NewValue.Object != OldValue.Object) continue;
674
675 /* We are done */
676 break;
677 }
678
679 /* Return success */
680 return TRUE;
681 }
682
683 FORCEINLINE
684 BOOLEAN
ExReleaseFastReference(IN PEX_FAST_REF FastRef,IN PVOID Object)685 ExReleaseFastReference(IN PEX_FAST_REF FastRef,
686 IN PVOID Object)
687 {
688 EX_FAST_REF OldValue, NewValue;
689
690 /* Sanity checks */
691 ASSERT(Object != NULL);
692 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
693
694 /* Start reference loop */
695 for (;;)
696 {
697 /* Get the current reference count */
698 OldValue = *FastRef;
699
700 /* Check if we're full if if the pointer changed */
701 if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
702
703 /* Decrease the reference count */
704 NewValue.Value = OldValue.Value + 1;
705 NewValue.Object = ExpChangePushlock(&FastRef->Object,
706 NewValue.Object,
707 OldValue.Object);
708 if (NewValue.Object != OldValue.Object) continue;
709
710 /* We are done */
711 break;
712 }
713
714 /* Return success */
715 return TRUE;
716 }
717
718 FORCEINLINE
719 EX_FAST_REF
ExSwapFastReference(IN PEX_FAST_REF FastRef,IN PVOID Object)720 ExSwapFastReference(IN PEX_FAST_REF FastRef,
721 IN PVOID Object)
722 {
723 EX_FAST_REF NewValue, OldValue;
724
725 /* Sanity check */
726 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
727
728 /* Check if an object is being set */
729 if (!Object)
730 {
731 /* Clear the field */
732 NewValue.Object = NULL;
733 }
734 else
735 {
736 /* Otherwise, we assume the object was referenced and is ready */
737 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
738 }
739
740 /* Update the object */
741 OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
742 return OldValue;
743 }
744
745 FORCEINLINE
746 EX_FAST_REF
ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,IN PVOID Object,IN PVOID OldObject)747 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,
748 IN PVOID Object,
749 IN PVOID OldObject)
750 {
751 EX_FAST_REF OldValue, NewValue;
752
753 /* Sanity check and start swap loop */
754 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
755 for (;;)
756 {
757 /* Get the current value */
758 OldValue = *FastRef;
759
760 /* Make sure there's enough references to swap */
761 if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
762
763 /* Check if we have an object to swap */
764 if (Object)
765 {
766 /* Set up the value with maximum fast references */
767 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
768 }
769 else
770 {
771 /* Write the object address itself (which is empty) */
772 NewValue.Value = (ULONG_PTR)Object;
773 }
774
775 /* Do the actual compare exchange */
776 NewValue.Object = ExpChangePushlock(&FastRef->Object,
777 NewValue.Object,
778 OldValue.Object);
779 if (NewValue.Object != OldValue.Object) continue;
780
781 /* All done */
782 break;
783 }
784
785 /* Return the old value */
786 return OldValue;
787 }
788
789 /* RUNDOWN *******************************************************************/
790
791 FORCEINLINE
792 PEX_RUNDOWN_REF
ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,IN ULONG ProcNumber)793 ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
794 IN ULONG ProcNumber)
795 {
796 return (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs +
797 RunRefCacheAware->RunRefSize *
798 (ProcNumber % RunRefCacheAware->Number));
799 }
800
801 /*++
802 * @name ExfAcquireRundownProtection
803 * INTERNAL MACRO
804 *
805 * The ExfAcquireRundownProtection routine acquires rundown protection for
806 * the specified descriptor.
807 *
808 * @param RunRef
809 * Pointer to a rundown reference descriptor.
810 *
811 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
812 *
813 * @remarks This is the internal macro for system use only.In case the rundown
814 * was active, then the slow-path will be called through the exported
815 * function.
816 *
817 *--*/
818 FORCEINLINE
819 BOOLEAN
_ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)820 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
821 {
822 ULONG_PTR Value, NewValue;
823
824 /* Get the current value and mask the active bit */
825 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
826
827 /* Add a reference */
828 NewValue = Value + EX_RUNDOWN_COUNT_INC;
829
830 /* Change the value */
831 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
832 if (NewValue != Value)
833 {
834 /* Rundown was active, use long path */
835 return ExfAcquireRundownProtection(RunRef);
836 }
837
838 /* Success */
839 return TRUE;
840 }
841
842 /*++
843 * @name ExReleaseRundownProtection
844 * INTERNAL MACRO
845 *
846 * The ExReleaseRundownProtection routine releases rundown protection for
847 * the specified descriptor.
848 *
849 * @param RunRef
850 * Pointer to a rundown reference descriptor.
851 *
852 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
853 *
854 * @remarks This is the internal macro for system use only.In case the rundown
855 * was active, then the slow-path will be called through the exported
856 * function.
857 *
858 *--*/
859 FORCEINLINE
860 VOID
_ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)861 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
862 {
863 ULONG_PTR Value, NewValue;
864
865 /* Get the current value and mask the active bit */
866 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
867
868 /* Remove a reference */
869 NewValue = Value - EX_RUNDOWN_COUNT_INC;
870
871 /* Change the value */
872 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
873
874 /* Check if the rundown was active */
875 if (NewValue != Value)
876 {
877 /* Rundown was active, use long path */
878 ExfReleaseRundownProtection(RunRef);
879 }
880 else
881 {
882 /* Sanity check */
883 ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
884 }
885 }
886
887 /*++
888 * @name ExInitializeRundownProtection
889 * INTERNAL MACRO
890 *
891 * The ExInitializeRundownProtection routine initializes a rundown
892 * protection descriptor.
893 *
894 * @param RunRef
895 * Pointer to a rundown reference descriptor.
896 *
897 * @return None.
898 *
899 * @remarks This is the internal macro for system use only.
900 *
901 *--*/
902 FORCEINLINE
903 VOID
_ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)904 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
905 {
906 /* Set the count to zero */
907 RunRef->Count = 0;
908 }
909
910 /*++
911 * @name ExWaitForRundownProtectionRelease
912 * INTERNAL MACRO
913 *
914 * The ExWaitForRundownProtectionRelease routine waits until the specified
915 * rundown descriptor has been released.
916 *
917 * @param RunRef
918 * Pointer to a rundown reference descriptor.
919 *
920 * @return None.
921 *
922 * @remarks This is the internal macro for system use only. If a wait is actually
923 * necessary, then the slow path is taken through the exported function.
924 *
925 *--*/
926 FORCEINLINE
927 VOID
_ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)928 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
929 {
930 ULONG_PTR Value;
931
932 /* Set the active bit */
933 Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
934 if ((Value) && (Value != EX_RUNDOWN_ACTIVE))
935 {
936 /* If the the rundown wasn't already active, then take the long path */
937 ExfWaitForRundownProtectionRelease(RunRef);
938 }
939 }
940
941 /*++
942 * @name ExRundownCompleted
943 * INTERNAL MACRO
944 *
945 * The ExRundownCompleted routine completes the rundown of the specified
946 * descriptor by setting the active bit.
947 *
948 * @param RunRef
949 * Pointer to a rundown reference descriptor.
950 *
951 * @return None.
952 *
953 * @remarks This is the internal macro for system use only.
954 *
955 *--*/
956 FORCEINLINE
957 VOID
_ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)958 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
959 {
960 /* Sanity check */
961 ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
962
963 /* Mark the counter as active */
964 ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
965 }
966
967 /* PUSHLOCKS *****************************************************************/
968
969 /* FIXME: VERIFY THESE! */
970
971 VOID
972 FASTCALL
973 ExBlockPushLock(
974 IN PEX_PUSH_LOCK PushLock,
975 IN PVOID WaitBlock
976 );
977
978 VOID
979 FASTCALL
980 ExfUnblockPushLock(
981 IN PEX_PUSH_LOCK PushLock,
982 IN PVOID CurrentWaitBlock
983 );
984
985 VOID
986 FASTCALL
987 ExWaitForUnblockPushLock(
988 IN PEX_PUSH_LOCK PushLock,
989 IN PVOID WaitBlock
990 );
991
992 /*++
993 * @name _ExInitializePushLock
994 * INTERNAL MACRO
995 *
996 * The _ExInitializePushLock macro initializes a PushLock.
997 *
998 * @params PushLock
999 * Pointer to the pushlock which is to be initialized.
1000 *
1001 * @return None.
1002 *
1003 * @remarks None.
1004 *
1005 *--*/
1006 FORCEINLINE
1007 VOID
_ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock)1008 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock)
1009 {
1010 /* Set the value to 0 */
1011 PushLock->Ptr = 0;
1012 }
1013 #define ExInitializePushLock _ExInitializePushLock
1014
1015 /*++
1016 * @name ExAcquirePushLockExclusive
1017 * INTERNAL MACRO
1018 *
1019 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
1020 *
1021 * @params PushLock
1022 * Pointer to the pushlock which is to be acquired.
1023 *
1024 * @return None.
1025 *
1026 * @remarks The function attempts the quickest route to acquire the lock, which is
1027 * to simply set the lock bit.
1028 * However, if the pushlock is already shared, the slower path is taken.
1029 *
1030 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1031 * This macro should usually be paired up with KeAcquireCriticalRegion.
1032 *
1033 *--*/
1034 FORCEINLINE
1035 VOID
ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)1036 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
1037 {
1038 /* Try acquiring the lock */
1039 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
1040 {
1041 /* Someone changed it, use the slow path */
1042 ExfAcquirePushLockExclusive(PushLock);
1043 }
1044
1045 /* Sanity check */
1046 ASSERT(PushLock->Locked);
1047 }
1048
1049 /*++
1050 * @name ExTryToAcquirePushLockExclusive
1051 * INTERNAL MACRO
1052 *
1053 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
1054 *
1055 * @params PushLock
1056 * Pointer to the pushlock which is to be acquired.
1057 *
1058 * @return None.
1059 *
1060 * @remarks The function attempts the quickest route to acquire the lock, which is
1061 * to simply set the lock bit.
1062 * However, if the pushlock is already shared, the slower path is taken.
1063 *
1064 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1065 * This macro should usually be paired up with KeAcquireCriticalRegion.
1066 *
1067 *--*/
1068 FORCEINLINE
1069 BOOLEAN
ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)1070 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
1071 {
1072 /* Try acquiring the lock */
1073 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
1074 {
1075 /* Can't acquire */
1076 return FALSE;
1077 }
1078
1079 /* Got acquired */
1080 ASSERT (PushLock->Locked);
1081 return TRUE;
1082 }
1083
1084 /*++
1085 * @name ExAcquirePushLockShared
1086 * INTERNAL MACRO
1087 *
1088 * The ExAcquirePushLockShared macro acquires a shared PushLock.
1089 *
1090 * @params PushLock
1091 * Pointer to the pushlock which is to be acquired.
1092 *
1093 * @return None.
1094 *
1095 * @remarks The function attempts the quickest route to acquire the lock, which is
1096 * to simply set the lock bit and set the share count to one.
1097 * However, if the pushlock is already shared, the slower path is taken.
1098 *
1099 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1100 * This macro should usually be paired up with KeAcquireCriticalRegion.
1101 *
1102 *--*/
1103 FORCEINLINE
1104 VOID
ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)1105 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
1106 {
1107 EX_PUSH_LOCK NewValue;
1108
1109 /* Try acquiring the lock */
1110 NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1111 if (ExpChangePushlock(PushLock, NewValue.Ptr, 0))
1112 {
1113 /* Someone changed it, use the slow path */
1114 ExfAcquirePushLockShared(PushLock);
1115 }
1116
1117 /* Sanity checks */
1118 ASSERT(PushLock->Locked);
1119 }
1120
1121 /*++
1122 * @name ExConvertPushLockSharedToExclusive
1123 * INTERNAL MACRO
1124 *
1125 * The ExConvertPushLockSharedToExclusive macro converts an exclusive
1126 * pushlock to a shared pushlock.
1127 *
1128 * @params PushLock
1129 * Pointer to the pushlock which is to be converted.
1130 *
1131 * @return FALSE if conversion failed, TRUE otherwise.
1132 *
1133 * @remarks The function attempts the quickest route to convert the lock, which is
1134 * to simply set the lock bit and remove any other bits.
1135 *
1136 *--*/
1137 FORCEINLINE
1138 BOOLEAN
ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)1139 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)
1140 {
1141 EX_PUSH_LOCK OldValue;
1142
1143 /* Set the expected old value */
1144 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1145
1146 /* Try converting the lock */
1147 if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) !=
1148 OldValue.Ptr)
1149 {
1150 /* Conversion failed */
1151 return FALSE;
1152 }
1153
1154 /* Sanity check */
1155 ASSERT(PushLock->Locked);
1156 return TRUE;
1157 }
1158
1159 /*++
1160 * @name ExWaitOnPushLock
1161 * INTERNAL MACRO
1162 *
1163 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1164 *
1165 * @params PushLock
1166 * Pointer to a pushlock.
1167 *
1168 * @return None.
1169 *
1170 * @remarks The function attempts to get any exclusive waiters out of their slow
1171 * path by forcing an instant acquire/release operation.
1172 *
1173 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1174 *
1175 *--*/
1176 FORCEINLINE
1177 VOID
ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)1178 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)
1179 {
1180 /* Check if we're locked */
1181 if (PushLock->Locked)
1182 {
1183 /* Acquire the lock */
1184 ExfAcquirePushLockExclusive(PushLock);
1185 ASSERT(PushLock->Locked);
1186
1187 /* Release it */
1188 ExfReleasePushLockExclusive(PushLock);
1189 }
1190 }
1191
1192 /*++
1193 * @name ExReleasePushLockShared
1194 * INTERNAL MACRO
1195 *
1196 * The ExReleasePushLockShared macro releases a previously acquired PushLock.
1197 *
1198 * @params PushLock
1199 * Pointer to a previously acquired pushlock.
1200 *
1201 * @return None.
1202 *
1203 * @remarks The function attempts the quickest route to release the lock, which is
1204 * to simply decrease the share count and remove the lock bit.
1205 * However, if the pushlock is being waited on then the long path is taken.
1206 *
1207 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1208 * This macro should usually be paired up with KeLeaveCriticalRegion.
1209 *
1210 *--*/
1211 FORCEINLINE
1212 VOID
ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)1213 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)
1214 {
1215 EX_PUSH_LOCK OldValue;
1216
1217 /* Sanity checks */
1218 ASSERT(PushLock->Locked);
1219
1220 /* Try to clear the pushlock */
1221 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1222 if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr)
1223 {
1224 /* There are still other people waiting on it */
1225 ExfReleasePushLockShared(PushLock);
1226 }
1227 }
1228
1229 /*++
1230 * @name ExReleasePushLockExclusive
1231 * INTERNAL MACRO
1232 *
1233 * The ExReleasePushLockExclusive macro releases a previously
1234 * exclusively acquired PushLock.
1235 *
1236 * @params PushLock
1237 * Pointer to a previously acquired pushlock.
1238 *
1239 * @return None.
1240 *
1241 * @remarks The function attempts the quickest route to release the lock, which is
1242 * to simply clear the locked bit.
1243 * However, if the pushlock is being waited on, the slow path is taken
1244 * in an attempt to wake up the lock.
1245 *
1246 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1247 * This macro should usually be paired up with KeLeaveCriticalRegion.
1248 *
1249 *--*/
1250 FORCEINLINE
1251 VOID
ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)1252 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
1253 {
1254 EX_PUSH_LOCK OldValue;
1255
1256 /* Sanity checks */
1257 ASSERT(PushLock->Locked);
1258
1259 /* Unlock the pushlock */
1260 OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock,
1261 -(SSIZE_T)EX_PUSH_LOCK_LOCK);
1262
1263 /* Sanity checks */
1264 ASSERT(OldValue.Locked);
1265 ASSERT(OldValue.Waiting || OldValue.Shared == 0);
1266
1267 /* Check if anyone is waiting on it and it's not already waking*/
1268 if ((OldValue.Waiting) && !(OldValue.Waking))
1269 {
1270 /* Wake it up */
1271 ExfTryToWakePushLock(PushLock);
1272 }
1273 }
1274
1275 /*++
1276 * @name ExReleasePushLock
1277 * INTERNAL MACRO
1278 *
1279 * The ExReleasePushLock macro releases a previously acquired PushLock.
1280 *
1281 * @params PushLock
1282 * Pointer to a previously acquired pushlock.
1283 *
1284 * @return None.
1285 *
1286 * @remarks The function attempts the quickest route to release the lock, which is
1287 * to simply clear all the fields and decrease the share count if required.
1288 * However, if the pushlock is being waited on then the long path is taken.
1289 *
1290 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1291 * This macro should usually be paired up with KeLeaveCriticalRegion.
1292 *
1293 *--*/
1294 FORCEINLINE
1295 VOID
ExReleasePushLock(PEX_PUSH_LOCK PushLock)1296 ExReleasePushLock(PEX_PUSH_LOCK PushLock)
1297 {
1298 EX_PUSH_LOCK OldValue = *PushLock;
1299 EX_PUSH_LOCK NewValue;
1300
1301 /* Sanity checks */
1302 ASSERT(OldValue.Locked);
1303
1304 /* Check if the pushlock is shared */
1305 if (OldValue.Shared > 1)
1306 {
1307 /* Decrease the share count */
1308 NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC;
1309 }
1310 else
1311 {
1312 /* Clear the pushlock entirely */
1313 NewValue.Value = 0;
1314 }
1315
1316 /* Check if nobody is waiting on us and try clearing the lock here */
1317 if ((OldValue.Waiting) ||
1318 (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) !=
1319 OldValue.Ptr))
1320 {
1321 /* We have waiters, use the long path */
1322 ExfReleasePushLock(PushLock);
1323 }
1324 }
1325
1326 /* FAST MUTEX INLINES *********************************************************/
1327
1328 FORCEINLINE
1329 VOID
_ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)1330 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
1331 {
1332 PKTHREAD Thread = KeGetCurrentThread();
1333
1334 /* Sanity check */
1335 ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1336 (Thread->CombinedApcDisable != 0) ||
1337 (Thread->Teb == NULL) ||
1338 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1339 ASSERT(FastMutex->Owner != Thread);
1340
1341 /* Decrease the count */
1342 if (InterlockedDecrement(&FastMutex->Count))
1343 {
1344 /* Someone is still holding it, use slow path */
1345 KiAcquireFastMutex(FastMutex);
1346 }
1347
1348 /* Set the owner */
1349 FastMutex->Owner = Thread;
1350 }
1351
1352 FORCEINLINE
1353 VOID
_ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)1354 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1355 {
1356 ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1357 (KeGetCurrentThread()->CombinedApcDisable != 0) ||
1358 (KeGetCurrentThread()->Teb == NULL) ||
1359 (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1360 ASSERT(FastMutex->Owner == KeGetCurrentThread());
1361
1362 /* Erase the owner */
1363 FastMutex->Owner = NULL;
1364
1365 /* Increase the count */
1366 if (InterlockedIncrement(&FastMutex->Count) <= 0)
1367 {
1368 /* Someone was waiting for it, signal the waiter */
1369 KeSetEventBoostPriority(&FastMutex->Event, NULL);
1370 }
1371 }
1372
1373 FORCEINLINE
1374 VOID
_ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)1375 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)
1376 {
1377 KIRQL OldIrql;
1378 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1379
1380 /* Raise IRQL to APC */
1381 KeRaiseIrql(APC_LEVEL, &OldIrql);
1382
1383 /* Decrease the count */
1384 if (InterlockedDecrement(&FastMutex->Count))
1385 {
1386 /* Someone is still holding it, use slow path */
1387 KiAcquireFastMutex(FastMutex);
1388 }
1389
1390 /* Set the owner and IRQL */
1391 FastMutex->Owner = KeGetCurrentThread();
1392 FastMutex->OldIrql = OldIrql;
1393 }
1394
1395 FORCEINLINE
1396 VOID
_ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)1397 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
1398 {
1399 KIRQL OldIrql;
1400 ASSERT(KeGetCurrentIrql() == APC_LEVEL);
1401
1402 /* Erase the owner */
1403 FastMutex->Owner = NULL;
1404 OldIrql = (KIRQL)FastMutex->OldIrql;
1405
1406 /* Increase the count */
1407 if (InterlockedIncrement(&FastMutex->Count) <= 0)
1408 {
1409 /* Someone was waiting for it, signal the waiter */
1410 KeSetEventBoostPriority(&FastMutex->Event, NULL);
1411 }
1412
1413 /* Lower IRQL back */
1414 KeLowerIrql(OldIrql);
1415 }
1416
1417 FORCEINLINE
1418 BOOLEAN
_ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)1419 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
1420 {
1421 KIRQL OldIrql;
1422 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1423
1424 /* Raise to APC_LEVEL */
1425 KeRaiseIrql(APC_LEVEL, &OldIrql);
1426
1427 /* Check if we can quickly acquire it */
1428 if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
1429 {
1430 /* We have, set us as owners */
1431 FastMutex->Owner = KeGetCurrentThread();
1432 FastMutex->OldIrql = OldIrql;
1433 return TRUE;
1434 }
1435 else
1436 {
1437 /* Acquire attempt failed */
1438 KeLowerIrql(OldIrql);
1439 YieldProcessor();
1440 return FALSE;
1441 }
1442 }
1443
1444 FORCEINLINE
1445 VOID
_ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)1446 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1447 {
1448 /* Enter the Critical Region */
1449 KeEnterCriticalRegion();
1450
1451 /* Acquire the mutex unsafely */
1452 _ExAcquireFastMutexUnsafe(FastMutex);
1453 }
1454
1455 FORCEINLINE
1456 VOID
_ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)1457 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
1458 {
1459 /* Release the mutex unsafely */
1460 _ExReleaseFastMutexUnsafe(FastMutex);
1461
1462 /* Leave the critical region */
1463 KeLeaveCriticalRegion();
1464 }
1465
1466 /* OTHER FUNCTIONS **********************************************************/
1467
1468 BOOLEAN
1469 NTAPI
1470 ExTryToAcquireResourceExclusiveLite(
1471 IN PERESOURCE Resource
1472 );
1473
1474 NTSTATUS
1475 ExpSetTimeZoneInformation(
1476 IN PRTL_TIME_ZONE_INFORMATION TimeZoneInformation
1477 );
1478
1479 BOOLEAN
1480 NTAPI
1481 ExAcquireTimeRefreshLock(
1482 IN BOOLEAN Wait
1483 );
1484
1485 VOID
1486 NTAPI
1487 ExReleaseTimeRefreshLock(
1488 VOID
1489 );
1490
1491 VOID
1492 NTAPI
1493 ExUpdateSystemTimeFromCmos(
1494 IN BOOLEAN UpdateInterruptTime,
1495 IN ULONG MaxSepInSeconds
1496 );
1497
1498 VOID
1499 NTAPI
1500 ExAllocateLocallyUniqueId(
1501 OUT LUID *LocallyUniqueId
1502 );
1503
1504 VOID
1505 NTAPI
1506 ExTimerRundown(
1507 VOID
1508 );
1509
1510 CODE_SEG("INIT")
1511 VOID
1512 NTAPI
1513 HeadlessInit(
1514 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1515 );
1516
1517 CODE_SEG("INIT")
1518 VOID
1519 NTAPI
1520 XIPInit(
1521 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1522 );
1523
1524 #define InterlockedDecrementUL(Addend) \
1525 (ULONG)InterlockedDecrement((PLONG)(Addend))
1526
1527 #define InterlockedIncrementUL(Addend) \
1528 (ULONG)InterlockedIncrement((PLONG)(Addend))
1529
1530 #define InterlockedExchangeUL(Target, Value) \
1531 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1532
1533 #define InterlockedExchangeAddUL(Addend, Value) \
1534 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1535
1536 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1537 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1538
1539 #define InterlockedCompareExchangeSizeT(Destination, Exchange, Comperand) \
1540 (SIZE_T)InterlockedCompareExchangePointer((PVOID*)(Destination), (PVOID)(SIZE_T)(Exchange), (PVOID)(SIZE_T)(Comperand))
1541
1542 #ifdef _WIN64
1543 #define InterlockedExchangeSizeT(Target, Value) \
1544 (SIZE_T)InterlockedExchange64((PLONG64)(Target), (LONG64)(Value))
1545 #else
1546 #define InterlockedExchangeSizeT(Target, Value) \
1547 (SIZE_T)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1548 #endif
1549
1550 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1551 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))
1552
1553 #ifdef __cplusplus
1554 } // extern "C"
1555 #endif
1556