xref: /reactos/ntoskrnl/mm/ARM3/miarm.h (revision 8a978a17) 
1 /*
2  * PROJECT:         ReactOS Kernel
3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
4  * FILE:            ntoskrnl/mm/ARM3/miarm.h
5  * PURPOSE:         ARM Memory Manager Header
6  * PROGRAMMERS:     ReactOS Portable Systems Group
7  */
8 
9 #pragma once
10 
11 #define MI_LOWEST_VAD_ADDRESS                   (PVOID)MM_LOWEST_USER_ADDRESS
12 
13 /* Make the code cleaner with some definitions for size multiples */
14 #define _1KB (1024u)
15 #define _1MB (1024 * _1KB)
16 #define _1GB (1024 * _1MB)
17 
18 /* Everyone loves 64K */
19 #define _64K (64 * _1KB)
20 
21 /* Size of a page table */
22 #define PT_SIZE  (PTE_PER_PAGE * sizeof(MMPTE))
23 
24 /* Size of a page directory */
25 #define PD_SIZE  (PDE_PER_PAGE * sizeof(MMPDE))
26 
27 /* Size of all page directories for a process */
28 #define SYSTEM_PD_SIZE (PPE_PER_PAGE * PD_SIZE)
29 #ifdef _M_IX86
30 C_ASSERT(SYSTEM_PD_SIZE == PAGE_SIZE);
31 #endif
32 
33 //
34 // Protection Bits part of the internal memory manager Protection Mask, from:
35 // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel
36 // https://www.reactos.org/wiki/Techwiki:Memory_Protection_constants
37 // and public assertions.
38 //
39 #define MM_ZERO_ACCESS         0
40 #define MM_READONLY            1
41 #define MM_EXECUTE             2
42 #define MM_EXECUTE_READ        3
43 #define MM_READWRITE           4
44 #define MM_WRITECOPY           5
45 #define MM_EXECUTE_READWRITE   6
46 #define MM_EXECUTE_WRITECOPY   7
47 #define MM_PROTECT_ACCESS      7
48 
49 //
50 // These are flags on top of the actual protection mask
51 //
52 #define MM_NOCACHE            0x08
53 #define MM_GUARDPAGE          0x10
54 #define MM_WRITECOMBINE       0x18
55 #define MM_PROTECT_SPECIAL    0x18
56 
57 //
58 // These are special cases
59 //
60 #define MM_DECOMMIT           (MM_ZERO_ACCESS | MM_GUARDPAGE)
61 #define MM_NOACCESS           (MM_ZERO_ACCESS | MM_WRITECOMBINE)
62 #define MM_OUTSWAPPED_KSTACK  (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE)
63 #define MM_INVALID_PROTECTION  0xFFFFFFFF
64 
65 //
66 // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits
67 // The Memory Manager's definition define the attributes that must be preserved
68 // and these PTE definitions describe the attributes in the hardware sense. This
69 // helps deal with hardware differences between the actual boolean expression of
70 // the argument.
71 //
72 // For example, in the logical attributes, we want to express read-only as a flag
73 // but on x86, it is writability that must be set. On the other hand, on x86, just
74 // like in the kernel, it is disabling the caches that requires a special flag,
75 // while on certain architectures such as ARM, it is enabling the cache which
76 // requires a flag.
77 //
78 #if defined(_M_IX86)
79 //
80 // Access Flags
81 //
82 #define PTE_READONLY            0 // Doesn't exist on x86
83 #define PTE_EXECUTE             0 // Not worrying about NX yet
84 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
85 #define PTE_READWRITE           0x2
86 #define PTE_WRITECOPY           0x200
87 #define PTE_EXECUTE_READWRITE   0x2 // Not worrying about NX yet
88 #define PTE_EXECUTE_WRITECOPY   0x200
89 #define PTE_PROTOTYPE           0x400
90 
91 //
92 // State Flags
93 //
94 #define PTE_VALID               0x1
95 #define PTE_ACCESSED            0x20
96 #define PTE_DIRTY               0x40
97 
98 //
99 // Cache flags
100 //
101 #define PTE_ENABLE_CACHE        0
102 #define PTE_DISABLE_CACHE       0x10
103 #define PTE_WRITECOMBINED_CACHE 0x10
104 #define PTE_PROTECT_MASK        0x612
105 #elif defined(_M_AMD64)
106 //
107 // Access Flags
108 //
109 #define PTE_READONLY            0x8000000000000000ULL
110 #define PTE_EXECUTE             0x0000000000000000ULL
111 #define PTE_EXECUTE_READ        PTE_EXECUTE /* EXECUTE implies READ on x64 */
112 #define PTE_READWRITE           0x8000000000000002ULL
113 #define PTE_WRITECOPY           0x8000000000000200ULL
114 #define PTE_EXECUTE_READWRITE   0x0000000000000002ULL
115 #define PTE_EXECUTE_WRITECOPY   0x0000000000000200ULL
116 #define PTE_PROTOTYPE           0x0000000000000400ULL
117 
118 //
119 // State Flags
120 //
121 #define PTE_VALID               0x0000000000000001ULL
122 #define PTE_ACCESSED            0x0000000000000020ULL
123 #define PTE_DIRTY               0x0000000000000040ULL
124 
125 //
126 // Cache flags
127 //
128 #define PTE_ENABLE_CACHE        0x0000000000000000ULL
129 #define PTE_DISABLE_CACHE       0x0000000000000010ULL
130 #define PTE_WRITECOMBINED_CACHE 0x0000000000000010ULL
131 #define PTE_PROTECT_MASK        0x8000000000000612ULL
132 #elif defined(_M_ARM)
133 #define PTE_READONLY            0x200
134 #define PTE_EXECUTE             0 // Not worrying about NX yet
135 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
136 #define PTE_READWRITE           0 // Doesn't exist on ARM
137 #define PTE_WRITECOPY           0 // Doesn't exist on ARM
138 #define PTE_EXECUTE_READWRITE   0 // Not worrying about NX yet
139 #define PTE_EXECUTE_WRITECOPY   0 // Not worrying about NX yet
140 #define PTE_PROTOTYPE           0x400 // Using the Shared bit
141 
142 //
143 // Cache flags
144 //
145 #define PTE_ENABLE_CACHE        0
146 #define PTE_DISABLE_CACHE       0x10
147 #define PTE_WRITECOMBINED_CACHE 0x10
148 #define PTE_PROTECT_MASK        0x610
149 #else
150 #error Define these please!
151 #endif
152 
153 //
154 // Mask for image section page protection
155 //
156 #define IMAGE_SCN_PROTECTION_MASK (IMAGE_SCN_MEM_WRITE | IMAGE_SCN_MEM_READ | IMAGE_SCN_MEM_EXECUTE)
157 
158 extern const ULONG_PTR MmProtectToPteMask[32];
159 extern const ULONG MmProtectToValue[32];
160 
161 //
162 // Assertions for session images, addresses, and PTEs
163 //
164 #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \
165     (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))
166 
167 #define MI_IS_SESSION_ADDRESS(Address) \
168     (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))
169 
170 #define MI_IS_SESSION_PTE(Pte) \
171     ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))
172 
173 #define MI_IS_PAGE_TABLE_ADDRESS(Address) \
174     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))
175 
176 #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \
177     (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))
178 
179 #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \
180     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))
181 
182 //
183 // Creates a software PTE with the given protection
184 //
185 #define MI_MAKE_SOFTWARE_PTE(p, x)          ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))
186 
187 //
188 // Marks a PTE as deleted
189 //
190 #define MI_SET_PFN_DELETED(x)               ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1))
191 #define MI_IS_PFN_DELETED(x)                ((ULONG_PTR)((x)->PteAddress) & 1)
192 
193 //
194 // Special values for LoadedImports
195 //
196 #ifdef _WIN64
197 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFFFFFFFFFEULL
198 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFFFFFFFFFFULL
199 #else
200 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFE
201 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFF
202 #endif
203 #define MM_SYSLDR_SINGLE_ENTRY 0x1
204 
205 //
206 // Number of initial session IDs
207 //
208 #define MI_INITIAL_SESSION_IDS  64
209 
210 #if defined(_M_IX86) || defined(_M_ARM)
211 //
212 // PFN List Sentinel
213 //
214 #define LIST_HEAD 0xFFFFFFFF
215 
216 //
217 // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits,
218 // we need a manual definition suited to the number of bits in the PteFrame.
219 // This is used as a LIST_HEAD for the colored list
220 //
221 #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF
222 #elif defined(_M_AMD64)
223 #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL
224 #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL
225 #else
226 #error Define these please!
227 #endif
228 
229 //
230 // Special IRQL value (found in assertions)
231 //
232 #define MM_NOIRQL (KIRQL)0xFFFFFFFF
233 
234 //
235 // Returns the color of a page
236 //
237 #define MI_GET_PAGE_COLOR(x)                ((x) & MmSecondaryColorMask)
238 #define MI_GET_NEXT_COLOR()                 (MI_GET_PAGE_COLOR(++MmSystemPageColor))
239 #define MI_GET_NEXT_PROCESS_COLOR(x)        (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
240 
241 //
242 // Prototype PTEs that don't yet have a pagefile association
243 //
244 #ifdef _WIN64
245 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL
246 #else
247 #define MI_PTE_LOOKUP_NEEDED 0xFFFFF
248 #endif
249 
250 //
251 // Number of session data and tag pages
252 //
253 #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
254 #define MI_SESSION_TAG_PAGES_MAXIMUM  (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
255 
256 //
257 // Used by MiCheckSecuredVad
258 //
259 #define MM_READ_WRITE_ALLOWED   11
260 #define MM_READ_ONLY_ALLOWED    10
261 #define MM_NO_ACCESS_ALLOWED    01
262 #define MM_DELETE_CHECK         85
263 
264 //
265 // System views are binned into 64K chunks
266 //
267 #define MI_SYSTEM_VIEW_BUCKET_SIZE  _64K
268 
269 //
270 // FIXFIX: These should go in ex.h after the pool merge
271 //
272 #ifdef _WIN64
273 #define POOL_BLOCK_SIZE 16
274 #else
275 #define POOL_BLOCK_SIZE  8
276 #endif
277 #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)
278 #define BASE_POOL_TYPE_MASK 1
279 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))
280 
281 //
282 // Pool debugging/analysis/tracing flags
283 //
284 #define POOL_FLAG_CHECK_TIMERS 0x1
285 #define POOL_FLAG_CHECK_WORKERS 0x2
286 #define POOL_FLAG_CHECK_RESOURCES 0x4
287 #define POOL_FLAG_VERIFIER 0x8
288 #define POOL_FLAG_CHECK_DEADLOCK 0x10
289 #define POOL_FLAG_SPECIAL_POOL 0x20
290 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40
291 #define POOL_FLAG_CRASH_ON_FAILURE 0x80
292 
293 //
294 // BAD_POOL_HEADER codes during pool bugcheck
295 //
296 #define POOL_CORRUPTED_LIST 3
297 #define POOL_SIZE_OR_INDEX_MISMATCH 5
298 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6
299 #define POOL_HEADER_NOT_ALIGNED 7
300 #define POOL_HEADER_IS_ZERO 8
301 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9
302 #define POOL_ENTRY_NOT_FOUND 10
303 
304 //
305 // BAD_POOL_CALLER codes during pool bugcheck
306 //
307 #define POOL_ENTRY_CORRUPTED 1
308 #define POOL_ENTRY_ALREADY_FREE 6
309 #define POOL_ENTRY_NOT_ALLOCATED 7
310 #define POOL_ALLOC_IRQL_INVALID 8
311 #define POOL_FREE_IRQL_INVALID 9
312 #define POOL_BILLED_PROCESS_INVALID 13
313 #define POOL_HEADER_SIZE_INVALID 32
314 
315 typedef struct _POOL_DESCRIPTOR
316 {
317     POOL_TYPE PoolType;
318     ULONG PoolIndex;
319     ULONG RunningAllocs;
320     ULONG RunningDeAllocs;
321     ULONG TotalPages;
322     ULONG TotalBigPages;
323     ULONG Threshold;
324     PVOID LockAddress;
325     PVOID PendingFrees;
326     LONG PendingFreeDepth;
327     SIZE_T TotalBytes;
328     SIZE_T Spare0;
329     LIST_ENTRY ListHeads[POOL_LISTS_PER_PAGE];
330 } POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
331 
332 typedef struct _POOL_HEADER
333 {
334     union
335     {
336         struct
337         {
338 #ifdef _WIN64
339             USHORT PreviousSize:8;
340             USHORT PoolIndex:8;
341             USHORT BlockSize:8;
342             USHORT PoolType:8;
343 #else
344             USHORT PreviousSize:9;
345             USHORT PoolIndex:7;
346             USHORT BlockSize:9;
347             USHORT PoolType:7;
348 #endif
349         };
350         ULONG Ulong1;
351     };
352 #ifdef _WIN64
353     ULONG PoolTag;
354 #endif
355     union
356     {
357 #ifdef _WIN64
358         PEPROCESS ProcessBilled;
359 #else
360         ULONG PoolTag;
361 #endif
362         struct
363         {
364             USHORT AllocatorBackTraceIndex;
365             USHORT PoolTagHash;
366         };
367     };
368 } POOL_HEADER, *PPOOL_HEADER;
369 
370 C_ASSERT(sizeof(POOL_HEADER) == POOL_BLOCK_SIZE);
371 C_ASSERT(POOL_BLOCK_SIZE == sizeof(LIST_ENTRY));
372 
373 typedef struct _POOL_TRACKER_TABLE
374 {
375     ULONG Key;
376     LONG NonPagedAllocs;
377     LONG NonPagedFrees;
378     SIZE_T NonPagedBytes;
379     LONG PagedAllocs;
380     LONG PagedFrees;
381     SIZE_T PagedBytes;
382 } POOL_TRACKER_TABLE, *PPOOL_TRACKER_TABLE;
383 
384 typedef struct _POOL_TRACKER_BIG_PAGES
385 {
386     PVOID Va;
387     ULONG Key;
388     ULONG NumberOfPages;
389     PVOID QuotaObject;
390 } POOL_TRACKER_BIG_PAGES, *PPOOL_TRACKER_BIG_PAGES;
391 
392 extern ULONG ExpNumberOfPagedPools;
393 extern POOL_DESCRIPTOR NonPagedPoolDescriptor;
394 extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16 + 1];
395 extern PPOOL_TRACKER_TABLE PoolTrackTable;
396 
397 //
398 // END FIXFIX
399 //
400 
401 typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY
402 {
403     LIST_ENTRY Links;
404     UNICODE_STRING BaseName;
405 } MI_LARGE_PAGE_DRIVER_ENTRY, *PMI_LARGE_PAGE_DRIVER_ENTRY;
406 
407 typedef enum _MMSYSTEM_PTE_POOL_TYPE
408 {
409     SystemPteSpace,
410     NonPagedPoolExpansion,
411     MaximumPtePoolTypes
412 } MMSYSTEM_PTE_POOL_TYPE;
413 
414 typedef enum _MI_PFN_CACHE_ATTRIBUTE
415 {
416     MiNonCached,
417     MiCached,
418     MiWriteCombined,
419     MiNotMapped
420 } MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;
421 
422 typedef struct _PHYSICAL_MEMORY_RUN
423 {
424     PFN_NUMBER BasePage;
425     PFN_NUMBER PageCount;
426 } PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN;
427 
428 typedef struct _PHYSICAL_MEMORY_DESCRIPTOR
429 {
430     ULONG NumberOfRuns;
431     PFN_NUMBER NumberOfPages;
432     PHYSICAL_MEMORY_RUN Run[1];
433 } PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR;
434 
435 typedef struct _MMCOLOR_TABLES
436 {
437     PFN_NUMBER Flink;
438     PVOID Blink;
439     PFN_NUMBER Count;
440 } MMCOLOR_TABLES, *PMMCOLOR_TABLES;
441 
442 typedef struct _MI_LARGE_PAGE_RANGES
443 {
444     PFN_NUMBER StartFrame;
445     PFN_NUMBER LastFrame;
446 } MI_LARGE_PAGE_RANGES, *PMI_LARGE_PAGE_RANGES;
447 
448 typedef struct _MMVIEW
449 {
450     ULONG_PTR Entry;
451     PCONTROL_AREA ControlArea;
452 } MMVIEW, *PMMVIEW;
453 
454 typedef struct _MMSESSION
455 {
456     KGUARDED_MUTEX SystemSpaceViewLock;
457     PKGUARDED_MUTEX SystemSpaceViewLockPointer;
458     PCHAR SystemSpaceViewStart;
459     PMMVIEW SystemSpaceViewTable;
460     ULONG SystemSpaceHashSize;
461     ULONG SystemSpaceHashEntries;
462     ULONG SystemSpaceHashKey;
463     ULONG BitmapFailures;
464     PRTL_BITMAP SystemSpaceBitMap;
465 } MMSESSION, *PMMSESSION;
466 
467 typedef struct _MM_SESSION_SPACE_FLAGS
468 {
469     ULONG Initialized:1;
470     ULONG DeletePending:1;
471     ULONG Filler:30;
472 } MM_SESSION_SPACE_FLAGS;
473 
474 typedef struct _MM_SESSION_SPACE
475 {
476     struct _MM_SESSION_SPACE *GlobalVirtualAddress;
477     LONG ReferenceCount;
478     union
479     {
480         ULONG LongFlags;
481         MM_SESSION_SPACE_FLAGS Flags;
482     } u;
483     ULONG SessionId;
484     LIST_ENTRY ProcessList;
485     LARGE_INTEGER LastProcessSwappedOutTime;
486     PFN_NUMBER SessionPageDirectoryIndex;
487     SIZE_T NonPageablePages;
488     SIZE_T CommittedPages;
489     PVOID PagedPoolStart;
490     PVOID PagedPoolEnd;
491     PMMPDE PagedPoolBasePde;
492     ULONG Color;
493     LONG ResidentProcessCount;
494     ULONG SessionPoolAllocationFailures[4];
495     LIST_ENTRY ImageList;
496     LCID LocaleId;
497     ULONG AttachCount;
498     KEVENT AttachEvent;
499     PEPROCESS LastProcess;
500     LONG ProcessReferenceToSession;
501     LIST_ENTRY WsListEntry;
502     GENERAL_LOOKASIDE Lookaside[SESSION_POOL_LOOKASIDES];
503     MMSESSION Session;
504     KGUARDED_MUTEX PagedPoolMutex;
505     MM_PAGED_POOL_INFO PagedPoolInfo;
506     MMSUPPORT Vm;
507     PMMWSLE Wsle;
508     PDRIVER_UNLOAD Win32KDriverUnload;
509     POOL_DESCRIPTOR PagedPool;
510 #if defined (_M_AMD64)
511     MMPDE PageDirectory;
512 #else
513     PMMPDE PageTables;
514 #endif
515 #if defined (_M_AMD64)
516     PMMPTE SpecialPoolFirstPte;
517     PMMPTE SpecialPoolLastPte;
518     PMMPTE NextPdeForSpecialPoolExpansion;
519     PMMPTE LastPdeForSpecialPoolExpansion;
520     PFN_NUMBER SpecialPagesInUse;
521 #endif
522     LONG ImageLoadingCount;
523 } MM_SESSION_SPACE, *PMM_SESSION_SPACE;
524 
525 extern PMM_SESSION_SPACE MmSessionSpace;
526 extern MMPTE HyperTemplatePte;
527 extern MMPDE ValidKernelPde;
528 extern MMPTE ValidKernelPte;
529 extern MMPDE ValidKernelPdeLocal;
530 extern MMPTE ValidKernelPteLocal;
531 extern MMPDE DemandZeroPde;
532 extern MMPTE DemandZeroPte;
533 extern MMPTE PrototypePte;
534 extern MMPTE MmDecommittedPte;
535 extern BOOLEAN MmLargeSystemCache;
536 extern BOOLEAN MmZeroPageFile;
537 extern BOOLEAN MmProtectFreedNonPagedPool;
538 extern BOOLEAN MmTrackLockedPages;
539 extern BOOLEAN MmTrackPtes;
540 extern BOOLEAN MmDynamicPfn;
541 extern BOOLEAN MmMirroring;
542 extern BOOLEAN MmMakeLowMemory;
543 extern BOOLEAN MmEnforceWriteProtection;
544 extern SIZE_T MmAllocationFragment;
545 extern ULONG MmConsumedPoolPercentage;
546 extern ULONG MmVerifyDriverBufferType;
547 extern ULONG MmVerifyDriverLevel;
548 extern WCHAR MmVerifyDriverBuffer[512];
549 extern WCHAR MmLargePageDriverBuffer[512];
550 extern LIST_ENTRY MiLargePageDriverList;
551 extern BOOLEAN MiLargePageAllDrivers;
552 extern ULONG MmVerifyDriverBufferLength;
553 extern ULONG MmLargePageDriverBufferLength;
554 extern SIZE_T MmSizeOfNonPagedPoolInBytes;
555 extern SIZE_T MmMaximumNonPagedPoolInBytes;
556 extern PFN_NUMBER MmMaximumNonPagedPoolInPages;
557 extern PFN_NUMBER MmSizeOfPagedPoolInPages;
558 extern PVOID MmNonPagedSystemStart;
559 extern PVOID MmNonPagedPoolStart;
560 extern PVOID MmNonPagedPoolExpansionStart;
561 extern PVOID MmNonPagedPoolEnd;
562 extern SIZE_T MmSizeOfPagedPoolInBytes;
563 extern PVOID MmPagedPoolStart;
564 extern PVOID MmPagedPoolEnd;
565 extern PVOID MmSessionBase;
566 extern SIZE_T MmSessionSize;
567 extern PMMPTE MmFirstReservedMappingPte, MmLastReservedMappingPte;
568 extern PMMPTE MiFirstReservedZeroingPte;
569 extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2][MmMaximumCacheType];
570 extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock;
571 extern SIZE_T MmBootImageSize;
572 extern PMMPTE MmSystemPtesStart[MaximumPtePoolTypes];
573 extern PMMPTE MmSystemPtesEnd[MaximumPtePoolTypes];
574 extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor;
575 extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor;
576 extern ULONG_PTR MxPfnAllocation;
577 extern MM_PAGED_POOL_INFO MmPagedPoolInfo;
578 extern RTL_BITMAP MiPfnBitMap;
579 extern KGUARDED_MUTEX MmPagedPoolMutex;
580 extern KGUARDED_MUTEX MmSectionCommitMutex;
581 extern PVOID MmPagedPoolStart;
582 extern PVOID MmPagedPoolEnd;
583 extern PVOID MmNonPagedSystemStart;
584 extern PVOID MiSystemViewStart;
585 extern SIZE_T MmSystemViewSize;
586 extern PVOID MmSessionBase;
587 extern PVOID MiSessionSpaceEnd;
588 extern PMMPTE MiSessionImagePteStart;
589 extern PMMPTE MiSessionImagePteEnd;
590 extern PMMPTE MiSessionBasePte;
591 extern PMMPTE MiSessionLastPte;
592 extern SIZE_T MmSizeOfPagedPoolInBytes;
593 extern PMMPDE MmSystemPagePtes;
594 extern PVOID MmSystemCacheStart;
595 extern PVOID MmSystemCacheEnd;
596 extern MMSUPPORT MmSystemCacheWs;
597 extern SIZE_T MmAllocatedNonPagedPool;
598 extern ULONG MmSpecialPoolTag;
599 extern PVOID MmHyperSpaceEnd;
600 extern PMMWSL MmSystemCacheWorkingSetList;
601 extern SIZE_T MmMinimumNonPagedPoolSize;
602 extern ULONG MmMinAdditionNonPagedPoolPerMb;
603 extern SIZE_T MmDefaultMaximumNonPagedPool;
604 extern ULONG MmMaxAdditionNonPagedPoolPerMb;
605 extern ULONG MmSecondaryColors;
606 extern ULONG MmSecondaryColorMask;
607 extern ULONG MmNumberOfSystemPtes;
608 extern ULONG MmMaximumNonPagedPoolPercent;
609 extern ULONG MmLargeStackSize;
610 extern PMMCOLOR_TABLES MmFreePagesByColor[FreePageList + 1];
611 extern MMPFNLIST MmStandbyPageListByPriority[8];
612 extern ULONG MmProductType;
613 extern MM_SYSTEMSIZE MmSystemSize;
614 extern PKEVENT MiLowMemoryEvent;
615 extern PKEVENT MiHighMemoryEvent;
616 extern PKEVENT MiLowPagedPoolEvent;
617 extern PKEVENT MiHighPagedPoolEvent;
618 extern PKEVENT MiLowNonPagedPoolEvent;
619 extern PKEVENT MiHighNonPagedPoolEvent;
620 extern PFN_NUMBER MmLowMemoryThreshold;
621 extern PFN_NUMBER MmHighMemoryThreshold;
622 extern PFN_NUMBER MiLowPagedPoolThreshold;
623 extern PFN_NUMBER MiHighPagedPoolThreshold;
624 extern PFN_NUMBER MiLowNonPagedPoolThreshold;
625 extern PFN_NUMBER MiHighNonPagedPoolThreshold;
626 extern PFN_NUMBER MmMinimumFreePages;
627 extern PFN_NUMBER MmPlentyFreePages;
628 extern SIZE_T MmMinimumStackCommitInBytes;
629 extern PFN_COUNT MiExpansionPoolPagesInitialCharge;
630 extern PFN_NUMBER MmResidentAvailablePages;
631 extern PFN_NUMBER MmResidentAvailableAtInit;
632 extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
633 extern PFN_NUMBER MmTotalSystemDriverPages;
634 extern ULONG MmCritsectTimeoutSeconds;
635 extern PVOID MiSessionImageStart;
636 extern PVOID MiSessionImageEnd;
637 extern PMMPTE MiHighestUserPte;
638 extern PMMPDE MiHighestUserPde;
639 extern PFN_NUMBER MmSystemPageDirectory[PPE_PER_PAGE];
640 extern PMMPTE MmSharedUserDataPte;
641 extern LIST_ENTRY MmProcessList;
642 extern KEVENT MmZeroingPageEvent;
643 extern ULONG MmSystemPageColor;
644 extern ULONG MmProcessColorSeed;
645 extern PMMWSL MmWorkingSetList;
646 extern PFN_NUMBER MiNumberOfFreePages;
647 extern SIZE_T MmSessionViewSize;
648 extern SIZE_T MmSessionPoolSize;
649 extern SIZE_T MmSessionImageSize;
650 extern PVOID MiSystemViewStart;
651 extern PVOID MiSessionPoolEnd;     // 0xBE000000
652 extern PVOID MiSessionPoolStart;   // 0xBD000000
653 extern PVOID MiSessionViewStart;   // 0xBE000000
654 extern PVOID MiSessionSpaceWs;
655 extern ULONG MmMaximumDeadKernelStacks;
656 extern SLIST_HEADER MmDeadStackSListHead;
657 extern MM_AVL_TABLE MmSectionBasedRoot;
658 extern KGUARDED_MUTEX MmSectionBasedMutex;
659 extern PVOID MmHighSectionBase;
660 extern SIZE_T MmSystemLockPagesCount;
661 extern ULONG_PTR MmSubsectionBase;
662 extern LARGE_INTEGER MmCriticalSectionTimeout;
663 extern LIST_ENTRY MmWorkingSetExpansionHead;
664 extern KSPIN_LOCK MmExpansionLock;
665 extern PETHREAD MiExpansionLockOwner;
666 
667 FORCEINLINE
668 BOOLEAN
669 MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType)
670 {
671     return ((MemoryType == LoaderFree) ||
672             (MemoryType == LoaderLoadedProgram) ||
673             (MemoryType == LoaderFirmwareTemporary) ||
674             (MemoryType == LoaderOsloaderStack));
675 }
676 
677 FORCEINLINE
678 BOOLEAN
679 MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType)
680 {
681     return ((MemoryType == LoaderFirmwarePermanent) ||
682             (MemoryType == LoaderSpecialMemory) ||
683             (MemoryType == LoaderHALCachedMemory) ||
684             (MemoryType == LoaderBBTMemory));
685 }
686 
687 #ifdef _M_AMD64
688 FORCEINLINE
689 BOOLEAN
690 MiIsUserPxe(PVOID Address)
691 {
692     return ((ULONG_PTR)Address >> 7) == 0x1FFFFEDF6FB7DA0ULL;
693 }
694 
695 FORCEINLINE
696 BOOLEAN
697 MiIsUserPpe(PVOID Address)
698 {
699     return ((ULONG_PTR)Address >> 16) == 0xFFFFF6FB7DA0ULL;
700 }
701 
702 FORCEINLINE
703 BOOLEAN
704 MiIsUserPde(PVOID Address)
705 {
706     return ((ULONG_PTR)Address >> 25) == 0x7FFFFB7DA0ULL;
707 }
708 
709 FORCEINLINE
710 BOOLEAN
711 MiIsUserPte(PVOID Address)
712 {
713     return ((ULONG_PTR)Address >> 34) == 0x3FFFFDA0ULL;
714 }
715 #else
716 FORCEINLINE
717 BOOLEAN
718 MiIsUserPde(PVOID Address)
719 {
720     return ((Address >= (PVOID)MiAddressToPde(NULL)) &&
721             (Address <= (PVOID)MiHighestUserPde));
722 }
723 
724 FORCEINLINE
725 BOOLEAN
726 MiIsUserPte(PVOID Address)
727 {
728     return (Address <= (PVOID)MiHighestUserPte);
729 }
730 #endif
731 
732 //
733 // Figures out the hardware bits for a PTE
734 //
735 FORCEINLINE
736 ULONG_PTR
737 MiDetermineUserGlobalPteMask(IN PVOID PointerPte)
738 {
739     MMPTE TempPte;
740 
741     /* Start fresh */
742     TempPte.u.Long = 0;
743 
744     /* Make it valid and accessed */
745     TempPte.u.Hard.Valid = TRUE;
746     MI_MAKE_ACCESSED_PAGE(&TempPte);
747 
748     /* Is this for user-mode? */
749     if (
750 #if (_MI_PAGING_LEVELS == 4)
751         MiIsUserPxe(PointerPte) ||
752 #endif
753 #if (_MI_PAGING_LEVELS >= 3)
754         MiIsUserPpe(PointerPte) ||
755 #endif
756         MiIsUserPde(PointerPte) ||
757         MiIsUserPte(PointerPte))
758     {
759         /* Set the owner bit */
760         MI_MAKE_OWNER_PAGE(&TempPte);
761     }
762 
763     /* FIXME: We should also set the global bit */
764 
765     /* Return the protection */
766     return TempPte.u.Long;
767 }
768 
769 //
770 // Creates a valid kernel PTE with the given protection
771 //
772 FORCEINLINE
773 VOID
774 MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte,
775                             IN PMMPTE MappingPte,
776                             IN ULONG_PTR ProtectionMask,
777                             IN PFN_NUMBER PageFrameNumber)
778 {
779     /* Only valid for kernel, non-session PTEs */
780     ASSERT(MappingPte > MiHighestUserPte);
781     ASSERT(!MI_IS_SESSION_PTE(MappingPte));
782     ASSERT((MappingPte < (PMMPTE)PDE_BASE) || (MappingPte > (PMMPTE)PDE_TOP));
783 
784     /* Start fresh */
785     *NewPte = ValidKernelPte;
786 
787     /* Set the protection and page */
788     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
789     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
790 }
791 
792 //
793 // Creates a valid PTE with the given protection
794 //
795 FORCEINLINE
796 VOID
797 MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte,
798                      IN PMMPTE MappingPte,
799                      IN ULONG_PTR ProtectionMask,
800                      IN PFN_NUMBER PageFrameNumber)
801 {
802     /* Set the protection and page */
803     NewPte->u.Long = MiDetermineUserGlobalPteMask(MappingPte);
804     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
805     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
806 }
807 
808 //
809 // Creates a valid user PTE with the given protection
810 //
811 FORCEINLINE
812 VOID
813 MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte,
814                           IN PMMPTE MappingPte,
815                           IN ULONG_PTR ProtectionMask,
816                           IN PFN_NUMBER PageFrameNumber)
817 {
818     /* Only valid for kernel, non-session PTEs */
819     ASSERT(MappingPte <= MiHighestUserPte);
820 
821     /* Start fresh */
822     NewPte->u.Long = 0;
823 
824     /* Set the protection and page */
825     NewPte->u.Hard.Valid = TRUE;
826     NewPte->u.Hard.Owner = TRUE;
827     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
828     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
829 }
830 
831 #ifndef _M_AMD64
832 //
833 // Builds a Prototype PTE for the address of the PTE
834 //
835 FORCEINLINE
836 VOID
837 MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte,
838                       IN PMMPTE PointerPte)
839 {
840     ULONG_PTR Offset;
841 
842     /* Mark this as a prototype */
843     NewPte->u.Long = 0;
844     NewPte->u.Proto.Prototype = 1;
845 
846     /*
847      * Prototype PTEs are only valid in paged pool by design, this little trick
848      * lets us only use 30 bits for the adress of the PTE, as long as the area
849      * stays 1024MB At most.
850      */
851     Offset = (ULONG_PTR)PointerPte - (ULONG_PTR)MmPagedPoolStart;
852 
853     /*
854      * 7 bits go in the "low" (but we assume the bottom 2 are zero)
855      * and the other 21 bits go in the "high"
856      */
857     NewPte->u.Proto.ProtoAddressLow = (Offset & 0x1FC) >> 2;
858     NewPte->u.Proto.ProtoAddressHigh = (Offset & 0x3FFFFE00) >> 9;
859 }
860 
861 //
862 // Builds a Subsection PTE for the address of the Segment
863 //
864 FORCEINLINE
865 VOID
866 MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte,
867                        IN PVOID Segment)
868 {
869     ULONG_PTR Offset;
870 
871     /* Mark this as a prototype */
872     NewPte->u.Long = 0;
873     NewPte->u.Subsect.Prototype = 1;
874 
875     /*
876      * Segments are only valid either in nonpaged pool. We store the 20 bit
877      * difference either from the top or bottom of nonpaged pool, giving a
878      * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed
879      * 256MB.
880      */
881     if ((ULONG_PTR)Segment < ((ULONG_PTR)MmSubsectionBase + (128 * _1MB)))
882     {
883         Offset = (ULONG_PTR)Segment - (ULONG_PTR)MmSubsectionBase;
884         NewPte->u.Subsect.WhichPool = PagedPool;
885     }
886     else
887     {
888         Offset = (ULONG_PTR)MmNonPagedPoolEnd - (ULONG_PTR)Segment;
889         NewPte->u.Subsect.WhichPool = NonPagedPool;
890     }
891 
892     /*
893      * 4 bits go in the "low" (but we assume the bottom 3 are zero)
894      * and the other 20 bits go in the "high"
895      */
896     NewPte->u.Subsect.SubsectionAddressLow = (Offset & 0x78) >> 3;
897     NewPte->u.Subsect.SubsectionAddressHigh = (Offset & 0xFFFFF80) >> 7;
898 }
899 
900 FORCEINLINE
901 BOOLEAN
902 MI_IS_MAPPED_PTE(PMMPTE PointerPte)
903 {
904     /// \todo Make this reasonable code, this is UGLY!
905     return ((PointerPte->u.Long & 0xFFFFFC01) != 0);
906 }
907 
908 #endif
909 
910 FORCEINLINE
911 VOID
912 MI_MAKE_TRANSITION_PTE(_Out_ PMMPTE NewPte,
913                        _In_ PFN_NUMBER Page,
914                        _In_ ULONG Protection)
915 {
916     NewPte->u.Long = 0;
917     NewPte->u.Trans.Transition = 1;
918     NewPte->u.Trans.Protection = Protection;
919     NewPte->u.Trans.PageFrameNumber = Page;
920 }
921 
922 //
923 // Returns if the page is physically resident (ie: a large page)
924 // FIXFIX: CISC/x86 only?
925 //
926 FORCEINLINE
927 BOOLEAN
928 MI_IS_PHYSICAL_ADDRESS(IN PVOID Address)
929 {
930     PMMPDE PointerPde;
931 
932     /* Large pages are never paged out, always physically resident */
933     PointerPde = MiAddressToPde(Address);
934     return ((PointerPde->u.Hard.LargePage) && (PointerPde->u.Hard.Valid));
935 }
936 
937 //
938 // Writes a valid PTE
939 //
940 FORCEINLINE
941 VOID
942 MI_WRITE_VALID_PTE(IN PMMPTE PointerPte,
943                    IN MMPTE TempPte)
944 {
945     /* Write the valid PTE */
946     ASSERT(PointerPte->u.Hard.Valid == 0);
947     ASSERT(TempPte.u.Hard.Valid == 1);
948 #if _M_AMD64
949     ASSERT(!MI_IS_PAGE_TABLE_ADDRESS(MiPteToAddress(PointerPte)) ||
950            (TempPte.u.Hard.NoExecute == 0));
951 #endif
952     *PointerPte = TempPte;
953 }
954 
955 //
956 // Updates a valid PTE
957 //
958 FORCEINLINE
959 VOID
960 MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte,
961                    IN MMPTE TempPte)
962 {
963     /* Write the valid PTE */
964     ASSERT(PointerPte->u.Hard.Valid == 1);
965     ASSERT(TempPte.u.Hard.Valid == 1);
966     ASSERT(PointerPte->u.Hard.PageFrameNumber == TempPte.u.Hard.PageFrameNumber);
967     *PointerPte = TempPte;
968 }
969 
970 //
971 // Writes an invalid PTE
972 //
973 FORCEINLINE
974 VOID
975 MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte,
976                      IN MMPTE InvalidPte)
977 {
978     /* Write the invalid PTE */
979     ASSERT(InvalidPte.u.Hard.Valid == 0);
980     ASSERT(InvalidPte.u.Long != 0);
981     *PointerPte = InvalidPte;
982 }
983 
984 //
985 // Erase the PTE completely
986 //
987 FORCEINLINE
988 VOID
989 MI_ERASE_PTE(IN PMMPTE PointerPte)
990 {
991     /* Zero out the PTE */
992     ASSERT(PointerPte->u.Long != 0);
993     PointerPte->u.Long = 0;
994 }
995 
996 //
997 // Writes a valid PDE
998 //
999 FORCEINLINE
1000 VOID
1001 MI_WRITE_VALID_PDE(IN PMMPDE PointerPde,
1002                    IN MMPDE TempPde)
1003 {
1004     /* Write the valid PDE */
1005     ASSERT(PointerPde->u.Hard.Valid == 0);
1006 #ifdef _M_AMD64
1007     ASSERT(PointerPde->u.Hard.NoExecute == 0);
1008 #endif
1009     ASSERT(TempPde.u.Hard.Valid == 1);
1010     *PointerPde = TempPde;
1011 }
1012 
1013 //
1014 // Writes an invalid PDE
1015 //
1016 FORCEINLINE
1017 VOID
1018 MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde,
1019                      IN MMPDE InvalidPde)
1020 {
1021     /* Write the invalid PDE */
1022     ASSERT(InvalidPde.u.Hard.Valid == 0);
1023     ASSERT(InvalidPde.u.Long != 0);
1024 #ifdef _M_AMD64
1025     ASSERT(InvalidPde.u.Soft.Protection == MM_EXECUTE_READWRITE);
1026 #endif
1027     *PointerPde = InvalidPde;
1028 }
1029 
1030 //
1031 // Checks if the thread already owns a working set
1032 //
1033 FORCEINLINE
1034 BOOLEAN
1035 MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread)
1036 {
1037     /* If any of these are held, return TRUE */
1038     return ((Thread->OwnsProcessWorkingSetExclusive) ||
1039             (Thread->OwnsProcessWorkingSetShared) ||
1040             (Thread->OwnsSystemWorkingSetExclusive) ||
1041             (Thread->OwnsSystemWorkingSetShared) ||
1042             (Thread->OwnsSessionWorkingSetExclusive) ||
1043             (Thread->OwnsSessionWorkingSetShared));
1044 }
1045 
1046 //
1047 // Checks if the process owns the working set lock
1048 //
1049 FORCEINLINE
1050 BOOLEAN
1051 MI_WS_OWNER(IN PEPROCESS Process)
1052 {
1053     /* Check if this process is the owner, and that the thread owns the WS */
1054     if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive == 0)
1055     {
1056         DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread());
1057     }
1058     if (KeGetCurrentThread()->ApcState.Process != &Process->Pcb)
1059     {
1060         DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process);
1061     }
1062     return ((KeGetCurrentThread()->ApcState.Process == &Process->Pcb) &&
1063             ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive) ||
1064              (PsGetCurrentThread()->OwnsProcessWorkingSetShared)));
1065 }
1066 
1067 //
1068 // New ARM3<->RosMM PAGE Architecture
1069 //
1070 FORCEINLINE
1071 BOOLEAN
1072 MiIsRosSectionObject(IN PVOID Section)
1073 {
1074     PROS_SECTION_OBJECT RosSection = Section;
1075     if ((RosSection->Type == 'SC') && (RosSection->Size == 'TN')) return TRUE;
1076     return FALSE;
1077 }
1078 
1079 #define MI_IS_ROS_PFN(x)     ((x)->u4.AweAllocation == TRUE)
1080 
1081 VOID
1082 NTAPI
1083 MiDecrementReferenceCount(
1084     IN PMMPFN Pfn1,
1085     IN PFN_NUMBER PageFrameIndex
1086 );
1087 
1088 FORCEINLINE
1089 BOOLEAN
1090 MI_IS_WS_UNSAFE(IN PEPROCESS Process)
1091 {
1092     return (Process->Vm.Flags.AcquiredUnsafe == TRUE);
1093 }
1094 
1095 //
1096 // Locks the working set for the given process
1097 //
1098 FORCEINLINE
1099 VOID
1100 MiLockProcessWorkingSet(IN PEPROCESS Process,
1101                         IN PETHREAD Thread)
1102 {
1103     /* Shouldn't already be owning the process working set */
1104     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1105     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1106 
1107     /* Block APCs, make sure that still nothing is already held */
1108     KeEnterGuardedRegion();
1109     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1110 
1111     /* Lock the working set */
1112     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1113 
1114     /* Now claim that we own the lock */
1115     ASSERT(!MI_IS_WS_UNSAFE(Process));
1116     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1117     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1118 }
1119 
1120 FORCEINLINE
1121 VOID
1122 MiLockProcessWorkingSetShared(IN PEPROCESS Process,
1123                               IN PETHREAD Thread)
1124 {
1125     /* Shouldn't already be owning the process working set */
1126     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1127     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1128 
1129     /* Block APCs, make sure that still nothing is already held */
1130     KeEnterGuardedRegion();
1131     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1132 
1133     /* Lock the working set */
1134     ExAcquirePushLockShared(&Process->Vm.WorkingSetMutex);
1135 
1136     /* Now claim that we own the lock */
1137     ASSERT(!MI_IS_WS_UNSAFE(Process));
1138     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1139     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1140     Thread->OwnsProcessWorkingSetShared = TRUE;
1141 }
1142 
1143 FORCEINLINE
1144 VOID
1145 MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1146                               IN PETHREAD Thread)
1147 {
1148     /* Shouldn't already be owning the process working set */
1149     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1150 
1151     /* APCs must be blocked, make sure that still nothing is already held */
1152     ASSERT(KeAreAllApcsDisabled() == TRUE);
1153     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1154 
1155     /* Lock the working set */
1156     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1157 
1158     /* Now claim that we own the lock */
1159     ASSERT(!MI_IS_WS_UNSAFE(Process));
1160     Process->Vm.Flags.AcquiredUnsafe = 1;
1161     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1162     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1163 }
1164 
1165 //
1166 // Unlocks the working set for the given process
1167 //
1168 FORCEINLINE
1169 VOID
1170 MiUnlockProcessWorkingSet(IN PEPROCESS Process,
1171                           IN PETHREAD Thread)
1172 {
1173     /* Make sure we are the owner of a safe acquisition */
1174     ASSERT(MI_WS_OWNER(Process));
1175     ASSERT(!MI_IS_WS_UNSAFE(Process));
1176 
1177     /* The thread doesn't own it anymore */
1178     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1179     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1180 
1181     /* Release the lock and re-enable APCs */
1182     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1183     KeLeaveGuardedRegion();
1184 }
1185 
1186 //
1187 // Unlocks the working set for the given process
1188 //
1189 FORCEINLINE
1190 VOID
1191 MiUnlockProcessWorkingSetShared(IN PEPROCESS Process,
1192                                 IN PETHREAD Thread)
1193 {
1194     /* Make sure we are the owner of a safe acquisition (because shared) */
1195     ASSERT(MI_WS_OWNER(Process));
1196     ASSERT(!MI_IS_WS_UNSAFE(Process));
1197 
1198     /* Ensure we are in a shared acquisition */
1199     ASSERT(Thread->OwnsProcessWorkingSetShared == TRUE);
1200     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1201 
1202     /* Don't claim the lock anylonger */
1203     Thread->OwnsProcessWorkingSetShared = FALSE;
1204 
1205     /* Release the lock and re-enable APCs */
1206     ExReleasePushLockShared(&Process->Vm.WorkingSetMutex);
1207     KeLeaveGuardedRegion();
1208 }
1209 
1210 //
1211 // Unlocks the working set for the given process
1212 //
1213 FORCEINLINE
1214 VOID
1215 MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1216                                 IN PETHREAD Thread)
1217 {
1218     /* Make sure we are the owner of an unsafe acquisition */
1219     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1220     ASSERT(KeAreAllApcsDisabled() == TRUE);
1221     ASSERT(MI_WS_OWNER(Process));
1222     ASSERT(MI_IS_WS_UNSAFE(Process));
1223 
1224     /* No longer unsafe */
1225     Process->Vm.Flags.AcquiredUnsafe = 0;
1226 
1227     /* The thread doesn't own it anymore */
1228     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1229     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1230 
1231     /* Release the lock but don't touch APC state */
1232     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1233     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1234 }
1235 
1236 //
1237 // Locks the working set
1238 //
1239 FORCEINLINE
1240 VOID
1241 MiLockWorkingSet(IN PETHREAD Thread,
1242                  IN PMMSUPPORT WorkingSet)
1243 {
1244     /* Block APCs */
1245     KeEnterGuardedRegion();
1246 
1247     /* Working set should be in global memory */
1248     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1249 
1250     /* Thread shouldn't already be owning something */
1251     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1252 
1253     /* Lock this working set */
1254     ExAcquirePushLockExclusive(&WorkingSet->WorkingSetMutex);
1255 
1256     /* Which working set is this? */
1257     if (WorkingSet == &MmSystemCacheWs)
1258     {
1259         /* Own the system working set */
1260         ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1261                (Thread->OwnsSystemWorkingSetShared == FALSE));
1262         Thread->OwnsSystemWorkingSetExclusive = TRUE;
1263     }
1264     else if (WorkingSet->Flags.SessionSpace)
1265     {
1266         /* Own the session working set */
1267         ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1268                (Thread->OwnsSessionWorkingSetShared == FALSE));
1269         Thread->OwnsSessionWorkingSetExclusive = TRUE;
1270     }
1271     else
1272     {
1273         /* Own the process working set */
1274         ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1275                (Thread->OwnsProcessWorkingSetShared == FALSE));
1276         Thread->OwnsProcessWorkingSetExclusive = TRUE;
1277     }
1278 }
1279 
1280 //
1281 // Unlocks the working set
1282 //
1283 FORCEINLINE
1284 VOID
1285 MiUnlockWorkingSet(IN PETHREAD Thread,
1286                    IN PMMSUPPORT WorkingSet)
1287 {
1288     /* Working set should be in global memory */
1289     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1290 
1291     /* Which working set is this? */
1292     if (WorkingSet == &MmSystemCacheWs)
1293     {
1294         /* Release the system working set */
1295         ASSERT((Thread->OwnsSystemWorkingSetExclusive == TRUE) ||
1296                (Thread->OwnsSystemWorkingSetShared == TRUE));
1297         Thread->OwnsSystemWorkingSetExclusive = FALSE;
1298     }
1299     else if (WorkingSet->Flags.SessionSpace)
1300     {
1301         /* Release the session working set */
1302         ASSERT((Thread->OwnsSessionWorkingSetExclusive == TRUE) ||
1303                (Thread->OwnsSessionWorkingSetShared == TRUE));
1304         Thread->OwnsSessionWorkingSetExclusive = 0;
1305     }
1306     else
1307     {
1308         /* Release the process working set */
1309         ASSERT((Thread->OwnsProcessWorkingSetExclusive) ||
1310                (Thread->OwnsProcessWorkingSetShared));
1311         Thread->OwnsProcessWorkingSetExclusive = FALSE;
1312     }
1313 
1314     /* Release the working set lock */
1315     ExReleasePushLockExclusive(&WorkingSet->WorkingSetMutex);
1316 
1317     /* Unblock APCs */
1318     KeLeaveGuardedRegion();
1319 }
1320 
1321 FORCEINLINE
1322 VOID
1323 MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process,
1324                                   IN PETHREAD Thread,
1325                                   OUT PBOOLEAN Safe,
1326                                   OUT PBOOLEAN Shared)
1327 {
1328     ASSERT(MI_WS_OWNER(Process));
1329 
1330     /* Check if the current owner is unsafe */
1331     if (MI_IS_WS_UNSAFE(Process))
1332     {
1333         /* Release unsafely */
1334         MiUnlockProcessWorkingSetUnsafe(Process, Thread);
1335         *Safe = FALSE;
1336         *Shared = FALSE;
1337     }
1338     else if (Thread->OwnsProcessWorkingSetExclusive == 1)
1339     {
1340         /* Owner is safe and exclusive, release normally */
1341         MiUnlockProcessWorkingSet(Process, Thread);
1342         *Safe = TRUE;
1343         *Shared = FALSE;
1344     }
1345     else
1346     {
1347         /* Owner is shared (implies safe), release normally */
1348         MiUnlockProcessWorkingSetShared(Process, Thread);
1349         *Safe = TRUE;
1350         *Shared = TRUE;
1351     }
1352 }
1353 
1354 FORCEINLINE
1355 VOID
1356 MiLockProcessWorkingSetForFault(IN PEPROCESS Process,
1357                                 IN PETHREAD Thread,
1358                                 IN BOOLEAN Safe,
1359                                 IN BOOLEAN Shared)
1360 {
1361     /* Check if this was a safe lock or not */
1362     if (Safe)
1363     {
1364         if (Shared)
1365         {
1366             /* Reacquire safely & shared */
1367             MiLockProcessWorkingSetShared(Process, Thread);
1368         }
1369         else
1370         {
1371             /* Reacquire safely */
1372             MiLockProcessWorkingSet(Process, Thread);
1373         }
1374     }
1375     else
1376     {
1377         /* Unsafe lock cannot be shared */
1378         ASSERT(Shared == FALSE);
1379         /* Reacquire unsafely */
1380         MiLockProcessWorkingSetUnsafe(Process, Thread);
1381     }
1382 }
1383 
1384 FORCEINLINE
1385 KIRQL
1386 MiAcquireExpansionLock(VOID)
1387 {
1388     KIRQL OldIrql;
1389 
1390     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1391     KeAcquireSpinLock(&MmExpansionLock, &OldIrql);
1392     ASSERT(MiExpansionLockOwner == NULL);
1393     MiExpansionLockOwner = PsGetCurrentThread();
1394     return OldIrql;
1395 }
1396 
1397 FORCEINLINE
1398 VOID
1399 MiReleaseExpansionLock(KIRQL OldIrql)
1400 {
1401     ASSERT(MiExpansionLockOwner == PsGetCurrentThread());
1402     MiExpansionLockOwner = NULL;
1403     KeReleaseSpinLock(&MmExpansionLock, OldIrql);
1404     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1405 }
1406 
1407 //
1408 // Returns the ProtoPTE inside a VAD for the given VPN
1409 //
1410 FORCEINLINE
1411 PMMPTE
1412 MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad,
1413                              IN ULONG_PTR Vpn)
1414 {
1415     PMMPTE ProtoPte;
1416 
1417     /* Find the offset within the VAD's prototype PTEs */
1418     ProtoPte = Vad->FirstPrototypePte + (Vpn - Vad->StartingVpn);
1419     ASSERT(ProtoPte <= Vad->LastContiguousPte);
1420     return ProtoPte;
1421 }
1422 
1423 //
1424 // Returns the PFN Database entry for the given page number
1425 // Warning: This is not necessarily a valid PFN database entry!
1426 //
1427 FORCEINLINE
1428 PMMPFN
1429 MI_PFN_ELEMENT(IN PFN_NUMBER Pfn)
1430 {
1431     /* Get the entry */
1432     return &MmPfnDatabase[Pfn];
1433 };
1434 
1435 //
1436 // Drops a locked page without dereferencing it
1437 //
1438 FORCEINLINE
1439 VOID
1440 MiDropLockCount(IN PMMPFN Pfn1)
1441 {
1442     /* This page shouldn't be locked, but it should be valid */
1443     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1444     ASSERT(Pfn1->u2.ShareCount == 0);
1445 
1446     /* Is this the last reference to the page */
1447     if (Pfn1->u3.e2.ReferenceCount == 1)
1448     {
1449         /* It better not be valid */
1450         ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1451 
1452         /* Is it a prototype PTE? */
1453         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1454             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1455         {
1456             /* FIXME: We should return commit */
1457             DPRINT1("Not returning commit for prototype PTE\n");
1458         }
1459 
1460         /* Update the counter */
1461         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1462     }
1463 }
1464 
1465 //
1466 // Drops a locked page and dereferences it
1467 //
1468 FORCEINLINE
1469 VOID
1470 MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1)
1471 {
1472     USHORT RefCount, OldRefCount;
1473     PFN_NUMBER PageFrameIndex;
1474 
1475     /* Loop while we decrement the page successfully */
1476     do
1477     {
1478         /* There should be at least one reference */
1479         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1480         ASSERT(OldRefCount != 0);
1481 
1482         /* Are we the last one */
1483         if (OldRefCount == 1)
1484         {
1485             /* The page shoudln't be shared not active at this point */
1486             ASSERT(Pfn1->u3.e2.ReferenceCount == 1);
1487             ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1488             ASSERT(Pfn1->u2.ShareCount == 0);
1489 
1490             /* Is it a prototype PTE? */
1491             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1492                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1493             {
1494                 /* FIXME: We should return commit */
1495                 DPRINT1("Not returning commit for prototype PTE\n");
1496             }
1497 
1498             /* Update the counter, and drop a reference the long way */
1499             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1500             PageFrameIndex = MiGetPfnEntryIndex(Pfn1);
1501             MiDecrementReferenceCount(Pfn1, PageFrameIndex);
1502             return;
1503         }
1504 
1505         /* Drop a reference the short way, and that's it */
1506         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1507                                                 OldRefCount - 1,
1508                                                 OldRefCount);
1509         ASSERT(RefCount != 0);
1510     } while (OldRefCount != RefCount);
1511 
1512     /* If we got here, there should be more than one reference */
1513     ASSERT(RefCount > 1);
1514     if (RefCount == 2)
1515     {
1516         /* Is it still being shared? */
1517         if (Pfn1->u2.ShareCount >= 1)
1518         {
1519             /* Then it should be valid */
1520             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1521 
1522             /* Is it a prototype PTE? */
1523             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1524                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1525             {
1526                 /* We don't handle ethis */
1527                 ASSERT(FALSE);
1528             }
1529 
1530             /* Update the counter */
1531             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1532         }
1533     }
1534 }
1535 
1536 //
1537 // References a locked page and updates the counter
1538 // Used in MmProbeAndLockPages to handle different edge cases
1539 //
1540 FORCEINLINE
1541 VOID
1542 MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1)
1543 {
1544     USHORT RefCount, OldRefCount;
1545 
1546     /* Sanity check */
1547     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1548 
1549     /* Does ARM3 own the page? */
1550     if (MI_IS_ROS_PFN(Pfn1))
1551     {
1552         /* ReactOS Mm doesn't track share count */
1553         ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1554     }
1555     else
1556     {
1557         /* On ARM3 pages, we should see a valid share count */
1558         ASSERT((Pfn1->u2.ShareCount != 0) && (Pfn1->u3.e1.PageLocation == ActiveAndValid));
1559 
1560         /* Is it a prototype PTE? */
1561         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1562             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1563         {
1564             /* FIXME: We should charge commit */
1565             DPRINT1("Not charging commit for prototype PTE\n");
1566         }
1567     }
1568 
1569     /* More locked pages! */
1570     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1571 
1572     /* Loop trying to update the reference count */
1573     do
1574     {
1575         /* Get the current reference count, make sure it's valid */
1576         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1577         ASSERT(OldRefCount != 0);
1578         ASSERT(OldRefCount < 2500);
1579 
1580         /* Bump it up by one */
1581         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1582                                                 OldRefCount + 1,
1583                                                 OldRefCount);
1584         ASSERT(RefCount != 0);
1585     } while (OldRefCount != RefCount);
1586 
1587     /* Was this the first lock attempt? If not, undo our bump */
1588     if (OldRefCount != 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1589 }
1590 
1591 //
1592 // References a locked page and updates the counter
1593 // Used in all other cases except MmProbeAndLockPages
1594 //
1595 FORCEINLINE
1596 VOID
1597 MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1)
1598 {
1599     USHORT NewRefCount;
1600 
1601     /* Is it a prototype PTE? */
1602     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1603         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1604     {
1605         /* FIXME: We should charge commit */
1606         DPRINT1("Not charging commit for prototype PTE\n");
1607     }
1608 
1609     /* More locked pages! */
1610     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1611 
1612     /* Update the reference count */
1613     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1614     if (NewRefCount == 2)
1615     {
1616         /* Is it locked or shared? */
1617         if (Pfn1->u2.ShareCount)
1618         {
1619             /* It's shared, so make sure it's active */
1620             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1621         }
1622         else
1623         {
1624             /* It's locked, so we shouldn't lock again */
1625             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1626         }
1627     }
1628     else
1629     {
1630         /* Someone had already locked the page, so undo our bump */
1631         ASSERT(NewRefCount < 2500);
1632         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1633     }
1634 }
1635 
1636 //
1637 // References a locked page and updates the counter
1638 // Used in all other cases except MmProbeAndLockPages
1639 //
1640 FORCEINLINE
1641 VOID
1642 MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1)
1643 {
1644     USHORT NewRefCount;
1645 
1646     /* Make sure the page isn't used yet */
1647     ASSERT(Pfn1->u2.ShareCount == 0);
1648     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1649 
1650     /* Is it a prototype PTE? */
1651     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1652         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1653     {
1654         /* FIXME: We should charge commit */
1655         DPRINT1("Not charging commit for prototype PTE\n");
1656     }
1657 
1658     /* More locked pages! */
1659     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1660 
1661     /* Update the reference count */
1662     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1663     if (NewRefCount != 1)
1664     {
1665         /* Someone had already locked the page, so undo our bump */
1666         ASSERT(NewRefCount < 2500);
1667         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1668     }
1669 }
1670 
1671 FORCEINLINE
1672 VOID
1673 MiIncrementPageTableReferences(IN PVOID Address)
1674 {
1675     PUSHORT RefCount;
1676 
1677     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1678 
1679     *RefCount += 1;
1680     ASSERT(*RefCount <= PTE_PER_PAGE);
1681 }
1682 
1683 FORCEINLINE
1684 VOID
1685 MiDecrementPageTableReferences(IN PVOID Address)
1686 {
1687     PUSHORT RefCount;
1688 
1689     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1690 
1691     *RefCount -= 1;
1692     ASSERT(*RefCount < PTE_PER_PAGE);
1693 }
1694 
1695 FORCEINLINE
1696 USHORT
1697 MiQueryPageTableReferences(IN PVOID Address)
1698 {
1699     PUSHORT RefCount;
1700 
1701     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1702 
1703     return *RefCount;
1704 }
1705 
1706 BOOLEAN
1707 NTAPI
1708 MmArmInitSystem(
1709     IN ULONG Phase,
1710     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1711 );
1712 
1713 VOID
1714 NTAPI
1715 MiInitializeSessionSpaceLayout(VOID);
1716 
1717 NTSTATUS
1718 NTAPI
1719 MiInitMachineDependent(
1720     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1721 );
1722 
1723 VOID
1724 NTAPI
1725 MiComputeColorInformation(
1726     VOID
1727 );
1728 
1729 VOID
1730 NTAPI
1731 MiMapPfnDatabase(
1732     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1733 );
1734 
1735 VOID
1736 NTAPI
1737 MiInitializeColorTables(
1738     VOID
1739 );
1740 
1741 VOID
1742 NTAPI
1743 MiInitializePfnDatabase(
1744     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1745 );
1746 
1747 VOID
1748 NTAPI
1749 MiInitializeSessionWsSupport(
1750     VOID
1751 );
1752 
1753 VOID
1754 NTAPI
1755 MiInitializeSessionIds(
1756     VOID
1757 );
1758 
1759 BOOLEAN
1760 NTAPI
1761 MiInitializeMemoryEvents(
1762     VOID
1763 );
1764 
1765 PFN_NUMBER
1766 NTAPI
1767 MxGetNextPage(
1768     IN PFN_NUMBER PageCount
1769 );
1770 
1771 PPHYSICAL_MEMORY_DESCRIPTOR
1772 NTAPI
1773 MmInitializeMemoryLimits(
1774     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1775     IN PBOOLEAN IncludeType
1776 );
1777 
1778 PFN_NUMBER
1779 NTAPI
1780 MiPagesInLoaderBlock(
1781     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1782     IN PBOOLEAN IncludeType
1783 );
1784 
1785 VOID
1786 FASTCALL
1787 MiSyncARM3WithROS(
1788     IN PVOID AddressStart,
1789     IN PVOID AddressEnd
1790 );
1791 
1792 NTSTATUS
1793 NTAPI
1794 MiRosProtectVirtualMemory(
1795     IN PEPROCESS Process,
1796     IN OUT PVOID *BaseAddress,
1797     IN OUT PSIZE_T NumberOfBytesToProtect,
1798     IN ULONG NewAccessProtection,
1799     OUT PULONG OldAccessProtection OPTIONAL
1800 );
1801 
1802 NTSTATUS
1803 NTAPI
1804 MmArmAccessFault(
1805     IN ULONG FaultCode,
1806     IN PVOID Address,
1807     IN KPROCESSOR_MODE Mode,
1808     IN PVOID TrapInformation
1809 );
1810 
1811 NTSTATUS
1812 FASTCALL
1813 MiCheckPdeForPagedPool(
1814     IN PVOID Address
1815 );
1816 
1817 VOID
1818 NTAPI
1819 MiInitializeNonPagedPool(
1820     VOID
1821 );
1822 
1823 VOID
1824 NTAPI
1825 MiInitializeNonPagedPoolThresholds(
1826     VOID
1827 );
1828 
1829 VOID
1830 NTAPI
1831 MiInitializePoolEvents(
1832     VOID
1833 );
1834 
1835 VOID                      //
1836 NTAPI                     //
1837 InitializePool(           //
1838     IN POOL_TYPE PoolType,// FIXFIX: This should go in ex.h after the pool merge
1839     IN ULONG Threshold    //
1840 );                        //
1841 
1842 // FIXFIX: THIS ONE TOO
1843 VOID
1844 NTAPI
1845 ExInitializePoolDescriptor(
1846     IN PPOOL_DESCRIPTOR PoolDescriptor,
1847     IN POOL_TYPE PoolType,
1848     IN ULONG PoolIndex,
1849     IN ULONG Threshold,
1850     IN PVOID PoolLock
1851 );
1852 
1853 NTSTATUS
1854 NTAPI
1855 MiInitializeSessionPool(
1856     VOID
1857 );
1858 
1859 VOID
1860 NTAPI
1861 MiInitializeSystemPtes(
1862     IN PMMPTE StartingPte,
1863     IN ULONG NumberOfPtes,
1864     IN MMSYSTEM_PTE_POOL_TYPE PoolType
1865 );
1866 
1867 PMMPTE
1868 NTAPI
1869 MiReserveSystemPtes(
1870     IN ULONG NumberOfPtes,
1871     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1872 );
1873 
1874 VOID
1875 NTAPI
1876 MiReleaseSystemPtes(
1877     IN PMMPTE StartingPte,
1878     IN ULONG NumberOfPtes,
1879     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1880 );
1881 
1882 
1883 PFN_NUMBER
1884 NTAPI
1885 MiFindContiguousPages(
1886     IN PFN_NUMBER LowestPfn,
1887     IN PFN_NUMBER HighestPfn,
1888     IN PFN_NUMBER BoundaryPfn,
1889     IN PFN_NUMBER SizeInPages,
1890     IN MEMORY_CACHING_TYPE CacheType
1891 );
1892 
1893 PVOID
1894 NTAPI
1895 MiCheckForContiguousMemory(
1896     IN PVOID BaseAddress,
1897     IN PFN_NUMBER BaseAddressPages,
1898     IN PFN_NUMBER SizeInPages,
1899     IN PFN_NUMBER LowestPfn,
1900     IN PFN_NUMBER HighestPfn,
1901     IN PFN_NUMBER BoundaryPfn,
1902     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
1903 );
1904 
1905 PMDL
1906 NTAPI
1907 MiAllocatePagesForMdl(
1908     IN PHYSICAL_ADDRESS LowAddress,
1909     IN PHYSICAL_ADDRESS HighAddress,
1910     IN PHYSICAL_ADDRESS SkipBytes,
1911     IN SIZE_T TotalBytes,
1912     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
1913     IN ULONG Flags
1914 );
1915 
1916 VOID
1917 NTAPI
1918 MiInsertPageInList(
1919     IN PMMPFNLIST ListHead,
1920     IN PFN_NUMBER PageFrameIndex
1921 );
1922 
1923 VOID
1924 NTAPI
1925 MiUnlinkFreeOrZeroedPage(
1926     IN PMMPFN Entry
1927 );
1928 
1929 VOID
1930 NTAPI
1931 MiUnlinkPageFromList(
1932     IN PMMPFN Pfn
1933 );
1934 
1935 VOID
1936 NTAPI
1937 MiInitializePfn(
1938     IN PFN_NUMBER PageFrameIndex,
1939     IN PMMPTE PointerPte,
1940     IN BOOLEAN Modified
1941 );
1942 
1943 NTSTATUS
1944 NTAPI
1945 MiInitializeAndChargePfn(
1946     OUT PPFN_NUMBER PageFrameIndex,
1947     IN PMMPDE PointerPde,
1948     IN PFN_NUMBER ContainingPageFrame,
1949     IN BOOLEAN SessionAllocation
1950 );
1951 
1952 VOID
1953 NTAPI
1954 MiInitializePfnAndMakePteValid(
1955     IN PFN_NUMBER PageFrameIndex,
1956     IN PMMPTE PointerPte,
1957     IN MMPTE TempPte
1958 );
1959 
1960 VOID
1961 NTAPI
1962 MiInitializePfnForOtherProcess(
1963     IN PFN_NUMBER PageFrameIndex,
1964     IN PVOID PteAddress,
1965     IN PFN_NUMBER PteFrame
1966 );
1967 
1968 VOID
1969 NTAPI
1970 MiDecrementShareCount(
1971     IN PMMPFN Pfn1,
1972     IN PFN_NUMBER PageFrameIndex
1973 );
1974 
1975 PFN_NUMBER
1976 NTAPI
1977 MiRemoveAnyPage(
1978     IN ULONG Color
1979 );
1980 
1981 PFN_NUMBER
1982 NTAPI
1983 MiRemoveZeroPage(
1984     IN ULONG Color
1985 );
1986 
1987 VOID
1988 NTAPI
1989 MiZeroPhysicalPage(
1990     IN PFN_NUMBER PageFrameIndex
1991 );
1992 
1993 VOID
1994 NTAPI
1995 MiInsertPageInFreeList(
1996     IN PFN_NUMBER PageFrameIndex
1997 );
1998 
1999 PFN_COUNT
2000 NTAPI
2001 MiDeleteSystemPageableVm(
2002     IN PMMPTE PointerPte,
2003     IN PFN_NUMBER PageCount,
2004     IN ULONG Flags,
2005     OUT PPFN_NUMBER ValidPages
2006 );
2007 
2008 ULONG
2009 NTAPI
2010 MiGetPageProtection(
2011     IN PMMPTE PointerPte
2012 );
2013 
2014 PLDR_DATA_TABLE_ENTRY
2015 NTAPI
2016 MiLookupDataTableEntry(
2017     IN PVOID Address
2018 );
2019 
2020 VOID
2021 NTAPI
2022 MiInitializeDriverLargePageList(
2023     VOID
2024 );
2025 
2026 VOID
2027 NTAPI
2028 MiInitializeLargePageSupport(
2029     VOID
2030 );
2031 
2032 VOID
2033 NTAPI
2034 MiSyncCachedRanges(
2035     VOID
2036 );
2037 
2038 BOOLEAN
2039 NTAPI
2040 MiIsPfnInUse(
2041     IN PMMPFN Pfn1
2042 );
2043 
2044 PMMVAD
2045 NTAPI
2046 MiLocateAddress(
2047     IN PVOID VirtualAddress
2048 );
2049 
2050 TABLE_SEARCH_RESULT
2051 NTAPI
2052 MiCheckForConflictingNode(
2053     IN ULONG_PTR StartVpn,
2054     IN ULONG_PTR EndVpn,
2055     IN PMM_AVL_TABLE Table,
2056     OUT PMMADDRESS_NODE *NodeOrParent
2057 );
2058 
2059 TABLE_SEARCH_RESULT
2060 NTAPI
2061 MiFindEmptyAddressRangeDownTree(
2062     IN SIZE_T Length,
2063     IN ULONG_PTR BoundaryAddress,
2064     IN ULONG_PTR Alignment,
2065     IN PMM_AVL_TABLE Table,
2066     OUT PULONG_PTR Base,
2067     OUT PMMADDRESS_NODE *Parent
2068 );
2069 
2070 NTSTATUS
2071 NTAPI
2072 MiFindEmptyAddressRangeDownBasedTree(
2073     IN SIZE_T Length,
2074     IN ULONG_PTR BoundaryAddress,
2075     IN ULONG_PTR Alignment,
2076     IN PMM_AVL_TABLE Table,
2077     OUT PULONG_PTR Base
2078 );
2079 
2080 TABLE_SEARCH_RESULT
2081 NTAPI
2082 MiFindEmptyAddressRangeInTree(
2083     IN SIZE_T Length,
2084     IN ULONG_PTR Alignment,
2085     IN PMM_AVL_TABLE Table,
2086     OUT PMMADDRESS_NODE *PreviousVad,
2087     OUT PULONG_PTR Base
2088 );
2089 
2090 NTSTATUS
2091 NTAPI
2092 MiCheckSecuredVad(
2093     IN PMMVAD Vad,
2094     IN PVOID Base,
2095     IN SIZE_T Size,
2096     IN ULONG ProtectionMask
2097 );
2098 
2099 VOID
2100 NTAPI
2101 MiInsertVad(
2102     _Inout_ PMMVAD Vad,
2103     _Inout_ PMM_AVL_TABLE VadRoot);
2104 
2105 NTSTATUS
2106 NTAPI
2107 MiInsertVadEx(
2108     _Inout_ PMMVAD Vad,
2109     _In_ ULONG_PTR *BaseAddress,
2110     _In_ SIZE_T ViewSize,
2111     _In_ ULONG_PTR HighestAddress,
2112     _In_ ULONG_PTR Alignment,
2113     _In_ ULONG AllocationType);
2114 
2115 VOID
2116 NTAPI
2117 MiInsertBasedSection(
2118     IN PSECTION Section
2119 );
2120 
2121 NTSTATUS
2122 NTAPI
2123 MiUnmapViewOfSection(
2124     IN PEPROCESS Process,
2125     IN PVOID BaseAddress,
2126     IN ULONG Flags
2127 );
2128 
2129 NTSTATUS
2130 NTAPI
2131 MiRosUnmapViewOfSection(
2132     IN PEPROCESS Process,
2133     IN PVOID BaseAddress,
2134     IN BOOLEAN SkipDebuggerNotify
2135 );
2136 
2137 VOID
2138 NTAPI
2139 MiInsertNode(
2140     IN PMM_AVL_TABLE Table,
2141     IN PMMADDRESS_NODE NewNode,
2142     PMMADDRESS_NODE Parent,
2143     TABLE_SEARCH_RESULT Result
2144 );
2145 
2146 VOID
2147 NTAPI
2148 MiRemoveNode(
2149     IN PMMADDRESS_NODE Node,
2150     IN PMM_AVL_TABLE Table
2151 );
2152 
2153 PMMADDRESS_NODE
2154 NTAPI
2155 MiGetPreviousNode(
2156     IN PMMADDRESS_NODE Node
2157 );
2158 
2159 PMMADDRESS_NODE
2160 NTAPI
2161 MiGetNextNode(
2162     IN PMMADDRESS_NODE Node
2163 );
2164 
2165 BOOLEAN
2166 NTAPI
2167 MiInitializeSystemSpaceMap(
2168     IN PMMSESSION InputSession OPTIONAL
2169 );
2170 
2171 VOID
2172 NTAPI
2173 MiSessionRemoveProcess(
2174     VOID
2175 );
2176 
2177 VOID
2178 NTAPI
2179 MiReleaseProcessReferenceToSessionDataPage(
2180     IN PMM_SESSION_SPACE SessionGlobal
2181 );
2182 
2183 VOID
2184 NTAPI
2185 MiSessionAddProcess(
2186     IN PEPROCESS NewProcess
2187 );
2188 
2189 NTSTATUS
2190 NTAPI
2191 MiSessionCommitPageTables(
2192     IN PVOID StartVa,
2193     IN PVOID EndVa
2194 );
2195 
2196 ULONG
2197 NTAPI
2198 MiMakeProtectionMask(
2199     IN ULONG Protect
2200 );
2201 
2202 VOID
2203 NTAPI
2204 MiDeleteVirtualAddresses(
2205     IN ULONG_PTR Va,
2206     IN ULONG_PTR EndingAddress,
2207     IN PMMVAD Vad
2208 );
2209 
2210 VOID
2211 NTAPI
2212 MiDeletePte(
2213     IN PMMPTE PointerPte,
2214     IN PVOID VirtualAddress,
2215     IN PEPROCESS CurrentProcess,
2216     IN PMMPTE PrototypePte
2217 );
2218 
2219 ULONG
2220 NTAPI
2221 MiMakeSystemAddressValid(
2222     IN PVOID PageTableVirtualAddress,
2223     IN PEPROCESS CurrentProcess
2224 );
2225 
2226 ULONG
2227 NTAPI
2228 MiMakeSystemAddressValidPfn(
2229     IN PVOID VirtualAddress,
2230     IN KIRQL OldIrql
2231 );
2232 
2233 VOID
2234 NTAPI
2235 MiRemoveMappedView(
2236     IN PEPROCESS CurrentProcess,
2237     IN PMMVAD Vad
2238 );
2239 
2240 PSUBSECTION
2241 NTAPI
2242 MiLocateSubsection(
2243     IN PMMVAD Vad,
2244     IN ULONG_PTR Vpn
2245 );
2246 
2247 VOID
2248 NTAPI
2249 MiDeleteARM3Section(
2250     PVOID ObjectBody
2251 );
2252 
2253 NTSTATUS
2254 NTAPI
2255 MiQueryMemorySectionName(
2256     IN HANDLE ProcessHandle,
2257     IN PVOID BaseAddress,
2258     OUT PVOID MemoryInformation,
2259     IN SIZE_T MemoryInformationLength,
2260     OUT PSIZE_T ReturnLength
2261 );
2262 
2263 NTSTATUS
2264 NTAPI
2265 MiRosUnmapViewInSystemSpace(
2266     IN PVOID MappedBase
2267 );
2268 
2269 POOL_TYPE
2270 NTAPI
2271 MmDeterminePoolType(
2272     IN PVOID PoolAddress
2273 );
2274 
2275 VOID
2276 NTAPI
2277 MiMakePdeExistAndMakeValid(
2278     IN PMMPDE PointerPde,
2279     IN PEPROCESS TargetProcess,
2280     IN KIRQL OldIrql
2281 );
2282 
2283 VOID
2284 NTAPI
2285 MiWriteProtectSystemImage(
2286     _In_ PVOID ImageBase);
2287 
2288 //
2289 // MiRemoveZeroPage will use inline code to zero out the page manually if only
2290 // free pages are available. In some scenarios, we don't/can't run that piece of
2291 // code and would rather only have a real zero page. If we can't have a zero page,
2292 // then we'd like to have our own code to grab a free page and zero it out, by
2293 // using MiRemoveAnyPage. This macro implements this.
2294 //
2295 FORCEINLINE
2296 PFN_NUMBER
2297 MiRemoveZeroPageSafe(IN ULONG Color)
2298 {
2299     if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color);
2300     return 0;
2301 }
2302 
2303 #if (_MI_PAGING_LEVELS == 2)
2304 FORCEINLINE
2305 BOOLEAN
2306 MiSynchronizeSystemPde(PMMPDE PointerPde)
2307 {
2308     MMPDE SystemPde;
2309     ULONG Index;
2310 
2311     /* Get the Index from the PDE */
2312     Index = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE);
2313 
2314     /* Copy the PDE from the double-mapped system page directory */
2315     SystemPde = MmSystemPagePtes[Index];
2316     *PointerPde = SystemPde;
2317 
2318     /* Make sure we re-read the PDE and PTE */
2319     KeMemoryBarrierWithoutFence();
2320 
2321     /* Return, if we had success */
2322     return SystemPde.u.Hard.Valid != 0;
2323 }
2324 #endif
2325 
2326 /* EOF */
2327