xref: /reactos/ntoskrnl/mm/ARM3/miarm.h (revision 92dfec21) 
1 /*
2  * PROJECT:         ReactOS Kernel
3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
4  * FILE:            ntoskrnl/mm/ARM3/miarm.h
5  * PURPOSE:         ARM Memory Manager Header
6  * PROGRAMMERS:     ReactOS Portable Systems Group
7  */
8 
9 #pragma once
10 
11 #define MI_LOWEST_VAD_ADDRESS                   (PVOID)MM_LOWEST_USER_ADDRESS
12 
13 /* Make the code cleaner with some definitions for size multiples */
14 #define _1KB (1024u)
15 #define _1MB (1024 * _1KB)
16 #define _1GB (1024 * _1MB)
17 
18 /* Everyone loves 64K */
19 #define _64K (64 * _1KB)
20 
21 /* Size of a page table */
22 #define PT_SIZE  (PTE_PER_PAGE * sizeof(MMPTE))
23 
24 /* Size of a page directory */
25 #define PD_SIZE  (PDE_PER_PAGE * sizeof(MMPDE))
26 
27 /* Size of all page directories for a process */
28 #define SYSTEM_PD_SIZE (PPE_PER_PAGE * PD_SIZE)
29 #ifdef _M_IX86
30 C_ASSERT(SYSTEM_PD_SIZE == PAGE_SIZE);
31 #endif
32 
33 //
34 // Protection Bits part of the internal memory manager Protection Mask, from:
35 // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel
36 // https://www.reactos.org/wiki/Techwiki:Memory_Protection_constants
37 // and public assertions.
38 //
39 #define MM_ZERO_ACCESS         0
40 #define MM_READONLY            1
41 #define MM_EXECUTE             2
42 #define MM_EXECUTE_READ        3
43 #define MM_READWRITE           4
44 #define MM_WRITECOPY           5
45 #define MM_EXECUTE_READWRITE   6
46 #define MM_EXECUTE_WRITECOPY   7
47 #define MM_PROTECT_ACCESS      7
48 
49 //
50 // These are flags on top of the actual protection mask
51 //
52 #define MM_NOCACHE            0x08
53 #define MM_GUARDPAGE          0x10
54 #define MM_WRITECOMBINE       0x18
55 #define MM_PROTECT_SPECIAL    0x18
56 
57 //
58 // These are special cases
59 //
60 #define MM_DECOMMIT           (MM_ZERO_ACCESS | MM_GUARDPAGE)
61 #define MM_NOACCESS           (MM_ZERO_ACCESS | MM_WRITECOMBINE)
62 #define MM_OUTSWAPPED_KSTACK  (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE)
63 #define MM_INVALID_PROTECTION  0xFFFFFFFF
64 
65 //
66 // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits
67 // The Memory Manager's definition define the attributes that must be preserved
68 // and these PTE definitions describe the attributes in the hardware sense. This
69 // helps deal with hardware differences between the actual boolean expression of
70 // the argument.
71 //
72 // For example, in the logical attributes, we want to express read-only as a flag
73 // but on x86, it is writability that must be set. On the other hand, on x86, just
74 // like in the kernel, it is disabling the caches that requires a special flag,
75 // while on certain architectures such as ARM, it is enabling the cache which
76 // requires a flag.
77 //
78 #if defined(_M_IX86)
79 //
80 // Access Flags
81 //
82 #define PTE_READONLY            0 // Doesn't exist on x86
83 #define PTE_EXECUTE             0 // Not worrying about NX yet
84 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
85 #define PTE_READWRITE           0x2
86 #define PTE_WRITECOPY           0x200
87 #define PTE_EXECUTE_READWRITE   0x2 // Not worrying about NX yet
88 #define PTE_EXECUTE_WRITECOPY   0x200
89 #define PTE_PROTOTYPE           0x400
90 
91 //
92 // State Flags
93 //
94 #define PTE_VALID               0x1
95 #define PTE_ACCESSED            0x20
96 #define PTE_DIRTY               0x40
97 
98 //
99 // Cache flags
100 //
101 #define PTE_ENABLE_CACHE        0
102 #define PTE_DISABLE_CACHE       0x10
103 #define PTE_WRITECOMBINED_CACHE 0x10
104 #define PTE_PROTECT_MASK        0x612
105 #elif defined(_M_AMD64)
106 //
107 // Access Flags
108 //
109 #define PTE_READONLY            0x8000000000000000ULL
110 #define PTE_EXECUTE             0x0000000000000000ULL
111 #define PTE_EXECUTE_READ        PTE_EXECUTE /* EXECUTE implies READ on x64 */
112 #define PTE_READWRITE           0x8000000000000002ULL
113 #define PTE_WRITECOPY           0x8000000000000200ULL
114 #define PTE_EXECUTE_READWRITE   0x0000000000000002ULL
115 #define PTE_EXECUTE_WRITECOPY   0x0000000000000200ULL
116 #define PTE_PROTOTYPE           0x0000000000000400ULL
117 
118 //
119 // State Flags
120 //
121 #define PTE_VALID               0x0000000000000001ULL
122 #define PTE_ACCESSED            0x0000000000000020ULL
123 #define PTE_DIRTY               0x0000000000000040ULL
124 
125 //
126 // Cache flags
127 //
128 #define PTE_ENABLE_CACHE        0x0000000000000000ULL
129 #define PTE_DISABLE_CACHE       0x0000000000000010ULL
130 #define PTE_WRITECOMBINED_CACHE 0x0000000000000010ULL
131 #define PTE_PROTECT_MASK        0x8000000000000612ULL
132 #elif defined(_M_ARM)
133 #define PTE_READONLY            0x200
134 #define PTE_EXECUTE             0 // Not worrying about NX yet
135 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
136 #define PTE_READWRITE           0 // Doesn't exist on ARM
137 #define PTE_WRITECOPY           0 // Doesn't exist on ARM
138 #define PTE_EXECUTE_READWRITE   0 // Not worrying about NX yet
139 #define PTE_EXECUTE_WRITECOPY   0 // Not worrying about NX yet
140 #define PTE_PROTOTYPE           0x400 // Using the Shared bit
141 
142 //
143 // Cache flags
144 //
145 #define PTE_ENABLE_CACHE        0
146 #define PTE_DISABLE_CACHE       0x10
147 #define PTE_WRITECOMBINED_CACHE 0x10
148 #define PTE_PROTECT_MASK        0x610
149 #else
150 #error Define these please!
151 #endif
152 
153 //
154 // Mask for image section page protection
155 //
156 #define IMAGE_SCN_PROTECTION_MASK (IMAGE_SCN_MEM_WRITE | IMAGE_SCN_MEM_READ | IMAGE_SCN_MEM_EXECUTE)
157 
158 extern const ULONG_PTR MmProtectToPteMask[32];
159 extern const ULONG MmProtectToValue[32];
160 
161 //
162 // Assertions for session images, addresses, and PTEs
163 //
164 #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \
165     (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))
166 
167 #define MI_IS_SESSION_ADDRESS(Address) \
168     (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))
169 
170 #define MI_IS_SESSION_PTE(Pte) \
171     ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))
172 
173 #define MI_IS_PAGE_TABLE_ADDRESS(Address) \
174     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))
175 
176 #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \
177     (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))
178 
179 #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \
180     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))
181 
182 //
183 // Creates a software PTE with the given protection
184 //
185 #define MI_MAKE_SOFTWARE_PTE(p, x)          ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))
186 
187 //
188 // Marks a PTE as deleted
189 //
190 #define MI_SET_PFN_DELETED(x)               ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1))
191 #define MI_IS_PFN_DELETED(x)                ((ULONG_PTR)((x)->PteAddress) & 1)
192 
193 //
194 // Special values for LoadedImports
195 //
196 #ifdef _WIN64
197 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFFFFFFFFFEULL
198 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFFFFFFFFFFULL
199 #else
200 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFE
201 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFF
202 #endif
203 #define MM_SYSLDR_SINGLE_ENTRY 0x1
204 
205 //
206 // Number of initial session IDs
207 //
208 #define MI_INITIAL_SESSION_IDS  64
209 
210 #if defined(_M_IX86) || defined(_M_ARM)
211 //
212 // PFN List Sentinel
213 //
214 #define LIST_HEAD 0xFFFFFFFF
215 
216 //
217 // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits,
218 // we need a manual definition suited to the number of bits in the PteFrame.
219 // This is used as a LIST_HEAD for the colored list
220 //
221 #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF
222 #elif defined(_M_AMD64)
223 #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL
224 #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL
225 #else
226 #error Define these please!
227 #endif
228 
229 //
230 // Special IRQL value (found in assertions)
231 //
232 #define MM_NOIRQL (KIRQL)0xFFFFFFFF
233 
234 //
235 // Returns the color of a page
236 //
237 #define MI_GET_PAGE_COLOR(x)                ((x) & MmSecondaryColorMask)
238 #define MI_GET_NEXT_COLOR()                 (MI_GET_PAGE_COLOR(++MmSystemPageColor))
239 #define MI_GET_NEXT_PROCESS_COLOR(x)        (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
240 
241 //
242 // Prototype PTEs that don't yet have a pagefile association
243 //
244 #ifdef _WIN64
245 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL
246 #else
247 #define MI_PTE_LOOKUP_NEEDED 0xFFFFF
248 #endif
249 
250 //
251 // Number of session data and tag pages
252 //
253 #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
254 #define MI_SESSION_TAG_PAGES_MAXIMUM  (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
255 
256 //
257 // Used by MiCheckSecuredVad
258 //
259 #define MM_READ_WRITE_ALLOWED   11
260 #define MM_READ_ONLY_ALLOWED    10
261 #define MM_NO_ACCESS_ALLOWED    01
262 #define MM_DELETE_CHECK         85
263 
264 //
265 // System views are binned into 64K chunks
266 //
267 #define MI_SYSTEM_VIEW_BUCKET_SIZE  _64K
268 
269 //
270 // FIXFIX: These should go in ex.h after the pool merge
271 //
272 #ifdef _WIN64
273 #define POOL_BLOCK_SIZE 16
274 #else
275 #define POOL_BLOCK_SIZE  8
276 #endif
277 #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)
278 #define BASE_POOL_TYPE_MASK 1
279 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))
280 
281 //
282 // Pool debugging/analysis/tracing flags
283 //
284 #define POOL_FLAG_CHECK_TIMERS 0x1
285 #define POOL_FLAG_CHECK_WORKERS 0x2
286 #define POOL_FLAG_CHECK_RESOURCES 0x4
287 #define POOL_FLAG_VERIFIER 0x8
288 #define POOL_FLAG_CHECK_DEADLOCK 0x10
289 #define POOL_FLAG_SPECIAL_POOL 0x20
290 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40
291 #define POOL_FLAG_CRASH_ON_FAILURE 0x80
292 
293 //
294 // BAD_POOL_HEADER codes during pool bugcheck
295 //
296 #define POOL_CORRUPTED_LIST 3
297 #define POOL_SIZE_OR_INDEX_MISMATCH 5
298 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6
299 #define POOL_HEADER_NOT_ALIGNED 7
300 #define POOL_HEADER_IS_ZERO 8
301 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9
302 #define POOL_ENTRY_NOT_FOUND 10
303 
304 //
305 // BAD_POOL_CALLER codes during pool bugcheck
306 //
307 #define POOL_ENTRY_CORRUPTED 1
308 #define POOL_ENTRY_ALREADY_FREE 6
309 #define POOL_ENTRY_NOT_ALLOCATED 7
310 #define POOL_ALLOC_IRQL_INVALID 8
311 #define POOL_FREE_IRQL_INVALID 9
312 #define POOL_BILLED_PROCESS_INVALID 13
313 #define POOL_HEADER_SIZE_INVALID 32
314 
315 typedef struct _POOL_DESCRIPTOR
316 {
317     POOL_TYPE PoolType;
318     ULONG PoolIndex;
319     ULONG RunningAllocs;
320     ULONG RunningDeAllocs;
321     ULONG TotalPages;
322     ULONG TotalBigPages;
323     ULONG Threshold;
324     PVOID LockAddress;
325     PVOID PendingFrees;
326     LONG PendingFreeDepth;
327     SIZE_T TotalBytes;
328     SIZE_T Spare0;
329     LIST_ENTRY ListHeads[POOL_LISTS_PER_PAGE];
330 } POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
331 
332 typedef struct _POOL_HEADER
333 {
334     union
335     {
336         struct
337         {
338 #ifdef _WIN64
339             USHORT PreviousSize:8;
340             USHORT PoolIndex:8;
341             USHORT BlockSize:8;
342             USHORT PoolType:8;
343 #else
344             USHORT PreviousSize:9;
345             USHORT PoolIndex:7;
346             USHORT BlockSize:9;
347             USHORT PoolType:7;
348 #endif
349         };
350         ULONG Ulong1;
351     };
352 #ifdef _WIN64
353     ULONG PoolTag;
354 #endif
355     union
356     {
357 #ifdef _WIN64
358         PEPROCESS ProcessBilled;
359 #else
360         ULONG PoolTag;
361 #endif
362         struct
363         {
364             USHORT AllocatorBackTraceIndex;
365             USHORT PoolTagHash;
366         };
367     };
368 } POOL_HEADER, *PPOOL_HEADER;
369 
370 C_ASSERT(sizeof(POOL_HEADER) == POOL_BLOCK_SIZE);
371 C_ASSERT(POOL_BLOCK_SIZE == sizeof(LIST_ENTRY));
372 
373 typedef struct _POOL_TRACKER_TABLE
374 {
375     ULONG Key;
376     LONG NonPagedAllocs;
377     LONG NonPagedFrees;
378     SIZE_T NonPagedBytes;
379     LONG PagedAllocs;
380     LONG PagedFrees;
381     SIZE_T PagedBytes;
382 } POOL_TRACKER_TABLE, *PPOOL_TRACKER_TABLE;
383 
384 typedef struct _POOL_TRACKER_BIG_PAGES
385 {
386     PVOID Va;
387     ULONG Key;
388     ULONG NumberOfPages;
389     PVOID QuotaObject;
390 } POOL_TRACKER_BIG_PAGES, *PPOOL_TRACKER_BIG_PAGES;
391 
392 extern ULONG ExpNumberOfPagedPools;
393 extern POOL_DESCRIPTOR NonPagedPoolDescriptor;
394 extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16 + 1];
395 extern PPOOL_TRACKER_TABLE PoolTrackTable;
396 
397 //
398 // END FIXFIX
399 //
400 
401 typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY
402 {
403     LIST_ENTRY Links;
404     UNICODE_STRING BaseName;
405 } MI_LARGE_PAGE_DRIVER_ENTRY, *PMI_LARGE_PAGE_DRIVER_ENTRY;
406 
407 typedef enum _MMSYSTEM_PTE_POOL_TYPE
408 {
409     SystemPteSpace,
410     NonPagedPoolExpansion,
411     MaximumPtePoolTypes
412 } MMSYSTEM_PTE_POOL_TYPE;
413 
414 typedef enum _MI_PFN_CACHE_ATTRIBUTE
415 {
416     MiNonCached,
417     MiCached,
418     MiWriteCombined,
419     MiNotMapped
420 } MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;
421 
422 typedef struct _PHYSICAL_MEMORY_RUN
423 {
424     PFN_NUMBER BasePage;
425     PFN_NUMBER PageCount;
426 } PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN;
427 
428 typedef struct _PHYSICAL_MEMORY_DESCRIPTOR
429 {
430     ULONG NumberOfRuns;
431     PFN_NUMBER NumberOfPages;
432     PHYSICAL_MEMORY_RUN Run[1];
433 } PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR;
434 
435 typedef struct _MMCOLOR_TABLES
436 {
437     PFN_NUMBER Flink;
438     PVOID Blink;
439     PFN_NUMBER Count;
440 } MMCOLOR_TABLES, *PMMCOLOR_TABLES;
441 
442 typedef struct _MI_LARGE_PAGE_RANGES
443 {
444     PFN_NUMBER StartFrame;
445     PFN_NUMBER LastFrame;
446 } MI_LARGE_PAGE_RANGES, *PMI_LARGE_PAGE_RANGES;
447 
448 typedef struct _MMVIEW
449 {
450     ULONG_PTR Entry;
451     PCONTROL_AREA ControlArea;
452 } MMVIEW, *PMMVIEW;
453 
454 typedef struct _MMSESSION
455 {
456     KGUARDED_MUTEX SystemSpaceViewLock;
457     PKGUARDED_MUTEX SystemSpaceViewLockPointer;
458     PCHAR SystemSpaceViewStart;
459     PMMVIEW SystemSpaceViewTable;
460     ULONG SystemSpaceHashSize;
461     ULONG SystemSpaceHashEntries;
462     ULONG SystemSpaceHashKey;
463     ULONG BitmapFailures;
464     PRTL_BITMAP SystemSpaceBitMap;
465 } MMSESSION, *PMMSESSION;
466 
467 typedef struct _MM_SESSION_SPACE_FLAGS
468 {
469     ULONG Initialized:1;
470     ULONG DeletePending:1;
471     ULONG Filler:30;
472 } MM_SESSION_SPACE_FLAGS;
473 
474 typedef struct _MM_SESSION_SPACE
475 {
476     struct _MM_SESSION_SPACE *GlobalVirtualAddress;
477     LONG ReferenceCount;
478     union
479     {
480         ULONG LongFlags;
481         MM_SESSION_SPACE_FLAGS Flags;
482     } u;
483     ULONG SessionId;
484     LIST_ENTRY ProcessList;
485     LARGE_INTEGER LastProcessSwappedOutTime;
486     PFN_NUMBER SessionPageDirectoryIndex;
487     SIZE_T NonPageablePages;
488     SIZE_T CommittedPages;
489     PVOID PagedPoolStart;
490     PVOID PagedPoolEnd;
491     PMMPDE PagedPoolBasePde;
492     ULONG Color;
493     LONG ResidentProcessCount;
494     ULONG SessionPoolAllocationFailures[4];
495     LIST_ENTRY ImageList;
496     LCID LocaleId;
497     ULONG AttachCount;
498     KEVENT AttachEvent;
499     PEPROCESS LastProcess;
500     LONG ProcessReferenceToSession;
501     LIST_ENTRY WsListEntry;
502     GENERAL_LOOKASIDE Lookaside[SESSION_POOL_LOOKASIDES];
503     MMSESSION Session;
504     KGUARDED_MUTEX PagedPoolMutex;
505     MM_PAGED_POOL_INFO PagedPoolInfo;
506     MMSUPPORT Vm;
507     PMMWSLE Wsle;
508     PDRIVER_UNLOAD Win32KDriverUnload;
509     POOL_DESCRIPTOR PagedPool;
510 #if defined (_M_AMD64)
511     MMPDE PageDirectory;
512 #else
513     PMMPDE PageTables;
514 #endif
515 #if defined (_M_AMD64)
516     PMMPTE SpecialPoolFirstPte;
517     PMMPTE SpecialPoolLastPte;
518     PMMPTE NextPdeForSpecialPoolExpansion;
519     PMMPTE LastPdeForSpecialPoolExpansion;
520     PFN_NUMBER SpecialPagesInUse;
521 #endif
522     LONG ImageLoadingCount;
523 } MM_SESSION_SPACE, *PMM_SESSION_SPACE;
524 
525 extern PMM_SESSION_SPACE MmSessionSpace;
526 extern MMPTE HyperTemplatePte;
527 extern MMPDE ValidKernelPde;
528 extern MMPTE ValidKernelPte;
529 extern MMPDE ValidKernelPdeLocal;
530 extern MMPTE ValidKernelPteLocal;
531 extern MMPDE DemandZeroPde;
532 extern MMPTE DemandZeroPte;
533 extern MMPTE PrototypePte;
534 extern MMPTE MmDecommittedPte;
535 extern BOOLEAN MmLargeSystemCache;
536 extern BOOLEAN MmZeroPageFile;
537 extern BOOLEAN MmProtectFreedNonPagedPool;
538 extern BOOLEAN MmTrackLockedPages;
539 extern BOOLEAN MmTrackPtes;
540 extern BOOLEAN MmDynamicPfn;
541 extern BOOLEAN MmMirroring;
542 extern BOOLEAN MmMakeLowMemory;
543 extern BOOLEAN MmEnforceWriteProtection;
544 extern SIZE_T MmAllocationFragment;
545 extern ULONG MmConsumedPoolPercentage;
546 extern ULONG MmVerifyDriverBufferType;
547 extern ULONG MmVerifyDriverLevel;
548 extern WCHAR MmVerifyDriverBuffer[512];
549 extern WCHAR MmLargePageDriverBuffer[512];
550 extern LIST_ENTRY MiLargePageDriverList;
551 extern BOOLEAN MiLargePageAllDrivers;
552 extern ULONG MmVerifyDriverBufferLength;
553 extern ULONG MmLargePageDriverBufferLength;
554 extern SIZE_T MmSizeOfNonPagedPoolInBytes;
555 extern SIZE_T MmMaximumNonPagedPoolInBytes;
556 extern PFN_NUMBER MmMaximumNonPagedPoolInPages;
557 extern PFN_NUMBER MmSizeOfPagedPoolInPages;
558 extern PVOID MmNonPagedSystemStart;
559 extern PVOID MmNonPagedPoolStart;
560 extern PVOID MmNonPagedPoolExpansionStart;
561 extern PVOID MmNonPagedPoolEnd;
562 extern SIZE_T MmSizeOfPagedPoolInBytes;
563 extern PVOID MmPagedPoolStart;
564 extern PVOID MmPagedPoolEnd;
565 extern PVOID MmSessionBase;
566 extern SIZE_T MmSessionSize;
567 extern PMMPTE MmFirstReservedMappingPte, MmLastReservedMappingPte;
568 extern PMMPTE MiFirstReservedZeroingPte;
569 extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2][MmMaximumCacheType];
570 extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock;
571 extern SIZE_T MmBootImageSize;
572 extern PMMPTE MmSystemPtesStart[MaximumPtePoolTypes];
573 extern PMMPTE MmSystemPtesEnd[MaximumPtePoolTypes];
574 extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor;
575 extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor;
576 extern ULONG_PTR MxPfnAllocation;
577 extern MM_PAGED_POOL_INFO MmPagedPoolInfo;
578 extern RTL_BITMAP MiPfnBitMap;
579 extern KGUARDED_MUTEX MmPagedPoolMutex;
580 extern KGUARDED_MUTEX MmSectionCommitMutex;
581 extern PVOID MmPagedPoolStart;
582 extern PVOID MmPagedPoolEnd;
583 extern PVOID MmNonPagedSystemStart;
584 extern PVOID MiSystemViewStart;
585 extern SIZE_T MmSystemViewSize;
586 extern PVOID MmSessionBase;
587 extern PVOID MiSessionSpaceEnd;
588 extern PMMPTE MiSessionImagePteStart;
589 extern PMMPTE MiSessionImagePteEnd;
590 extern PMMPTE MiSessionBasePte;
591 extern PMMPTE MiSessionLastPte;
592 extern SIZE_T MmSizeOfPagedPoolInBytes;
593 extern PMMPDE MmSystemPagePtes;
594 extern PVOID MmSystemCacheStart;
595 extern PVOID MmSystemCacheEnd;
596 extern MMSUPPORT MmSystemCacheWs;
597 extern SIZE_T MmAllocatedNonPagedPool;
598 extern ULONG MmSpecialPoolTag;
599 extern PVOID MmHyperSpaceEnd;
600 extern PMMWSL MmSystemCacheWorkingSetList;
601 extern SIZE_T MmMinimumNonPagedPoolSize;
602 extern ULONG MmMinAdditionNonPagedPoolPerMb;
603 extern SIZE_T MmDefaultMaximumNonPagedPool;
604 extern ULONG MmMaxAdditionNonPagedPoolPerMb;
605 extern ULONG MmSecondaryColors;
606 extern ULONG MmSecondaryColorMask;
607 extern ULONG MmNumberOfSystemPtes;
608 extern ULONG MmMaximumNonPagedPoolPercent;
609 extern ULONG MmLargeStackSize;
610 extern PMMCOLOR_TABLES MmFreePagesByColor[FreePageList + 1];
611 extern MMPFNLIST MmStandbyPageListByPriority[8];
612 extern ULONG MmProductType;
613 extern MM_SYSTEMSIZE MmSystemSize;
614 extern PKEVENT MiLowMemoryEvent;
615 extern PKEVENT MiHighMemoryEvent;
616 extern PKEVENT MiLowPagedPoolEvent;
617 extern PKEVENT MiHighPagedPoolEvent;
618 extern PKEVENT MiLowNonPagedPoolEvent;
619 extern PKEVENT MiHighNonPagedPoolEvent;
620 extern PFN_NUMBER MmLowMemoryThreshold;
621 extern PFN_NUMBER MmHighMemoryThreshold;
622 extern PFN_NUMBER MiLowPagedPoolThreshold;
623 extern PFN_NUMBER MiHighPagedPoolThreshold;
624 extern PFN_NUMBER MiLowNonPagedPoolThreshold;
625 extern PFN_NUMBER MiHighNonPagedPoolThreshold;
626 extern PFN_NUMBER MmMinimumFreePages;
627 extern PFN_NUMBER MmPlentyFreePages;
628 extern SIZE_T MmMinimumStackCommitInBytes;
629 extern PFN_COUNT MiExpansionPoolPagesInitialCharge;
630 extern PFN_NUMBER MmResidentAvailablePages;
631 extern PFN_NUMBER MmResidentAvailableAtInit;
632 extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
633 extern PFN_NUMBER MmTotalSystemDriverPages;
634 extern ULONG MmCritsectTimeoutSeconds;
635 extern PVOID MiSessionImageStart;
636 extern PVOID MiSessionImageEnd;
637 extern PMMPTE MiHighestUserPte;
638 extern PMMPDE MiHighestUserPde;
639 extern PFN_NUMBER MmSystemPageDirectory[PPE_PER_PAGE];
640 extern PMMPTE MmSharedUserDataPte;
641 extern LIST_ENTRY MmProcessList;
642 extern KEVENT MmZeroingPageEvent;
643 extern ULONG MmSystemPageColor;
644 extern ULONG MmProcessColorSeed;
645 extern PMMWSL MmWorkingSetList;
646 extern PFN_NUMBER MiNumberOfFreePages;
647 extern SIZE_T MmSessionViewSize;
648 extern SIZE_T MmSessionPoolSize;
649 extern SIZE_T MmSessionImageSize;
650 extern PVOID MiSystemViewStart;
651 extern PVOID MiSessionPoolEnd;     // 0xBE000000
652 extern PVOID MiSessionPoolStart;   // 0xBD000000
653 extern PVOID MiSessionViewStart;   // 0xBE000000
654 extern PVOID MiSessionSpaceWs;
655 extern ULONG MmMaximumDeadKernelStacks;
656 extern SLIST_HEADER MmDeadStackSListHead;
657 extern MM_AVL_TABLE MmSectionBasedRoot;
658 extern KGUARDED_MUTEX MmSectionBasedMutex;
659 extern PVOID MmHighSectionBase;
660 extern SIZE_T MmSystemLockPagesCount;
661 extern ULONG_PTR MmSubsectionBase;
662 extern LARGE_INTEGER MmCriticalSectionTimeout;
663 extern LIST_ENTRY MmWorkingSetExpansionHead;
664 extern KSPIN_LOCK MmExpansionLock;
665 extern PETHREAD MiExpansionLockOwner;
666 
667 FORCEINLINE
668 BOOLEAN
669 MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType)
670 {
671     return ((MemoryType == LoaderFree) ||
672             (MemoryType == LoaderLoadedProgram) ||
673             (MemoryType == LoaderFirmwareTemporary) ||
674             (MemoryType == LoaderOsloaderStack));
675 }
676 
677 FORCEINLINE
678 BOOLEAN
679 MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType)
680 {
681     return ((MemoryType == LoaderFirmwarePermanent) ||
682             (MemoryType == LoaderSpecialMemory) ||
683             (MemoryType == LoaderHALCachedMemory) ||
684             (MemoryType == LoaderBBTMemory));
685 }
686 
687 #ifdef _M_AMD64
688 FORCEINLINE
689 BOOLEAN
690 MiIsUserPxe(PVOID Address)
691 {
692     return ((ULONG_PTR)Address >> 7) == 0x1FFFFEDF6FB7DA0ULL;
693 }
694 
695 FORCEINLINE
696 BOOLEAN
697 MiIsUserPpe(PVOID Address)
698 {
699     return ((ULONG_PTR)Address >> 16) == 0xFFFFF6FB7DA0ULL;
700 }
701 
702 FORCEINLINE
703 BOOLEAN
704 MiIsUserPde(PVOID Address)
705 {
706     return ((ULONG_PTR)Address >> 25) == 0x7FFFFB7DA0ULL;
707 }
708 
709 FORCEINLINE
710 BOOLEAN
711 MiIsUserPte(PVOID Address)
712 {
713     return ((ULONG_PTR)Address >> 34) == 0x3FFFFDA0ULL;
714 }
715 #else
716 FORCEINLINE
717 BOOLEAN
718 MiIsUserPde(PVOID Address)
719 {
720     return ((Address >= (PVOID)MiAddressToPde(NULL)) &&
721             (Address <= (PVOID)MiHighestUserPde));
722 }
723 
724 FORCEINLINE
725 BOOLEAN
726 MiIsUserPte(PVOID Address)
727 {
728     return (Address <= (PVOID)MiHighestUserPte);
729 }
730 #endif
731 
732 //
733 // Figures out the hardware bits for a PTE
734 //
735 FORCEINLINE
736 ULONG_PTR
737 MiDetermineUserGlobalPteMask(IN PVOID PointerPte)
738 {
739     MMPTE TempPte;
740 
741     /* Start fresh */
742     TempPte.u.Long = 0;
743 
744     /* Make it valid and accessed */
745     TempPte.u.Hard.Valid = TRUE;
746     MI_MAKE_ACCESSED_PAGE(&TempPte);
747 
748     /* Is this for user-mode? */
749     if (
750 #if (_MI_PAGING_LEVELS == 4)
751         MiIsUserPxe(PointerPte) ||
752 #endif
753 #if (_MI_PAGING_LEVELS >= 3)
754         MiIsUserPpe(PointerPte) ||
755 #endif
756         MiIsUserPde(PointerPte) ||
757         MiIsUserPte(PointerPte))
758     {
759         /* Set the owner bit */
760         MI_MAKE_OWNER_PAGE(&TempPte);
761     }
762 
763     /* FIXME: We should also set the global bit */
764 
765     /* Return the protection */
766     return TempPte.u.Long;
767 }
768 
769 //
770 // Creates a valid kernel PTE with the given protection
771 //
772 FORCEINLINE
773 VOID
774 MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte,
775                             IN PMMPTE MappingPte,
776                             IN ULONG_PTR ProtectionMask,
777                             IN PFN_NUMBER PageFrameNumber)
778 {
779     /* Only valid for kernel, non-session PTEs */
780     ASSERT(MappingPte > MiHighestUserPte);
781     ASSERT(!MI_IS_SESSION_PTE(MappingPte));
782     ASSERT((MappingPte < (PMMPTE)PDE_BASE) || (MappingPte > (PMMPTE)PDE_TOP));
783 
784     /* Start fresh */
785     *NewPte = ValidKernelPte;
786 
787     /* Set the protection and page */
788     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
789     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
790 }
791 
792 //
793 // Creates a valid PTE with the given protection
794 //
795 FORCEINLINE
796 VOID
797 MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte,
798                      IN PMMPTE MappingPte,
799                      IN ULONG_PTR ProtectionMask,
800                      IN PFN_NUMBER PageFrameNumber)
801 {
802     /* Set the protection and page */
803     NewPte->u.Long = MiDetermineUserGlobalPteMask(MappingPte);
804     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
805     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
806 }
807 
808 //
809 // Creates a valid user PTE with the given protection
810 //
811 FORCEINLINE
812 VOID
813 MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte,
814                           IN PMMPTE MappingPte,
815                           IN ULONG_PTR ProtectionMask,
816                           IN PFN_NUMBER PageFrameNumber)
817 {
818     /* Only valid for kernel, non-session PTEs */
819     ASSERT(MappingPte <= MiHighestUserPte);
820 
821     /* Start fresh */
822     NewPte->u.Long = 0;
823 
824     /* Set the protection and page */
825     NewPte->u.Hard.Valid = TRUE;
826     NewPte->u.Hard.Owner = TRUE;
827     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
828     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
829 }
830 
831 #ifndef _M_AMD64
832 //
833 // Builds a Prototype PTE for the address of the PTE
834 //
835 FORCEINLINE
836 VOID
837 MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte,
838                       IN PMMPTE PointerPte)
839 {
840     ULONG_PTR Offset;
841 
842     /* Mark this as a prototype */
843     NewPte->u.Long = 0;
844     NewPte->u.Proto.Prototype = 1;
845 
846     /*
847      * Prototype PTEs are only valid in paged pool by design, this little trick
848      * lets us only use 30 bits for the adress of the PTE, as long as the area
849      * stays 1024MB At most.
850      */
851     Offset = (ULONG_PTR)PointerPte - (ULONG_PTR)MmPagedPoolStart;
852 
853     /*
854      * 7 bits go in the "low" (but we assume the bottom 2 are zero)
855      * and the other 21 bits go in the "high"
856      */
857     NewPte->u.Proto.ProtoAddressLow = (Offset & 0x1FC) >> 2;
858     NewPte->u.Proto.ProtoAddressHigh = (Offset & 0x3FFFFE00) >> 9;
859 }
860 
861 //
862 // Builds a Subsection PTE for the address of the Segment
863 //
864 FORCEINLINE
865 VOID
866 MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte,
867                        IN PVOID Segment)
868 {
869     ULONG_PTR Offset;
870 
871     /* Mark this as a prototype */
872     NewPte->u.Long = 0;
873     NewPte->u.Subsect.Prototype = 1;
874 
875     /*
876      * Segments are only valid either in nonpaged pool. We store the 20 bit
877      * difference either from the top or bottom of nonpaged pool, giving a
878      * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed
879      * 256MB.
880      */
881     if ((ULONG_PTR)Segment < ((ULONG_PTR)MmSubsectionBase + (128 * _1MB)))
882     {
883         Offset = (ULONG_PTR)Segment - (ULONG_PTR)MmSubsectionBase;
884         NewPte->u.Subsect.WhichPool = PagedPool;
885     }
886     else
887     {
888         Offset = (ULONG_PTR)MmNonPagedPoolEnd - (ULONG_PTR)Segment;
889         NewPte->u.Subsect.WhichPool = NonPagedPool;
890     }
891 
892     /*
893      * 4 bits go in the "low" (but we assume the bottom 3 are zero)
894      * and the other 20 bits go in the "high"
895      */
896     NewPte->u.Subsect.SubsectionAddressLow = (Offset & 0x78) >> 3;
897     NewPte->u.Subsect.SubsectionAddressHigh = (Offset & 0xFFFFF80) >> 7;
898 }
899 
900 FORCEINLINE
901 BOOLEAN
902 MI_IS_MAPPED_PTE(PMMPTE PointerPte)
903 {
904     /// \todo Make this reasonable code, this is UGLY!
905     return ((PointerPte->u.Long & 0xFFFFFC01) != 0);
906 }
907 
908 #endif
909 
910 FORCEINLINE
911 VOID
912 MI_MAKE_TRANSITION_PTE(_Out_ PMMPTE NewPte,
913                        _In_ PFN_NUMBER Page,
914                        _In_ ULONG Protection)
915 {
916     NewPte->u.Long = 0;
917     NewPte->u.Trans.Transition = 1;
918     NewPte->u.Trans.Protection = Protection;
919     NewPte->u.Trans.PageFrameNumber = Page;
920 }
921 
922 //
923 // Returns if the page is physically resident (ie: a large page)
924 // FIXFIX: CISC/x86 only?
925 //
926 FORCEINLINE
927 BOOLEAN
928 MI_IS_PHYSICAL_ADDRESS(IN PVOID Address)
929 {
930     PMMPDE PointerPde;
931 
932     /* Large pages are never paged out, always physically resident */
933     PointerPde = MiAddressToPde(Address);
934     return ((PointerPde->u.Hard.LargePage) && (PointerPde->u.Hard.Valid));
935 }
936 
937 //
938 // Writes a valid PTE
939 //
940 FORCEINLINE
941 VOID
942 MI_WRITE_VALID_PTE(IN PMMPTE PointerPte,
943                    IN MMPTE TempPte)
944 {
945     /* Write the valid PTE */
946     ASSERT(PointerPte->u.Hard.Valid == 0);
947     ASSERT(TempPte.u.Hard.Valid == 1);
948     *PointerPte = TempPte;
949 }
950 
951 //
952 // Updates a valid PTE
953 //
954 FORCEINLINE
955 VOID
956 MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte,
957                    IN MMPTE TempPte)
958 {
959     /* Write the valid PTE */
960     ASSERT(PointerPte->u.Hard.Valid == 1);
961     ASSERT(TempPte.u.Hard.Valid == 1);
962     ASSERT(PointerPte->u.Hard.PageFrameNumber == TempPte.u.Hard.PageFrameNumber);
963     *PointerPte = TempPte;
964 }
965 
966 //
967 // Writes an invalid PTE
968 //
969 FORCEINLINE
970 VOID
971 MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte,
972                      IN MMPTE InvalidPte)
973 {
974     /* Write the invalid PTE */
975     ASSERT(InvalidPte.u.Hard.Valid == 0);
976     ASSERT(InvalidPte.u.Long != 0);
977     *PointerPte = InvalidPte;
978 }
979 
980 //
981 // Erase the PTE completely
982 //
983 FORCEINLINE
984 VOID
985 MI_ERASE_PTE(IN PMMPTE PointerPte)
986 {
987     /* Zero out the PTE */
988     ASSERT(PointerPte->u.Long != 0);
989     PointerPte->u.Long = 0;
990 }
991 
992 //
993 // Writes a valid PDE
994 //
995 FORCEINLINE
996 VOID
997 MI_WRITE_VALID_PDE(IN PMMPDE PointerPde,
998                    IN MMPDE TempPde)
999 {
1000     /* Write the valid PDE */
1001     ASSERT(PointerPde->u.Hard.Valid == 0);
1002     ASSERT(TempPde.u.Hard.Valid == 1);
1003     *PointerPde = TempPde;
1004 }
1005 
1006 //
1007 // Writes an invalid PDE
1008 //
1009 FORCEINLINE
1010 VOID
1011 MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde,
1012                      IN MMPDE InvalidPde)
1013 {
1014     /* Write the invalid PDE */
1015     ASSERT(InvalidPde.u.Hard.Valid == 0);
1016     ASSERT(InvalidPde.u.Long != 0);
1017     *PointerPde = InvalidPde;
1018 }
1019 
1020 //
1021 // Checks if the thread already owns a working set
1022 //
1023 FORCEINLINE
1024 BOOLEAN
1025 MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread)
1026 {
1027     /* If any of these are held, return TRUE */
1028     return ((Thread->OwnsProcessWorkingSetExclusive) ||
1029             (Thread->OwnsProcessWorkingSetShared) ||
1030             (Thread->OwnsSystemWorkingSetExclusive) ||
1031             (Thread->OwnsSystemWorkingSetShared) ||
1032             (Thread->OwnsSessionWorkingSetExclusive) ||
1033             (Thread->OwnsSessionWorkingSetShared));
1034 }
1035 
1036 //
1037 // Checks if the process owns the working set lock
1038 //
1039 FORCEINLINE
1040 BOOLEAN
1041 MI_WS_OWNER(IN PEPROCESS Process)
1042 {
1043     /* Check if this process is the owner, and that the thread owns the WS */
1044     if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive == 0)
1045     {
1046         DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread());
1047     }
1048     if (KeGetCurrentThread()->ApcState.Process != &Process->Pcb)
1049     {
1050         DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process);
1051     }
1052     return ((KeGetCurrentThread()->ApcState.Process == &Process->Pcb) &&
1053             ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive) ||
1054              (PsGetCurrentThread()->OwnsProcessWorkingSetShared)));
1055 }
1056 
1057 //
1058 // New ARM3<->RosMM PAGE Architecture
1059 //
1060 FORCEINLINE
1061 BOOLEAN
1062 MiIsRosSectionObject(IN PVOID Section)
1063 {
1064     PROS_SECTION_OBJECT RosSection = Section;
1065     if ((RosSection->Type == 'SC') && (RosSection->Size == 'TN')) return TRUE;
1066     return FALSE;
1067 }
1068 
1069 #define MI_IS_ROS_PFN(x)     ((x)->u4.AweAllocation == TRUE)
1070 
1071 VOID
1072 NTAPI
1073 MiDecrementReferenceCount(
1074     IN PMMPFN Pfn1,
1075     IN PFN_NUMBER PageFrameIndex
1076 );
1077 
1078 FORCEINLINE
1079 BOOLEAN
1080 MI_IS_WS_UNSAFE(IN PEPROCESS Process)
1081 {
1082     return (Process->Vm.Flags.AcquiredUnsafe == TRUE);
1083 }
1084 
1085 //
1086 // Locks the working set for the given process
1087 //
1088 FORCEINLINE
1089 VOID
1090 MiLockProcessWorkingSet(IN PEPROCESS Process,
1091                         IN PETHREAD Thread)
1092 {
1093     /* Shouldn't already be owning the process working set */
1094     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1095     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1096 
1097     /* Block APCs, make sure that still nothing is already held */
1098     KeEnterGuardedRegion();
1099     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1100 
1101     /* Lock the working set */
1102     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1103 
1104     /* Now claim that we own the lock */
1105     ASSERT(!MI_IS_WS_UNSAFE(Process));
1106     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1107     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1108 }
1109 
1110 FORCEINLINE
1111 VOID
1112 MiLockProcessWorkingSetShared(IN PEPROCESS Process,
1113                               IN PETHREAD Thread)
1114 {
1115     /* Shouldn't already be owning the process working set */
1116     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1117     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1118 
1119     /* Block APCs, make sure that still nothing is already held */
1120     KeEnterGuardedRegion();
1121     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1122 
1123     /* Lock the working set */
1124     ExAcquirePushLockShared(&Process->Vm.WorkingSetMutex);
1125 
1126     /* Now claim that we own the lock */
1127     ASSERT(!MI_IS_WS_UNSAFE(Process));
1128     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1129     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1130     Thread->OwnsProcessWorkingSetShared = TRUE;
1131 }
1132 
1133 FORCEINLINE
1134 VOID
1135 MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1136                               IN PETHREAD Thread)
1137 {
1138     /* Shouldn't already be owning the process working set */
1139     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1140 
1141     /* APCs must be blocked, make sure that still nothing is already held */
1142     ASSERT(KeAreAllApcsDisabled() == TRUE);
1143     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1144 
1145     /* Lock the working set */
1146     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1147 
1148     /* Now claim that we own the lock */
1149     ASSERT(!MI_IS_WS_UNSAFE(Process));
1150     Process->Vm.Flags.AcquiredUnsafe = 1;
1151     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1152     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1153 }
1154 
1155 //
1156 // Unlocks the working set for the given process
1157 //
1158 FORCEINLINE
1159 VOID
1160 MiUnlockProcessWorkingSet(IN PEPROCESS Process,
1161                           IN PETHREAD Thread)
1162 {
1163     /* Make sure we are the owner of a safe acquisition */
1164     ASSERT(MI_WS_OWNER(Process));
1165     ASSERT(!MI_IS_WS_UNSAFE(Process));
1166 
1167     /* The thread doesn't own it anymore */
1168     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1169     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1170 
1171     /* Release the lock and re-enable APCs */
1172     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1173     KeLeaveGuardedRegion();
1174 }
1175 
1176 //
1177 // Unlocks the working set for the given process
1178 //
1179 FORCEINLINE
1180 VOID
1181 MiUnlockProcessWorkingSetShared(IN PEPROCESS Process,
1182                                 IN PETHREAD Thread)
1183 {
1184     /* Make sure we are the owner of a safe acquisition (because shared) */
1185     ASSERT(MI_WS_OWNER(Process));
1186     ASSERT(!MI_IS_WS_UNSAFE(Process));
1187 
1188     /* Ensure we are in a shared acquisition */
1189     ASSERT(Thread->OwnsProcessWorkingSetShared == TRUE);
1190     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1191 
1192     /* Don't claim the lock anylonger */
1193     Thread->OwnsProcessWorkingSetShared = FALSE;
1194 
1195     /* Release the lock and re-enable APCs */
1196     ExReleasePushLockShared(&Process->Vm.WorkingSetMutex);
1197     KeLeaveGuardedRegion();
1198 }
1199 
1200 //
1201 // Unlocks the working set for the given process
1202 //
1203 FORCEINLINE
1204 VOID
1205 MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1206                                 IN PETHREAD Thread)
1207 {
1208     /* Make sure we are the owner of an unsafe acquisition */
1209     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1210     ASSERT(KeAreAllApcsDisabled() == TRUE);
1211     ASSERT(MI_WS_OWNER(Process));
1212     ASSERT(MI_IS_WS_UNSAFE(Process));
1213 
1214     /* No longer unsafe */
1215     Process->Vm.Flags.AcquiredUnsafe = 0;
1216 
1217     /* The thread doesn't own it anymore */
1218     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1219     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1220 
1221     /* Release the lock but don't touch APC state */
1222     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1223     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1224 }
1225 
1226 //
1227 // Locks the working set
1228 //
1229 FORCEINLINE
1230 VOID
1231 MiLockWorkingSet(IN PETHREAD Thread,
1232                  IN PMMSUPPORT WorkingSet)
1233 {
1234     /* Block APCs */
1235     KeEnterGuardedRegion();
1236 
1237     /* Working set should be in global memory */
1238     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1239 
1240     /* Thread shouldn't already be owning something */
1241     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1242 
1243     /* Lock this working set */
1244     ExAcquirePushLockExclusive(&WorkingSet->WorkingSetMutex);
1245 
1246     /* Which working set is this? */
1247     if (WorkingSet == &MmSystemCacheWs)
1248     {
1249         /* Own the system working set */
1250         ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1251                (Thread->OwnsSystemWorkingSetShared == FALSE));
1252         Thread->OwnsSystemWorkingSetExclusive = TRUE;
1253     }
1254     else if (WorkingSet->Flags.SessionSpace)
1255     {
1256         /* Own the session working set */
1257         ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1258                (Thread->OwnsSessionWorkingSetShared == FALSE));
1259         Thread->OwnsSessionWorkingSetExclusive = TRUE;
1260     }
1261     else
1262     {
1263         /* Own the process working set */
1264         ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1265                (Thread->OwnsProcessWorkingSetShared == FALSE));
1266         Thread->OwnsProcessWorkingSetExclusive = TRUE;
1267     }
1268 }
1269 
1270 //
1271 // Unlocks the working set
1272 //
1273 FORCEINLINE
1274 VOID
1275 MiUnlockWorkingSet(IN PETHREAD Thread,
1276                    IN PMMSUPPORT WorkingSet)
1277 {
1278     /* Working set should be in global memory */
1279     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1280 
1281     /* Which working set is this? */
1282     if (WorkingSet == &MmSystemCacheWs)
1283     {
1284         /* Release the system working set */
1285         ASSERT((Thread->OwnsSystemWorkingSetExclusive == TRUE) ||
1286                (Thread->OwnsSystemWorkingSetShared == TRUE));
1287         Thread->OwnsSystemWorkingSetExclusive = FALSE;
1288     }
1289     else if (WorkingSet->Flags.SessionSpace)
1290     {
1291         /* Release the session working set */
1292         ASSERT((Thread->OwnsSessionWorkingSetExclusive == TRUE) ||
1293                (Thread->OwnsSessionWorkingSetShared == TRUE));
1294         Thread->OwnsSessionWorkingSetExclusive = 0;
1295     }
1296     else
1297     {
1298         /* Release the process working set */
1299         ASSERT((Thread->OwnsProcessWorkingSetExclusive) ||
1300                (Thread->OwnsProcessWorkingSetShared));
1301         Thread->OwnsProcessWorkingSetExclusive = FALSE;
1302     }
1303 
1304     /* Release the working set lock */
1305     ExReleasePushLockExclusive(&WorkingSet->WorkingSetMutex);
1306 
1307     /* Unblock APCs */
1308     KeLeaveGuardedRegion();
1309 }
1310 
1311 FORCEINLINE
1312 VOID
1313 MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process,
1314                                   IN PETHREAD Thread,
1315                                   OUT PBOOLEAN Safe,
1316                                   OUT PBOOLEAN Shared)
1317 {
1318     ASSERT(MI_WS_OWNER(Process));
1319 
1320     /* Check if the current owner is unsafe */
1321     if (MI_IS_WS_UNSAFE(Process))
1322     {
1323         /* Release unsafely */
1324         MiUnlockProcessWorkingSetUnsafe(Process, Thread);
1325         *Safe = FALSE;
1326         *Shared = FALSE;
1327     }
1328     else if (Thread->OwnsProcessWorkingSetExclusive == 1)
1329     {
1330         /* Owner is safe and exclusive, release normally */
1331         MiUnlockProcessWorkingSet(Process, Thread);
1332         *Safe = TRUE;
1333         *Shared = FALSE;
1334     }
1335     else
1336     {
1337         /* Owner is shared (implies safe), release normally */
1338         MiUnlockProcessWorkingSetShared(Process, Thread);
1339         *Safe = TRUE;
1340         *Shared = TRUE;
1341     }
1342 }
1343 
1344 FORCEINLINE
1345 VOID
1346 MiLockProcessWorkingSetForFault(IN PEPROCESS Process,
1347                                 IN PETHREAD Thread,
1348                                 IN BOOLEAN Safe,
1349                                 IN BOOLEAN Shared)
1350 {
1351     /* Check if this was a safe lock or not */
1352     if (Safe)
1353     {
1354         if (Shared)
1355         {
1356             /* Reacquire safely & shared */
1357             MiLockProcessWorkingSetShared(Process, Thread);
1358         }
1359         else
1360         {
1361             /* Reacquire safely */
1362             MiLockProcessWorkingSet(Process, Thread);
1363         }
1364     }
1365     else
1366     {
1367         /* Unsafe lock cannot be shared */
1368         ASSERT(Shared == FALSE);
1369         /* Reacquire unsafely */
1370         MiLockProcessWorkingSetUnsafe(Process, Thread);
1371     }
1372 }
1373 
1374 FORCEINLINE
1375 KIRQL
1376 MiAcquireExpansionLock(VOID)
1377 {
1378     KIRQL OldIrql;
1379 
1380     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1381     KeAcquireSpinLock(&MmExpansionLock, &OldIrql);
1382     ASSERT(MiExpansionLockOwner == NULL);
1383     MiExpansionLockOwner = PsGetCurrentThread();
1384     return OldIrql;
1385 }
1386 
1387 FORCEINLINE
1388 VOID
1389 MiReleaseExpansionLock(KIRQL OldIrql)
1390 {
1391     ASSERT(MiExpansionLockOwner == PsGetCurrentThread());
1392     MiExpansionLockOwner = NULL;
1393     KeReleaseSpinLock(&MmExpansionLock, OldIrql);
1394     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1395 }
1396 
1397 //
1398 // Returns the ProtoPTE inside a VAD for the given VPN
1399 //
1400 FORCEINLINE
1401 PMMPTE
1402 MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad,
1403                              IN ULONG_PTR Vpn)
1404 {
1405     PMMPTE ProtoPte;
1406 
1407     /* Find the offset within the VAD's prototype PTEs */
1408     ProtoPte = Vad->FirstPrototypePte + (Vpn - Vad->StartingVpn);
1409     ASSERT(ProtoPte <= Vad->LastContiguousPte);
1410     return ProtoPte;
1411 }
1412 
1413 //
1414 // Returns the PFN Database entry for the given page number
1415 // Warning: This is not necessarily a valid PFN database entry!
1416 //
1417 FORCEINLINE
1418 PMMPFN
1419 MI_PFN_ELEMENT(IN PFN_NUMBER Pfn)
1420 {
1421     /* Get the entry */
1422     return &MmPfnDatabase[Pfn];
1423 };
1424 
1425 //
1426 // Drops a locked page without dereferencing it
1427 //
1428 FORCEINLINE
1429 VOID
1430 MiDropLockCount(IN PMMPFN Pfn1)
1431 {
1432     /* This page shouldn't be locked, but it should be valid */
1433     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1434     ASSERT(Pfn1->u2.ShareCount == 0);
1435 
1436     /* Is this the last reference to the page */
1437     if (Pfn1->u3.e2.ReferenceCount == 1)
1438     {
1439         /* It better not be valid */
1440         ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1441 
1442         /* Is it a prototype PTE? */
1443         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1444             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1445         {
1446             /* FIXME: We should return commit */
1447             DPRINT1("Not returning commit for prototype PTE\n");
1448         }
1449 
1450         /* Update the counter */
1451         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1452     }
1453 }
1454 
1455 //
1456 // Drops a locked page and dereferences it
1457 //
1458 FORCEINLINE
1459 VOID
1460 MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1)
1461 {
1462     USHORT RefCount, OldRefCount;
1463     PFN_NUMBER PageFrameIndex;
1464 
1465     /* Loop while we decrement the page successfully */
1466     do
1467     {
1468         /* There should be at least one reference */
1469         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1470         ASSERT(OldRefCount != 0);
1471 
1472         /* Are we the last one */
1473         if (OldRefCount == 1)
1474         {
1475             /* The page shoudln't be shared not active at this point */
1476             ASSERT(Pfn1->u3.e2.ReferenceCount == 1);
1477             ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1478             ASSERT(Pfn1->u2.ShareCount == 0);
1479 
1480             /* Is it a prototype PTE? */
1481             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1482                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1483             {
1484                 /* FIXME: We should return commit */
1485                 DPRINT1("Not returning commit for prototype PTE\n");
1486             }
1487 
1488             /* Update the counter, and drop a reference the long way */
1489             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1490             PageFrameIndex = MiGetPfnEntryIndex(Pfn1);
1491             MiDecrementReferenceCount(Pfn1, PageFrameIndex);
1492             return;
1493         }
1494 
1495         /* Drop a reference the short way, and that's it */
1496         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1497                                                 OldRefCount - 1,
1498                                                 OldRefCount);
1499         ASSERT(RefCount != 0);
1500     } while (OldRefCount != RefCount);
1501 
1502     /* If we got here, there should be more than one reference */
1503     ASSERT(RefCount > 1);
1504     if (RefCount == 2)
1505     {
1506         /* Is it still being shared? */
1507         if (Pfn1->u2.ShareCount >= 1)
1508         {
1509             /* Then it should be valid */
1510             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1511 
1512             /* Is it a prototype PTE? */
1513             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1514                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1515             {
1516                 /* We don't handle ethis */
1517                 ASSERT(FALSE);
1518             }
1519 
1520             /* Update the counter */
1521             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1522         }
1523     }
1524 }
1525 
1526 //
1527 // References a locked page and updates the counter
1528 // Used in MmProbeAndLockPages to handle different edge cases
1529 //
1530 FORCEINLINE
1531 VOID
1532 MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1)
1533 {
1534     USHORT RefCount, OldRefCount;
1535 
1536     /* Sanity check */
1537     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1538 
1539     /* Does ARM3 own the page? */
1540     if (MI_IS_ROS_PFN(Pfn1))
1541     {
1542         /* ReactOS Mm doesn't track share count */
1543         ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1544     }
1545     else
1546     {
1547         /* On ARM3 pages, we should see a valid share count */
1548         ASSERT((Pfn1->u2.ShareCount != 0) && (Pfn1->u3.e1.PageLocation == ActiveAndValid));
1549 
1550         /* Is it a prototype PTE? */
1551         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1552             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1553         {
1554             /* FIXME: We should charge commit */
1555             DPRINT1("Not charging commit for prototype PTE\n");
1556         }
1557     }
1558 
1559     /* More locked pages! */
1560     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1561 
1562     /* Loop trying to update the reference count */
1563     do
1564     {
1565         /* Get the current reference count, make sure it's valid */
1566         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1567         ASSERT(OldRefCount != 0);
1568         ASSERT(OldRefCount < 2500);
1569 
1570         /* Bump it up by one */
1571         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1572                                                 OldRefCount + 1,
1573                                                 OldRefCount);
1574         ASSERT(RefCount != 0);
1575     } while (OldRefCount != RefCount);
1576 
1577     /* Was this the first lock attempt? If not, undo our bump */
1578     if (OldRefCount != 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1579 }
1580 
1581 //
1582 // References a locked page and updates the counter
1583 // Used in all other cases except MmProbeAndLockPages
1584 //
1585 FORCEINLINE
1586 VOID
1587 MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1)
1588 {
1589     USHORT NewRefCount;
1590 
1591     /* Is it a prototype PTE? */
1592     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1593         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1594     {
1595         /* FIXME: We should charge commit */
1596         DPRINT1("Not charging commit for prototype PTE\n");
1597     }
1598 
1599     /* More locked pages! */
1600     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1601 
1602     /* Update the reference count */
1603     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1604     if (NewRefCount == 2)
1605     {
1606         /* Is it locked or shared? */
1607         if (Pfn1->u2.ShareCount)
1608         {
1609             /* It's shared, so make sure it's active */
1610             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1611         }
1612         else
1613         {
1614             /* It's locked, so we shouldn't lock again */
1615             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1616         }
1617     }
1618     else
1619     {
1620         /* Someone had already locked the page, so undo our bump */
1621         ASSERT(NewRefCount < 2500);
1622         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1623     }
1624 }
1625 
1626 //
1627 // References a locked page and updates the counter
1628 // Used in all other cases except MmProbeAndLockPages
1629 //
1630 FORCEINLINE
1631 VOID
1632 MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1)
1633 {
1634     USHORT NewRefCount;
1635 
1636     /* Make sure the page isn't used yet */
1637     ASSERT(Pfn1->u2.ShareCount == 0);
1638     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1639 
1640     /* Is it a prototype PTE? */
1641     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1642         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1643     {
1644         /* FIXME: We should charge commit */
1645         DPRINT1("Not charging commit for prototype PTE\n");
1646     }
1647 
1648     /* More locked pages! */
1649     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1650 
1651     /* Update the reference count */
1652     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1653     if (NewRefCount != 1)
1654     {
1655         /* Someone had already locked the page, so undo our bump */
1656         ASSERT(NewRefCount < 2500);
1657         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1658     }
1659 }
1660 
1661 FORCEINLINE
1662 VOID
1663 MiIncrementPageTableReferences(IN PVOID Address)
1664 {
1665     PUSHORT RefCount;
1666 
1667     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1668 
1669     *RefCount += 1;
1670     ASSERT(*RefCount <= PTE_PER_PAGE);
1671 }
1672 
1673 FORCEINLINE
1674 VOID
1675 MiDecrementPageTableReferences(IN PVOID Address)
1676 {
1677     PUSHORT RefCount;
1678 
1679     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1680 
1681     *RefCount -= 1;
1682     ASSERT(*RefCount < PTE_PER_PAGE);
1683 }
1684 
1685 FORCEINLINE
1686 USHORT
1687 MiQueryPageTableReferences(IN PVOID Address)
1688 {
1689     PUSHORT RefCount;
1690 
1691     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1692 
1693     return *RefCount;
1694 }
1695 
1696 INIT_FUNCTION
1697 BOOLEAN
1698 NTAPI
1699 MmArmInitSystem(
1700     IN ULONG Phase,
1701     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1702 );
1703 
1704 INIT_FUNCTION
1705 VOID
1706 NTAPI
1707 MiInitializeSessionSpaceLayout(VOID);
1708 
1709 INIT_FUNCTION
1710 NTSTATUS
1711 NTAPI
1712 MiInitMachineDependent(
1713     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1714 );
1715 
1716 INIT_FUNCTION
1717 VOID
1718 NTAPI
1719 MiComputeColorInformation(
1720     VOID
1721 );
1722 
1723 INIT_FUNCTION
1724 VOID
1725 NTAPI
1726 MiMapPfnDatabase(
1727     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1728 );
1729 
1730 INIT_FUNCTION
1731 VOID
1732 NTAPI
1733 MiInitializeColorTables(
1734     VOID
1735 );
1736 
1737 INIT_FUNCTION
1738 VOID
1739 NTAPI
1740 MiInitializePfnDatabase(
1741     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1742 );
1743 
1744 VOID
1745 NTAPI
1746 MiInitializeSessionWsSupport(
1747     VOID
1748 );
1749 
1750 VOID
1751 NTAPI
1752 MiInitializeSessionIds(
1753     VOID
1754 );
1755 
1756 INIT_FUNCTION
1757 BOOLEAN
1758 NTAPI
1759 MiInitializeMemoryEvents(
1760     VOID
1761 );
1762 
1763 INIT_FUNCTION
1764 PFN_NUMBER
1765 NTAPI
1766 MxGetNextPage(
1767     IN PFN_NUMBER PageCount
1768 );
1769 
1770 INIT_FUNCTION
1771 PPHYSICAL_MEMORY_DESCRIPTOR
1772 NTAPI
1773 MmInitializeMemoryLimits(
1774     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1775     IN PBOOLEAN IncludeType
1776 );
1777 
1778 PFN_NUMBER
1779 NTAPI
1780 MiPagesInLoaderBlock(
1781     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1782     IN PBOOLEAN IncludeType
1783 );
1784 
1785 VOID
1786 FASTCALL
1787 MiSyncARM3WithROS(
1788     IN PVOID AddressStart,
1789     IN PVOID AddressEnd
1790 );
1791 
1792 NTSTATUS
1793 NTAPI
1794 MiRosProtectVirtualMemory(
1795     IN PEPROCESS Process,
1796     IN OUT PVOID *BaseAddress,
1797     IN OUT PSIZE_T NumberOfBytesToProtect,
1798     IN ULONG NewAccessProtection,
1799     OUT PULONG OldAccessProtection OPTIONAL
1800 );
1801 
1802 NTSTATUS
1803 NTAPI
1804 MmArmAccessFault(
1805     IN ULONG FaultCode,
1806     IN PVOID Address,
1807     IN KPROCESSOR_MODE Mode,
1808     IN PVOID TrapInformation
1809 );
1810 
1811 NTSTATUS
1812 FASTCALL
1813 MiCheckPdeForPagedPool(
1814     IN PVOID Address
1815 );
1816 
1817 INIT_FUNCTION
1818 VOID
1819 NTAPI
1820 MiInitializeNonPagedPool(
1821     VOID
1822 );
1823 
1824 INIT_FUNCTION
1825 VOID
1826 NTAPI
1827 MiInitializeNonPagedPoolThresholds(
1828     VOID
1829 );
1830 
1831 INIT_FUNCTION
1832 VOID
1833 NTAPI
1834 MiInitializePoolEvents(
1835     VOID
1836 );
1837 
1838 INIT_FUNCTION
1839 VOID                      //
1840 NTAPI                     //
1841 InitializePool(           //
1842     IN POOL_TYPE PoolType,// FIXFIX: This should go in ex.h after the pool merge
1843     IN ULONG Threshold    //
1844 );                        //
1845 
1846 // FIXFIX: THIS ONE TOO
1847 INIT_FUNCTION
1848 VOID
1849 NTAPI
1850 ExInitializePoolDescriptor(
1851     IN PPOOL_DESCRIPTOR PoolDescriptor,
1852     IN POOL_TYPE PoolType,
1853     IN ULONG PoolIndex,
1854     IN ULONG Threshold,
1855     IN PVOID PoolLock
1856 );
1857 
1858 NTSTATUS
1859 NTAPI
1860 MiInitializeSessionPool(
1861     VOID
1862 );
1863 
1864 INIT_FUNCTION
1865 VOID
1866 NTAPI
1867 MiInitializeSystemPtes(
1868     IN PMMPTE StartingPte,
1869     IN ULONG NumberOfPtes,
1870     IN MMSYSTEM_PTE_POOL_TYPE PoolType
1871 );
1872 
1873 PMMPTE
1874 NTAPI
1875 MiReserveSystemPtes(
1876     IN ULONG NumberOfPtes,
1877     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1878 );
1879 
1880 VOID
1881 NTAPI
1882 MiReleaseSystemPtes(
1883     IN PMMPTE StartingPte,
1884     IN ULONG NumberOfPtes,
1885     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1886 );
1887 
1888 
1889 PFN_NUMBER
1890 NTAPI
1891 MiFindContiguousPages(
1892     IN PFN_NUMBER LowestPfn,
1893     IN PFN_NUMBER HighestPfn,
1894     IN PFN_NUMBER BoundaryPfn,
1895     IN PFN_NUMBER SizeInPages,
1896     IN MEMORY_CACHING_TYPE CacheType
1897 );
1898 
1899 PVOID
1900 NTAPI
1901 MiCheckForContiguousMemory(
1902     IN PVOID BaseAddress,
1903     IN PFN_NUMBER BaseAddressPages,
1904     IN PFN_NUMBER SizeInPages,
1905     IN PFN_NUMBER LowestPfn,
1906     IN PFN_NUMBER HighestPfn,
1907     IN PFN_NUMBER BoundaryPfn,
1908     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
1909 );
1910 
1911 PMDL
1912 NTAPI
1913 MiAllocatePagesForMdl(
1914     IN PHYSICAL_ADDRESS LowAddress,
1915     IN PHYSICAL_ADDRESS HighAddress,
1916     IN PHYSICAL_ADDRESS SkipBytes,
1917     IN SIZE_T TotalBytes,
1918     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
1919     IN ULONG Flags
1920 );
1921 
1922 VOID
1923 NTAPI
1924 MiInsertPageInList(
1925     IN PMMPFNLIST ListHead,
1926     IN PFN_NUMBER PageFrameIndex
1927 );
1928 
1929 VOID
1930 NTAPI
1931 MiUnlinkFreeOrZeroedPage(
1932     IN PMMPFN Entry
1933 );
1934 
1935 VOID
1936 NTAPI
1937 MiUnlinkPageFromList(
1938     IN PMMPFN Pfn
1939 );
1940 
1941 VOID
1942 NTAPI
1943 MiInitializePfn(
1944     IN PFN_NUMBER PageFrameIndex,
1945     IN PMMPTE PointerPte,
1946     IN BOOLEAN Modified
1947 );
1948 
1949 NTSTATUS
1950 NTAPI
1951 MiInitializeAndChargePfn(
1952     OUT PPFN_NUMBER PageFrameIndex,
1953     IN PMMPDE PointerPde,
1954     IN PFN_NUMBER ContainingPageFrame,
1955     IN BOOLEAN SessionAllocation
1956 );
1957 
1958 VOID
1959 NTAPI
1960 MiInitializePfnAndMakePteValid(
1961     IN PFN_NUMBER PageFrameIndex,
1962     IN PMMPTE PointerPte,
1963     IN MMPTE TempPte
1964 );
1965 
1966 VOID
1967 NTAPI
1968 MiInitializePfnForOtherProcess(
1969     IN PFN_NUMBER PageFrameIndex,
1970     IN PVOID PteAddress,
1971     IN PFN_NUMBER PteFrame
1972 );
1973 
1974 VOID
1975 NTAPI
1976 MiDecrementShareCount(
1977     IN PMMPFN Pfn1,
1978     IN PFN_NUMBER PageFrameIndex
1979 );
1980 
1981 PFN_NUMBER
1982 NTAPI
1983 MiRemoveAnyPage(
1984     IN ULONG Color
1985 );
1986 
1987 PFN_NUMBER
1988 NTAPI
1989 MiRemoveZeroPage(
1990     IN ULONG Color
1991 );
1992 
1993 VOID
1994 NTAPI
1995 MiZeroPhysicalPage(
1996     IN PFN_NUMBER PageFrameIndex
1997 );
1998 
1999 VOID
2000 NTAPI
2001 MiInsertPageInFreeList(
2002     IN PFN_NUMBER PageFrameIndex
2003 );
2004 
2005 PFN_COUNT
2006 NTAPI
2007 MiDeleteSystemPageableVm(
2008     IN PMMPTE PointerPte,
2009     IN PFN_NUMBER PageCount,
2010     IN ULONG Flags,
2011     OUT PPFN_NUMBER ValidPages
2012 );
2013 
2014 ULONG
2015 NTAPI
2016 MiGetPageProtection(
2017     IN PMMPTE PointerPte
2018 );
2019 
2020 PLDR_DATA_TABLE_ENTRY
2021 NTAPI
2022 MiLookupDataTableEntry(
2023     IN PVOID Address
2024 );
2025 
2026 INIT_FUNCTION
2027 VOID
2028 NTAPI
2029 MiInitializeDriverLargePageList(
2030     VOID
2031 );
2032 
2033 INIT_FUNCTION
2034 VOID
2035 NTAPI
2036 MiInitializeLargePageSupport(
2037     VOID
2038 );
2039 
2040 INIT_FUNCTION
2041 VOID
2042 NTAPI
2043 MiSyncCachedRanges(
2044     VOID
2045 );
2046 
2047 BOOLEAN
2048 NTAPI
2049 MiIsPfnInUse(
2050     IN PMMPFN Pfn1
2051 );
2052 
2053 PMMVAD
2054 NTAPI
2055 MiLocateAddress(
2056     IN PVOID VirtualAddress
2057 );
2058 
2059 TABLE_SEARCH_RESULT
2060 NTAPI
2061 MiCheckForConflictingNode(
2062     IN ULONG_PTR StartVpn,
2063     IN ULONG_PTR EndVpn,
2064     IN PMM_AVL_TABLE Table,
2065     OUT PMMADDRESS_NODE *NodeOrParent
2066 );
2067 
2068 TABLE_SEARCH_RESULT
2069 NTAPI
2070 MiFindEmptyAddressRangeDownTree(
2071     IN SIZE_T Length,
2072     IN ULONG_PTR BoundaryAddress,
2073     IN ULONG_PTR Alignment,
2074     IN PMM_AVL_TABLE Table,
2075     OUT PULONG_PTR Base,
2076     OUT PMMADDRESS_NODE *Parent
2077 );
2078 
2079 NTSTATUS
2080 NTAPI
2081 MiFindEmptyAddressRangeDownBasedTree(
2082     IN SIZE_T Length,
2083     IN ULONG_PTR BoundaryAddress,
2084     IN ULONG_PTR Alignment,
2085     IN PMM_AVL_TABLE Table,
2086     OUT PULONG_PTR Base
2087 );
2088 
2089 TABLE_SEARCH_RESULT
2090 NTAPI
2091 MiFindEmptyAddressRangeInTree(
2092     IN SIZE_T Length,
2093     IN ULONG_PTR Alignment,
2094     IN PMM_AVL_TABLE Table,
2095     OUT PMMADDRESS_NODE *PreviousVad,
2096     OUT PULONG_PTR Base
2097 );
2098 
2099 NTSTATUS
2100 NTAPI
2101 MiCheckSecuredVad(
2102     IN PMMVAD Vad,
2103     IN PVOID Base,
2104     IN SIZE_T Size,
2105     IN ULONG ProtectionMask
2106 );
2107 
2108 VOID
2109 NTAPI
2110 MiInsertVad(
2111     _Inout_ PMMVAD Vad,
2112     _Inout_ PMM_AVL_TABLE VadRoot);
2113 
2114 NTSTATUS
2115 NTAPI
2116 MiInsertVadEx(
2117     _Inout_ PMMVAD Vad,
2118     _In_ ULONG_PTR *BaseAddress,
2119     _In_ SIZE_T ViewSize,
2120     _In_ ULONG_PTR HighestAddress,
2121     _In_ ULONG_PTR Alignment,
2122     _In_ ULONG AllocationType);
2123 
2124 VOID
2125 NTAPI
2126 MiInsertBasedSection(
2127     IN PSECTION Section
2128 );
2129 
2130 NTSTATUS
2131 NTAPI
2132 MiUnmapViewOfSection(
2133     IN PEPROCESS Process,
2134     IN PVOID BaseAddress,
2135     IN ULONG Flags
2136 );
2137 
2138 NTSTATUS
2139 NTAPI
2140 MiRosUnmapViewOfSection(
2141     IN PEPROCESS Process,
2142     IN PVOID BaseAddress,
2143     IN BOOLEAN SkipDebuggerNotify
2144 );
2145 
2146 VOID
2147 NTAPI
2148 MiInsertNode(
2149     IN PMM_AVL_TABLE Table,
2150     IN PMMADDRESS_NODE NewNode,
2151     PMMADDRESS_NODE Parent,
2152     TABLE_SEARCH_RESULT Result
2153 );
2154 
2155 VOID
2156 NTAPI
2157 MiRemoveNode(
2158     IN PMMADDRESS_NODE Node,
2159     IN PMM_AVL_TABLE Table
2160 );
2161 
2162 PMMADDRESS_NODE
2163 NTAPI
2164 MiGetPreviousNode(
2165     IN PMMADDRESS_NODE Node
2166 );
2167 
2168 PMMADDRESS_NODE
2169 NTAPI
2170 MiGetNextNode(
2171     IN PMMADDRESS_NODE Node
2172 );
2173 
2174 BOOLEAN
2175 NTAPI
2176 MiInitializeSystemSpaceMap(
2177     IN PMMSESSION InputSession OPTIONAL
2178 );
2179 
2180 VOID
2181 NTAPI
2182 MiSessionRemoveProcess(
2183     VOID
2184 );
2185 
2186 VOID
2187 NTAPI
2188 MiReleaseProcessReferenceToSessionDataPage(
2189     IN PMM_SESSION_SPACE SessionGlobal
2190 );
2191 
2192 VOID
2193 NTAPI
2194 MiSessionAddProcess(
2195     IN PEPROCESS NewProcess
2196 );
2197 
2198 NTSTATUS
2199 NTAPI
2200 MiSessionCommitPageTables(
2201     IN PVOID StartVa,
2202     IN PVOID EndVa
2203 );
2204 
2205 ULONG
2206 NTAPI
2207 MiMakeProtectionMask(
2208     IN ULONG Protect
2209 );
2210 
2211 VOID
2212 NTAPI
2213 MiDeleteVirtualAddresses(
2214     IN ULONG_PTR Va,
2215     IN ULONG_PTR EndingAddress,
2216     IN PMMVAD Vad
2217 );
2218 
2219 VOID
2220 NTAPI
2221 MiDeletePte(
2222     IN PMMPTE PointerPte,
2223     IN PVOID VirtualAddress,
2224     IN PEPROCESS CurrentProcess,
2225     IN PMMPTE PrototypePte
2226 );
2227 
2228 ULONG
2229 NTAPI
2230 MiMakeSystemAddressValid(
2231     IN PVOID PageTableVirtualAddress,
2232     IN PEPROCESS CurrentProcess
2233 );
2234 
2235 ULONG
2236 NTAPI
2237 MiMakeSystemAddressValidPfn(
2238     IN PVOID VirtualAddress,
2239     IN KIRQL OldIrql
2240 );
2241 
2242 VOID
2243 NTAPI
2244 MiRemoveMappedView(
2245     IN PEPROCESS CurrentProcess,
2246     IN PMMVAD Vad
2247 );
2248 
2249 PSUBSECTION
2250 NTAPI
2251 MiLocateSubsection(
2252     IN PMMVAD Vad,
2253     IN ULONG_PTR Vpn
2254 );
2255 
2256 VOID
2257 NTAPI
2258 MiDeleteARM3Section(
2259     PVOID ObjectBody
2260 );
2261 
2262 NTSTATUS
2263 NTAPI
2264 MiQueryMemorySectionName(
2265     IN HANDLE ProcessHandle,
2266     IN PVOID BaseAddress,
2267     OUT PVOID MemoryInformation,
2268     IN SIZE_T MemoryInformationLength,
2269     OUT PSIZE_T ReturnLength
2270 );
2271 
2272 NTSTATUS
2273 NTAPI
2274 MiRosUnmapViewInSystemSpace(
2275     IN PVOID MappedBase
2276 );
2277 
2278 POOL_TYPE
2279 NTAPI
2280 MmDeterminePoolType(
2281     IN PVOID PoolAddress
2282 );
2283 
2284 VOID
2285 NTAPI
2286 MiMakePdeExistAndMakeValid(
2287     IN PMMPDE PointerPde,
2288     IN PEPROCESS TargetProcess,
2289     IN KIRQL OldIrql
2290 );
2291 
2292 VOID
2293 NTAPI
2294 MiWriteProtectSystemImage(
2295     _In_ PVOID ImageBase);
2296 
2297 //
2298 // MiRemoveZeroPage will use inline code to zero out the page manually if only
2299 // free pages are available. In some scenarios, we don't/can't run that piece of
2300 // code and would rather only have a real zero page. If we can't have a zero page,
2301 // then we'd like to have our own code to grab a free page and zero it out, by
2302 // using MiRemoveAnyPage. This macro implements this.
2303 //
2304 FORCEINLINE
2305 PFN_NUMBER
2306 MiRemoveZeroPageSafe(IN ULONG Color)
2307 {
2308     if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color);
2309     return 0;
2310 }
2311 
2312 #if (_MI_PAGING_LEVELS == 2)
2313 FORCEINLINE
2314 BOOLEAN
2315 MiSynchronizeSystemPde(PMMPDE PointerPde)
2316 {
2317     MMPDE SystemPde;
2318     ULONG Index;
2319 
2320     /* Get the Index from the PDE */
2321     Index = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE);
2322 
2323     /* Copy the PDE from the double-mapped system page directory */
2324     SystemPde = MmSystemPagePtes[Index];
2325     *PointerPde = SystemPde;
2326 
2327     /* Make sure we re-read the PDE and PTE */
2328     KeMemoryBarrierWithoutFence();
2329 
2330     /* Return, if we had success */
2331     return SystemPde.u.Hard.Valid != 0;
2332 }
2333 #endif
2334 
2335 /* EOF */
2336