xref: /reactos/ntoskrnl/mm/ARM3/miarm.h (revision 71fefa32) 
1 /*
2  * PROJECT:         ReactOS Kernel
3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
4  * FILE:            ntoskrnl/mm/ARM3/miarm.h
5  * PURPOSE:         ARM Memory Manager Header
6  * PROGRAMMERS:     ReactOS Portable Systems Group
7  */
8 
9 #pragma once
10 
11 #define MI_LOWEST_VAD_ADDRESS                   (PVOID)MM_LOWEST_USER_ADDRESS
12 
13 /* Make the code cleaner with some definitions for size multiples */
14 #define _1KB (1024u)
15 #define _1MB (1024 * _1KB)
16 #define _1GB (1024 * _1MB)
17 
18 /* Everyone loves 64K */
19 #define _64K (64 * _1KB)
20 
21 /* Area mapped by a PDE */
22 #define PDE_MAPPED_VA  (PTE_COUNT * PAGE_SIZE)
23 
24 /* Size of a page table */
25 #define PT_SIZE  (PTE_COUNT * sizeof(MMPTE))
26 
27 /* Size of a page directory */
28 #define PD_SIZE  (PDE_COUNT * sizeof(MMPDE))
29 
30 /* Stop using these! */
31 #define PD_COUNT  PPE_PER_PAGE
32 #define PDE_COUNT PDE_PER_PAGE
33 #define PTE_COUNT PTE_PER_PAGE
34 
35 /* Size of all page directories for a process */
36 #define SYSTEM_PD_SIZE (PD_COUNT * PD_SIZE)
37 #ifdef _M_IX86
38 C_ASSERT(SYSTEM_PD_SIZE == PAGE_SIZE);
39 #endif
40 
41 //
42 // Protection Bits part of the internal memory manager Protection Mask, from:
43 // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel
44 // https://www.reactos.org/wiki/Techwiki:Memory_Protection_constants
45 // and public assertions.
46 //
47 #define MM_ZERO_ACCESS         0
48 #define MM_READONLY            1
49 #define MM_EXECUTE             2
50 #define MM_EXECUTE_READ        3
51 #define MM_READWRITE           4
52 #define MM_WRITECOPY           5
53 #define MM_EXECUTE_READWRITE   6
54 #define MM_EXECUTE_WRITECOPY   7
55 #define MM_PROTECT_ACCESS      7
56 
57 //
58 // These are flags on top of the actual protection mask
59 //
60 #define MM_NOCACHE            0x08
61 #define MM_GUARDPAGE          0x10
62 #define MM_WRITECOMBINE       0x18
63 #define MM_PROTECT_SPECIAL    0x18
64 
65 //
66 // These are special cases
67 //
68 #define MM_DECOMMIT           (MM_ZERO_ACCESS | MM_GUARDPAGE)
69 #define MM_NOACCESS           (MM_ZERO_ACCESS | MM_WRITECOMBINE)
70 #define MM_OUTSWAPPED_KSTACK  (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE)
71 #define MM_INVALID_PROTECTION  0xFFFFFFFF
72 
73 //
74 // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits
75 // The Memory Manager's definition define the attributes that must be preserved
76 // and these PTE definitions describe the attributes in the hardware sense. This
77 // helps deal with hardware differences between the actual boolean expression of
78 // the argument.
79 //
80 // For example, in the logical attributes, we want to express read-only as a flag
81 // but on x86, it is writability that must be set. On the other hand, on x86, just
82 // like in the kernel, it is disabling the caches that requires a special flag,
83 // while on certain architectures such as ARM, it is enabling the cache which
84 // requires a flag.
85 //
86 #if defined(_M_IX86) || defined(_M_AMD64)
87 //
88 // Access Flags
89 //
90 #define PTE_READONLY            0 // Doesn't exist on x86
91 #define PTE_EXECUTE             0 // Not worrying about NX yet
92 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
93 #define PTE_READWRITE           0x2
94 #define PTE_WRITECOPY           0x200
95 #define PTE_EXECUTE_READWRITE   0x2 // Not worrying about NX yet
96 #define PTE_EXECUTE_WRITECOPY   0x200
97 #define PTE_PROTOTYPE           0x400
98 
99 //
100 // State Flags
101 //
102 #define PTE_VALID               0x1
103 #define PTE_ACCESSED            0x20
104 #define PTE_DIRTY               0x40
105 
106 //
107 // Cache flags
108 //
109 #define PTE_ENABLE_CACHE        0
110 #define PTE_DISABLE_CACHE       0x10
111 #define PTE_WRITECOMBINED_CACHE 0x10
112 #elif defined(_M_ARM)
113 #define PTE_READONLY            0x200
114 #define PTE_EXECUTE             0 // Not worrying about NX yet
115 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
116 #define PTE_READWRITE           0 // Doesn't exist on ARM
117 #define PTE_WRITECOPY           0 // Doesn't exist on ARM
118 #define PTE_EXECUTE_READWRITE   0 // Not worrying about NX yet
119 #define PTE_EXECUTE_WRITECOPY   0 // Not worrying about NX yet
120 #define PTE_PROTOTYPE           0x400 // Using the Shared bit
121 //
122 // Cache flags
123 //
124 #define PTE_ENABLE_CACHE        0
125 #define PTE_DISABLE_CACHE       0x10
126 #define PTE_WRITECOMBINED_CACHE 0x10
127 #else
128 #error Define these please!
129 #endif
130 
131 extern const ULONG_PTR MmProtectToPteMask[32];
132 extern const ULONG MmProtectToValue[32];
133 
134 //
135 // Assertions for session images, addresses, and PTEs
136 //
137 #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \
138     (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))
139 
140 #define MI_IS_SESSION_ADDRESS(Address) \
141     (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))
142 
143 #define MI_IS_SESSION_PTE(Pte) \
144     ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))
145 
146 #define MI_IS_PAGE_TABLE_ADDRESS(Address) \
147     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))
148 
149 #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \
150     (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))
151 
152 #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \
153     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))
154 
155 //
156 // Creates a software PTE with the given protection
157 //
158 #define MI_MAKE_SOFTWARE_PTE(p, x)          ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))
159 
160 //
161 // Marks a PTE as deleted
162 //
163 #define MI_SET_PFN_DELETED(x)               ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1))
164 #define MI_IS_PFN_DELETED(x)                ((ULONG_PTR)((x)->PteAddress) & 1)
165 
166 //
167 // Special values for LoadedImports
168 //
169 #ifdef _WIN64
170 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFFFFFFFFFEULL
171 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFFFFFFFFFFULL
172 #else
173 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFE
174 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFF
175 #endif
176 #define MM_SYSLDR_SINGLE_ENTRY 0x1
177 
178 //
179 // Number of initial session IDs
180 //
181 #define MI_INITIAL_SESSION_IDS  64
182 
183 #if defined(_M_IX86) || defined(_M_ARM)
184 //
185 // PFN List Sentinel
186 //
187 #define LIST_HEAD 0xFFFFFFFF
188 
189 //
190 // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits,
191 // we need a manual definition suited to the number of bits in the PteFrame.
192 // This is used as a LIST_HEAD for the colored list
193 //
194 #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF
195 #elif defined(_M_AMD64)
196 #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL
197 #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL
198 #else
199 #error Define these please!
200 #endif
201 
202 //
203 // Special IRQL value (found in assertions)
204 //
205 #define MM_NOIRQL (KIRQL)0xFFFFFFFF
206 
207 //
208 // Returns the color of a page
209 //
210 #define MI_GET_PAGE_COLOR(x)                ((x) & MmSecondaryColorMask)
211 #define MI_GET_NEXT_COLOR()                 (MI_GET_PAGE_COLOR(++MmSystemPageColor))
212 #define MI_GET_NEXT_PROCESS_COLOR(x)        (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
213 
214 //
215 // Prototype PTEs that don't yet have a pagefile association
216 //
217 #ifdef _WIN64
218 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL
219 #else
220 #define MI_PTE_LOOKUP_NEEDED 0xFFFFF
221 #endif
222 
223 //
224 // Number of session data and tag pages
225 //
226 #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
227 #define MI_SESSION_TAG_PAGES_MAXIMUM  (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
228 
229 //
230 // Used by MiCheckSecuredVad
231 //
232 #define MM_READ_WRITE_ALLOWED   11
233 #define MM_READ_ONLY_ALLOWED    10
234 #define MM_NO_ACCESS_ALLOWED    01
235 #define MM_DELETE_CHECK         85
236 
237 //
238 // System views are binned into 64K chunks
239 //
240 #define MI_SYSTEM_VIEW_BUCKET_SIZE  _64K
241 
242 //
243 // FIXFIX: These should go in ex.h after the pool merge
244 //
245 #ifdef _WIN64
246 #define POOL_BLOCK_SIZE 16
247 #else
248 #define POOL_BLOCK_SIZE  8
249 #endif
250 #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)
251 #define BASE_POOL_TYPE_MASK 1
252 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))
253 
254 //
255 // Pool debugging/analysis/tracing flags
256 //
257 #define POOL_FLAG_CHECK_TIMERS 0x1
258 #define POOL_FLAG_CHECK_WORKERS 0x2
259 #define POOL_FLAG_CHECK_RESOURCES 0x4
260 #define POOL_FLAG_VERIFIER 0x8
261 #define POOL_FLAG_CHECK_DEADLOCK 0x10
262 #define POOL_FLAG_SPECIAL_POOL 0x20
263 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40
264 #define POOL_FLAG_CRASH_ON_FAILURE 0x80
265 
266 //
267 // BAD_POOL_HEADER codes during pool bugcheck
268 //
269 #define POOL_CORRUPTED_LIST 3
270 #define POOL_SIZE_OR_INDEX_MISMATCH 5
271 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6
272 #define POOL_HEADER_NOT_ALIGNED 7
273 #define POOL_HEADER_IS_ZERO 8
274 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9
275 #define POOL_ENTRY_NOT_FOUND 10
276 
277 //
278 // BAD_POOL_CALLER codes during pool bugcheck
279 //
280 #define POOL_ENTRY_CORRUPTED 1
281 #define POOL_ENTRY_ALREADY_FREE 6
282 #define POOL_ENTRY_NOT_ALLOCATED 7
283 #define POOL_ALLOC_IRQL_INVALID 8
284 #define POOL_FREE_IRQL_INVALID 9
285 #define POOL_BILLED_PROCESS_INVALID 13
286 #define POOL_HEADER_SIZE_INVALID 32
287 
288 typedef struct _POOL_DESCRIPTOR
289 {
290     POOL_TYPE PoolType;
291     ULONG PoolIndex;
292     ULONG RunningAllocs;
293     ULONG RunningDeAllocs;
294     ULONG TotalPages;
295     ULONG TotalBigPages;
296     ULONG Threshold;
297     PVOID LockAddress;
298     PVOID PendingFrees;
299     LONG PendingFreeDepth;
300     SIZE_T TotalBytes;
301     SIZE_T Spare0;
302     LIST_ENTRY ListHeads[POOL_LISTS_PER_PAGE];
303 } POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
304 
305 typedef struct _POOL_HEADER
306 {
307     union
308     {
309         struct
310         {
311 #ifdef _WIN64
312             USHORT PreviousSize:8;
313             USHORT PoolIndex:8;
314             USHORT BlockSize:8;
315             USHORT PoolType:8;
316 #else
317             USHORT PreviousSize:9;
318             USHORT PoolIndex:7;
319             USHORT BlockSize:9;
320             USHORT PoolType:7;
321 #endif
322         };
323         ULONG Ulong1;
324     };
325 #ifdef _WIN64
326     ULONG PoolTag;
327 #endif
328     union
329     {
330 #ifdef _WIN64
331         PEPROCESS ProcessBilled;
332 #else
333         ULONG PoolTag;
334 #endif
335         struct
336         {
337             USHORT AllocatorBackTraceIndex;
338             USHORT PoolTagHash;
339         };
340     };
341 } POOL_HEADER, *PPOOL_HEADER;
342 
343 C_ASSERT(sizeof(POOL_HEADER) == POOL_BLOCK_SIZE);
344 C_ASSERT(POOL_BLOCK_SIZE == sizeof(LIST_ENTRY));
345 
346 typedef struct _POOL_TRACKER_TABLE
347 {
348     ULONG Key;
349     LONG NonPagedAllocs;
350     LONG NonPagedFrees;
351     SIZE_T NonPagedBytes;
352     LONG PagedAllocs;
353     LONG PagedFrees;
354     SIZE_T PagedBytes;
355 } POOL_TRACKER_TABLE, *PPOOL_TRACKER_TABLE;
356 
357 typedef struct _POOL_TRACKER_BIG_PAGES
358 {
359     PVOID Va;
360     ULONG Key;
361     ULONG NumberOfPages;
362     PVOID QuotaObject;
363 } POOL_TRACKER_BIG_PAGES, *PPOOL_TRACKER_BIG_PAGES;
364 
365 extern ULONG ExpNumberOfPagedPools;
366 extern POOL_DESCRIPTOR NonPagedPoolDescriptor;
367 extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16 + 1];
368 extern PPOOL_TRACKER_TABLE PoolTrackTable;
369 
370 //
371 // END FIXFIX
372 //
373 
374 typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY
375 {
376     LIST_ENTRY Links;
377     UNICODE_STRING BaseName;
378 } MI_LARGE_PAGE_DRIVER_ENTRY, *PMI_LARGE_PAGE_DRIVER_ENTRY;
379 
380 typedef enum _MMSYSTEM_PTE_POOL_TYPE
381 {
382     SystemPteSpace,
383     NonPagedPoolExpansion,
384     MaximumPtePoolTypes
385 } MMSYSTEM_PTE_POOL_TYPE;
386 
387 typedef enum _MI_PFN_CACHE_ATTRIBUTE
388 {
389     MiNonCached,
390     MiCached,
391     MiWriteCombined,
392     MiNotMapped
393 } MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;
394 
395 typedef struct _PHYSICAL_MEMORY_RUN
396 {
397     PFN_NUMBER BasePage;
398     PFN_NUMBER PageCount;
399 } PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN;
400 
401 typedef struct _PHYSICAL_MEMORY_DESCRIPTOR
402 {
403     ULONG NumberOfRuns;
404     PFN_NUMBER NumberOfPages;
405     PHYSICAL_MEMORY_RUN Run[1];
406 } PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR;
407 
408 typedef struct _MMCOLOR_TABLES
409 {
410     PFN_NUMBER Flink;
411     PVOID Blink;
412     PFN_NUMBER Count;
413 } MMCOLOR_TABLES, *PMMCOLOR_TABLES;
414 
415 typedef struct _MI_LARGE_PAGE_RANGES
416 {
417     PFN_NUMBER StartFrame;
418     PFN_NUMBER LastFrame;
419 } MI_LARGE_PAGE_RANGES, *PMI_LARGE_PAGE_RANGES;
420 
421 typedef struct _MMVIEW
422 {
423     ULONG_PTR Entry;
424     PCONTROL_AREA ControlArea;
425 } MMVIEW, *PMMVIEW;
426 
427 typedef struct _MMSESSION
428 {
429     KGUARDED_MUTEX SystemSpaceViewLock;
430     PKGUARDED_MUTEX SystemSpaceViewLockPointer;
431     PCHAR SystemSpaceViewStart;
432     PMMVIEW SystemSpaceViewTable;
433     ULONG SystemSpaceHashSize;
434     ULONG SystemSpaceHashEntries;
435     ULONG SystemSpaceHashKey;
436     ULONG BitmapFailures;
437     PRTL_BITMAP SystemSpaceBitMap;
438 } MMSESSION, *PMMSESSION;
439 
440 typedef struct _MM_SESSION_SPACE_FLAGS
441 {
442     ULONG Initialized:1;
443     ULONG DeletePending:1;
444     ULONG Filler:30;
445 } MM_SESSION_SPACE_FLAGS;
446 
447 typedef struct _MM_SESSION_SPACE
448 {
449     struct _MM_SESSION_SPACE *GlobalVirtualAddress;
450     LONG ReferenceCount;
451     union
452     {
453         ULONG LongFlags;
454         MM_SESSION_SPACE_FLAGS Flags;
455     } u;
456     ULONG SessionId;
457     LIST_ENTRY ProcessList;
458     LARGE_INTEGER LastProcessSwappedOutTime;
459     PFN_NUMBER SessionPageDirectoryIndex;
460     SIZE_T NonPageablePages;
461     SIZE_T CommittedPages;
462     PVOID PagedPoolStart;
463     PVOID PagedPoolEnd;
464     PMMPDE PagedPoolBasePde;
465     ULONG Color;
466     LONG ResidentProcessCount;
467     ULONG SessionPoolAllocationFailures[4];
468     LIST_ENTRY ImageList;
469     LCID LocaleId;
470     ULONG AttachCount;
471     KEVENT AttachEvent;
472     PEPROCESS LastProcess;
473     LONG ProcessReferenceToSession;
474     LIST_ENTRY WsListEntry;
475     GENERAL_LOOKASIDE Lookaside[SESSION_POOL_LOOKASIDES];
476     MMSESSION Session;
477     KGUARDED_MUTEX PagedPoolMutex;
478     MM_PAGED_POOL_INFO PagedPoolInfo;
479     MMSUPPORT Vm;
480     PMMWSLE Wsle;
481     PDRIVER_UNLOAD Win32KDriverUnload;
482     POOL_DESCRIPTOR PagedPool;
483 #if defined (_M_AMD64)
484     MMPDE PageDirectory;
485 #else
486     PMMPDE PageTables;
487 #endif
488 #if defined (_M_AMD64)
489     PMMPTE SpecialPoolFirstPte;
490     PMMPTE SpecialPoolLastPte;
491     PMMPTE NextPdeForSpecialPoolExpansion;
492     PMMPTE LastPdeForSpecialPoolExpansion;
493     PFN_NUMBER SpecialPagesInUse;
494 #endif
495     LONG ImageLoadingCount;
496 } MM_SESSION_SPACE, *PMM_SESSION_SPACE;
497 
498 extern PMM_SESSION_SPACE MmSessionSpace;
499 extern MMPTE HyperTemplatePte;
500 extern MMPDE ValidKernelPde;
501 extern MMPTE ValidKernelPte;
502 extern MMPDE ValidKernelPdeLocal;
503 extern MMPTE ValidKernelPteLocal;
504 extern MMPDE DemandZeroPde;
505 extern MMPTE DemandZeroPte;
506 extern MMPTE PrototypePte;
507 extern MMPTE MmDecommittedPte;
508 extern BOOLEAN MmLargeSystemCache;
509 extern BOOLEAN MmZeroPageFile;
510 extern BOOLEAN MmProtectFreedNonPagedPool;
511 extern BOOLEAN MmTrackLockedPages;
512 extern BOOLEAN MmTrackPtes;
513 extern BOOLEAN MmDynamicPfn;
514 extern BOOLEAN MmMirroring;
515 extern BOOLEAN MmMakeLowMemory;
516 extern BOOLEAN MmEnforceWriteProtection;
517 extern SIZE_T MmAllocationFragment;
518 extern ULONG MmConsumedPoolPercentage;
519 extern ULONG MmVerifyDriverBufferType;
520 extern ULONG MmVerifyDriverLevel;
521 extern WCHAR MmVerifyDriverBuffer[512];
522 extern WCHAR MmLargePageDriverBuffer[512];
523 extern LIST_ENTRY MiLargePageDriverList;
524 extern BOOLEAN MiLargePageAllDrivers;
525 extern ULONG MmVerifyDriverBufferLength;
526 extern ULONG MmLargePageDriverBufferLength;
527 extern SIZE_T MmSizeOfNonPagedPoolInBytes;
528 extern SIZE_T MmMaximumNonPagedPoolInBytes;
529 extern PFN_NUMBER MmMaximumNonPagedPoolInPages;
530 extern PFN_NUMBER MmSizeOfPagedPoolInPages;
531 extern PVOID MmNonPagedSystemStart;
532 extern PVOID MmNonPagedPoolStart;
533 extern PVOID MmNonPagedPoolExpansionStart;
534 extern PVOID MmNonPagedPoolEnd;
535 extern SIZE_T MmSizeOfPagedPoolInBytes;
536 extern PVOID MmPagedPoolStart;
537 extern PVOID MmPagedPoolEnd;
538 extern PVOID MmSessionBase;
539 extern SIZE_T MmSessionSize;
540 extern PMMPTE MmFirstReservedMappingPte, MmLastReservedMappingPte;
541 extern PMMPTE MiFirstReservedZeroingPte;
542 extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2][MmMaximumCacheType];
543 extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock;
544 extern SIZE_T MmBootImageSize;
545 extern PMMPTE MmSystemPtesStart[MaximumPtePoolTypes];
546 extern PMMPTE MmSystemPtesEnd[MaximumPtePoolTypes];
547 extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor;
548 extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor;
549 extern ULONG_PTR MxPfnAllocation;
550 extern MM_PAGED_POOL_INFO MmPagedPoolInfo;
551 extern RTL_BITMAP MiPfnBitMap;
552 extern KGUARDED_MUTEX MmPagedPoolMutex;
553 extern KGUARDED_MUTEX MmSectionCommitMutex;
554 extern PVOID MmPagedPoolStart;
555 extern PVOID MmPagedPoolEnd;
556 extern PVOID MmNonPagedSystemStart;
557 extern PVOID MiSystemViewStart;
558 extern SIZE_T MmSystemViewSize;
559 extern PVOID MmSessionBase;
560 extern PVOID MiSessionSpaceEnd;
561 extern PMMPTE MiSessionImagePteStart;
562 extern PMMPTE MiSessionImagePteEnd;
563 extern PMMPTE MiSessionBasePte;
564 extern PMMPTE MiSessionLastPte;
565 extern SIZE_T MmSizeOfPagedPoolInBytes;
566 extern PMMPDE MmSystemPagePtes;
567 extern PVOID MmSystemCacheStart;
568 extern PVOID MmSystemCacheEnd;
569 extern MMSUPPORT MmSystemCacheWs;
570 extern SIZE_T MmAllocatedNonPagedPool;
571 extern ULONG MmSpecialPoolTag;
572 extern PVOID MmHyperSpaceEnd;
573 extern PMMWSL MmSystemCacheWorkingSetList;
574 extern SIZE_T MmMinimumNonPagedPoolSize;
575 extern ULONG MmMinAdditionNonPagedPoolPerMb;
576 extern SIZE_T MmDefaultMaximumNonPagedPool;
577 extern ULONG MmMaxAdditionNonPagedPoolPerMb;
578 extern ULONG MmSecondaryColors;
579 extern ULONG MmSecondaryColorMask;
580 extern ULONG MmNumberOfSystemPtes;
581 extern ULONG MmMaximumNonPagedPoolPercent;
582 extern ULONG MmLargeStackSize;
583 extern PMMCOLOR_TABLES MmFreePagesByColor[FreePageList + 1];
584 extern MMPFNLIST MmStandbyPageListByPriority[8];
585 extern ULONG MmProductType;
586 extern MM_SYSTEMSIZE MmSystemSize;
587 extern PKEVENT MiLowMemoryEvent;
588 extern PKEVENT MiHighMemoryEvent;
589 extern PKEVENT MiLowPagedPoolEvent;
590 extern PKEVENT MiHighPagedPoolEvent;
591 extern PKEVENT MiLowNonPagedPoolEvent;
592 extern PKEVENT MiHighNonPagedPoolEvent;
593 extern PFN_NUMBER MmLowMemoryThreshold;
594 extern PFN_NUMBER MmHighMemoryThreshold;
595 extern PFN_NUMBER MiLowPagedPoolThreshold;
596 extern PFN_NUMBER MiHighPagedPoolThreshold;
597 extern PFN_NUMBER MiLowNonPagedPoolThreshold;
598 extern PFN_NUMBER MiHighNonPagedPoolThreshold;
599 extern PFN_NUMBER MmMinimumFreePages;
600 extern PFN_NUMBER MmPlentyFreePages;
601 extern SIZE_T MmMinimumStackCommitInBytes;
602 extern PFN_COUNT MiExpansionPoolPagesInitialCharge;
603 extern PFN_NUMBER MmResidentAvailablePages;
604 extern PFN_NUMBER MmResidentAvailableAtInit;
605 extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
606 extern PFN_NUMBER MmTotalSystemDriverPages;
607 extern ULONG MmCritsectTimeoutSeconds;
608 extern PVOID MiSessionImageStart;
609 extern PVOID MiSessionImageEnd;
610 extern PMMPTE MiHighestUserPte;
611 extern PMMPDE MiHighestUserPde;
612 extern PFN_NUMBER MmSystemPageDirectory[PD_COUNT];
613 extern PMMPTE MmSharedUserDataPte;
614 extern LIST_ENTRY MmProcessList;
615 extern BOOLEAN MmZeroingPageThreadActive;
616 extern KEVENT MmZeroingPageEvent;
617 extern ULONG MmSystemPageColor;
618 extern ULONG MmProcessColorSeed;
619 extern PMMWSL MmWorkingSetList;
620 extern PFN_NUMBER MiNumberOfFreePages;
621 extern SIZE_T MmSessionViewSize;
622 extern SIZE_T MmSessionPoolSize;
623 extern SIZE_T MmSessionImageSize;
624 extern PVOID MiSystemViewStart;
625 extern PVOID MiSessionPoolEnd;     // 0xBE000000
626 extern PVOID MiSessionPoolStart;   // 0xBD000000
627 extern PVOID MiSessionViewStart;   // 0xBE000000
628 extern PVOID MiSessionSpaceWs;
629 extern ULONG MmMaximumDeadKernelStacks;
630 extern SLIST_HEADER MmDeadStackSListHead;
631 extern MM_AVL_TABLE MmSectionBasedRoot;
632 extern KGUARDED_MUTEX MmSectionBasedMutex;
633 extern PVOID MmHighSectionBase;
634 extern SIZE_T MmSystemLockPagesCount;
635 extern ULONG_PTR MmSubsectionBase;
636 extern LARGE_INTEGER MmCriticalSectionTimeout;
637 extern LIST_ENTRY MmWorkingSetExpansionHead;
638 extern KSPIN_LOCK MmExpansionLock;
639 extern PETHREAD MiExpansionLockOwner;
640 
641 FORCEINLINE
642 BOOLEAN
643 MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType)
644 {
645     return ((MemoryType == LoaderFree) ||
646             (MemoryType == LoaderLoadedProgram) ||
647             (MemoryType == LoaderFirmwareTemporary) ||
648             (MemoryType == LoaderOsloaderStack));
649 }
650 
651 FORCEINLINE
652 BOOLEAN
653 MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType)
654 {
655     return ((MemoryType == LoaderFirmwarePermanent) ||
656             (MemoryType == LoaderSpecialMemory) ||
657             (MemoryType == LoaderHALCachedMemory) ||
658             (MemoryType == LoaderBBTMemory));
659 }
660 
661 #ifdef _M_AMD64
662 FORCEINLINE
663 BOOLEAN
664 MiIsUserPxe(PVOID Address)
665 {
666     return ((ULONG_PTR)Address >> 7) == 0x1FFFFEDF6FB7DA0ULL;
667 }
668 
669 FORCEINLINE
670 BOOLEAN
671 MiIsUserPpe(PVOID Address)
672 {
673     return ((ULONG_PTR)Address >> 16) == 0xFFFFF6FB7DA0ULL;
674 }
675 
676 FORCEINLINE
677 BOOLEAN
678 MiIsUserPde(PVOID Address)
679 {
680     return ((ULONG_PTR)Address >> 25) == 0x7FFFFB7DA0ULL;
681 }
682 
683 FORCEINLINE
684 BOOLEAN
685 MiIsUserPte(PVOID Address)
686 {
687     return ((ULONG_PTR)Address >> 34) == 0x3FFFFDA0ULL;
688 }
689 #else
690 FORCEINLINE
691 BOOLEAN
692 MiIsUserPde(PVOID Address)
693 {
694     return ((Address >= (PVOID)MiAddressToPde(NULL)) &&
695             (Address <= (PVOID)MiHighestUserPde));
696 }
697 
698 FORCEINLINE
699 BOOLEAN
700 MiIsUserPte(PVOID Address)
701 {
702     return (Address <= (PVOID)MiHighestUserPte);
703 }
704 #endif
705 
706 //
707 // Figures out the hardware bits for a PTE
708 //
709 FORCEINLINE
710 ULONG_PTR
711 MiDetermineUserGlobalPteMask(IN PVOID PointerPte)
712 {
713     MMPTE TempPte;
714 
715     /* Start fresh */
716     TempPte.u.Long = 0;
717 
718     /* Make it valid and accessed */
719     TempPte.u.Hard.Valid = TRUE;
720     MI_MAKE_ACCESSED_PAGE(&TempPte);
721 
722     /* Is this for user-mode? */
723     if (
724 #if (_MI_PAGING_LEVELS == 4)
725         MiIsUserPxe(PointerPte) ||
726 #endif
727 #if (_MI_PAGING_LEVELS >= 3)
728         MiIsUserPpe(PointerPte) ||
729 #endif
730         MiIsUserPde(PointerPte) ||
731         MiIsUserPte(PointerPte))
732     {
733         /* Set the owner bit */
734         MI_MAKE_OWNER_PAGE(&TempPte);
735     }
736 
737     /* FIXME: We should also set the global bit */
738 
739     /* Return the protection */
740     return TempPte.u.Long;
741 }
742 
743 //
744 // Creates a valid kernel PTE with the given protection
745 //
746 FORCEINLINE
747 VOID
748 MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte,
749                             IN PMMPTE MappingPte,
750                             IN ULONG_PTR ProtectionMask,
751                             IN PFN_NUMBER PageFrameNumber)
752 {
753     /* Only valid for kernel, non-session PTEs */
754     ASSERT(MappingPte > MiHighestUserPte);
755     ASSERT(!MI_IS_SESSION_PTE(MappingPte));
756     ASSERT((MappingPte < (PMMPTE)PDE_BASE) || (MappingPte > (PMMPTE)PDE_TOP));
757 
758     /* Start fresh */
759     *NewPte = ValidKernelPte;
760 
761     /* Set the protection and page */
762     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
763     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
764 }
765 
766 //
767 // Creates a valid PTE with the given protection
768 //
769 FORCEINLINE
770 VOID
771 MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte,
772                      IN PMMPTE MappingPte,
773                      IN ULONG_PTR ProtectionMask,
774                      IN PFN_NUMBER PageFrameNumber)
775 {
776     /* Set the protection and page */
777     NewPte->u.Long = MiDetermineUserGlobalPteMask(MappingPte);
778     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
779     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
780 }
781 
782 //
783 // Creates a valid user PTE with the given protection
784 //
785 FORCEINLINE
786 VOID
787 MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte,
788                           IN PMMPTE MappingPte,
789                           IN ULONG_PTR ProtectionMask,
790                           IN PFN_NUMBER PageFrameNumber)
791 {
792     /* Only valid for kernel, non-session PTEs */
793     ASSERT(MappingPte <= MiHighestUserPte);
794 
795     /* Start fresh */
796     NewPte->u.Long = 0;
797 
798     /* Set the protection and page */
799     NewPte->u.Hard.Valid = TRUE;
800     NewPte->u.Hard.Owner = TRUE;
801     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
802     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
803 }
804 
805 #ifndef _M_AMD64
806 //
807 // Builds a Prototype PTE for the address of the PTE
808 //
809 FORCEINLINE
810 VOID
811 MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte,
812                       IN PMMPTE PointerPte)
813 {
814     ULONG_PTR Offset;
815 
816     /* Mark this as a prototype */
817     NewPte->u.Long = 0;
818     NewPte->u.Proto.Prototype = 1;
819 
820     /*
821      * Prototype PTEs are only valid in paged pool by design, this little trick
822      * lets us only use 30 bits for the adress of the PTE, as long as the area
823      * stays 1024MB At most.
824      */
825     Offset = (ULONG_PTR)PointerPte - (ULONG_PTR)MmPagedPoolStart;
826 
827     /*
828      * 7 bits go in the "low" (but we assume the bottom 2 are zero)
829      * and the other 21 bits go in the "high"
830      */
831     NewPte->u.Proto.ProtoAddressLow = (Offset & 0x1FC) >> 2;
832     NewPte->u.Proto.ProtoAddressHigh = (Offset & 0x3FFFFE00) >> 9;
833 }
834 
835 //
836 // Builds a Subsection PTE for the address of the Segment
837 //
838 FORCEINLINE
839 VOID
840 MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte,
841                        IN PVOID Segment)
842 {
843     ULONG_PTR Offset;
844 
845     /* Mark this as a prototype */
846     NewPte->u.Long = 0;
847     NewPte->u.Subsect.Prototype = 1;
848 
849     /*
850      * Segments are only valid either in nonpaged pool. We store the 20 bit
851      * difference either from the top or bottom of nonpaged pool, giving a
852      * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed
853      * 256MB.
854      */
855     if ((ULONG_PTR)Segment < ((ULONG_PTR)MmSubsectionBase + (128 * _1MB)))
856     {
857         Offset = (ULONG_PTR)Segment - (ULONG_PTR)MmSubsectionBase;
858         NewPte->u.Subsect.WhichPool = PagedPool;
859     }
860     else
861     {
862         Offset = (ULONG_PTR)MmNonPagedPoolEnd - (ULONG_PTR)Segment;
863         NewPte->u.Subsect.WhichPool = NonPagedPool;
864     }
865 
866     /*
867      * 4 bits go in the "low" (but we assume the bottom 3 are zero)
868      * and the other 20 bits go in the "high"
869      */
870     NewPte->u.Subsect.SubsectionAddressLow = (Offset & 0x78) >> 3;
871     NewPte->u.Subsect.SubsectionAddressHigh = (Offset & 0xFFFFF80) >> 7;
872 }
873 
874 FORCEINLINE
875 BOOLEAN
876 MI_IS_MAPPED_PTE(PMMPTE PointerPte)
877 {
878     /// \todo Make this reasonable code, this is UGLY!
879     return ((PointerPte->u.Long & 0xFFFFFC01) != 0);
880 }
881 
882 #endif
883 
884 FORCEINLINE
885 VOID
886 MI_MAKE_TRANSITION_PTE(_Out_ PMMPTE NewPte,
887                        _In_ PFN_NUMBER Page,
888                        _In_ ULONG Protection)
889 {
890     NewPte->u.Long = 0;
891     NewPte->u.Trans.Transition = 1;
892     NewPte->u.Trans.Protection = Protection;
893     NewPte->u.Trans.PageFrameNumber = Page;
894 }
895 
896 //
897 // Returns if the page is physically resident (ie: a large page)
898 // FIXFIX: CISC/x86 only?
899 //
900 FORCEINLINE
901 BOOLEAN
902 MI_IS_PHYSICAL_ADDRESS(IN PVOID Address)
903 {
904     PMMPDE PointerPde;
905 
906     /* Large pages are never paged out, always physically resident */
907     PointerPde = MiAddressToPde(Address);
908     return ((PointerPde->u.Hard.LargePage) && (PointerPde->u.Hard.Valid));
909 }
910 
911 //
912 // Writes a valid PTE
913 //
914 FORCEINLINE
915 VOID
916 MI_WRITE_VALID_PTE(IN PMMPTE PointerPte,
917                    IN MMPTE TempPte)
918 {
919     /* Write the valid PTE */
920     ASSERT(PointerPte->u.Hard.Valid == 0);
921     ASSERT(TempPte.u.Hard.Valid == 1);
922     *PointerPte = TempPte;
923 }
924 
925 //
926 // Updates a valid PTE
927 //
928 FORCEINLINE
929 VOID
930 MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte,
931                    IN MMPTE TempPte)
932 {
933     /* Write the valid PTE */
934     ASSERT(PointerPte->u.Hard.Valid == 1);
935     ASSERT(TempPte.u.Hard.Valid == 1);
936     ASSERT(PointerPte->u.Hard.PageFrameNumber == TempPte.u.Hard.PageFrameNumber);
937     *PointerPte = TempPte;
938 }
939 
940 //
941 // Writes an invalid PTE
942 //
943 FORCEINLINE
944 VOID
945 MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte,
946                      IN MMPTE InvalidPte)
947 {
948     /* Write the invalid PTE */
949     ASSERT(InvalidPte.u.Hard.Valid == 0);
950     ASSERT(InvalidPte.u.Long != 0);
951     *PointerPte = InvalidPte;
952 }
953 
954 //
955 // Erase the PTE completely
956 //
957 FORCEINLINE
958 VOID
959 MI_ERASE_PTE(IN PMMPTE PointerPte)
960 {
961     /* Zero out the PTE */
962     ASSERT(PointerPte->u.Long != 0);
963     PointerPte->u.Long = 0;
964 }
965 
966 //
967 // Writes a valid PDE
968 //
969 FORCEINLINE
970 VOID
971 MI_WRITE_VALID_PDE(IN PMMPDE PointerPde,
972                    IN MMPDE TempPde)
973 {
974     /* Write the valid PDE */
975     ASSERT(PointerPde->u.Hard.Valid == 0);
976     ASSERT(TempPde.u.Hard.Valid == 1);
977     *PointerPde = TempPde;
978 }
979 
980 //
981 // Writes an invalid PDE
982 //
983 FORCEINLINE
984 VOID
985 MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde,
986                      IN MMPDE InvalidPde)
987 {
988     /* Write the invalid PDE */
989     ASSERT(InvalidPde.u.Hard.Valid == 0);
990     ASSERT(InvalidPde.u.Long != 0);
991     *PointerPde = InvalidPde;
992 }
993 
994 //
995 // Checks if the thread already owns a working set
996 //
997 FORCEINLINE
998 BOOLEAN
999 MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread)
1000 {
1001     /* If any of these are held, return TRUE */
1002     return ((Thread->OwnsProcessWorkingSetExclusive) ||
1003             (Thread->OwnsProcessWorkingSetShared) ||
1004             (Thread->OwnsSystemWorkingSetExclusive) ||
1005             (Thread->OwnsSystemWorkingSetShared) ||
1006             (Thread->OwnsSessionWorkingSetExclusive) ||
1007             (Thread->OwnsSessionWorkingSetShared));
1008 }
1009 
1010 //
1011 // Checks if the process owns the working set lock
1012 //
1013 FORCEINLINE
1014 BOOLEAN
1015 MI_WS_OWNER(IN PEPROCESS Process)
1016 {
1017     /* Check if this process is the owner, and that the thread owns the WS */
1018     if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive == 0)
1019     {
1020         DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread());
1021     }
1022     if (KeGetCurrentThread()->ApcState.Process != &Process->Pcb)
1023     {
1024         DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process);
1025     }
1026     return ((KeGetCurrentThread()->ApcState.Process == &Process->Pcb) &&
1027             ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive) ||
1028              (PsGetCurrentThread()->OwnsProcessWorkingSetShared)));
1029 }
1030 
1031 //
1032 // New ARM3<->RosMM PAGE Architecture
1033 //
1034 FORCEINLINE
1035 BOOLEAN
1036 MiIsRosSectionObject(IN PVOID Section)
1037 {
1038     PROS_SECTION_OBJECT RosSection = Section;
1039     if ((RosSection->Type == 'SC') && (RosSection->Size == 'TN')) return TRUE;
1040     return FALSE;
1041 }
1042 
1043 #define MI_IS_ROS_PFN(x)     ((x)->u4.AweAllocation == TRUE)
1044 
1045 VOID
1046 NTAPI
1047 MiDecrementReferenceCount(
1048     IN PMMPFN Pfn1,
1049     IN PFN_NUMBER PageFrameIndex
1050 );
1051 
1052 FORCEINLINE
1053 BOOLEAN
1054 MI_IS_WS_UNSAFE(IN PEPROCESS Process)
1055 {
1056     return (Process->Vm.Flags.AcquiredUnsafe == TRUE);
1057 }
1058 
1059 //
1060 // Locks the working set for the given process
1061 //
1062 FORCEINLINE
1063 VOID
1064 MiLockProcessWorkingSet(IN PEPROCESS Process,
1065                         IN PETHREAD Thread)
1066 {
1067     /* Shouldn't already be owning the process working set */
1068     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1069     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1070 
1071     /* Block APCs, make sure that still nothing is already held */
1072     KeEnterGuardedRegion();
1073     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1074 
1075     /* Lock the working set */
1076     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1077 
1078     /* Now claim that we own the lock */
1079     ASSERT(!MI_IS_WS_UNSAFE(Process));
1080     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1081     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1082 }
1083 
1084 FORCEINLINE
1085 VOID
1086 MiLockProcessWorkingSetShared(IN PEPROCESS Process,
1087                               IN PETHREAD Thread)
1088 {
1089     /* Shouldn't already be owning the process working set */
1090     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1091     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1092 
1093     /* Block APCs, make sure that still nothing is already held */
1094     KeEnterGuardedRegion();
1095     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1096 
1097     /* Lock the working set */
1098     ExAcquirePushLockShared(&Process->Vm.WorkingSetMutex);
1099 
1100     /* Now claim that we own the lock */
1101     ASSERT(!MI_IS_WS_UNSAFE(Process));
1102     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1103     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1104     Thread->OwnsProcessWorkingSetShared = TRUE;
1105 }
1106 
1107 FORCEINLINE
1108 VOID
1109 MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1110                               IN PETHREAD Thread)
1111 {
1112     /* Shouldn't already be owning the process working set */
1113     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1114 
1115     /* APCs must be blocked, make sure that still nothing is already held */
1116     ASSERT(KeAreAllApcsDisabled() == TRUE);
1117     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1118 
1119     /* Lock the working set */
1120     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1121 
1122     /* Now claim that we own the lock */
1123     ASSERT(!MI_IS_WS_UNSAFE(Process));
1124     Process->Vm.Flags.AcquiredUnsafe = 1;
1125     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1126     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1127 }
1128 
1129 //
1130 // Unlocks the working set for the given process
1131 //
1132 FORCEINLINE
1133 VOID
1134 MiUnlockProcessWorkingSet(IN PEPROCESS Process,
1135                           IN PETHREAD Thread)
1136 {
1137     /* Make sure we are the owner of a safe acquisition */
1138     ASSERT(MI_WS_OWNER(Process));
1139     ASSERT(!MI_IS_WS_UNSAFE(Process));
1140 
1141     /* The thread doesn't own it anymore */
1142     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1143     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1144 
1145     /* Release the lock and re-enable APCs */
1146     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1147     KeLeaveGuardedRegion();
1148 }
1149 
1150 //
1151 // Unlocks the working set for the given process
1152 //
1153 FORCEINLINE
1154 VOID
1155 MiUnlockProcessWorkingSetShared(IN PEPROCESS Process,
1156                                 IN PETHREAD Thread)
1157 {
1158     /* Make sure we are the owner of a safe acquisition (because shared) */
1159     ASSERT(MI_WS_OWNER(Process));
1160     ASSERT(!MI_IS_WS_UNSAFE(Process));
1161 
1162     /* Ensure we are in a shared acquisition */
1163     ASSERT(Thread->OwnsProcessWorkingSetShared == TRUE);
1164     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1165 
1166     /* Don't claim the lock anylonger */
1167     Thread->OwnsProcessWorkingSetShared = FALSE;
1168 
1169     /* Release the lock and re-enable APCs */
1170     ExReleasePushLockShared(&Process->Vm.WorkingSetMutex);
1171     KeLeaveGuardedRegion();
1172 }
1173 
1174 //
1175 // Unlocks the working set for the given process
1176 //
1177 FORCEINLINE
1178 VOID
1179 MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1180                                 IN PETHREAD Thread)
1181 {
1182     /* Make sure we are the owner of an unsafe acquisition */
1183     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1184     ASSERT(KeAreAllApcsDisabled() == TRUE);
1185     ASSERT(MI_WS_OWNER(Process));
1186     ASSERT(MI_IS_WS_UNSAFE(Process));
1187 
1188     /* No longer unsafe */
1189     Process->Vm.Flags.AcquiredUnsafe = 0;
1190 
1191     /* The thread doesn't own it anymore */
1192     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1193     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1194 
1195     /* Release the lock but don't touch APC state */
1196     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1197     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1198 }
1199 
1200 //
1201 // Locks the working set
1202 //
1203 FORCEINLINE
1204 VOID
1205 MiLockWorkingSet(IN PETHREAD Thread,
1206                  IN PMMSUPPORT WorkingSet)
1207 {
1208     /* Block APCs */
1209     KeEnterGuardedRegion();
1210 
1211     /* Working set should be in global memory */
1212     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1213 
1214     /* Thread shouldn't already be owning something */
1215     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1216 
1217     /* Lock this working set */
1218     ExAcquirePushLockExclusive(&WorkingSet->WorkingSetMutex);
1219 
1220     /* Which working set is this? */
1221     if (WorkingSet == &MmSystemCacheWs)
1222     {
1223         /* Own the system working set */
1224         ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1225                (Thread->OwnsSystemWorkingSetShared == FALSE));
1226         Thread->OwnsSystemWorkingSetExclusive = TRUE;
1227     }
1228     else if (WorkingSet->Flags.SessionSpace)
1229     {
1230         /* Own the session working set */
1231         ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1232                (Thread->OwnsSessionWorkingSetShared == FALSE));
1233         Thread->OwnsSessionWorkingSetExclusive = TRUE;
1234     }
1235     else
1236     {
1237         /* Own the process working set */
1238         ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1239                (Thread->OwnsProcessWorkingSetShared == FALSE));
1240         Thread->OwnsProcessWorkingSetExclusive = TRUE;
1241     }
1242 }
1243 
1244 //
1245 // Unlocks the working set
1246 //
1247 FORCEINLINE
1248 VOID
1249 MiUnlockWorkingSet(IN PETHREAD Thread,
1250                    IN PMMSUPPORT WorkingSet)
1251 {
1252     /* Working set should be in global memory */
1253     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1254 
1255     /* Which working set is this? */
1256     if (WorkingSet == &MmSystemCacheWs)
1257     {
1258         /* Release the system working set */
1259         ASSERT((Thread->OwnsSystemWorkingSetExclusive == TRUE) ||
1260                (Thread->OwnsSystemWorkingSetShared == TRUE));
1261         Thread->OwnsSystemWorkingSetExclusive = FALSE;
1262     }
1263     else if (WorkingSet->Flags.SessionSpace)
1264     {
1265         /* Release the session working set */
1266         ASSERT((Thread->OwnsSessionWorkingSetExclusive == TRUE) ||
1267                (Thread->OwnsSessionWorkingSetShared == TRUE));
1268         Thread->OwnsSessionWorkingSetExclusive = 0;
1269     }
1270     else
1271     {
1272         /* Release the process working set */
1273         ASSERT((Thread->OwnsProcessWorkingSetExclusive) ||
1274                (Thread->OwnsProcessWorkingSetShared));
1275         Thread->OwnsProcessWorkingSetExclusive = FALSE;
1276     }
1277 
1278     /* Release the working set lock */
1279     ExReleasePushLockExclusive(&WorkingSet->WorkingSetMutex);
1280 
1281     /* Unblock APCs */
1282     KeLeaveGuardedRegion();
1283 }
1284 
1285 FORCEINLINE
1286 VOID
1287 MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process,
1288                                   IN PETHREAD Thread,
1289                                   OUT PBOOLEAN Safe,
1290                                   OUT PBOOLEAN Shared)
1291 {
1292     ASSERT(MI_WS_OWNER(Process));
1293 
1294     /* Check if the current owner is unsafe */
1295     if (MI_IS_WS_UNSAFE(Process))
1296     {
1297         /* Release unsafely */
1298         MiUnlockProcessWorkingSetUnsafe(Process, Thread);
1299         *Safe = FALSE;
1300         *Shared = FALSE;
1301     }
1302     else if (Thread->OwnsProcessWorkingSetExclusive == 1)
1303     {
1304         /* Owner is safe and exclusive, release normally */
1305         MiUnlockProcessWorkingSet(Process, Thread);
1306         *Safe = TRUE;
1307         *Shared = FALSE;
1308     }
1309     else
1310     {
1311         /* Owner is shared (implies safe), release normally */
1312         MiUnlockProcessWorkingSetShared(Process, Thread);
1313         *Safe = TRUE;
1314         *Shared = TRUE;
1315     }
1316 }
1317 
1318 FORCEINLINE
1319 VOID
1320 MiLockProcessWorkingSetForFault(IN PEPROCESS Process,
1321                                 IN PETHREAD Thread,
1322                                 IN BOOLEAN Safe,
1323                                 IN BOOLEAN Shared)
1324 {
1325     /* Check if this was a safe lock or not */
1326     if (Safe)
1327     {
1328         if (Shared)
1329         {
1330             /* Reacquire safely & shared */
1331             MiLockProcessWorkingSetShared(Process, Thread);
1332         }
1333         else
1334         {
1335             /* Reacquire safely */
1336             MiLockProcessWorkingSet(Process, Thread);
1337         }
1338     }
1339     else
1340     {
1341         /* Unsafe lock cannot be shared */
1342         ASSERT(Shared == FALSE);
1343         /* Reacquire unsafely */
1344         MiLockProcessWorkingSetUnsafe(Process, Thread);
1345     }
1346 }
1347 
1348 FORCEINLINE
1349 KIRQL
1350 MiAcquireExpansionLock(VOID)
1351 {
1352     KIRQL OldIrql;
1353 
1354     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1355     KeAcquireSpinLock(&MmExpansionLock, &OldIrql);
1356     ASSERT(MiExpansionLockOwner == NULL);
1357     MiExpansionLockOwner = PsGetCurrentThread();
1358     return OldIrql;
1359 }
1360 
1361 FORCEINLINE
1362 VOID
1363 MiReleaseExpansionLock(KIRQL OldIrql)
1364 {
1365     ASSERT(MiExpansionLockOwner == PsGetCurrentThread());
1366     MiExpansionLockOwner = NULL;
1367     KeReleaseSpinLock(&MmExpansionLock, OldIrql);
1368     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1369 }
1370 
1371 //
1372 // Returns the ProtoPTE inside a VAD for the given VPN
1373 //
1374 FORCEINLINE
1375 PMMPTE
1376 MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad,
1377                              IN ULONG_PTR Vpn)
1378 {
1379     PMMPTE ProtoPte;
1380 
1381     /* Find the offset within the VAD's prototype PTEs */
1382     ProtoPte = Vad->FirstPrototypePte + (Vpn - Vad->StartingVpn);
1383     ASSERT(ProtoPte <= Vad->LastContiguousPte);
1384     return ProtoPte;
1385 }
1386 
1387 //
1388 // Returns the PFN Database entry for the given page number
1389 // Warning: This is not necessarily a valid PFN database entry!
1390 //
1391 FORCEINLINE
1392 PMMPFN
1393 MI_PFN_ELEMENT(IN PFN_NUMBER Pfn)
1394 {
1395     /* Get the entry */
1396     return &MmPfnDatabase[Pfn];
1397 };
1398 
1399 //
1400 // Drops a locked page without dereferencing it
1401 //
1402 FORCEINLINE
1403 VOID
1404 MiDropLockCount(IN PMMPFN Pfn1)
1405 {
1406     /* This page shouldn't be locked, but it should be valid */
1407     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1408     ASSERT(Pfn1->u2.ShareCount == 0);
1409 
1410     /* Is this the last reference to the page */
1411     if (Pfn1->u3.e2.ReferenceCount == 1)
1412     {
1413         /* It better not be valid */
1414         ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1415 
1416         /* Is it a prototype PTE? */
1417         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1418             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1419         {
1420             /* FIXME: We should return commit */
1421             DPRINT1("Not returning commit for prototype PTE\n");
1422         }
1423 
1424         /* Update the counter */
1425         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1426     }
1427 }
1428 
1429 //
1430 // Drops a locked page and dereferences it
1431 //
1432 FORCEINLINE
1433 VOID
1434 MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1)
1435 {
1436     USHORT RefCount, OldRefCount;
1437     PFN_NUMBER PageFrameIndex;
1438 
1439     /* Loop while we decrement the page successfully */
1440     do
1441     {
1442         /* There should be at least one reference */
1443         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1444         ASSERT(OldRefCount != 0);
1445 
1446         /* Are we the last one */
1447         if (OldRefCount == 1)
1448         {
1449             /* The page shoudln't be shared not active at this point */
1450             ASSERT(Pfn1->u3.e2.ReferenceCount == 1);
1451             ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1452             ASSERT(Pfn1->u2.ShareCount == 0);
1453 
1454             /* Is it a prototype PTE? */
1455             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1456                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1457             {
1458                 /* FIXME: We should return commit */
1459                 DPRINT1("Not returning commit for prototype PTE\n");
1460             }
1461 
1462             /* Update the counter, and drop a reference the long way */
1463             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1464             PageFrameIndex = MiGetPfnEntryIndex(Pfn1);
1465             MiDecrementReferenceCount(Pfn1, PageFrameIndex);
1466             return;
1467         }
1468 
1469         /* Drop a reference the short way, and that's it */
1470         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1471                                                 OldRefCount - 1,
1472                                                 OldRefCount);
1473         ASSERT(RefCount != 0);
1474     } while (OldRefCount != RefCount);
1475 
1476     /* If we got here, there should be more than one reference */
1477     ASSERT(RefCount > 1);
1478     if (RefCount == 2)
1479     {
1480         /* Is it still being shared? */
1481         if (Pfn1->u2.ShareCount >= 1)
1482         {
1483             /* Then it should be valid */
1484             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1485 
1486             /* Is it a prototype PTE? */
1487             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1488                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1489             {
1490                 /* We don't handle ethis */
1491                 ASSERT(FALSE);
1492             }
1493 
1494             /* Update the counter */
1495             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1496         }
1497     }
1498 }
1499 
1500 //
1501 // References a locked page and updates the counter
1502 // Used in MmProbeAndLockPages to handle different edge cases
1503 //
1504 FORCEINLINE
1505 VOID
1506 MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1)
1507 {
1508     USHORT RefCount, OldRefCount;
1509 
1510     /* Sanity check */
1511     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1512 
1513     /* Does ARM3 own the page? */
1514     if (MI_IS_ROS_PFN(Pfn1))
1515     {
1516         /* ReactOS Mm doesn't track share count */
1517         ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1518     }
1519     else
1520     {
1521         /* On ARM3 pages, we should see a valid share count */
1522         ASSERT((Pfn1->u2.ShareCount != 0) && (Pfn1->u3.e1.PageLocation == ActiveAndValid));
1523 
1524         /* Is it a prototype PTE? */
1525         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1526             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1527         {
1528             /* FIXME: We should charge commit */
1529             DPRINT1("Not charging commit for prototype PTE\n");
1530         }
1531     }
1532 
1533     /* More locked pages! */
1534     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1535 
1536     /* Loop trying to update the reference count */
1537     do
1538     {
1539         /* Get the current reference count, make sure it's valid */
1540         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1541         ASSERT(OldRefCount != 0);
1542         ASSERT(OldRefCount < 2500);
1543 
1544         /* Bump it up by one */
1545         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1546                                                 OldRefCount + 1,
1547                                                 OldRefCount);
1548         ASSERT(RefCount != 0);
1549     } while (OldRefCount != RefCount);
1550 
1551     /* Was this the first lock attempt? If not, undo our bump */
1552     if (OldRefCount != 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1553 }
1554 
1555 //
1556 // References a locked page and updates the counter
1557 // Used in all other cases except MmProbeAndLockPages
1558 //
1559 FORCEINLINE
1560 VOID
1561 MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1)
1562 {
1563     USHORT NewRefCount;
1564 
1565     /* Is it a prototype PTE? */
1566     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1567         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1568     {
1569         /* FIXME: We should charge commit */
1570         DPRINT1("Not charging commit for prototype PTE\n");
1571     }
1572 
1573     /* More locked pages! */
1574     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1575 
1576     /* Update the reference count */
1577     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1578     if (NewRefCount == 2)
1579     {
1580         /* Is it locked or shared? */
1581         if (Pfn1->u2.ShareCount)
1582         {
1583             /* It's shared, so make sure it's active */
1584             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1585         }
1586         else
1587         {
1588             /* It's locked, so we shouldn't lock again */
1589             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1590         }
1591     }
1592     else
1593     {
1594         /* Someone had already locked the page, so undo our bump */
1595         ASSERT(NewRefCount < 2500);
1596         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1597     }
1598 }
1599 
1600 //
1601 // References a locked page and updates the counter
1602 // Used in all other cases except MmProbeAndLockPages
1603 //
1604 FORCEINLINE
1605 VOID
1606 MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1)
1607 {
1608     USHORT NewRefCount;
1609 
1610     /* Make sure the page isn't used yet */
1611     ASSERT(Pfn1->u2.ShareCount == 0);
1612     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1613 
1614     /* Is it a prototype PTE? */
1615     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1616         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1617     {
1618         /* FIXME: We should charge commit */
1619         DPRINT1("Not charging commit for prototype PTE\n");
1620     }
1621 
1622     /* More locked pages! */
1623     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1624 
1625     /* Update the reference count */
1626     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1627     if (NewRefCount != 1)
1628     {
1629         /* Someone had already locked the page, so undo our bump */
1630         ASSERT(NewRefCount < 2500);
1631         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1632     }
1633 }
1634 
1635 FORCEINLINE
1636 VOID
1637 MiIncrementPageTableReferences(IN PVOID Address)
1638 {
1639     PUSHORT RefCount;
1640 
1641     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1642 
1643     *RefCount += 1;
1644     ASSERT(*RefCount <= PTE_PER_PAGE);
1645 }
1646 
1647 FORCEINLINE
1648 VOID
1649 MiDecrementPageTableReferences(IN PVOID Address)
1650 {
1651     PUSHORT RefCount;
1652 
1653     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1654 
1655     *RefCount -= 1;
1656     ASSERT(*RefCount < PTE_PER_PAGE);
1657 }
1658 
1659 FORCEINLINE
1660 USHORT
1661 MiQueryPageTableReferences(IN PVOID Address)
1662 {
1663     PUSHORT RefCount;
1664 
1665     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1666 
1667     return *RefCount;
1668 }
1669 
1670 INIT_FUNCTION
1671 BOOLEAN
1672 NTAPI
1673 MmArmInitSystem(
1674     IN ULONG Phase,
1675     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1676 );
1677 
1678 INIT_FUNCTION
1679 VOID
1680 NTAPI
1681 MiInitializeSessionSpaceLayout(VOID);
1682 
1683 INIT_FUNCTION
1684 NTSTATUS
1685 NTAPI
1686 MiInitMachineDependent(
1687     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1688 );
1689 
1690 INIT_FUNCTION
1691 VOID
1692 NTAPI
1693 MiComputeColorInformation(
1694     VOID
1695 );
1696 
1697 INIT_FUNCTION
1698 VOID
1699 NTAPI
1700 MiMapPfnDatabase(
1701     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1702 );
1703 
1704 INIT_FUNCTION
1705 VOID
1706 NTAPI
1707 MiInitializeColorTables(
1708     VOID
1709 );
1710 
1711 INIT_FUNCTION
1712 VOID
1713 NTAPI
1714 MiInitializePfnDatabase(
1715     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1716 );
1717 
1718 VOID
1719 NTAPI
1720 MiInitializeSessionWsSupport(
1721     VOID
1722 );
1723 
1724 VOID
1725 NTAPI
1726 MiInitializeSessionIds(
1727     VOID
1728 );
1729 
1730 INIT_FUNCTION
1731 BOOLEAN
1732 NTAPI
1733 MiInitializeMemoryEvents(
1734     VOID
1735 );
1736 
1737 INIT_FUNCTION
1738 PFN_NUMBER
1739 NTAPI
1740 MxGetNextPage(
1741     IN PFN_NUMBER PageCount
1742 );
1743 
1744 INIT_FUNCTION
1745 PPHYSICAL_MEMORY_DESCRIPTOR
1746 NTAPI
1747 MmInitializeMemoryLimits(
1748     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1749     IN PBOOLEAN IncludeType
1750 );
1751 
1752 PFN_NUMBER
1753 NTAPI
1754 MiPagesInLoaderBlock(
1755     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1756     IN PBOOLEAN IncludeType
1757 );
1758 
1759 VOID
1760 FASTCALL
1761 MiSyncARM3WithROS(
1762     IN PVOID AddressStart,
1763     IN PVOID AddressEnd
1764 );
1765 
1766 NTSTATUS
1767 NTAPI
1768 MiRosProtectVirtualMemory(
1769     IN PEPROCESS Process,
1770     IN OUT PVOID *BaseAddress,
1771     IN OUT PSIZE_T NumberOfBytesToProtect,
1772     IN ULONG NewAccessProtection,
1773     OUT PULONG OldAccessProtection OPTIONAL
1774 );
1775 
1776 NTSTATUS
1777 NTAPI
1778 MmArmAccessFault(
1779     IN ULONG FaultCode,
1780     IN PVOID Address,
1781     IN KPROCESSOR_MODE Mode,
1782     IN PVOID TrapInformation
1783 );
1784 
1785 NTSTATUS
1786 FASTCALL
1787 MiCheckPdeForPagedPool(
1788     IN PVOID Address
1789 );
1790 
1791 INIT_FUNCTION
1792 VOID
1793 NTAPI
1794 MiInitializeNonPagedPool(
1795     VOID
1796 );
1797 
1798 INIT_FUNCTION
1799 VOID
1800 NTAPI
1801 MiInitializeNonPagedPoolThresholds(
1802     VOID
1803 );
1804 
1805 INIT_FUNCTION
1806 VOID
1807 NTAPI
1808 MiInitializePoolEvents(
1809     VOID
1810 );
1811 
1812 INIT_FUNCTION
1813 VOID                      //
1814 NTAPI                     //
1815 InitializePool(           //
1816     IN POOL_TYPE PoolType,// FIXFIX: This should go in ex.h after the pool merge
1817     IN ULONG Threshold    //
1818 );                        //
1819 
1820 // FIXFIX: THIS ONE TOO
1821 INIT_FUNCTION
1822 VOID
1823 NTAPI
1824 ExInitializePoolDescriptor(
1825     IN PPOOL_DESCRIPTOR PoolDescriptor,
1826     IN POOL_TYPE PoolType,
1827     IN ULONG PoolIndex,
1828     IN ULONG Threshold,
1829     IN PVOID PoolLock
1830 );
1831 
1832 NTSTATUS
1833 NTAPI
1834 MiInitializeSessionPool(
1835     VOID
1836 );
1837 
1838 INIT_FUNCTION
1839 VOID
1840 NTAPI
1841 MiInitializeSystemPtes(
1842     IN PMMPTE StartingPte,
1843     IN ULONG NumberOfPtes,
1844     IN MMSYSTEM_PTE_POOL_TYPE PoolType
1845 );
1846 
1847 PMMPTE
1848 NTAPI
1849 MiReserveSystemPtes(
1850     IN ULONG NumberOfPtes,
1851     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1852 );
1853 
1854 VOID
1855 NTAPI
1856 MiReleaseSystemPtes(
1857     IN PMMPTE StartingPte,
1858     IN ULONG NumberOfPtes,
1859     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1860 );
1861 
1862 
1863 PFN_NUMBER
1864 NTAPI
1865 MiFindContiguousPages(
1866     IN PFN_NUMBER LowestPfn,
1867     IN PFN_NUMBER HighestPfn,
1868     IN PFN_NUMBER BoundaryPfn,
1869     IN PFN_NUMBER SizeInPages,
1870     IN MEMORY_CACHING_TYPE CacheType
1871 );
1872 
1873 PVOID
1874 NTAPI
1875 MiCheckForContiguousMemory(
1876     IN PVOID BaseAddress,
1877     IN PFN_NUMBER BaseAddressPages,
1878     IN PFN_NUMBER SizeInPages,
1879     IN PFN_NUMBER LowestPfn,
1880     IN PFN_NUMBER HighestPfn,
1881     IN PFN_NUMBER BoundaryPfn,
1882     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
1883 );
1884 
1885 PMDL
1886 NTAPI
1887 MiAllocatePagesForMdl(
1888     IN PHYSICAL_ADDRESS LowAddress,
1889     IN PHYSICAL_ADDRESS HighAddress,
1890     IN PHYSICAL_ADDRESS SkipBytes,
1891     IN SIZE_T TotalBytes,
1892     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
1893     IN ULONG Flags
1894 );
1895 
1896 VOID
1897 NTAPI
1898 MiInsertPageInList(
1899     IN PMMPFNLIST ListHead,
1900     IN PFN_NUMBER PageFrameIndex
1901 );
1902 
1903 VOID
1904 NTAPI
1905 MiUnlinkFreeOrZeroedPage(
1906     IN PMMPFN Entry
1907 );
1908 
1909 VOID
1910 NTAPI
1911 MiUnlinkPageFromList(
1912     IN PMMPFN Pfn
1913 );
1914 
1915 VOID
1916 NTAPI
1917 MiInitializePfn(
1918     IN PFN_NUMBER PageFrameIndex,
1919     IN PMMPTE PointerPte,
1920     IN BOOLEAN Modified
1921 );
1922 
1923 NTSTATUS
1924 NTAPI
1925 MiInitializeAndChargePfn(
1926     OUT PPFN_NUMBER PageFrameIndex,
1927     IN PMMPDE PointerPde,
1928     IN PFN_NUMBER ContainingPageFrame,
1929     IN BOOLEAN SessionAllocation
1930 );
1931 
1932 VOID
1933 NTAPI
1934 MiInitializePfnAndMakePteValid(
1935     IN PFN_NUMBER PageFrameIndex,
1936     IN PMMPTE PointerPte,
1937     IN MMPTE TempPte
1938 );
1939 
1940 VOID
1941 NTAPI
1942 MiInitializePfnForOtherProcess(
1943     IN PFN_NUMBER PageFrameIndex,
1944     IN PVOID PteAddress,
1945     IN PFN_NUMBER PteFrame
1946 );
1947 
1948 VOID
1949 NTAPI
1950 MiDecrementShareCount(
1951     IN PMMPFN Pfn1,
1952     IN PFN_NUMBER PageFrameIndex
1953 );
1954 
1955 PFN_NUMBER
1956 NTAPI
1957 MiRemoveAnyPage(
1958     IN ULONG Color
1959 );
1960 
1961 PFN_NUMBER
1962 NTAPI
1963 MiRemoveZeroPage(
1964     IN ULONG Color
1965 );
1966 
1967 VOID
1968 NTAPI
1969 MiZeroPhysicalPage(
1970     IN PFN_NUMBER PageFrameIndex
1971 );
1972 
1973 VOID
1974 NTAPI
1975 MiInsertPageInFreeList(
1976     IN PFN_NUMBER PageFrameIndex
1977 );
1978 
1979 PFN_COUNT
1980 NTAPI
1981 MiDeleteSystemPageableVm(
1982     IN PMMPTE PointerPte,
1983     IN PFN_NUMBER PageCount,
1984     IN ULONG Flags,
1985     OUT PPFN_NUMBER ValidPages
1986 );
1987 
1988 ULONG
1989 NTAPI
1990 MiGetPageProtection(
1991     IN PMMPTE PointerPte
1992 );
1993 
1994 PLDR_DATA_TABLE_ENTRY
1995 NTAPI
1996 MiLookupDataTableEntry(
1997     IN PVOID Address
1998 );
1999 
2000 INIT_FUNCTION
2001 VOID
2002 NTAPI
2003 MiInitializeDriverLargePageList(
2004     VOID
2005 );
2006 
2007 INIT_FUNCTION
2008 VOID
2009 NTAPI
2010 MiInitializeLargePageSupport(
2011     VOID
2012 );
2013 
2014 INIT_FUNCTION
2015 VOID
2016 NTAPI
2017 MiSyncCachedRanges(
2018     VOID
2019 );
2020 
2021 BOOLEAN
2022 NTAPI
2023 MiIsPfnInUse(
2024     IN PMMPFN Pfn1
2025 );
2026 
2027 PMMVAD
2028 NTAPI
2029 MiLocateAddress(
2030     IN PVOID VirtualAddress
2031 );
2032 
2033 TABLE_SEARCH_RESULT
2034 NTAPI
2035 MiCheckForConflictingNode(
2036     IN ULONG_PTR StartVpn,
2037     IN ULONG_PTR EndVpn,
2038     IN PMM_AVL_TABLE Table,
2039     OUT PMMADDRESS_NODE *NodeOrParent
2040 );
2041 
2042 TABLE_SEARCH_RESULT
2043 NTAPI
2044 MiFindEmptyAddressRangeDownTree(
2045     IN SIZE_T Length,
2046     IN ULONG_PTR BoundaryAddress,
2047     IN ULONG_PTR Alignment,
2048     IN PMM_AVL_TABLE Table,
2049     OUT PULONG_PTR Base,
2050     OUT PMMADDRESS_NODE *Parent
2051 );
2052 
2053 NTSTATUS
2054 NTAPI
2055 MiFindEmptyAddressRangeDownBasedTree(
2056     IN SIZE_T Length,
2057     IN ULONG_PTR BoundaryAddress,
2058     IN ULONG_PTR Alignment,
2059     IN PMM_AVL_TABLE Table,
2060     OUT PULONG_PTR Base
2061 );
2062 
2063 TABLE_SEARCH_RESULT
2064 NTAPI
2065 MiFindEmptyAddressRangeInTree(
2066     IN SIZE_T Length,
2067     IN ULONG_PTR Alignment,
2068     IN PMM_AVL_TABLE Table,
2069     OUT PMMADDRESS_NODE *PreviousVad,
2070     OUT PULONG_PTR Base
2071 );
2072 
2073 NTSTATUS
2074 NTAPI
2075 MiCheckSecuredVad(
2076     IN PMMVAD Vad,
2077     IN PVOID Base,
2078     IN SIZE_T Size,
2079     IN ULONG ProtectionMask
2080 );
2081 
2082 VOID
2083 NTAPI
2084 MiInsertVad(
2085     _Inout_ PMMVAD Vad,
2086     _Inout_ PMM_AVL_TABLE VadRoot);
2087 
2088 NTSTATUS
2089 NTAPI
2090 MiInsertVadEx(
2091     _Inout_ PMMVAD Vad,
2092     _In_ ULONG_PTR *BaseAddress,
2093     _In_ SIZE_T ViewSize,
2094     _In_ ULONG_PTR HighestAddress,
2095     _In_ ULONG_PTR Alignment,
2096     _In_ ULONG AllocationType);
2097 
2098 VOID
2099 NTAPI
2100 MiInsertBasedSection(
2101     IN PSECTION Section
2102 );
2103 
2104 NTSTATUS
2105 NTAPI
2106 MiUnmapViewOfSection(
2107     IN PEPROCESS Process,
2108     IN PVOID BaseAddress,
2109     IN ULONG Flags
2110 );
2111 
2112 NTSTATUS
2113 NTAPI
2114 MiRosUnmapViewOfSection(
2115     IN PEPROCESS Process,
2116     IN PVOID BaseAddress,
2117     IN BOOLEAN SkipDebuggerNotify
2118 );
2119 
2120 VOID
2121 NTAPI
2122 MiInsertNode(
2123     IN PMM_AVL_TABLE Table,
2124     IN PMMADDRESS_NODE NewNode,
2125     PMMADDRESS_NODE Parent,
2126     TABLE_SEARCH_RESULT Result
2127 );
2128 
2129 VOID
2130 NTAPI
2131 MiRemoveNode(
2132     IN PMMADDRESS_NODE Node,
2133     IN PMM_AVL_TABLE Table
2134 );
2135 
2136 PMMADDRESS_NODE
2137 NTAPI
2138 MiGetPreviousNode(
2139     IN PMMADDRESS_NODE Node
2140 );
2141 
2142 PMMADDRESS_NODE
2143 NTAPI
2144 MiGetNextNode(
2145     IN PMMADDRESS_NODE Node
2146 );
2147 
2148 BOOLEAN
2149 NTAPI
2150 MiInitializeSystemSpaceMap(
2151     IN PMMSESSION InputSession OPTIONAL
2152 );
2153 
2154 VOID
2155 NTAPI
2156 MiSessionRemoveProcess(
2157     VOID
2158 );
2159 
2160 VOID
2161 NTAPI
2162 MiReleaseProcessReferenceToSessionDataPage(
2163     IN PMM_SESSION_SPACE SessionGlobal
2164 );
2165 
2166 VOID
2167 NTAPI
2168 MiSessionAddProcess(
2169     IN PEPROCESS NewProcess
2170 );
2171 
2172 NTSTATUS
2173 NTAPI
2174 MiSessionCommitPageTables(
2175     IN PVOID StartVa,
2176     IN PVOID EndVa
2177 );
2178 
2179 ULONG
2180 NTAPI
2181 MiMakeProtectionMask(
2182     IN ULONG Protect
2183 );
2184 
2185 VOID
2186 NTAPI
2187 MiDeleteVirtualAddresses(
2188     IN ULONG_PTR Va,
2189     IN ULONG_PTR EndingAddress,
2190     IN PMMVAD Vad
2191 );
2192 
2193 VOID
2194 NTAPI
2195 MiDeletePte(
2196     IN PMMPTE PointerPte,
2197     IN PVOID VirtualAddress,
2198     IN PEPROCESS CurrentProcess,
2199     IN PMMPTE PrototypePte
2200 );
2201 
2202 ULONG
2203 NTAPI
2204 MiMakeSystemAddressValid(
2205     IN PVOID PageTableVirtualAddress,
2206     IN PEPROCESS CurrentProcess
2207 );
2208 
2209 ULONG
2210 NTAPI
2211 MiMakeSystemAddressValidPfn(
2212     IN PVOID VirtualAddress,
2213     IN KIRQL OldIrql
2214 );
2215 
2216 VOID
2217 NTAPI
2218 MiRemoveMappedView(
2219     IN PEPROCESS CurrentProcess,
2220     IN PMMVAD Vad
2221 );
2222 
2223 PSUBSECTION
2224 NTAPI
2225 MiLocateSubsection(
2226     IN PMMVAD Vad,
2227     IN ULONG_PTR Vpn
2228 );
2229 
2230 VOID
2231 NTAPI
2232 MiDeleteARM3Section(
2233     PVOID ObjectBody
2234 );
2235 
2236 NTSTATUS
2237 NTAPI
2238 MiQueryMemorySectionName(
2239     IN HANDLE ProcessHandle,
2240     IN PVOID BaseAddress,
2241     OUT PVOID MemoryInformation,
2242     IN SIZE_T MemoryInformationLength,
2243     OUT PSIZE_T ReturnLength
2244 );
2245 
2246 NTSTATUS
2247 NTAPI
2248 MiRosUnmapViewInSystemSpace(
2249     IN PVOID MappedBase
2250 );
2251 
2252 POOL_TYPE
2253 NTAPI
2254 MmDeterminePoolType(
2255     IN PVOID PoolAddress
2256 );
2257 
2258 VOID
2259 NTAPI
2260 MiMakePdeExistAndMakeValid(
2261     IN PMMPDE PointerPde,
2262     IN PEPROCESS TargetProcess,
2263     IN KIRQL OldIrql
2264 );
2265 
2266 //
2267 // MiRemoveZeroPage will use inline code to zero out the page manually if only
2268 // free pages are available. In some scenarios, we don't/can't run that piece of
2269 // code and would rather only have a real zero page. If we can't have a zero page,
2270 // then we'd like to have our own code to grab a free page and zero it out, by
2271 // using MiRemoveAnyPage. This macro implements this.
2272 //
2273 FORCEINLINE
2274 PFN_NUMBER
2275 MiRemoveZeroPageSafe(IN ULONG Color)
2276 {
2277     if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color);
2278     return 0;
2279 }
2280 
2281 #if (_MI_PAGING_LEVELS == 2)
2282 FORCEINLINE
2283 BOOLEAN
2284 MiSynchronizeSystemPde(PMMPDE PointerPde)
2285 {
2286     MMPDE SystemPde;
2287     ULONG Index;
2288 
2289     /* Get the Index from the PDE */
2290     Index = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE);
2291 
2292     /* Copy the PDE from the double-mapped system page directory */
2293     SystemPde = MmSystemPagePtes[Index];
2294     *PointerPde = SystemPde;
2295 
2296     /* Make sure we re-read the PDE and PTE */
2297     KeMemoryBarrierWithoutFence();
2298 
2299     /* Return, if we had success */
2300     return SystemPde.u.Hard.Valid != 0;
2301 }
2302 #endif
2303 
2304 /* EOF */
2305