1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/mm/ARM3/miarm.h
5 * PURPOSE: ARM Memory Manager Header
6 * PROGRAMMERS: ReactOS Portable Systems Group
7 */
8
9 #pragma once
10
11 #ifdef __cplusplus
12 extern "C" {
13 #endif
14
15 #define MI_LOWEST_VAD_ADDRESS (PVOID)MM_LOWEST_USER_ADDRESS
16
17 /* Make the code cleaner with some definitions for size multiples */
18 #define _1KB (1024u)
19 #define _1MB (1024 * _1KB)
20 #define _1GB (1024 * _1MB)
21
22 /* Everyone loves 64K */
23 #define _64K (64 * _1KB)
24
25 /* Size of a page table */
26 #define PT_SIZE (PTE_PER_PAGE * sizeof(MMPTE))
27
28 /* Size of a page directory */
29 #define PD_SIZE (PDE_PER_PAGE * sizeof(MMPDE))
30
31 /* Size of all page directories for a process */
32 #define SYSTEM_PD_SIZE (PPE_PER_PAGE * PD_SIZE)
33 #ifdef _M_IX86
34 C_ASSERT(SYSTEM_PD_SIZE == PAGE_SIZE);
35 #endif
36
37 //
38 // Protection Bits part of the internal memory manager Protection Mask, from:
39 // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel
40 // https://www.reactos.org/wiki/Techwiki:Memory_Protection_constants
41 // and public assertions.
42 //
43 #define MM_ZERO_ACCESS 0
44 #define MM_READONLY 1
45 #define MM_EXECUTE 2
46 #define MM_EXECUTE_READ 3
47 #define MM_READWRITE 4
48 #define MM_WRITECOPY 5
49 #define MM_EXECUTE_READWRITE 6
50 #define MM_EXECUTE_WRITECOPY 7
51 #define MM_PROTECT_ACCESS 7
52
53 //
54 // These are flags on top of the actual protection mask
55 //
56 #define MM_NOCACHE 0x08
57 #define MM_GUARDPAGE 0x10
58 #define MM_WRITECOMBINE 0x18
59 #define MM_PROTECT_SPECIAL 0x18
60
61 //
62 // These are special cases
63 //
64 #define MM_DECOMMIT (MM_ZERO_ACCESS | MM_GUARDPAGE)
65 #define MM_NOACCESS (MM_ZERO_ACCESS | MM_WRITECOMBINE)
66 #define MM_OUTSWAPPED_KSTACK (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE)
67 #define MM_INVALID_PROTECTION 0xFFFFFFFF
68
69 //
70 // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits
71 // The Memory Manager's definition define the attributes that must be preserved
72 // and these PTE definitions describe the attributes in the hardware sense. This
73 // helps deal with hardware differences between the actual boolean expression of
74 // the argument.
75 //
76 // For example, in the logical attributes, we want to express read-only as a flag
77 // but on x86, it is writability that must be set. On the other hand, on x86, just
78 // like in the kernel, it is disabling the caches that requires a special flag,
79 // while on certain architectures such as ARM, it is enabling the cache which
80 // requires a flag.
81 //
82 #if defined(_M_IX86)
83 //
84 // Access Flags
85 //
86 #define PTE_READONLY 0 // Doesn't exist on x86
87 #define PTE_EXECUTE 0 // Not worrying about NX yet
88 #define PTE_EXECUTE_READ 0 // Not worrying about NX yet
89 #define PTE_READWRITE 0x2
90 #define PTE_WRITECOPY 0x200
91 #define PTE_EXECUTE_READWRITE 0x2 // Not worrying about NX yet
92 #define PTE_EXECUTE_WRITECOPY 0x200
93 #define PTE_PROTOTYPE 0x400
94
95 //
96 // State Flags
97 //
98 #define PTE_VALID 0x1
99 #define PTE_ACCESSED 0x20
100 #define PTE_DIRTY 0x40
101
102 //
103 // Cache flags
104 //
105 #define PTE_ENABLE_CACHE 0
106 #define PTE_DISABLE_CACHE 0x10
107 #define PTE_WRITECOMBINED_CACHE 0x10
108 #define PTE_PROTECT_MASK 0x612
109 #elif defined(_M_AMD64)
110 //
111 // Access Flags
112 //
113 #define PTE_READONLY 0x8000000000000000ULL
114 #define PTE_EXECUTE 0x0000000000000000ULL
115 #define PTE_EXECUTE_READ PTE_EXECUTE /* EXECUTE implies READ on x64 */
116 #define PTE_READWRITE 0x8000000000000002ULL
117 #define PTE_WRITECOPY 0x8000000000000200ULL
118 #define PTE_EXECUTE_READWRITE 0x0000000000000002ULL
119 #define PTE_EXECUTE_WRITECOPY 0x0000000000000200ULL
120 #define PTE_PROTOTYPE 0x0000000000000400ULL
121
122 //
123 // State Flags
124 //
125 #define PTE_VALID 0x0000000000000001ULL
126 #define PTE_ACCESSED 0x0000000000000020ULL
127 #define PTE_DIRTY 0x0000000000000040ULL
128
129 //
130 // Cache flags
131 //
132 #define PTE_ENABLE_CACHE 0x0000000000000000ULL
133 #define PTE_DISABLE_CACHE 0x0000000000000010ULL
134 #define PTE_WRITECOMBINED_CACHE 0x0000000000000010ULL
135 #define PTE_PROTECT_MASK 0x8000000000000612ULL
136 #elif defined(_M_ARM)
137 #define PTE_READONLY 0x200
138 #define PTE_EXECUTE 0 // Not worrying about NX yet
139 #define PTE_EXECUTE_READ 0 // Not worrying about NX yet
140 #define PTE_READWRITE 0 // Doesn't exist on ARM
141 #define PTE_WRITECOPY 0 // Doesn't exist on ARM
142 #define PTE_EXECUTE_READWRITE 0 // Not worrying about NX yet
143 #define PTE_EXECUTE_WRITECOPY 0 // Not worrying about NX yet
144 #define PTE_PROTOTYPE 0x400 // Using the Shared bit
145
146 //
147 // Cache flags
148 //
149 #define PTE_ENABLE_CACHE 0
150 #define PTE_DISABLE_CACHE 0x10
151 #define PTE_WRITECOMBINED_CACHE 0x10
152 #define PTE_PROTECT_MASK 0x610
153 #else
154 #error Define these please!
155 #endif
156
157 //
158 // Some internal SYSTEM_PTE_MISUSE bugcheck subcodes
159 //
160 #define PTE_MAPPING_NONE 0x100
161 #define PTE_MAPPING_NOT_OWNED 0x101
162 #define PTE_MAPPING_EMPTY 0x102
163 #define PTE_MAPPING_RESERVED 0x103
164
165 //
166 // Mask for image section page protection
167 //
168 #define IMAGE_SCN_PROTECTION_MASK (IMAGE_SCN_MEM_WRITE | IMAGE_SCN_MEM_READ | IMAGE_SCN_MEM_EXECUTE)
169
170 extern const ULONG_PTR MmProtectToPteMask[32];
171 extern const ULONG MmProtectToValue[32];
172
173 //
174 // Assertions for session images, addresses, and PTEs
175 //
176 #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \
177 (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))
178
179 #define MI_IS_SESSION_ADDRESS(Address) \
180 (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))
181
182 #define MI_IS_SESSION_PTE(Pte) \
183 ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))
184
185 #define MI_IS_PAGE_TABLE_ADDRESS(Address) \
186 (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))
187
188 #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \
189 (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))
190
191 #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \
192 (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))
193
194 //
195 // Creates a software PTE with the given protection
196 //
197 #define MI_MAKE_SOFTWARE_PTE(p, x) ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))
198
199 //
200 // Marks a PTE as deleted
201 //
202 #define MI_SET_PFN_DELETED(x) ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1))
203 #define MI_IS_PFN_DELETED(x) ((ULONG_PTR)((x)->PteAddress) & 1)
204
205 //
206 // Special values for LoadedImports
207 //
208 #define MM_SYSLDR_NO_IMPORTS ((PVOID)(ULONG_PTR)-2)
209 #define MM_SYSLDR_BOOT_LOADED ((PVOID)(ULONG_PTR)-1)
210 #define MM_SYSLDR_SINGLE_ENTRY 0x1
211
212 //
213 // Number of initial session IDs
214 //
215 #define MI_INITIAL_SESSION_IDS 64
216
217 #if defined(_M_IX86) || defined(_M_ARM)
218 //
219 // PFN List Sentinel
220 //
221 #define LIST_HEAD 0xFFFFFFFF
222
223 //
224 // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits,
225 // we need a manual definition suited to the number of bits in the PteFrame.
226 // This is used as a LIST_HEAD for the colored list
227 //
228 #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF
229 #elif defined(_M_AMD64)
230 #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL
231 #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL
232 #else
233 #error Define these please!
234 #endif
235
236 //
237 // Returns the color of a page
238 //
239 #define MI_GET_PAGE_COLOR(x) ((x) & MmSecondaryColorMask)
240 #define MI_GET_NEXT_COLOR() (MI_GET_PAGE_COLOR(++MmSystemPageColor))
241 #define MI_GET_NEXT_PROCESS_COLOR(x) (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
242
243 //
244 // Prototype PTEs that don't yet have a pagefile association
245 //
246 #ifdef _WIN64
247 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL
248 #else
249 #define MI_PTE_LOOKUP_NEEDED 0xFFFFF
250 #endif
251
252 //
253 // Number of session data and tag pages
254 //
255 #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
256 #define MI_SESSION_TAG_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
257
258 //
259 // Used by MiCheckSecuredVad
260 //
261 #define MM_READ_WRITE_ALLOWED 11
262 #define MM_READ_ONLY_ALLOWED 10
263 #define MM_NO_ACCESS_ALLOWED 01
264 #define MM_DELETE_CHECK 85
265
266 //
267 // System views are binned into 64K chunks
268 //
269 #define MI_SYSTEM_VIEW_BUCKET_SIZE _64K
270
271 //
272 // FIXFIX: These should go in ex.h after the pool merge
273 //
274 #ifdef _WIN64
275 #define POOL_BLOCK_SIZE 16
276 #else
277 #define POOL_BLOCK_SIZE 8
278 #endif
279 #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)
280 #define BASE_POOL_TYPE_MASK 1
281 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))
282
283 //
284 // Pool debugging/analysis/tracing flags
285 //
286 #define POOL_FLAG_CHECK_TIMERS 0x1
287 #define POOL_FLAG_CHECK_WORKERS 0x2
288 #define POOL_FLAG_CHECK_RESOURCES 0x4
289 #define POOL_FLAG_VERIFIER 0x8
290 #define POOL_FLAG_CHECK_DEADLOCK 0x10
291 #define POOL_FLAG_SPECIAL_POOL 0x20
292 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40
293 #define POOL_FLAG_CRASH_ON_FAILURE 0x80
294
295 //
296 // BAD_POOL_HEADER codes during pool bugcheck
297 //
298 #define POOL_CORRUPTED_LIST 3
299 #define POOL_SIZE_OR_INDEX_MISMATCH 5
300 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6
301 #define POOL_HEADER_NOT_ALIGNED 7
302 #define POOL_HEADER_IS_ZERO 8
303 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9
304 #define POOL_ENTRY_NOT_FOUND 10
305
306 //
307 // BAD_POOL_CALLER codes during pool bugcheck
308 //
309 #define POOL_ENTRY_CORRUPTED 1
310 #define POOL_ENTRY_ALREADY_FREE 6
311 #define POOL_ENTRY_NOT_ALLOCATED 7
312 #define POOL_ALLOC_IRQL_INVALID 8
313 #define POOL_FREE_IRQL_INVALID 9
314 #define POOL_BILLED_PROCESS_INVALID 13
315 #define POOL_HEADER_SIZE_INVALID 32
316
317 typedef struct _POOL_DESCRIPTOR
318 {
319 POOL_TYPE PoolType;
320 ULONG PoolIndex;
321 ULONG RunningAllocs;
322 ULONG RunningDeAllocs;
323 ULONG TotalPages;
324 ULONG TotalBigPages;
325 ULONG Threshold;
326 PVOID LockAddress;
327 PVOID PendingFrees;
328 LONG PendingFreeDepth;
329 SIZE_T TotalBytes;
330 SIZE_T Spare0;
331 LIST_ENTRY ListHeads[POOL_LISTS_PER_PAGE];
332 } POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
333
334 typedef struct _POOL_HEADER
335 {
336 union
337 {
338 struct
339 {
340 #ifdef _WIN64
341 USHORT PreviousSize:8;
342 USHORT PoolIndex:8;
343 USHORT BlockSize:8;
344 USHORT PoolType:8;
345 #else
346 USHORT PreviousSize:9;
347 USHORT PoolIndex:7;
348 USHORT BlockSize:9;
349 USHORT PoolType:7;
350 #endif
351 };
352 ULONG Ulong1;
353 };
354 #ifdef _WIN64
355 ULONG PoolTag;
356 #endif
357 union
358 {
359 #ifdef _WIN64
360 PEPROCESS ProcessBilled;
361 #else
362 ULONG PoolTag;
363 #endif
364 struct
365 {
366 USHORT AllocatorBackTraceIndex;
367 USHORT PoolTagHash;
368 };
369 };
370 } POOL_HEADER, *PPOOL_HEADER;
371
372 C_ASSERT(sizeof(POOL_HEADER) == POOL_BLOCK_SIZE);
373 C_ASSERT(POOL_BLOCK_SIZE == sizeof(LIST_ENTRY));
374
375 typedef struct _POOL_TRACKER_TABLE
376 {
377 ULONG Key;
378 LONG NonPagedAllocs;
379 LONG NonPagedFrees;
380 SIZE_T NonPagedBytes;
381 LONG PagedAllocs;
382 LONG PagedFrees;
383 SIZE_T PagedBytes;
384 } POOL_TRACKER_TABLE, *PPOOL_TRACKER_TABLE;
385
386 typedef struct _POOL_TRACKER_BIG_PAGES
387 {
388 PVOID Va;
389 ULONG Key;
390 ULONG NumberOfPages;
391 PVOID QuotaObject;
392 } POOL_TRACKER_BIG_PAGES, *PPOOL_TRACKER_BIG_PAGES;
393
394 extern ULONG ExpNumberOfPagedPools;
395 extern POOL_DESCRIPTOR NonPagedPoolDescriptor;
396 extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16 + 1];
397 extern PPOOL_TRACKER_TABLE PoolTrackTable;
398
399 //
400 // END FIXFIX
401 //
402
403 typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY
404 {
405 LIST_ENTRY Links;
406 UNICODE_STRING BaseName;
407 } MI_LARGE_PAGE_DRIVER_ENTRY, *PMI_LARGE_PAGE_DRIVER_ENTRY;
408
409 typedef enum _MMSYSTEM_PTE_POOL_TYPE
410 {
411 SystemPteSpace,
412 NonPagedPoolExpansion,
413 MaximumPtePoolTypes
414 } MMSYSTEM_PTE_POOL_TYPE;
415
416 typedef enum _MI_PFN_CACHE_ATTRIBUTE
417 {
418 MiNonCached,
419 MiCached,
420 MiWriteCombined,
421 MiNotMapped
422 } MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;
423
424 typedef struct _PHYSICAL_MEMORY_RUN
425 {
426 PFN_NUMBER BasePage;
427 PFN_NUMBER PageCount;
428 } PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN;
429
430 typedef struct _PHYSICAL_MEMORY_DESCRIPTOR
431 {
432 ULONG NumberOfRuns;
433 PFN_NUMBER NumberOfPages;
434 PHYSICAL_MEMORY_RUN Run[1];
435 } PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR;
436
437 typedef struct _MMCOLOR_TABLES
438 {
439 PFN_NUMBER Flink;
440 PVOID Blink;
441 PFN_NUMBER Count;
442 } MMCOLOR_TABLES, *PMMCOLOR_TABLES;
443
444 typedef struct _MI_LARGE_PAGE_RANGES
445 {
446 PFN_NUMBER StartFrame;
447 PFN_NUMBER LastFrame;
448 } MI_LARGE_PAGE_RANGES, *PMI_LARGE_PAGE_RANGES;
449
450 typedef struct _MMVIEW
451 {
452 ULONG_PTR Entry;
453 PCONTROL_AREA ControlArea;
454 } MMVIEW, *PMMVIEW;
455
456 typedef struct _MMSESSION
457 {
458 KGUARDED_MUTEX SystemSpaceViewLock;
459 PKGUARDED_MUTEX SystemSpaceViewLockPointer;
460 PCHAR SystemSpaceViewStart;
461 PMMVIEW SystemSpaceViewTable;
462 ULONG SystemSpaceHashSize;
463 ULONG SystemSpaceHashEntries;
464 ULONG SystemSpaceHashKey;
465 ULONG BitmapFailures;
466 PRTL_BITMAP SystemSpaceBitMap;
467 } MMSESSION, *PMMSESSION;
468
469 typedef struct _MM_SESSION_SPACE_FLAGS
470 {
471 ULONG Initialized:1;
472 ULONG DeletePending:1;
473 ULONG Filler:30;
474 } MM_SESSION_SPACE_FLAGS;
475
476 typedef struct _MM_SESSION_SPACE
477 {
478 struct _MM_SESSION_SPACE *GlobalVirtualAddress;
479 LONG ReferenceCount;
480 union
481 {
482 ULONG LongFlags;
483 MM_SESSION_SPACE_FLAGS Flags;
484 } u;
485 ULONG SessionId;
486 LIST_ENTRY ProcessList;
487 LARGE_INTEGER LastProcessSwappedOutTime;
488 PFN_NUMBER SessionPageDirectoryIndex;
489 SIZE_T NonPageablePages;
490 SIZE_T CommittedPages;
491 PVOID PagedPoolStart;
492 PVOID PagedPoolEnd;
493 PMMPDE PagedPoolBasePde;
494 ULONG Color;
495 LONG ResidentProcessCount;
496 ULONG SessionPoolAllocationFailures[4];
497 LIST_ENTRY ImageList;
498 LCID LocaleId;
499 ULONG AttachCount;
500 KEVENT AttachEvent;
501 PEPROCESS LastProcess;
502 LONG ProcessReferenceToSession;
503 LIST_ENTRY WsListEntry;
504 GENERAL_LOOKASIDE Lookaside[SESSION_POOL_LOOKASIDES];
505 MMSESSION Session;
506 KGUARDED_MUTEX PagedPoolMutex;
507 MM_PAGED_POOL_INFO PagedPoolInfo;
508 MMSUPPORT Vm;
509 PMMWSLE Wsle;
510 PDRIVER_UNLOAD Win32KDriverUnload;
511 POOL_DESCRIPTOR PagedPool;
512 #if defined (_M_AMD64)
513 MMPDE PageDirectory;
514 #else
515 PMMPDE PageTables;
516 #endif
517 #if defined (_M_AMD64)
518 PMMPTE SpecialPoolFirstPte;
519 PMMPTE SpecialPoolLastPte;
520 PMMPTE NextPdeForSpecialPoolExpansion;
521 PMMPTE LastPdeForSpecialPoolExpansion;
522 PFN_NUMBER SpecialPagesInUse;
523 #endif
524 LONG ImageLoadingCount;
525 } MM_SESSION_SPACE, *PMM_SESSION_SPACE;
526
527 extern PMM_SESSION_SPACE MmSessionSpace;
528 extern MMPTE HyperTemplatePte;
529 extern MMPDE ValidKernelPde;
530 extern MMPTE ValidKernelPte;
531 extern MMPDE ValidKernelPdeLocal;
532 extern MMPTE ValidKernelPteLocal;
533 extern MMPDE DemandZeroPde;
534 extern MMPTE DemandZeroPte;
535 extern MMPTE PrototypePte;
536 extern MMPTE MmDecommittedPte;
537 extern BOOLEAN MmLargeSystemCache;
538 extern BOOLEAN MmZeroPageFile;
539 extern BOOLEAN MmProtectFreedNonPagedPool;
540 extern BOOLEAN MmTrackLockedPages;
541 extern BOOLEAN MmTrackPtes;
542 extern BOOLEAN MmDynamicPfn;
543 extern BOOLEAN MmMirroring;
544 extern BOOLEAN MmMakeLowMemory;
545 extern BOOLEAN MmEnforceWriteProtection;
546 extern SIZE_T MmAllocationFragment;
547 extern ULONG MmConsumedPoolPercentage;
548 extern ULONG MmVerifyDriverBufferType;
549 extern ULONG MmVerifyDriverLevel;
550 extern WCHAR MmVerifyDriverBuffer[512];
551 extern WCHAR MmLargePageDriverBuffer[512];
552 extern LIST_ENTRY MiLargePageDriverList;
553 extern BOOLEAN MiLargePageAllDrivers;
554 extern ULONG MmVerifyDriverBufferLength;
555 extern ULONG MmLargePageDriverBufferLength;
556 extern SIZE_T MmSizeOfNonPagedPoolInBytes;
557 extern SIZE_T MmMaximumNonPagedPoolInBytes;
558 extern PFN_NUMBER MmMaximumNonPagedPoolInPages;
559 extern PFN_NUMBER MmSizeOfPagedPoolInPages;
560 extern PVOID MmNonPagedSystemStart;
561 extern PVOID MmNonPagedPoolStart;
562 extern PVOID MmNonPagedPoolExpansionStart;
563 extern PVOID MmNonPagedPoolEnd;
564 extern SIZE_T MmSizeOfPagedPoolInBytes;
565 extern PVOID MmPagedPoolStart;
566 extern PVOID MmPagedPoolEnd;
567 extern PVOID MmSessionBase;
568 extern SIZE_T MmSessionSize;
569 extern PMMPTE MmFirstReservedMappingPte, MmLastReservedMappingPte;
570 extern PMMPTE MiFirstReservedZeroingPte;
571 extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2][MmMaximumCacheType];
572 extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock;
573 extern SIZE_T MmBootImageSize;
574 extern PMMPTE MmSystemPtesStart[MaximumPtePoolTypes];
575 extern PMMPTE MmSystemPtesEnd[MaximumPtePoolTypes];
576 extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor;
577 extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor;
578 extern ULONG_PTR MxPfnAllocation;
579 extern MM_PAGED_POOL_INFO MmPagedPoolInfo;
580 extern KGUARDED_MUTEX MmPagedPoolMutex;
581 extern KGUARDED_MUTEX MmSectionCommitMutex;
582 extern PVOID MmPagedPoolStart;
583 extern PVOID MmPagedPoolEnd;
584 extern PVOID MmNonPagedSystemStart;
585 extern PVOID MiSystemViewStart;
586 extern SIZE_T MmSystemViewSize;
587 extern PVOID MmSessionBase;
588 extern PVOID MiSessionSpaceEnd;
589 extern PMMPTE MiSessionImagePteStart;
590 extern PMMPTE MiSessionImagePteEnd;
591 extern PMMPTE MiSessionBasePte;
592 extern PMMPTE MiSessionLastPte;
593 extern SIZE_T MmSizeOfPagedPoolInBytes;
594 extern PMMPDE MmSystemPagePtes;
595 extern PVOID MmSystemCacheStart;
596 extern PVOID MmSystemCacheEnd;
597 extern MMSUPPORT MmSystemCacheWs;
598 extern SIZE_T MmAllocatedNonPagedPool;
599 extern ULONG MmSpecialPoolTag;
600 extern PVOID MmHyperSpaceEnd;
601 extern PMMWSL MmSystemCacheWorkingSetList;
602 extern SIZE_T MmMinimumNonPagedPoolSize;
603 extern ULONG MmMinAdditionNonPagedPoolPerMb;
604 extern SIZE_T MmDefaultMaximumNonPagedPool;
605 extern ULONG MmMaxAdditionNonPagedPoolPerMb;
606 extern ULONG MmSecondaryColors;
607 extern ULONG MmSecondaryColorMask;
608 extern ULONG MmNumberOfSystemPtes;
609 extern ULONG MmMaximumNonPagedPoolPercent;
610 extern ULONG MmLargeStackSize;
611 extern PMMCOLOR_TABLES MmFreePagesByColor[FreePageList + 1];
612 extern MMPFNLIST MmStandbyPageListByPriority[8];
613 extern ULONG MmProductType;
614 extern MM_SYSTEMSIZE MmSystemSize;
615 extern PKEVENT MiLowMemoryEvent;
616 extern PKEVENT MiHighMemoryEvent;
617 extern PKEVENT MiLowPagedPoolEvent;
618 extern PKEVENT MiHighPagedPoolEvent;
619 extern PKEVENT MiLowNonPagedPoolEvent;
620 extern PKEVENT MiHighNonPagedPoolEvent;
621 extern PFN_NUMBER MmLowMemoryThreshold;
622 extern PFN_NUMBER MmHighMemoryThreshold;
623 extern PFN_NUMBER MiLowPagedPoolThreshold;
624 extern PFN_NUMBER MiHighPagedPoolThreshold;
625 extern PFN_NUMBER MiLowNonPagedPoolThreshold;
626 extern PFN_NUMBER MiHighNonPagedPoolThreshold;
627 extern PFN_NUMBER MmMinimumFreePages;
628 extern PFN_NUMBER MmPlentyFreePages;
629 extern SIZE_T MmMinimumStackCommitInBytes;
630 extern PFN_COUNT MiExpansionPoolPagesInitialCharge;
631 extern PFN_NUMBER MmResidentAvailableAtInit;
632 extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
633 extern PFN_NUMBER MmTotalSystemDriverPages;
634 extern ULONG MmCritsectTimeoutSeconds;
635 extern PVOID MiSessionImageStart;
636 extern PVOID MiSessionImageEnd;
637 extern PMMPTE MiHighestUserPte;
638 extern PMMPDE MiHighestUserPde;
639 extern PFN_NUMBER MmSystemPageDirectory[PPE_PER_PAGE];
640 extern PMMPTE MmSharedUserDataPte;
641 extern LIST_ENTRY MmProcessList;
642 extern KEVENT MmZeroingPageEvent;
643 extern ULONG MmSystemPageColor;
644 extern ULONG MmProcessColorSeed;
645 extern PMMWSL MmWorkingSetList;
646 extern PFN_NUMBER MiNumberOfFreePages;
647 extern SIZE_T MmSessionViewSize;
648 extern SIZE_T MmSessionPoolSize;
649 extern SIZE_T MmSessionImageSize;
650 extern PVOID MiSystemViewStart;
651 extern PVOID MiSessionPoolEnd; // 0xBE000000
652 extern PVOID MiSessionPoolStart; // 0xBD000000
653 extern PVOID MiSessionViewStart; // 0xBE000000
654 extern PVOID MiSessionSpaceWs;
655 extern ULONG MmMaximumDeadKernelStacks;
656 extern SLIST_HEADER MmDeadStackSListHead;
657 extern MM_AVL_TABLE MmSectionBasedRoot;
658 extern KGUARDED_MUTEX MmSectionBasedMutex;
659 extern PVOID MmHighSectionBase;
660 extern SIZE_T MmSystemLockPagesCount;
661 extern ULONG_PTR MmSubsectionBase;
662 extern LARGE_INTEGER MmCriticalSectionTimeout;
663 extern LIST_ENTRY MmWorkingSetExpansionHead;
664 extern KSPIN_LOCK MmExpansionLock;
665 extern PETHREAD MiExpansionLockOwner;
666
667 FORCEINLINE
668 BOOLEAN
MI_IS_PROCESS_WORKING_SET(PMMSUPPORT WorkingSet)669 MI_IS_PROCESS_WORKING_SET(PMMSUPPORT WorkingSet)
670 {
671 return (WorkingSet != &MmSystemCacheWs) && !WorkingSet->Flags.SessionSpace;
672 }
673
674 FORCEINLINE
675 BOOLEAN
MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType)676 MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType)
677 {
678 return ((MemoryType == LoaderFree) ||
679 (MemoryType == LoaderLoadedProgram) ||
680 (MemoryType == LoaderFirmwareTemporary) ||
681 (MemoryType == LoaderOsloaderStack));
682 }
683
684 FORCEINLINE
685 BOOLEAN
MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType)686 MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType)
687 {
688 return ((MemoryType == LoaderFirmwarePermanent) ||
689 (MemoryType == LoaderSpecialMemory) ||
690 (MemoryType == LoaderHALCachedMemory) ||
691 (MemoryType == LoaderBBTMemory));
692 }
693
694 #ifdef _M_AMD64
695 FORCEINLINE
696 BOOLEAN
MiIsUserPxe(PVOID Address)697 MiIsUserPxe(PVOID Address)
698 {
699 return ((ULONG_PTR)Address >> 7) == 0x1FFFFEDF6FB7DA0ULL;
700 }
701
702 FORCEINLINE
703 BOOLEAN
MiIsUserPpe(PVOID Address)704 MiIsUserPpe(PVOID Address)
705 {
706 return ((ULONG_PTR)Address >> 16) == 0xFFFFF6FB7DA0ULL;
707 }
708
709 FORCEINLINE
710 BOOLEAN
MiIsUserPde(PVOID Address)711 MiIsUserPde(PVOID Address)
712 {
713 return ((ULONG_PTR)Address >> 25) == 0x7FFFFB7DA0ULL;
714 }
715
716 FORCEINLINE
717 BOOLEAN
MiIsUserPte(PVOID Address)718 MiIsUserPte(PVOID Address)
719 {
720 return ((ULONG_PTR)Address >> 34) == 0x3FFFFDA0ULL;
721 }
722 #else
723 FORCEINLINE
724 BOOLEAN
MiIsUserPde(PVOID Address)725 MiIsUserPde(PVOID Address)
726 {
727 return ((Address >= (PVOID)MiAddressToPde(NULL)) &&
728 (Address <= (PVOID)MiHighestUserPde));
729 }
730
731 FORCEINLINE
732 BOOLEAN
MiIsUserPte(PVOID Address)733 MiIsUserPte(PVOID Address)
734 {
735 return (Address >= (PVOID)PTE_BASE) && (Address <= (PVOID)MiHighestUserPte);
736 }
737 #endif
738
739 //
740 // Figures out the hardware bits for a PTE
741 //
742 FORCEINLINE
743 ULONG_PTR
MiDetermineUserGlobalPteMask(IN PVOID PointerPte)744 MiDetermineUserGlobalPteMask(IN PVOID PointerPte)
745 {
746 MMPTE TempPte;
747
748 /* Start fresh */
749 TempPte.u.Long = 0;
750
751 /* Make it valid and accessed */
752 TempPte.u.Hard.Valid = TRUE;
753 MI_MAKE_ACCESSED_PAGE(&TempPte);
754
755 /* Is this for user-mode? */
756 if (
757 #if (_MI_PAGING_LEVELS == 4)
758 MiIsUserPxe(PointerPte) ||
759 #endif
760 #if (_MI_PAGING_LEVELS >= 3)
761 MiIsUserPpe(PointerPte) ||
762 #endif
763 MiIsUserPde(PointerPte) ||
764 MiIsUserPte(PointerPte))
765 {
766 /* Set the owner bit */
767 MI_MAKE_OWNER_PAGE(&TempPte);
768 }
769
770 /* FIXME: We should also set the global bit */
771
772 /* Return the protection */
773 return TempPte.u.Long;
774 }
775
776 //
777 // Creates a valid kernel PTE with the given protection
778 //
779 FORCEINLINE
780 VOID
MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte,IN PMMPTE MappingPte,IN ULONG_PTR ProtectionMask,IN PFN_NUMBER PageFrameNumber)781 MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte,
782 IN PMMPTE MappingPte,
783 IN ULONG_PTR ProtectionMask,
784 IN PFN_NUMBER PageFrameNumber)
785 {
786 /* Only valid for kernel, non-session PTEs */
787 ASSERT(MappingPte > MiHighestUserPte);
788 ASSERT(!MI_IS_SESSION_PTE(MappingPte));
789 ASSERT((MappingPte < (PMMPTE)PDE_BASE) || (MappingPte > (PMMPTE)PDE_TOP));
790
791 /* Check that we are not setting valid a page that should not be */
792 ASSERT(ProtectionMask & MM_PROTECT_ACCESS);
793 ASSERT((ProtectionMask & MM_GUARDPAGE) == 0);
794
795 /* Start fresh */
796 NewPte->u.Long = 0;
797
798 /* Set the protection and page */
799 NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
800 NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
801
802 /* Make this valid & global */
803 #ifdef _GLOBAL_PAGES_ARE_AWESOME_
804 if (KeFeatureBits & KF_GLOBAL_PAGE)
805 NewPte->u.Hard.Global = 1;
806 #endif
807 NewPte->u.Hard.Valid = 1;
808 }
809
810 //
811 // Creates a valid PTE with the given protection
812 //
813 FORCEINLINE
814 VOID
MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte,IN PMMPTE MappingPte,IN ULONG_PTR ProtectionMask,IN PFN_NUMBER PageFrameNumber)815 MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte,
816 IN PMMPTE MappingPte,
817 IN ULONG_PTR ProtectionMask,
818 IN PFN_NUMBER PageFrameNumber)
819 {
820 /* Check that we are not setting valid a page that should not be */
821 ASSERT(ProtectionMask & MM_PROTECT_ACCESS);
822 ASSERT((ProtectionMask & MM_GUARDPAGE) == 0);
823
824 /* Set the protection and page */
825 NewPte->u.Long = MiDetermineUserGlobalPteMask(MappingPte);
826 NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
827 NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
828 }
829
830 //
831 // Creates a valid user PTE with the given protection
832 //
833 FORCEINLINE
834 VOID
MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte,IN PMMPTE MappingPte,IN ULONG_PTR ProtectionMask,IN PFN_NUMBER PageFrameNumber)835 MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte,
836 IN PMMPTE MappingPte,
837 IN ULONG_PTR ProtectionMask,
838 IN PFN_NUMBER PageFrameNumber)
839 {
840 /* Only valid for kernel, non-session PTEs */
841 ASSERT(MappingPte <= MiHighestUserPte);
842
843 /* Start fresh */
844 NewPte->u.Long = 0;
845
846 /* Check that we are not setting valid a page that should not be */
847 ASSERT(ProtectionMask & MM_PROTECT_ACCESS);
848 ASSERT((ProtectionMask & MM_GUARDPAGE) == 0);
849
850 NewPte->u.Hard.Valid = TRUE;
851 NewPte->u.Hard.Owner = TRUE;
852 NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
853 NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
854 }
855
856 #ifndef _M_AMD64
857 //
858 // Builds a Prototype PTE for the address of the PTE
859 //
860 FORCEINLINE
861 VOID
MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte,IN PMMPTE PointerPte)862 MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte,
863 IN PMMPTE PointerPte)
864 {
865 ULONG_PTR Offset;
866
867 /* Mark this as a prototype */
868 NewPte->u.Long = 0;
869 NewPte->u.Proto.Prototype = 1;
870
871 /*
872 * Prototype PTEs are only valid in paged pool by design, this little trick
873 * lets us only use 30 bits for the address of the PTE, as long as the area
874 * stays 1024MB At most.
875 */
876 Offset = (ULONG_PTR)PointerPte - (ULONG_PTR)MmPagedPoolStart;
877
878 /*
879 * 7 bits go in the "low" (but we assume the bottom 2 are zero)
880 * and the other 21 bits go in the "high"
881 */
882 NewPte->u.Proto.ProtoAddressLow = (Offset & 0x1FC) >> 2;
883 NewPte->u.Proto.ProtoAddressHigh = (Offset & 0x3FFFFE00) >> 9;
884 }
885
886 //
887 // Builds a Subsection PTE for the address of the Segment
888 //
889 FORCEINLINE
890 VOID
MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte,IN PVOID Segment)891 MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte,
892 IN PVOID Segment)
893 {
894 ULONG_PTR Offset;
895
896 /* Mark this as a prototype */
897 NewPte->u.Long = 0;
898 NewPte->u.Subsect.Prototype = 1;
899
900 /*
901 * Segments are only valid either in nonpaged pool. We store the 20 bit
902 * difference either from the top or bottom of nonpaged pool, giving a
903 * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed
904 * 256MB.
905 */
906 if ((ULONG_PTR)Segment < ((ULONG_PTR)MmSubsectionBase + (128 * _1MB)))
907 {
908 Offset = (ULONG_PTR)Segment - (ULONG_PTR)MmSubsectionBase;
909 NewPte->u.Subsect.WhichPool = PagedPool;
910 }
911 else
912 {
913 Offset = (ULONG_PTR)MmNonPagedPoolEnd - (ULONG_PTR)Segment;
914 NewPte->u.Subsect.WhichPool = NonPagedPool;
915 }
916
917 /*
918 * 4 bits go in the "low" (but we assume the bottom 3 are zero)
919 * and the other 20 bits go in the "high"
920 */
921 NewPte->u.Subsect.SubsectionAddressLow = (Offset & 0x78) >> 3;
922 NewPte->u.Subsect.SubsectionAddressHigh = (Offset & 0xFFFFF80) >> 7;
923 }
924
925 FORCEINLINE
926 BOOLEAN
MI_IS_MAPPED_PTE(PMMPTE PointerPte)927 MI_IS_MAPPED_PTE(PMMPTE PointerPte)
928 {
929 /// \todo Make this reasonable code, this is UGLY!
930 return ((PointerPte->u.Long & 0xFFFFFC01) != 0);
931 }
932
933 #endif
934
935 FORCEINLINE
936 VOID
MI_MAKE_TRANSITION_PTE(_Out_ PMMPTE NewPte,_In_ PFN_NUMBER Page,_In_ ULONG Protection)937 MI_MAKE_TRANSITION_PTE(_Out_ PMMPTE NewPte,
938 _In_ PFN_NUMBER Page,
939 _In_ ULONG Protection)
940 {
941 NewPte->u.Long = 0;
942 NewPte->u.Trans.Transition = 1;
943 NewPte->u.Trans.Protection = Protection;
944 NewPte->u.Trans.PageFrameNumber = Page;
945 }
946
947 //
948 // Returns if the page is physically resident (ie: a large page)
949 // FIXFIX: CISC/x86 only?
950 //
951 FORCEINLINE
952 BOOLEAN
MI_IS_PHYSICAL_ADDRESS(IN PVOID Address)953 MI_IS_PHYSICAL_ADDRESS(IN PVOID Address)
954 {
955 PMMPDE PointerPde;
956
957 /* Large pages are never paged out, always physically resident */
958 PointerPde = MiAddressToPde(Address);
959 return ((PointerPde->u.Hard.LargePage) && (PointerPde->u.Hard.Valid));
960 }
961
962 //
963 // Writes a valid PTE
964 //
965 FORCEINLINE
966 VOID
MI_WRITE_VALID_PTE(IN PMMPTE PointerPte,IN MMPTE TempPte)967 MI_WRITE_VALID_PTE(IN PMMPTE PointerPte,
968 IN MMPTE TempPte)
969 {
970 /* Write the valid PTE */
971 ASSERT(PointerPte->u.Hard.Valid == 0);
972 ASSERT(TempPte.u.Hard.Valid == 1);
973 #if _M_AMD64
974 ASSERT(!MI_IS_PAGE_TABLE_ADDRESS(MiPteToAddress(PointerPte)) ||
975 (TempPte.u.Hard.NoExecute == 0));
976 #endif
977 *PointerPte = TempPte;
978 }
979
980 //
981 // Updates a valid PTE
982 //
983 FORCEINLINE
984 VOID
MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte,IN MMPTE TempPte)985 MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte,
986 IN MMPTE TempPte)
987 {
988 /* Write the valid PTE */
989 ASSERT(PointerPte->u.Hard.Valid == 1);
990 ASSERT(TempPte.u.Hard.Valid == 1);
991 ASSERT(PointerPte->u.Hard.PageFrameNumber == TempPte.u.Hard.PageFrameNumber);
992 *PointerPte = TempPte;
993 }
994
995 //
996 // Writes an invalid PTE
997 //
998 FORCEINLINE
999 VOID
MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte,IN MMPTE InvalidPte)1000 MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte,
1001 IN MMPTE InvalidPte)
1002 {
1003 /* Write the invalid PTE */
1004 ASSERT(InvalidPte.u.Hard.Valid == 0);
1005 ASSERT(InvalidPte.u.Long != 0);
1006 *PointerPte = InvalidPte;
1007 }
1008
1009 //
1010 // Erase the PTE completely
1011 //
1012 FORCEINLINE
1013 VOID
MI_ERASE_PTE(IN PMMPTE PointerPte)1014 MI_ERASE_PTE(IN PMMPTE PointerPte)
1015 {
1016 /* Zero out the PTE */
1017 ASSERT(PointerPte->u.Long != 0);
1018 PointerPte->u.Long = 0;
1019 }
1020
1021 //
1022 // Writes a valid PDE
1023 //
1024 FORCEINLINE
1025 VOID
MI_WRITE_VALID_PDE(IN PMMPDE PointerPde,IN MMPDE TempPde)1026 MI_WRITE_VALID_PDE(IN PMMPDE PointerPde,
1027 IN MMPDE TempPde)
1028 {
1029 /* Write the valid PDE */
1030 ASSERT(PointerPde->u.Hard.Valid == 0);
1031 #ifdef _M_AMD64
1032 ASSERT(PointerPde->u.Hard.NoExecute == 0);
1033 #endif
1034 ASSERT(TempPde.u.Hard.Valid == 1);
1035 *PointerPde = TempPde;
1036 }
1037
1038 //
1039 // Writes an invalid PDE
1040 //
1041 FORCEINLINE
1042 VOID
MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde,IN MMPDE InvalidPde)1043 MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde,
1044 IN MMPDE InvalidPde)
1045 {
1046 /* Write the invalid PDE */
1047 ASSERT(InvalidPde.u.Hard.Valid == 0);
1048 ASSERT(InvalidPde.u.Long != 0);
1049 #ifdef _M_AMD64
1050 ASSERT(InvalidPde.u.Soft.Protection == MM_EXECUTE_READWRITE);
1051 #endif
1052 *PointerPde = InvalidPde;
1053 }
1054
1055 //
1056 // Checks if the thread already owns a working set
1057 //
1058 FORCEINLINE
1059 BOOLEAN
MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread)1060 MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread)
1061 {
1062 /* If any of these are held, return TRUE */
1063 return ((Thread->OwnsProcessWorkingSetExclusive) ||
1064 (Thread->OwnsProcessWorkingSetShared) ||
1065 (Thread->OwnsSystemWorkingSetExclusive) ||
1066 (Thread->OwnsSystemWorkingSetShared) ||
1067 (Thread->OwnsSessionWorkingSetExclusive) ||
1068 (Thread->OwnsSessionWorkingSetShared));
1069 }
1070
1071 FORCEINLINE
1072 BOOLEAN
MM_ANY_WS_LOCK_HELD_EXCLUSIVE(_In_ PETHREAD Thread)1073 MM_ANY_WS_LOCK_HELD_EXCLUSIVE(_In_ PETHREAD Thread)
1074 {
1075 return ((Thread->OwnsProcessWorkingSetExclusive) ||
1076 (Thread->OwnsSystemWorkingSetExclusive) ||
1077 (Thread->OwnsSessionWorkingSetExclusive));
1078 }
1079
1080 //
1081 // Checks if the process owns the working set lock
1082 //
1083 FORCEINLINE
1084 BOOLEAN
MI_WS_OWNER(IN PEPROCESS Process)1085 MI_WS_OWNER(IN PEPROCESS Process)
1086 {
1087 /* Check if this process is the owner, and that the thread owns the WS */
1088 if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive == 0)
1089 {
1090 DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread());
1091 }
1092 if (KeGetCurrentThread()->ApcState.Process != &Process->Pcb)
1093 {
1094 DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process);
1095 }
1096 return ((KeGetCurrentThread()->ApcState.Process == &Process->Pcb) &&
1097 ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive) ||
1098 (PsGetCurrentThread()->OwnsProcessWorkingSetShared)));
1099 }
1100
1101 //
1102 // New ARM3<->RosMM PAGE Architecture
1103 //
1104 FORCEINLINE
1105 BOOLEAN
MiIsRosSectionObject(IN PSECTION Section)1106 MiIsRosSectionObject(IN PSECTION Section)
1107 {
1108 return Section->u.Flags.filler;
1109 }
1110
1111 #define MI_IS_ROS_PFN(x) ((x)->u4.AweAllocation == TRUE)
1112
1113 VOID
1114 NTAPI
1115 MiDecrementReferenceCount(
1116 IN PMMPFN Pfn1,
1117 IN PFN_NUMBER PageFrameIndex
1118 );
1119
1120 FORCEINLINE
1121 BOOLEAN
MI_IS_WS_UNSAFE(IN PEPROCESS Process)1122 MI_IS_WS_UNSAFE(IN PEPROCESS Process)
1123 {
1124 return (Process->Vm.Flags.AcquiredUnsafe == TRUE);
1125 }
1126
1127 //
1128 // Locks the working set for the given process
1129 //
1130 FORCEINLINE
1131 VOID
MiLockProcessWorkingSet(IN PEPROCESS Process,IN PETHREAD Thread)1132 MiLockProcessWorkingSet(IN PEPROCESS Process,
1133 IN PETHREAD Thread)
1134 {
1135 /* Shouldn't already be owning the process working set */
1136 ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1137 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1138
1139 /* Block APCs, make sure that still nothing is already held */
1140 KeEnterGuardedRegion();
1141 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1142
1143 /* Lock the working set */
1144 ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1145
1146 /* Now claim that we own the lock */
1147 ASSERT(!MI_IS_WS_UNSAFE(Process));
1148 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1149 Thread->OwnsProcessWorkingSetExclusive = TRUE;
1150 }
1151
1152 FORCEINLINE
1153 VOID
MiLockProcessWorkingSetShared(IN PEPROCESS Process,IN PETHREAD Thread)1154 MiLockProcessWorkingSetShared(IN PEPROCESS Process,
1155 IN PETHREAD Thread)
1156 {
1157 /* Shouldn't already be owning the process working set */
1158 ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1159 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1160
1161 /* Block APCs, make sure that still nothing is already held */
1162 KeEnterGuardedRegion();
1163 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1164
1165 /* Lock the working set */
1166 ExAcquirePushLockShared(&Process->Vm.WorkingSetMutex);
1167
1168 /* Now claim that we own the lock */
1169 ASSERT(!MI_IS_WS_UNSAFE(Process));
1170 ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1171 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1172 Thread->OwnsProcessWorkingSetShared = TRUE;
1173 }
1174
1175 FORCEINLINE
1176 VOID
MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process,IN PETHREAD Thread)1177 MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1178 IN PETHREAD Thread)
1179 {
1180 /* Shouldn't already be owning the process working set */
1181 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1182
1183 /* APCs must be blocked, make sure that still nothing is already held */
1184 ASSERT(KeAreAllApcsDisabled() == TRUE);
1185 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1186
1187 /* Lock the working set */
1188 ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1189
1190 /* Now claim that we own the lock */
1191 ASSERT(!MI_IS_WS_UNSAFE(Process));
1192 Process->Vm.Flags.AcquiredUnsafe = 1;
1193 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1194 Thread->OwnsProcessWorkingSetExclusive = TRUE;
1195 }
1196
1197 //
1198 // Unlocks the working set for the given process
1199 //
1200 FORCEINLINE
1201 VOID
MiUnlockProcessWorkingSet(IN PEPROCESS Process,IN PETHREAD Thread)1202 MiUnlockProcessWorkingSet(IN PEPROCESS Process,
1203 IN PETHREAD Thread)
1204 {
1205 /* Make sure we are the owner of a safe acquisition */
1206 ASSERT(MI_WS_OWNER(Process));
1207 ASSERT(!MI_IS_WS_UNSAFE(Process));
1208
1209 /* The thread doesn't own it anymore */
1210 ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1211 Thread->OwnsProcessWorkingSetExclusive = FALSE;
1212
1213 /* Release the lock and re-enable APCs */
1214 ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1215 KeLeaveGuardedRegion();
1216 }
1217
1218 //
1219 // Unlocks the working set for the given process
1220 //
1221 FORCEINLINE
1222 VOID
MiUnlockProcessWorkingSetShared(IN PEPROCESS Process,IN PETHREAD Thread)1223 MiUnlockProcessWorkingSetShared(IN PEPROCESS Process,
1224 IN PETHREAD Thread)
1225 {
1226 /* Make sure we are the owner of a safe acquisition (because shared) */
1227 ASSERT(MI_WS_OWNER(Process));
1228 ASSERT(!MI_IS_WS_UNSAFE(Process));
1229
1230 /* Ensure we are in a shared acquisition */
1231 ASSERT(Thread->OwnsProcessWorkingSetShared == TRUE);
1232 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1233
1234 /* Don't claim the lock anylonger */
1235 Thread->OwnsProcessWorkingSetShared = FALSE;
1236
1237 /* Release the lock and re-enable APCs */
1238 ExReleasePushLockShared(&Process->Vm.WorkingSetMutex);
1239 KeLeaveGuardedRegion();
1240 }
1241
1242 //
1243 // Unlocks the working set for the given process
1244 //
1245 FORCEINLINE
1246 VOID
MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process,IN PETHREAD Thread)1247 MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1248 IN PETHREAD Thread)
1249 {
1250 /* Make sure we are the owner of an unsafe acquisition */
1251 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1252 ASSERT(KeAreAllApcsDisabled() == TRUE);
1253 ASSERT(MI_WS_OWNER(Process));
1254 ASSERT(MI_IS_WS_UNSAFE(Process));
1255
1256 /* No longer unsafe */
1257 Process->Vm.Flags.AcquiredUnsafe = 0;
1258
1259 /* The thread doesn't own it anymore */
1260 ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1261 Thread->OwnsProcessWorkingSetExclusive = FALSE;
1262
1263 /* Release the lock but don't touch APC state */
1264 ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1265 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1266 }
1267
1268 //
1269 // Locks the working set
1270 //
1271 FORCEINLINE
1272 VOID
MiLockWorkingSet(IN PETHREAD Thread,IN PMMSUPPORT WorkingSet)1273 MiLockWorkingSet(IN PETHREAD Thread,
1274 IN PMMSUPPORT WorkingSet)
1275 {
1276 /* Block APCs */
1277 KeEnterGuardedRegion();
1278
1279 /* Working set should be in global memory */
1280 ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1281
1282 /* Thread shouldn't already be owning something */
1283 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1284
1285 /* Lock this working set */
1286 ExAcquirePushLockExclusive(&WorkingSet->WorkingSetMutex);
1287
1288 /* Which working set is this? */
1289 if (WorkingSet == &MmSystemCacheWs)
1290 {
1291 /* Own the system working set */
1292 ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1293 (Thread->OwnsSystemWorkingSetShared == FALSE));
1294 Thread->OwnsSystemWorkingSetExclusive = TRUE;
1295 }
1296 else if (WorkingSet->Flags.SessionSpace)
1297 {
1298 /* Own the session working set */
1299 ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1300 (Thread->OwnsSessionWorkingSetShared == FALSE));
1301 Thread->OwnsSessionWorkingSetExclusive = TRUE;
1302 }
1303 else
1304 {
1305 /* Own the process working set */
1306 ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1307 (Thread->OwnsProcessWorkingSetShared == FALSE));
1308 Thread->OwnsProcessWorkingSetExclusive = TRUE;
1309 }
1310 }
1311
1312 FORCEINLINE
1313 VOID
MiLockWorkingSetShared(_In_ PETHREAD Thread,_In_ PMMSUPPORT WorkingSet)1314 MiLockWorkingSetShared(
1315 _In_ PETHREAD Thread,
1316 _In_ PMMSUPPORT WorkingSet)
1317 {
1318 /* Block APCs */
1319 KeEnterGuardedRegion();
1320
1321 /* Working set should be in global memory */
1322 ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1323
1324 /* Thread shouldn't already be owning something */
1325 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1326
1327 /* Lock this working set */
1328 ExAcquirePushLockShared(&WorkingSet->WorkingSetMutex);
1329
1330 /* Which working set is this? */
1331 if (WorkingSet == &MmSystemCacheWs)
1332 {
1333 /* Own the system working set */
1334 ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1335 (Thread->OwnsSystemWorkingSetShared == FALSE));
1336 Thread->OwnsSystemWorkingSetShared = TRUE;
1337 }
1338 else if (WorkingSet->Flags.SessionSpace)
1339 {
1340 /* Own the session working set */
1341 ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1342 (Thread->OwnsSessionWorkingSetShared == FALSE));
1343 Thread->OwnsSessionWorkingSetShared = TRUE;
1344 }
1345 else
1346 {
1347 /* Own the process working set */
1348 ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1349 (Thread->OwnsProcessWorkingSetShared == FALSE));
1350 Thread->OwnsProcessWorkingSetShared = TRUE;
1351 }
1352 }
1353
1354 //
1355 // Unlocks the working set
1356 //
1357 FORCEINLINE
1358 VOID
MiUnlockWorkingSet(IN PETHREAD Thread,IN PMMSUPPORT WorkingSet)1359 MiUnlockWorkingSet(IN PETHREAD Thread,
1360 IN PMMSUPPORT WorkingSet)
1361 {
1362 /* Working set should be in global memory */
1363 ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1364
1365 /* Which working set is this? */
1366 if (WorkingSet == &MmSystemCacheWs)
1367 {
1368 /* Release the system working set */
1369 ASSERT((Thread->OwnsSystemWorkingSetExclusive == TRUE) &&
1370 (Thread->OwnsSystemWorkingSetShared == FALSE));
1371 Thread->OwnsSystemWorkingSetExclusive = FALSE;
1372 }
1373 else if (WorkingSet->Flags.SessionSpace)
1374 {
1375 /* Release the session working set */
1376 ASSERT((Thread->OwnsSessionWorkingSetExclusive == TRUE) &&
1377 (Thread->OwnsSessionWorkingSetShared == FALSE));
1378 Thread->OwnsSessionWorkingSetExclusive = FALSE;
1379 }
1380 else
1381 {
1382 /* Release the process working set */
1383 ASSERT((Thread->OwnsProcessWorkingSetExclusive == TRUE) &&
1384 (Thread->OwnsProcessWorkingSetShared == FALSE));
1385 Thread->OwnsProcessWorkingSetExclusive = FALSE;
1386 }
1387
1388 /* Release the working set lock */
1389 ExReleasePushLockExclusive(&WorkingSet->WorkingSetMutex);
1390
1391 /* Unblock APCs */
1392 KeLeaveGuardedRegion();
1393 }
1394
1395 FORCEINLINE
1396 VOID
MiUnlockWorkingSetShared(_In_ PETHREAD Thread,_In_ PMMSUPPORT WorkingSet)1397 MiUnlockWorkingSetShared(
1398 _In_ PETHREAD Thread,
1399 _In_ PMMSUPPORT WorkingSet)
1400 {
1401 /* Working set should be in global memory */
1402 ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1403
1404 /* Which working set is this? */
1405 if (WorkingSet == &MmSystemCacheWs)
1406 {
1407 /* Release the system working set */
1408 ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1409 (Thread->OwnsSystemWorkingSetShared == TRUE));
1410 Thread->OwnsSystemWorkingSetShared = FALSE;
1411 }
1412 else if (WorkingSet->Flags.SessionSpace)
1413 {
1414 /* Release the session working set */
1415 ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1416 (Thread->OwnsSessionWorkingSetShared == TRUE));
1417 Thread->OwnsSessionWorkingSetShared = FALSE;
1418 }
1419 else
1420 {
1421 /* Release the process working set */
1422 ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1423 (Thread->OwnsProcessWorkingSetShared == TRUE));
1424 Thread->OwnsProcessWorkingSetShared = FALSE;
1425 }
1426
1427 /* Release the working set lock */
1428 ExReleasePushLockShared(&WorkingSet->WorkingSetMutex);
1429
1430 /* Unblock APCs */
1431 KeLeaveGuardedRegion();
1432 }
1433
1434 FORCEINLINE
1435 BOOLEAN
MiConvertSharedWorkingSetLockToExclusive(_In_ PETHREAD Thread,_In_ PMMSUPPORT Vm)1436 MiConvertSharedWorkingSetLockToExclusive(
1437 _In_ PETHREAD Thread,
1438 _In_ PMMSUPPORT Vm)
1439 {
1440 /* Sanity check: No exclusive lock. */
1441 ASSERT(!Thread->OwnsProcessWorkingSetExclusive);
1442 ASSERT(!Thread->OwnsSessionWorkingSetExclusive);
1443 ASSERT(!Thread->OwnsSystemWorkingSetExclusive);
1444
1445 /* And it should have one and only one shared lock */
1446 ASSERT((Thread->OwnsProcessWorkingSetShared + Thread->OwnsSessionWorkingSetShared + Thread->OwnsSystemWorkingSetShared) == 1);
1447
1448 /* Try. */
1449 if (!ExConvertPushLockSharedToExclusive(&Vm->WorkingSetMutex))
1450 return FALSE;
1451
1452 if (Vm == &MmSystemCacheWs)
1453 {
1454 ASSERT(Thread->OwnsSystemWorkingSetShared);
1455 Thread->OwnsSystemWorkingSetShared = FALSE;
1456 Thread->OwnsSystemWorkingSetExclusive = TRUE;
1457 }
1458 else if (Vm->Flags.SessionSpace)
1459 {
1460 ASSERT(Thread->OwnsSessionWorkingSetShared);
1461 Thread->OwnsSessionWorkingSetShared = FALSE;
1462 Thread->OwnsSessionWorkingSetExclusive = TRUE;
1463 }
1464 else
1465 {
1466 ASSERT(Thread->OwnsProcessWorkingSetShared);
1467 Thread->OwnsProcessWorkingSetShared = FALSE;
1468 Thread->OwnsProcessWorkingSetExclusive = TRUE;
1469 }
1470
1471 return TRUE;
1472 }
1473
1474 FORCEINLINE
1475 VOID
MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process,IN PETHREAD Thread,OUT PBOOLEAN Safe,OUT PBOOLEAN Shared)1476 MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process,
1477 IN PETHREAD Thread,
1478 OUT PBOOLEAN Safe,
1479 OUT PBOOLEAN Shared)
1480 {
1481 ASSERT(MI_WS_OWNER(Process));
1482
1483 /* Check if the current owner is unsafe */
1484 if (MI_IS_WS_UNSAFE(Process))
1485 {
1486 /* Release unsafely */
1487 MiUnlockProcessWorkingSetUnsafe(Process, Thread);
1488 *Safe = FALSE;
1489 *Shared = FALSE;
1490 }
1491 else if (Thread->OwnsProcessWorkingSetExclusive == 1)
1492 {
1493 /* Owner is safe and exclusive, release normally */
1494 MiUnlockProcessWorkingSet(Process, Thread);
1495 *Safe = TRUE;
1496 *Shared = FALSE;
1497 }
1498 else
1499 {
1500 /* Owner is shared (implies safe), release normally */
1501 MiUnlockProcessWorkingSetShared(Process, Thread);
1502 *Safe = TRUE;
1503 *Shared = TRUE;
1504 }
1505 }
1506
1507 FORCEINLINE
1508 VOID
MiLockProcessWorkingSetForFault(IN PEPROCESS Process,IN PETHREAD Thread,IN BOOLEAN Safe,IN BOOLEAN Shared)1509 MiLockProcessWorkingSetForFault(IN PEPROCESS Process,
1510 IN PETHREAD Thread,
1511 IN BOOLEAN Safe,
1512 IN BOOLEAN Shared)
1513 {
1514 /* Check if this was a safe lock or not */
1515 if (Safe)
1516 {
1517 if (Shared)
1518 {
1519 /* Reacquire safely & shared */
1520 MiLockProcessWorkingSetShared(Process, Thread);
1521 }
1522 else
1523 {
1524 /* Reacquire safely */
1525 MiLockProcessWorkingSet(Process, Thread);
1526 }
1527 }
1528 else
1529 {
1530 /* Unsafe lock cannot be shared */
1531 ASSERT(Shared == FALSE);
1532 /* Reacquire unsafely */
1533 MiLockProcessWorkingSetUnsafe(Process, Thread);
1534 }
1535 }
1536
1537 FORCEINLINE
1538 KIRQL
MiAcquireExpansionLock(VOID)1539 MiAcquireExpansionLock(VOID)
1540 {
1541 KIRQL OldIrql;
1542
1543 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1544 KeAcquireSpinLock(&MmExpansionLock, &OldIrql);
1545 ASSERT(MiExpansionLockOwner == NULL);
1546 MiExpansionLockOwner = PsGetCurrentThread();
1547 return OldIrql;
1548 }
1549
1550 FORCEINLINE
1551 VOID
MiReleaseExpansionLock(KIRQL OldIrql)1552 MiReleaseExpansionLock(KIRQL OldIrql)
1553 {
1554 ASSERT(MiExpansionLockOwner == PsGetCurrentThread());
1555 MiExpansionLockOwner = NULL;
1556 KeReleaseSpinLock(&MmExpansionLock, OldIrql);
1557 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1558 }
1559
1560 //
1561 // Returns the ProtoPTE inside a VAD for the given VPN
1562 //
1563 FORCEINLINE
1564 PMMPTE
MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad,IN ULONG_PTR Vpn)1565 MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad,
1566 IN ULONG_PTR Vpn)
1567 {
1568 PMMPTE ProtoPte;
1569
1570 /* Find the offset within the VAD's prototype PTEs */
1571 ProtoPte = Vad->FirstPrototypePte + (Vpn - Vad->StartingVpn);
1572 ASSERT(ProtoPte <= Vad->LastContiguousPte);
1573 return ProtoPte;
1574 }
1575
1576 //
1577 // Returns the PFN Database entry for the given page number
1578 // Warning: This is not necessarily a valid PFN database entry!
1579 //
1580 FORCEINLINE
1581 PMMPFN
MI_PFN_ELEMENT(IN PFN_NUMBER Pfn)1582 MI_PFN_ELEMENT(IN PFN_NUMBER Pfn)
1583 {
1584 /* Get the entry */
1585 return &MmPfnDatabase[Pfn];
1586 };
1587
1588 //
1589 // Drops a locked page without dereferencing it
1590 //
1591 FORCEINLINE
1592 VOID
MiDropLockCount(IN PMMPFN Pfn1)1593 MiDropLockCount(IN PMMPFN Pfn1)
1594 {
1595 /* This page shouldn't be locked, but it should be valid */
1596 ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1597 ASSERT(Pfn1->u2.ShareCount == 0);
1598
1599 /* Is this the last reference to the page */
1600 if (Pfn1->u3.e2.ReferenceCount == 1)
1601 {
1602 /* It better not be valid */
1603 ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1604
1605 /* Is it a prototype PTE? */
1606 if ((Pfn1->u3.e1.PrototypePte == 1) &&
1607 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1608 {
1609 /* FIXME: We should return commit */
1610 DPRINT1("Not returning commit for prototype PTE\n");
1611 }
1612
1613 /* Update the counter */
1614 InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1615 }
1616 }
1617
1618 //
1619 // Drops a locked page and dereferences it
1620 //
1621 FORCEINLINE
1622 VOID
MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1)1623 MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1)
1624 {
1625 USHORT RefCount, OldRefCount;
1626 PFN_NUMBER PageFrameIndex;
1627
1628 /* Loop while we decrement the page successfully */
1629 do
1630 {
1631 /* There should be at least one reference */
1632 OldRefCount = Pfn1->u3.e2.ReferenceCount;
1633 ASSERT(OldRefCount != 0);
1634
1635 /* Are we the last one */
1636 if (OldRefCount == 1)
1637 {
1638 /* The page shoudln't be shared not active at this point */
1639 ASSERT(Pfn1->u3.e2.ReferenceCount == 1);
1640 ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1641 ASSERT(Pfn1->u2.ShareCount == 0);
1642
1643 /* Is it a prototype PTE? */
1644 if ((Pfn1->u3.e1.PrototypePte == 1) &&
1645 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1646 {
1647 /* FIXME: We should return commit */
1648 DPRINT1("Not returning commit for prototype PTE\n");
1649 }
1650
1651 /* Update the counter, and drop a reference the long way */
1652 InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1653 PageFrameIndex = MiGetPfnEntryIndex(Pfn1);
1654 MiDecrementReferenceCount(Pfn1, PageFrameIndex);
1655 return;
1656 }
1657
1658 /* Drop a reference the short way, and that's it */
1659 RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1660 OldRefCount - 1,
1661 OldRefCount);
1662 ASSERT(RefCount != 0);
1663 } while (OldRefCount != RefCount);
1664
1665 /* If we got here, there should be more than one reference */
1666 ASSERT(RefCount > 1);
1667 if (RefCount == 2)
1668 {
1669 /* Is it still being shared? */
1670 if (Pfn1->u2.ShareCount >= 1)
1671 {
1672 /* Then it should be valid */
1673 ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1674
1675 /* Is it a prototype PTE? */
1676 if ((Pfn1->u3.e1.PrototypePte == 1) &&
1677 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1678 {
1679 /* We don't handle ethis */
1680 ASSERT(FALSE);
1681 }
1682
1683 /* Update the counter */
1684 InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1685 }
1686 }
1687 }
1688
1689 //
1690 // References a locked page and updates the counter
1691 // Used in MmProbeAndLockPages to handle different edge cases
1692 //
1693 FORCEINLINE
1694 VOID
MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1)1695 MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1)
1696 {
1697 USHORT RefCount, OldRefCount;
1698
1699 /* Sanity check */
1700 ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1701
1702 /* Does ARM3 own the page? */
1703 if (MI_IS_ROS_PFN(Pfn1))
1704 {
1705 /* ReactOS Mm doesn't track share count */
1706 ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1707 }
1708 else
1709 {
1710 /* On ARM3 pages, we should see a valid share count */
1711 ASSERT((Pfn1->u2.ShareCount != 0) && (Pfn1->u3.e1.PageLocation == ActiveAndValid));
1712
1713 /* Is it a prototype PTE? */
1714 if ((Pfn1->u3.e1.PrototypePte == 1) &&
1715 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1716 {
1717 /* FIXME: We should charge commit */
1718 DPRINT1("Not charging commit for prototype PTE\n");
1719 }
1720 }
1721
1722 /* More locked pages! */
1723 InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1724
1725 /* Loop trying to update the reference count */
1726 do
1727 {
1728 /* Get the current reference count, make sure it's valid */
1729 OldRefCount = Pfn1->u3.e2.ReferenceCount;
1730 ASSERT(OldRefCount != 0);
1731 ASSERT(OldRefCount < 2500);
1732
1733 /* Bump it up by one */
1734 RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1735 OldRefCount + 1,
1736 OldRefCount);
1737 ASSERT(RefCount != 0);
1738 } while (OldRefCount != RefCount);
1739
1740 /* Was this the first lock attempt? If not, undo our bump */
1741 if (OldRefCount != 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1742 }
1743
1744 //
1745 // References a locked page and updates the counter
1746 // Used in all other cases except MmProbeAndLockPages
1747 //
1748 FORCEINLINE
1749 VOID
MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1)1750 MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1)
1751 {
1752 USHORT NewRefCount;
1753
1754 /* Is it a prototype PTE? */
1755 if ((Pfn1->u3.e1.PrototypePte == 1) &&
1756 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1757 {
1758 /* FIXME: We should charge commit */
1759 DPRINT1("Not charging commit for prototype PTE\n");
1760 }
1761
1762 /* More locked pages! */
1763 InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1764
1765 /* Update the reference count */
1766 NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1767 if (NewRefCount == 2)
1768 {
1769 /* Is it locked or shared? */
1770 if (Pfn1->u2.ShareCount)
1771 {
1772 /* It's shared, so make sure it's active */
1773 ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1774 }
1775 else
1776 {
1777 /* It's locked, so we shouldn't lock again */
1778 InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1779 }
1780 }
1781 else
1782 {
1783 /* Someone had already locked the page, so undo our bump */
1784 ASSERT(NewRefCount < 2500);
1785 InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1786 }
1787 }
1788
1789 //
1790 // References a locked page and updates the counter
1791 // Used in all other cases except MmProbeAndLockPages
1792 //
1793 FORCEINLINE
1794 VOID
MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1)1795 MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1)
1796 {
1797 USHORT NewRefCount;
1798
1799 /* Make sure the page isn't used yet */
1800 ASSERT(Pfn1->u2.ShareCount == 0);
1801 ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1802
1803 /* Is it a prototype PTE? */
1804 if ((Pfn1->u3.e1.PrototypePte == 1) &&
1805 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1806 {
1807 /* FIXME: We should charge commit */
1808 DPRINT1("Not charging commit for prototype PTE\n");
1809 }
1810
1811 /* More locked pages! */
1812 InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1813
1814 /* Update the reference count */
1815 NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1816 if (NewRefCount != 1)
1817 {
1818 /* Someone had already locked the page, so undo our bump */
1819 ASSERT(NewRefCount < 2500);
1820 InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1821 }
1822 }
1823
1824
1825
1826 CODE_SEG("INIT")
1827 BOOLEAN
1828 NTAPI
1829 MmArmInitSystem(
1830 IN ULONG Phase,
1831 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1832 );
1833
1834 CODE_SEG("INIT")
1835 VOID
1836 NTAPI
1837 MiInitializeSessionSpaceLayout(VOID);
1838
1839 CODE_SEG("INIT")
1840 NTSTATUS
1841 NTAPI
1842 MiInitMachineDependent(
1843 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1844 );
1845
1846 CODE_SEG("INIT")
1847 VOID
1848 NTAPI
1849 MiComputeColorInformation(
1850 VOID
1851 );
1852
1853 CODE_SEG("INIT")
1854 VOID
1855 NTAPI
1856 MiMapPfnDatabase(
1857 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1858 );
1859
1860 CODE_SEG("INIT")
1861 VOID
1862 NTAPI
1863 MiInitializeColorTables(
1864 VOID
1865 );
1866
1867 CODE_SEG("INIT")
1868 VOID
1869 NTAPI
1870 MiInitializePfnDatabase(
1871 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1872 );
1873
1874 VOID
1875 NTAPI
1876 MiInitializeSessionWsSupport(
1877 VOID
1878 );
1879
1880 VOID
1881 NTAPI
1882 MiInitializeSessionIds(
1883 VOID
1884 );
1885
1886 CODE_SEG("INIT")
1887 BOOLEAN
1888 NTAPI
1889 MiInitializeMemoryEvents(
1890 VOID
1891 );
1892
1893 CODE_SEG("INIT")
1894 PFN_NUMBER
1895 NTAPI
1896 MxGetNextPage(
1897 IN PFN_NUMBER PageCount
1898 );
1899
1900 CODE_SEG("INIT")
1901 PPHYSICAL_MEMORY_DESCRIPTOR
1902 NTAPI
1903 MmInitializeMemoryLimits(
1904 IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1905 IN PBOOLEAN IncludeType
1906 );
1907
1908 PFN_NUMBER
1909 NTAPI
1910 MiPagesInLoaderBlock(
1911 IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1912 IN PBOOLEAN IncludeType
1913 );
1914
1915 VOID
1916 FASTCALL
1917 MiSyncARM3WithROS(
1918 IN PVOID AddressStart,
1919 IN PVOID AddressEnd
1920 );
1921
1922 NTSTATUS
1923 NTAPI
1924 MiRosProtectVirtualMemory(
1925 IN PEPROCESS Process,
1926 IN OUT PVOID *BaseAddress,
1927 IN OUT PSIZE_T NumberOfBytesToProtect,
1928 IN ULONG NewAccessProtection,
1929 OUT PULONG OldAccessProtection OPTIONAL
1930 );
1931
1932 NTSTATUS
1933 NTAPI
1934 MmArmAccessFault(
1935 IN ULONG FaultCode,
1936 IN PVOID Address,
1937 IN KPROCESSOR_MODE Mode,
1938 IN PVOID TrapInformation
1939 );
1940
1941 NTSTATUS
1942 FASTCALL
1943 MiCheckPdeForPagedPool(
1944 IN PVOID Address
1945 );
1946
1947 CODE_SEG("INIT")
1948 VOID
1949 NTAPI
1950 MiInitializeNonPagedPoolThresholds(
1951 VOID
1952 );
1953
1954 CODE_SEG("INIT")
1955 VOID
1956 NTAPI
1957 MiInitializePoolEvents(
1958 VOID
1959 );
1960
1961 CODE_SEG("INIT")
1962 VOID //
1963 NTAPI //
1964 InitializePool( //
1965 IN POOL_TYPE PoolType,// FIXFIX: This should go in ex.h after the pool merge
1966 IN ULONG Threshold //
1967 ); //
1968
1969 // FIXFIX: THIS ONE TOO
1970 CODE_SEG("INIT")
1971 VOID
1972 NTAPI
1973 ExInitializePoolDescriptor(
1974 IN PPOOL_DESCRIPTOR PoolDescriptor,
1975 IN POOL_TYPE PoolType,
1976 IN ULONG PoolIndex,
1977 IN ULONG Threshold,
1978 IN PVOID PoolLock
1979 );
1980
1981 NTSTATUS
1982 NTAPI
1983 MiInitializeSessionPool(
1984 VOID
1985 );
1986
1987 CODE_SEG("INIT")
1988 VOID
1989 NTAPI
1990 MiInitializeSystemPtes(
1991 IN PMMPTE StartingPte,
1992 IN ULONG NumberOfPtes,
1993 IN MMSYSTEM_PTE_POOL_TYPE PoolType
1994 );
1995
1996 PMMPTE
1997 NTAPI
1998 MiReserveSystemPtes(
1999 IN ULONG NumberOfPtes,
2000 IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
2001 );
2002
2003 VOID
2004 NTAPI
2005 MiReleaseSystemPtes(
2006 IN PMMPTE StartingPte,
2007 IN ULONG NumberOfPtes,
2008 IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
2009 );
2010
2011
2012 PFN_NUMBER
2013 NTAPI
2014 MiFindContiguousPages(
2015 IN PFN_NUMBER LowestPfn,
2016 IN PFN_NUMBER HighestPfn,
2017 IN PFN_NUMBER BoundaryPfn,
2018 IN PFN_NUMBER SizeInPages,
2019 IN MEMORY_CACHING_TYPE CacheType
2020 );
2021
2022 PVOID
2023 NTAPI
2024 MiCheckForContiguousMemory(
2025 IN PVOID BaseAddress,
2026 IN PFN_NUMBER BaseAddressPages,
2027 IN PFN_NUMBER SizeInPages,
2028 IN PFN_NUMBER LowestPfn,
2029 IN PFN_NUMBER HighestPfn,
2030 IN PFN_NUMBER BoundaryPfn,
2031 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
2032 );
2033
2034 PMDL
2035 NTAPI
2036 MiAllocatePagesForMdl(
2037 IN PHYSICAL_ADDRESS LowAddress,
2038 IN PHYSICAL_ADDRESS HighAddress,
2039 IN PHYSICAL_ADDRESS SkipBytes,
2040 IN SIZE_T TotalBytes,
2041 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
2042 IN ULONG Flags
2043 );
2044
2045 VOID
2046 NTAPI
2047 MiInsertPageInList(
2048 IN PMMPFNLIST ListHead,
2049 IN PFN_NUMBER PageFrameIndex
2050 );
2051
2052 VOID
2053 NTAPI
2054 MiUnlinkFreeOrZeroedPage(
2055 IN PMMPFN Entry
2056 );
2057
2058 VOID
2059 NTAPI
2060 MiUnlinkPageFromList(
2061 IN PMMPFN Pfn
2062 );
2063
2064 VOID
2065 NTAPI
2066 MiInitializePfn(
2067 IN PFN_NUMBER PageFrameIndex,
2068 IN PMMPTE PointerPte,
2069 IN BOOLEAN Modified
2070 );
2071
2072 NTSTATUS
2073 NTAPI
2074 MiInitializeAndChargePfn(
2075 OUT PPFN_NUMBER PageFrameIndex,
2076 IN PMMPDE PointerPde,
2077 IN PFN_NUMBER ContainingPageFrame,
2078 IN BOOLEAN SessionAllocation
2079 );
2080
2081 VOID
2082 NTAPI
2083 MiInitializePfnAndMakePteValid(
2084 IN PFN_NUMBER PageFrameIndex,
2085 IN PMMPTE PointerPte,
2086 IN MMPTE TempPte
2087 );
2088
2089 VOID
2090 NTAPI
2091 MiInitializePfnForOtherProcess(
2092 IN PFN_NUMBER PageFrameIndex,
2093 IN PVOID PteAddress,
2094 IN PFN_NUMBER PteFrame
2095 );
2096
2097 VOID
2098 NTAPI
2099 MiDecrementShareCount(
2100 IN PMMPFN Pfn1,
2101 IN PFN_NUMBER PageFrameIndex
2102 );
2103
2104 PFN_NUMBER
2105 NTAPI
2106 MiRemoveAnyPage(
2107 IN ULONG Color
2108 );
2109
2110 PFN_NUMBER
2111 NTAPI
2112 MiRemoveZeroPage(
2113 IN ULONG Color
2114 );
2115
2116 VOID
2117 NTAPI
2118 MiZeroPhysicalPage(
2119 IN PFN_NUMBER PageFrameIndex
2120 );
2121
2122 VOID
2123 NTAPI
2124 MiInsertPageInFreeList(
2125 IN PFN_NUMBER PageFrameIndex
2126 );
2127
2128 PFN_COUNT
2129 NTAPI
2130 MiDeleteSystemPageableVm(
2131 IN PMMPTE PointerPte,
2132 IN PFN_NUMBER PageCount,
2133 IN ULONG Flags,
2134 OUT PPFN_NUMBER ValidPages
2135 );
2136
2137 ULONG
2138 NTAPI
2139 MiGetPageProtection(
2140 IN PMMPTE PointerPte
2141 );
2142
2143 PLDR_DATA_TABLE_ENTRY
2144 NTAPI
2145 MiLookupDataTableEntry(
2146 IN PVOID Address
2147 );
2148
2149 CODE_SEG("INIT")
2150 VOID
2151 NTAPI
2152 MiInitializeDriverLargePageList(
2153 VOID
2154 );
2155
2156 CODE_SEG("INIT")
2157 VOID
2158 NTAPI
2159 MiInitializeLargePageSupport(
2160 VOID
2161 );
2162
2163 CODE_SEG("INIT")
2164 VOID
2165 NTAPI
2166 MiSyncCachedRanges(
2167 VOID
2168 );
2169
2170 BOOLEAN
2171 NTAPI
2172 MiIsPfnInUse(
2173 IN PMMPFN Pfn1
2174 );
2175
2176 PMMVAD
2177 NTAPI
2178 MiLocateAddress(
2179 IN PVOID VirtualAddress
2180 );
2181
2182 TABLE_SEARCH_RESULT
2183 NTAPI
2184 MiCheckForConflictingNode(
2185 IN ULONG_PTR StartVpn,
2186 IN ULONG_PTR EndVpn,
2187 IN PMM_AVL_TABLE Table,
2188 OUT PMMADDRESS_NODE *NodeOrParent
2189 );
2190
2191 TABLE_SEARCH_RESULT
2192 NTAPI
2193 MiFindEmptyAddressRangeDownTree(
2194 IN SIZE_T Length,
2195 IN ULONG_PTR BoundaryAddress,
2196 IN ULONG_PTR Alignment,
2197 IN PMM_AVL_TABLE Table,
2198 OUT PULONG_PTR Base,
2199 OUT PMMADDRESS_NODE *Parent
2200 );
2201
2202 NTSTATUS
2203 NTAPI
2204 MiFindEmptyAddressRangeDownBasedTree(
2205 IN SIZE_T Length,
2206 IN ULONG_PTR BoundaryAddress,
2207 IN ULONG_PTR Alignment,
2208 IN PMM_AVL_TABLE Table,
2209 OUT PULONG_PTR Base
2210 );
2211
2212 TABLE_SEARCH_RESULT
2213 NTAPI
2214 MiFindEmptyAddressRangeInTree(
2215 IN SIZE_T Length,
2216 IN ULONG_PTR Alignment,
2217 IN PMM_AVL_TABLE Table,
2218 OUT PMMADDRESS_NODE *PreviousVad,
2219 OUT PULONG_PTR Base
2220 );
2221
2222 NTSTATUS
2223 NTAPI
2224 MiCheckSecuredVad(
2225 IN PMMVAD Vad,
2226 IN PVOID Base,
2227 IN SIZE_T Size,
2228 IN ULONG ProtectionMask
2229 );
2230
2231 VOID
2232 NTAPI
2233 MiInsertVad(
2234 _Inout_ PMMVAD Vad,
2235 _Inout_ PMM_AVL_TABLE VadRoot);
2236
2237 NTSTATUS
2238 NTAPI
2239 MiInsertVadEx(
2240 _Inout_ PMMVAD Vad,
2241 _In_ ULONG_PTR *BaseAddress,
2242 _In_ SIZE_T ViewSize,
2243 _In_ ULONG_PTR HighestAddress,
2244 _In_ ULONG_PTR Alignment,
2245 _In_ ULONG AllocationType);
2246
2247 VOID
2248 NTAPI
2249 MiInsertBasedSection(
2250 IN PSECTION Section
2251 );
2252
2253 NTSTATUS
2254 NTAPI
2255 MiRosUnmapViewOfSection(
2256 IN PEPROCESS Process,
2257 IN PVOID BaseAddress,
2258 IN BOOLEAN SkipDebuggerNotify
2259 );
2260
2261 VOID
2262 NTAPI
2263 MiInsertNode(
2264 IN PMM_AVL_TABLE Table,
2265 IN PMMADDRESS_NODE NewNode,
2266 PMMADDRESS_NODE Parent,
2267 TABLE_SEARCH_RESULT Result
2268 );
2269
2270 VOID
2271 NTAPI
2272 MiRemoveNode(
2273 IN PMMADDRESS_NODE Node,
2274 IN PMM_AVL_TABLE Table
2275 );
2276
2277 PMMADDRESS_NODE
2278 NTAPI
2279 MiGetPreviousNode(
2280 IN PMMADDRESS_NODE Node
2281 );
2282
2283 PMMADDRESS_NODE
2284 NTAPI
2285 MiGetNextNode(
2286 IN PMMADDRESS_NODE Node
2287 );
2288
2289 BOOLEAN
2290 NTAPI
2291 MiInitializeSystemSpaceMap(
2292 IN PMMSESSION InputSession OPTIONAL
2293 );
2294
2295 VOID
2296 NTAPI
2297 MiSessionRemoveProcess(
2298 VOID
2299 );
2300
2301 VOID
2302 NTAPI
2303 MiReleaseProcessReferenceToSessionDataPage(
2304 IN PMM_SESSION_SPACE SessionGlobal
2305 );
2306
2307 VOID
2308 NTAPI
2309 MiSessionAddProcess(
2310 IN PEPROCESS NewProcess
2311 );
2312
2313 ULONG
2314 NTAPI
2315 MiMakeProtectionMask(
2316 IN ULONG Protect
2317 );
2318
2319 VOID
2320 NTAPI
2321 MiDeleteVirtualAddresses(
2322 IN ULONG_PTR Va,
2323 IN ULONG_PTR EndingAddress,
2324 IN PMMVAD Vad
2325 );
2326
2327 VOID
2328 NTAPI
2329 MiDeletePte(
2330 IN PMMPTE PointerPte,
2331 IN PVOID VirtualAddress,
2332 IN PEPROCESS CurrentProcess,
2333 IN PMMPTE PrototypePte
2334 );
2335
2336 ULONG
2337 NTAPI
2338 MiMakeSystemAddressValid(
2339 IN PVOID PageTableVirtualAddress,
2340 IN PEPROCESS CurrentProcess
2341 );
2342
2343 ULONG
2344 NTAPI
2345 MiMakeSystemAddressValidPfn(
2346 IN PVOID VirtualAddress,
2347 IN KIRQL OldIrql
2348 );
2349
2350 VOID
2351 NTAPI
2352 MiRemoveMappedView(
2353 IN PEPROCESS CurrentProcess,
2354 IN PMMVAD Vad
2355 );
2356
2357 PSUBSECTION
2358 NTAPI
2359 MiLocateSubsection(
2360 IN PMMVAD Vad,
2361 IN ULONG_PTR Vpn
2362 );
2363
2364 VOID
2365 NTAPI
2366 MiDeleteARM3Section(
2367 PVOID ObjectBody
2368 );
2369
2370 NTSTATUS
2371 NTAPI
2372 MiQueryMemorySectionName(
2373 IN HANDLE ProcessHandle,
2374 IN PVOID BaseAddress,
2375 OUT PVOID MemoryInformation,
2376 IN SIZE_T MemoryInformationLength,
2377 OUT PSIZE_T ReturnLength
2378 );
2379
2380 NTSTATUS
2381 NTAPI
2382 MiRosUnmapViewInSystemSpace(
2383 IN PVOID MappedBase
2384 );
2385
2386 VOID
2387 NTAPI
2388 MiMakePdeExistAndMakeValid(
2389 IN PMMPDE PointerPde,
2390 IN PEPROCESS TargetProcess,
2391 IN KIRQL OldIrql
2392 );
2393
2394 VOID
2395 NTAPI
2396 MiWriteProtectSystemImage(
2397 _In_ PVOID ImageBase);
2398
2399 //
2400 // MiRemoveZeroPage will use inline code to zero out the page manually if only
2401 // free pages are available. In some scenarios, we don't/can't run that piece of
2402 // code and would rather only have a real zero page. If we can't have a zero page,
2403 // then we'd like to have our own code to grab a free page and zero it out, by
2404 // using MiRemoveAnyPage. This macro implements this.
2405 //
2406 FORCEINLINE
2407 PFN_NUMBER
MiRemoveZeroPageSafe(IN ULONG Color)2408 MiRemoveZeroPageSafe(IN ULONG Color)
2409 {
2410 if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color);
2411 return 0;
2412 }
2413
2414 #if (_MI_PAGING_LEVELS == 2)
2415 FORCEINLINE
2416 BOOLEAN
MiSynchronizeSystemPde(PMMPDE PointerPde)2417 MiSynchronizeSystemPde(PMMPDE PointerPde)
2418 {
2419 ULONG Index;
2420
2421 /* Get the Index from the PDE */
2422 Index = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE);
2423 if (PointerPde->u.Hard.Valid != 0)
2424 {
2425 NT_ASSERT(PointerPde->u.Long == MmSystemPagePtes[Index].u.Long);
2426 return TRUE;
2427 }
2428
2429 if (MmSystemPagePtes[Index].u.Hard.Valid == 0)
2430 {
2431 return FALSE;
2432 }
2433
2434 /* Copy the PDE from the double-mapped system page directory */
2435 MI_WRITE_VALID_PDE(PointerPde, MmSystemPagePtes[Index]);
2436
2437 /* Make sure we re-read the PDE and PTE */
2438 KeMemoryBarrierWithoutFence();
2439
2440 /* Return success */
2441 return TRUE;
2442 }
2443 #endif
2444
2445 #if _MI_PAGING_LEVELS == 2
2446 FORCEINLINE
2447 USHORT
MiIncrementPageTableReferences(IN PVOID Address)2448 MiIncrementPageTableReferences(IN PVOID Address)
2449 {
2450 PUSHORT RefCount;
2451
2452 RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
2453
2454 *RefCount += 1;
2455 ASSERT(*RefCount <= PTE_PER_PAGE);
2456 return *RefCount;
2457 }
2458
2459 FORCEINLINE
2460 USHORT
MiDecrementPageTableReferences(IN PVOID Address)2461 MiDecrementPageTableReferences(IN PVOID Address)
2462 {
2463 PUSHORT RefCount;
2464
2465 RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
2466
2467 *RefCount -= 1;
2468 ASSERT(*RefCount < PTE_PER_PAGE);
2469 return *RefCount;
2470 }
2471 #else
2472 FORCEINLINE
2473 USHORT
MiIncrementPageTableReferences(IN PVOID Address)2474 MiIncrementPageTableReferences(IN PVOID Address)
2475 {
2476 PMMPDE PointerPde = MiAddressToPde(Address);
2477 PMMPFN Pfn;
2478
2479 /* We should not tinker with this one. */
2480 ASSERT(PointerPde != (PMMPDE)PXE_SELFMAP);
2481 DPRINT("Incrementing %p from %p\n", Address, _ReturnAddress());
2482
2483 /* Make sure we're locked */
2484 ASSERT(PsGetCurrentThread()->OwnsProcessWorkingSetExclusive);
2485
2486 /* If we're bumping refcount, then it must be valid! */
2487 ASSERT(PointerPde->u.Hard.Valid == 1);
2488
2489 /* This lies on the PFN */
2490 Pfn = MiGetPfnEntry(PFN_FROM_PDE(PointerPde));
2491 Pfn->OriginalPte.u.Soft.UsedPageTableEntries++;
2492
2493 ASSERT(Pfn->OriginalPte.u.Soft.UsedPageTableEntries <= PTE_PER_PAGE);
2494
2495 return Pfn->OriginalPte.u.Soft.UsedPageTableEntries;
2496 }
2497
2498 FORCEINLINE
2499 USHORT
MiDecrementPageTableReferences(IN PVOID Address)2500 MiDecrementPageTableReferences(IN PVOID Address)
2501 {
2502 PMMPDE PointerPde = MiAddressToPde(Address);
2503 PMMPFN Pfn;
2504
2505 /* We should not tinker with this one. */
2506 ASSERT(PointerPde != (PMMPDE)PXE_SELFMAP);
2507
2508 DPRINT("Decrementing %p from %p\n", PointerPde, _ReturnAddress());
2509
2510 /* Make sure we're locked */
2511 ASSERT(PsGetCurrentThread()->OwnsProcessWorkingSetExclusive);
2512
2513 /* If we're decreasing refcount, then it must be valid! */
2514 ASSERT(PointerPde->u.Hard.Valid == 1);
2515
2516 /* This lies on the PFN */
2517 Pfn = MiGetPfnEntry(PFN_FROM_PDE(PointerPde));
2518
2519 ASSERT(Pfn->OriginalPte.u.Soft.UsedPageTableEntries != 0);
2520 Pfn->OriginalPte.u.Soft.UsedPageTableEntries--;
2521
2522 ASSERT(Pfn->OriginalPte.u.Soft.UsedPageTableEntries < PTE_PER_PAGE);
2523
2524 return Pfn->OriginalPte.u.Soft.UsedPageTableEntries;
2525 }
2526 #endif
2527
2528 #ifdef __cplusplus
2529 } // extern "C"
2530 #endif
2531
2532 FORCEINLINE
2533 VOID
MiDeletePde(_In_ PMMPDE PointerPde,_In_ PEPROCESS CurrentProcess)2534 MiDeletePde(
2535 _In_ PMMPDE PointerPde,
2536 _In_ PEPROCESS CurrentProcess)
2537 {
2538 /* Only for user-mode ones */
2539 ASSERT(MiIsUserPde(PointerPde));
2540
2541 /* Kill this one as a PTE */
2542 MiDeletePte((PMMPTE)PointerPde, MiPdeToPte(PointerPde), CurrentProcess, NULL);
2543 #if _MI_PAGING_LEVELS >= 3
2544 /* Cascade down */
2545 if (MiDecrementPageTableReferences(MiPdeToPte(PointerPde)) == 0)
2546 {
2547 MiDeletePte(MiPdeToPpe(PointerPde), PointerPde, CurrentProcess, NULL);
2548 #if _MI_PAGING_LEVELS == 4
2549 if (MiDecrementPageTableReferences(PointerPde) == 0)
2550 {
2551 MiDeletePte(MiPdeToPxe(PointerPde), MiPdeToPpe(PointerPde), CurrentProcess, NULL);
2552 }
2553 #endif
2554 }
2555 #endif
2556 }
2557
2558 /* EOF */
2559