1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/mm/ARM3/miarm.h
5 * PURPOSE: ARM Memory Manager Header
6 * PROGRAMMERS: ReactOS Portable Systems Group
7 */
8
9 #pragma once
10
11 #ifdef __cplusplus
12 extern "C" {
13 #endif
14
15 #define MI_LOWEST_VAD_ADDRESS (PVOID)MM_LOWEST_USER_ADDRESS
16
17 /* Make the code cleaner with some definitions for size multiples */
18 #define _1KB (1024u)
19 #define _1MB (1024 * _1KB)
20 #define _1GB (1024 * _1MB)
21
22 /* Everyone loves 64K */
23 #define _64K (64 * _1KB)
24
25 /* Size of a page table */
26 #define PT_SIZE (PTE_PER_PAGE * sizeof(MMPTE))
27
28 /* Size of a page directory */
29 #define PD_SIZE (PDE_PER_PAGE * sizeof(MMPDE))
30
31 /* Size of all page directories for a process */
32 #define SYSTEM_PD_SIZE (PPE_PER_PAGE * PD_SIZE)
33 #ifdef _M_IX86
34 C_ASSERT(SYSTEM_PD_SIZE == PAGE_SIZE);
35 #endif
36
37 //
38 // Protection Bits part of the internal memory manager Protection Mask, from:
39 // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel
40 // https://www.reactos.org/wiki/Techwiki:Memory_Protection_constants
41 // and public assertions.
42 //
43 #define MM_ZERO_ACCESS 0
44 #define MM_READONLY 1
45 #define MM_EXECUTE 2
46 #define MM_EXECUTE_READ 3
47 #define MM_READWRITE 4
48 #define MM_WRITECOPY 5
49 #define MM_EXECUTE_READWRITE 6
50 #define MM_EXECUTE_WRITECOPY 7
51 #define MM_PROTECT_ACCESS 7
52
53 //
54 // These are flags on top of the actual protection mask
55 //
56 #define MM_NOCACHE 0x08
57 #define MM_GUARDPAGE 0x10
58 #define MM_WRITECOMBINE 0x18
59 #define MM_PROTECT_SPECIAL 0x18
60
61 //
62 // These are special cases
63 //
64 #define MM_DECOMMIT (MM_ZERO_ACCESS | MM_GUARDPAGE)
65 #define MM_NOACCESS (MM_ZERO_ACCESS | MM_WRITECOMBINE)
66 #define MM_OUTSWAPPED_KSTACK (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE)
67 #define MM_INVALID_PROTECTION 0xFFFFFFFF
68
69 //
70 // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits
71 // The Memory Manager's definition define the attributes that must be preserved
72 // and these PTE definitions describe the attributes in the hardware sense. This
73 // helps deal with hardware differences between the actual boolean expression of
74 // the argument.
75 //
76 // For example, in the logical attributes, we want to express read-only as a flag
77 // but on x86, it is writability that must be set. On the other hand, on x86, just
78 // like in the kernel, it is disabling the caches that requires a special flag,
79 // while on certain architectures such as ARM, it is enabling the cache which
80 // requires a flag.
81 //
82 #if defined(_M_IX86)
83 //
84 // Access Flags
85 //
86 #define PTE_READONLY 0 // Doesn't exist on x86
87 #define PTE_EXECUTE 0 // Not worrying about NX yet
88 #define PTE_EXECUTE_READ 0 // Not worrying about NX yet
89 #define PTE_READWRITE 0x2
90 #define PTE_WRITECOPY 0x200
91 #define PTE_EXECUTE_READWRITE 0x2 // Not worrying about NX yet
92 #define PTE_EXECUTE_WRITECOPY 0x200
93 #define PTE_PROTOTYPE 0x400
94
95 //
96 // State Flags
97 //
98 #define PTE_VALID 0x1
99 #define PTE_ACCESSED 0x20
100 #define PTE_DIRTY 0x40
101
102 //
103 // Cache flags
104 //
105 #define PTE_ENABLE_CACHE 0
106 #define PTE_DISABLE_CACHE 0x10
107 #define PTE_WRITECOMBINED_CACHE 0x10
108 #define PTE_PROTECT_MASK 0x612
109 #elif defined(_M_AMD64)
110 //
111 // Access Flags
112 //
113 #define PTE_READONLY 0x8000000000000000ULL
114 #define PTE_EXECUTE 0x0000000000000000ULL
115 #define PTE_EXECUTE_READ PTE_EXECUTE /* EXECUTE implies READ on x64 */
116 #define PTE_READWRITE 0x8000000000000002ULL
117 #define PTE_WRITECOPY 0x8000000000000200ULL
118 #define PTE_EXECUTE_READWRITE 0x0000000000000002ULL
119 #define PTE_EXECUTE_WRITECOPY 0x0000000000000200ULL
120 #define PTE_PROTOTYPE 0x0000000000000400ULL
121
122 //
123 // State Flags
124 //
125 #define PTE_VALID 0x0000000000000001ULL
126 #define PTE_ACCESSED 0x0000000000000020ULL
127 #define PTE_DIRTY 0x0000000000000040ULL
128
129 //
130 // Cache flags
131 //
132 #define PTE_ENABLE_CACHE 0x0000000000000000ULL
133 #define PTE_DISABLE_CACHE 0x0000000000000010ULL
134 #define PTE_WRITECOMBINED_CACHE 0x0000000000000010ULL
135 #define PTE_PROTECT_MASK 0x8000000000000612ULL
136 #elif defined(_M_ARM)
137 #define PTE_READONLY 0x200
138 #define PTE_EXECUTE 0 // Not worrying about NX yet
139 #define PTE_EXECUTE_READ 0 // Not worrying about NX yet
140 #define PTE_READWRITE 0 // Doesn't exist on ARM
141 #define PTE_WRITECOPY 0 // Doesn't exist on ARM
142 #define PTE_EXECUTE_READWRITE 0 // Not worrying about NX yet
143 #define PTE_EXECUTE_WRITECOPY 0 // Not worrying about NX yet
144 #define PTE_PROTOTYPE 0x400 // Using the Shared bit
145
146 //
147 // Cache flags
148 //
149 #define PTE_ENABLE_CACHE 0
150 #define PTE_DISABLE_CACHE 0x10
151 #define PTE_WRITECOMBINED_CACHE 0x10
152 #define PTE_PROTECT_MASK 0x610
153 #else
154 #error Define these please!
155 #endif
156
157 //
158 // Some internal SYSTEM_PTE_MISUSE bugcheck subcodes
159 // These names were created by Oleg Dubinskiy and Doug Lyons for ReactOS. For reference, see
160 // https://learn.microsoft.com/en-us/windows-hardware/drivers/debugger/bug-check-0xda--system-pte-misuse
161 //
162 #define PTE_MAPPING_NONE 0x100
163 #define PTE_MAPPING_NOT_OWNED 0x101
164 #define PTE_MAPPING_EMPTY 0x102
165 #define PTE_MAPPING_RESERVED 0x103
166 #define PTE_MAPPING_ADDRESS_NOT_OWNED 0x104
167 #define PTE_MAPPING_ADDRESS_INVALID 0x105
168 #define PTE_UNMAPPING_ADDRESS_NOT_OWNED 0x108
169 #define PTE_MAPPING_ADDRESS_EMPTY 0x109
170
171 //
172 // Mask for image section page protection
173 //
174 #define IMAGE_SCN_PROTECTION_MASK (IMAGE_SCN_MEM_WRITE | IMAGE_SCN_MEM_READ | IMAGE_SCN_MEM_EXECUTE)
175
176 extern const ULONG_PTR MmProtectToPteMask[32];
177 extern const ULONG MmProtectToValue[32];
178
179 //
180 // Assertions for session images, addresses, and PTEs
181 //
182 #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \
183 (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))
184
185 #define MI_IS_SESSION_ADDRESS(Address) \
186 (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))
187
188 #define MI_IS_SESSION_PTE(Pte) \
189 ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))
190
191 #define MI_IS_PAGE_TABLE_ADDRESS(Address) \
192 (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))
193
194 #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \
195 (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))
196
197 #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \
198 (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))
199
200 //
201 // Creates a software PTE with the given protection
202 //
203 #define MI_MAKE_SOFTWARE_PTE(p, x) ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))
204
205 //
206 // Marks a PTE as deleted
207 //
208 #define MI_SET_PFN_DELETED(x) ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1))
209 #define MI_IS_PFN_DELETED(x) ((ULONG_PTR)((x)->PteAddress) & 1)
210
211 //
212 // Special values for LoadedImports
213 //
214 #define MM_SYSLDR_NO_IMPORTS ((PVOID)(ULONG_PTR)-2)
215 #define MM_SYSLDR_BOOT_LOADED ((PVOID)(ULONG_PTR)-1)
216 #define MM_SYSLDR_SINGLE_ENTRY 0x1
217
218 //
219 // Number of initial session IDs
220 //
221 #define MI_INITIAL_SESSION_IDS 64
222
223 #if defined(_M_IX86) || defined(_M_ARM)
224 //
225 // PFN List Sentinel
226 //
227 #define LIST_HEAD 0xFFFFFFFF
228
229 //
230 // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits,
231 // we need a manual definition suited to the number of bits in the PteFrame.
232 // This is used as a LIST_HEAD for the colored list
233 //
234 #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF
235 #elif defined(_M_AMD64)
236 #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL
237 #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL
238 #else
239 #error Define these please!
240 #endif
241
242 //
243 // Returns the color of a page
244 //
245 #define MI_GET_PAGE_COLOR(x) ((x) & MmSecondaryColorMask)
246 #define MI_GET_NEXT_COLOR() (MI_GET_PAGE_COLOR(++MmSystemPageColor))
247 #define MI_GET_NEXT_PROCESS_COLOR(x) (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
248
249 //
250 // Prototype PTEs that don't yet have a pagefile association
251 //
252 #ifdef _WIN64
253 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL
254 #else
255 #define MI_PTE_LOOKUP_NEEDED 0xFFFFF
256 #endif
257
258 //
259 // Number of session data and tag pages
260 //
261 #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
262 #define MI_SESSION_TAG_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
263
264 //
265 // Used by MiCheckSecuredVad
266 //
267 #define MM_READ_WRITE_ALLOWED 11
268 #define MM_READ_ONLY_ALLOWED 10
269 #define MM_NO_ACCESS_ALLOWED 01
270 #define MM_DELETE_CHECK 85
271
272 //
273 // System views are binned into 64K chunks
274 //
275 #define MI_SYSTEM_VIEW_BUCKET_SIZE _64K
276
277 //
278 // FIXFIX: These should go in ex.h after the pool merge
279 //
280 #ifdef _WIN64
281 #define POOL_BLOCK_SIZE 16
282 #else
283 #define POOL_BLOCK_SIZE 8
284 #endif
285 #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)
286 #define BASE_POOL_TYPE_MASK 1
287 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))
288
289 //
290 // Pool debugging/analysis/tracing flags
291 //
292 #define POOL_FLAG_CHECK_TIMERS 0x1
293 #define POOL_FLAG_CHECK_WORKERS 0x2
294 #define POOL_FLAG_CHECK_RESOURCES 0x4
295 #define POOL_FLAG_VERIFIER 0x8
296 #define POOL_FLAG_CHECK_DEADLOCK 0x10
297 #define POOL_FLAG_SPECIAL_POOL 0x20
298 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40
299 #define POOL_FLAG_CRASH_ON_FAILURE 0x80
300
301 //
302 // BAD_POOL_HEADER codes during pool bugcheck
303 //
304 #define POOL_CORRUPTED_LIST 3
305 #define POOL_SIZE_OR_INDEX_MISMATCH 5
306 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6
307 #define POOL_HEADER_NOT_ALIGNED 7
308 #define POOL_HEADER_IS_ZERO 8
309 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9
310 #define POOL_ENTRY_NOT_FOUND 10
311
312 //
313 // BAD_POOL_CALLER codes during pool bugcheck
314 //
315 #define POOL_ENTRY_CORRUPTED 1
316 #define POOL_ENTRY_ALREADY_FREE 6
317 #define POOL_ENTRY_NOT_ALLOCATED 7
318 #define POOL_ALLOC_IRQL_INVALID 8
319 #define POOL_FREE_IRQL_INVALID 9
320 #define POOL_BILLED_PROCESS_INVALID 13
321 #define POOL_HEADER_SIZE_INVALID 32
322
323 typedef struct _POOL_DESCRIPTOR
324 {
325 POOL_TYPE PoolType;
326 ULONG PoolIndex;
327 ULONG RunningAllocs;
328 ULONG RunningDeAllocs;
329 ULONG TotalPages;
330 ULONG TotalBigPages;
331 ULONG Threshold;
332 PVOID LockAddress;
333 PVOID PendingFrees;
334 LONG PendingFreeDepth;
335 SIZE_T TotalBytes;
336 SIZE_T Spare0;
337 LIST_ENTRY ListHeads[POOL_LISTS_PER_PAGE];
338 } POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
339
340 typedef struct _POOL_HEADER
341 {
342 union
343 {
344 struct
345 {
346 #ifdef _WIN64
347 USHORT PreviousSize:8;
348 USHORT PoolIndex:8;
349 USHORT BlockSize:8;
350 USHORT PoolType:8;
351 #else
352 USHORT PreviousSize:9;
353 USHORT PoolIndex:7;
354 USHORT BlockSize:9;
355 USHORT PoolType:7;
356 #endif
357 };
358 ULONG Ulong1;
359 };
360 #ifdef _WIN64
361 ULONG PoolTag;
362 #endif
363 union
364 {
365 #ifdef _WIN64
366 PEPROCESS ProcessBilled;
367 #else
368 ULONG PoolTag;
369 #endif
370 struct
371 {
372 USHORT AllocatorBackTraceIndex;
373 USHORT PoolTagHash;
374 };
375 };
376 } POOL_HEADER, *PPOOL_HEADER;
377
378 C_ASSERT(sizeof(POOL_HEADER) == POOL_BLOCK_SIZE);
379 C_ASSERT(POOL_BLOCK_SIZE == sizeof(LIST_ENTRY));
380
381 typedef struct _POOL_TRACKER_TABLE
382 {
383 ULONG Key;
384 LONG NonPagedAllocs;
385 LONG NonPagedFrees;
386 SIZE_T NonPagedBytes;
387 LONG PagedAllocs;
388 LONG PagedFrees;
389 SIZE_T PagedBytes;
390 } POOL_TRACKER_TABLE, *PPOOL_TRACKER_TABLE;
391
392 typedef struct _POOL_TRACKER_BIG_PAGES
393 {
394 PVOID Va;
395 ULONG Key;
396 ULONG NumberOfPages;
397 PVOID QuotaObject;
398 } POOL_TRACKER_BIG_PAGES, *PPOOL_TRACKER_BIG_PAGES;
399
400 extern ULONG ExpNumberOfPagedPools;
401 extern POOL_DESCRIPTOR NonPagedPoolDescriptor;
402 extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16 + 1];
403 extern PPOOL_TRACKER_TABLE PoolTrackTable;
404
405 //
406 // END FIXFIX
407 //
408
409 typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY
410 {
411 LIST_ENTRY Links;
412 UNICODE_STRING BaseName;
413 } MI_LARGE_PAGE_DRIVER_ENTRY, *PMI_LARGE_PAGE_DRIVER_ENTRY;
414
415 typedef enum _MMSYSTEM_PTE_POOL_TYPE
416 {
417 SystemPteSpace,
418 NonPagedPoolExpansion,
419 MaximumPtePoolTypes
420 } MMSYSTEM_PTE_POOL_TYPE;
421
422 typedef enum _MI_PFN_CACHE_ATTRIBUTE
423 {
424 MiNonCached,
425 MiCached,
426 MiWriteCombined,
427 MiNotMapped
428 } MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;
429
430 typedef struct _PHYSICAL_MEMORY_RUN
431 {
432 PFN_NUMBER BasePage;
433 PFN_NUMBER PageCount;
434 } PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN;
435
436 typedef struct _PHYSICAL_MEMORY_DESCRIPTOR
437 {
438 ULONG NumberOfRuns;
439 PFN_NUMBER NumberOfPages;
440 PHYSICAL_MEMORY_RUN Run[1];
441 } PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR;
442
443 typedef struct _MMCOLOR_TABLES
444 {
445 PFN_NUMBER Flink;
446 PVOID Blink;
447 PFN_NUMBER Count;
448 } MMCOLOR_TABLES, *PMMCOLOR_TABLES;
449
450 typedef struct _MI_LARGE_PAGE_RANGES
451 {
452 PFN_NUMBER StartFrame;
453 PFN_NUMBER LastFrame;
454 } MI_LARGE_PAGE_RANGES, *PMI_LARGE_PAGE_RANGES;
455
456 typedef struct _MMVIEW
457 {
458 ULONG_PTR Entry;
459 PCONTROL_AREA ControlArea;
460 } MMVIEW, *PMMVIEW;
461
462 typedef struct _MMSESSION
463 {
464 KGUARDED_MUTEX SystemSpaceViewLock;
465 PKGUARDED_MUTEX SystemSpaceViewLockPointer;
466 PCHAR SystemSpaceViewStart;
467 PMMVIEW SystemSpaceViewTable;
468 ULONG SystemSpaceHashSize;
469 ULONG SystemSpaceHashEntries;
470 ULONG SystemSpaceHashKey;
471 ULONG BitmapFailures;
472 PRTL_BITMAP SystemSpaceBitMap;
473 } MMSESSION, *PMMSESSION;
474
475 typedef struct _MM_SESSION_SPACE_FLAGS
476 {
477 ULONG Initialized:1;
478 ULONG DeletePending:1;
479 ULONG Filler:30;
480 } MM_SESSION_SPACE_FLAGS;
481
482 typedef struct _MM_SESSION_SPACE
483 {
484 struct _MM_SESSION_SPACE *GlobalVirtualAddress;
485 LONG ReferenceCount;
486 union
487 {
488 ULONG LongFlags;
489 MM_SESSION_SPACE_FLAGS Flags;
490 } u;
491 ULONG SessionId;
492 LIST_ENTRY ProcessList;
493 LARGE_INTEGER LastProcessSwappedOutTime;
494 PFN_NUMBER SessionPageDirectoryIndex;
495 SIZE_T NonPageablePages;
496 SIZE_T CommittedPages;
497 PVOID PagedPoolStart;
498 PVOID PagedPoolEnd;
499 PMMPDE PagedPoolBasePde;
500 ULONG Color;
501 LONG ResidentProcessCount;
502 ULONG SessionPoolAllocationFailures[4];
503 LIST_ENTRY ImageList;
504 LCID LocaleId;
505 ULONG AttachCount;
506 KEVENT AttachEvent;
507 PEPROCESS LastProcess;
508 LONG ProcessReferenceToSession;
509 LIST_ENTRY WsListEntry;
510 GENERAL_LOOKASIDE Lookaside[SESSION_POOL_LOOKASIDES];
511 MMSESSION Session;
512 KGUARDED_MUTEX PagedPoolMutex;
513 MM_PAGED_POOL_INFO PagedPoolInfo;
514 MMSUPPORT Vm;
515 PMMWSLE Wsle;
516 PDRIVER_UNLOAD Win32KDriverUnload;
517 POOL_DESCRIPTOR PagedPool;
518 #if defined (_M_AMD64)
519 MMPDE PageDirectory;
520 #else
521 PMMPDE PageTables;
522 #endif
523 #if defined (_M_AMD64)
524 PMMPTE SpecialPoolFirstPte;
525 PMMPTE SpecialPoolLastPte;
526 PMMPTE NextPdeForSpecialPoolExpansion;
527 PMMPTE LastPdeForSpecialPoolExpansion;
528 PFN_NUMBER SpecialPagesInUse;
529 #endif
530 LONG ImageLoadingCount;
531 } MM_SESSION_SPACE, *PMM_SESSION_SPACE;
532
533 extern PMM_SESSION_SPACE MmSessionSpace;
534 extern MMPTE HyperTemplatePte;
535 extern MMPDE ValidKernelPde;
536 extern MMPTE ValidKernelPte;
537 extern MMPDE ValidKernelPdeLocal;
538 extern MMPTE ValidKernelPteLocal;
539 extern MMPDE DemandZeroPde;
540 extern MMPTE DemandZeroPte;
541 extern MMPTE PrototypePte;
542 extern MMPTE MmDecommittedPte;
543 extern BOOLEAN MmLargeSystemCache;
544 extern BOOLEAN MmZeroPageFile;
545 extern BOOLEAN MmProtectFreedNonPagedPool;
546 extern BOOLEAN MmTrackLockedPages;
547 extern BOOLEAN MmTrackPtes;
548 extern BOOLEAN MmDynamicPfn;
549 extern BOOLEAN MmMirroring;
550 extern BOOLEAN MmMakeLowMemory;
551 extern BOOLEAN MmEnforceWriteProtection;
552 extern SIZE_T MmAllocationFragment;
553 extern ULONG MmConsumedPoolPercentage;
554 extern ULONG MmVerifyDriverBufferType;
555 extern ULONG MmVerifyDriverLevel;
556 extern WCHAR MmVerifyDriverBuffer[512];
557 extern WCHAR MmLargePageDriverBuffer[512];
558 extern LIST_ENTRY MiLargePageDriverList;
559 extern BOOLEAN MiLargePageAllDrivers;
560 extern ULONG MmVerifyDriverBufferLength;
561 extern ULONG MmLargePageDriverBufferLength;
562 extern SIZE_T MmSizeOfNonPagedPoolInBytes;
563 extern SIZE_T MmMaximumNonPagedPoolInBytes;
564 extern PFN_NUMBER MmMaximumNonPagedPoolInPages;
565 extern PFN_NUMBER MmSizeOfPagedPoolInPages;
566 extern PVOID MmNonPagedSystemStart;
567 extern PVOID MmNonPagedPoolStart;
568 extern PVOID MmNonPagedPoolExpansionStart;
569 extern PVOID MmNonPagedPoolEnd;
570 extern SIZE_T MmSizeOfPagedPoolInBytes;
571 extern PVOID MmPagedPoolStart;
572 extern PVOID MmPagedPoolEnd;
573 extern PVOID MmSessionBase;
574 extern SIZE_T MmSessionSize;
575 extern PMMPTE MmFirstReservedMappingPte, MmLastReservedMappingPte;
576 extern PMMPTE MiFirstReservedZeroingPte;
577 extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2][MmMaximumCacheType];
578 extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock;
579 extern SIZE_T MmBootImageSize;
580 extern PMMPTE MmSystemPtesStart[MaximumPtePoolTypes];
581 extern PMMPTE MmSystemPtesEnd[MaximumPtePoolTypes];
582 extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor;
583 extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor;
584 extern ULONG_PTR MxPfnAllocation;
585 extern MM_PAGED_POOL_INFO MmPagedPoolInfo;
586 extern KGUARDED_MUTEX MmPagedPoolMutex;
587 extern KGUARDED_MUTEX MmSectionCommitMutex;
588 extern PVOID MmPagedPoolStart;
589 extern PVOID MmPagedPoolEnd;
590 extern PVOID MmNonPagedSystemStart;
591 extern PVOID MiSystemViewStart;
592 extern SIZE_T MmSystemViewSize;
593 extern PVOID MmSessionBase;
594 extern PVOID MiSessionSpaceEnd;
595 extern PMMPTE MiSessionImagePteStart;
596 extern PMMPTE MiSessionImagePteEnd;
597 extern PMMPTE MiSessionBasePte;
598 extern PMMPTE MiSessionLastPte;
599 extern SIZE_T MmSizeOfPagedPoolInBytes;
600 extern PMMPDE MmSystemPagePtes;
601 extern PVOID MmSystemCacheStart;
602 extern PVOID MmSystemCacheEnd;
603 extern MMSUPPORT MmSystemCacheWs;
604 extern SIZE_T MmAllocatedNonPagedPool;
605 extern ULONG MmSpecialPoolTag;
606 extern PVOID MmHyperSpaceEnd;
607 extern PMMWSL MmSystemCacheWorkingSetList;
608 extern SIZE_T MmMinimumNonPagedPoolSize;
609 extern ULONG MmMinAdditionNonPagedPoolPerMb;
610 extern SIZE_T MmDefaultMaximumNonPagedPool;
611 extern ULONG MmMaxAdditionNonPagedPoolPerMb;
612 extern ULONG MmSecondaryColors;
613 extern ULONG MmSecondaryColorMask;
614 extern ULONG MmNumberOfSystemPtes;
615 extern ULONG MmMaximumNonPagedPoolPercent;
616 extern ULONG MmLargeStackSize;
617 extern PMMCOLOR_TABLES MmFreePagesByColor[FreePageList + 1];
618 extern MMPFNLIST MmStandbyPageListByPriority[8];
619 extern ULONG MmProductType;
620 extern MM_SYSTEMSIZE MmSystemSize;
621 extern PKEVENT MiLowMemoryEvent;
622 extern PKEVENT MiHighMemoryEvent;
623 extern PKEVENT MiLowPagedPoolEvent;
624 extern PKEVENT MiHighPagedPoolEvent;
625 extern PKEVENT MiLowNonPagedPoolEvent;
626 extern PKEVENT MiHighNonPagedPoolEvent;
627 extern PFN_NUMBER MmLowMemoryThreshold;
628 extern PFN_NUMBER MmHighMemoryThreshold;
629 extern PFN_NUMBER MiLowPagedPoolThreshold;
630 extern PFN_NUMBER MiHighPagedPoolThreshold;
631 extern PFN_NUMBER MiLowNonPagedPoolThreshold;
632 extern PFN_NUMBER MiHighNonPagedPoolThreshold;
633 extern PFN_NUMBER MmMinimumFreePages;
634 extern PFN_NUMBER MmPlentyFreePages;
635 extern SIZE_T MmMinimumStackCommitInBytes;
636 extern PFN_COUNT MiExpansionPoolPagesInitialCharge;
637 extern PFN_NUMBER MmResidentAvailableAtInit;
638 extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
639 extern PFN_NUMBER MmTotalSystemDriverPages;
640 extern ULONG MmCritsectTimeoutSeconds;
641 extern PVOID MiSessionImageStart;
642 extern PVOID MiSessionImageEnd;
643 extern PMMPTE MiHighestUserPte;
644 extern PMMPDE MiHighestUserPde;
645 extern PFN_NUMBER MmSystemPageDirectory[PPE_PER_PAGE];
646 extern PMMPTE MmSharedUserDataPte;
647 extern LIST_ENTRY MmProcessList;
648 extern KEVENT MmZeroingPageEvent;
649 extern ULONG MmSystemPageColor;
650 extern ULONG MmProcessColorSeed;
651 extern PMMWSL MmWorkingSetList;
652 extern PFN_NUMBER MiNumberOfFreePages;
653 extern SIZE_T MmSessionViewSize;
654 extern SIZE_T MmSessionPoolSize;
655 extern SIZE_T MmSessionImageSize;
656 extern PVOID MiSystemViewStart;
657 extern PVOID MiSessionPoolEnd; // 0xBE000000
658 extern PVOID MiSessionPoolStart; // 0xBD000000
659 extern PVOID MiSessionViewStart; // 0xBE000000
660 extern PVOID MiSessionSpaceWs;
661 extern ULONG MmMaximumDeadKernelStacks;
662 extern SLIST_HEADER MmDeadStackSListHead;
663 extern MM_AVL_TABLE MmSectionBasedRoot;
664 extern KGUARDED_MUTEX MmSectionBasedMutex;
665 extern PVOID MmHighSectionBase;
666 extern SIZE_T MmSystemLockPagesCount;
667 extern ULONG_PTR MmSubsectionBase;
668 extern LARGE_INTEGER MmCriticalSectionTimeout;
669 extern LIST_ENTRY MmWorkingSetExpansionHead;
670 extern KSPIN_LOCK MmExpansionLock;
671 extern PETHREAD MiExpansionLockOwner;
672
673 FORCEINLINE
674 BOOLEAN
MI_IS_PROCESS_WORKING_SET(PMMSUPPORT WorkingSet)675 MI_IS_PROCESS_WORKING_SET(PMMSUPPORT WorkingSet)
676 {
677 return (WorkingSet != &MmSystemCacheWs) && !WorkingSet->Flags.SessionSpace;
678 }
679
680 FORCEINLINE
681 BOOLEAN
MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType)682 MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType)
683 {
684 return ((MemoryType == LoaderFree) ||
685 (MemoryType == LoaderLoadedProgram) ||
686 (MemoryType == LoaderFirmwareTemporary) ||
687 (MemoryType == LoaderOsloaderStack));
688 }
689
690 FORCEINLINE
691 BOOLEAN
MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType)692 MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType)
693 {
694 return ((MemoryType == LoaderFirmwarePermanent) ||
695 (MemoryType == LoaderSpecialMemory) ||
696 (MemoryType == LoaderHALCachedMemory) ||
697 (MemoryType == LoaderBBTMemory));
698 }
699
700 #ifdef _M_AMD64
701 FORCEINLINE
702 BOOLEAN
MiIsUserPxe(PVOID Address)703 MiIsUserPxe(PVOID Address)
704 {
705 return ((ULONG_PTR)Address >> 7) == 0x1FFFFEDF6FB7DA0ULL;
706 }
707
708 FORCEINLINE
709 BOOLEAN
MiIsUserPpe(PVOID Address)710 MiIsUserPpe(PVOID Address)
711 {
712 return ((ULONG_PTR)Address >> 16) == 0xFFFFF6FB7DA0ULL;
713 }
714
715 FORCEINLINE
716 BOOLEAN
MiIsUserPde(PVOID Address)717 MiIsUserPde(PVOID Address)
718 {
719 return ((ULONG_PTR)Address >> 25) == 0x7FFFFB7DA0ULL;
720 }
721
722 FORCEINLINE
723 BOOLEAN
MiIsUserPte(PVOID Address)724 MiIsUserPte(PVOID Address)
725 {
726 return ((ULONG_PTR)Address >> 34) == 0x3FFFFDA0ULL;
727 }
728 #else
729 FORCEINLINE
730 BOOLEAN
MiIsUserPde(PVOID Address)731 MiIsUserPde(PVOID Address)
732 {
733 return ((Address >= (PVOID)MiAddressToPde(NULL)) &&
734 (Address <= (PVOID)MiHighestUserPde));
735 }
736
737 FORCEINLINE
738 BOOLEAN
MiIsUserPte(PVOID Address)739 MiIsUserPte(PVOID Address)
740 {
741 return (Address >= (PVOID)PTE_BASE) && (Address <= (PVOID)MiHighestUserPte);
742 }
743 #endif
744
745 //
746 // Figures out the hardware bits for a PTE
747 //
748 FORCEINLINE
749 ULONG_PTR
MiDetermineUserGlobalPteMask(IN PVOID PointerPte)750 MiDetermineUserGlobalPteMask(IN PVOID PointerPte)
751 {
752 MMPTE TempPte;
753
754 /* Start fresh */
755 TempPte.u.Long = 0;
756
757 /* Make it valid and accessed */
758 TempPte.u.Hard.Valid = TRUE;
759 MI_MAKE_ACCESSED_PAGE(&TempPte);
760
761 /* Is this for user-mode? */
762 if (
763 #if (_MI_PAGING_LEVELS == 4)
764 MiIsUserPxe(PointerPte) ||
765 #endif
766 #if (_MI_PAGING_LEVELS >= 3)
767 MiIsUserPpe(PointerPte) ||
768 #endif
769 MiIsUserPde(PointerPte) ||
770 MiIsUserPte(PointerPte))
771 {
772 /* Set the owner bit */
773 MI_MAKE_OWNER_PAGE(&TempPte);
774 }
775
776 /* FIXME: We should also set the global bit */
777
778 /* Return the protection */
779 return TempPte.u.Long;
780 }
781
782 //
783 // Creates a valid kernel PTE with the given protection
784 //
785 FORCEINLINE
786 VOID
MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte,IN PMMPTE MappingPte,IN ULONG_PTR ProtectionMask,IN PFN_NUMBER PageFrameNumber)787 MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte,
788 IN PMMPTE MappingPte,
789 IN ULONG_PTR ProtectionMask,
790 IN PFN_NUMBER PageFrameNumber)
791 {
792 /* Only valid for kernel, non-session PTEs */
793 ASSERT(MappingPte > MiHighestUserPte);
794 ASSERT(!MI_IS_SESSION_PTE(MappingPte));
795 ASSERT((MappingPte < (PMMPTE)PDE_BASE) || (MappingPte > (PMMPTE)PDE_TOP));
796
797 /* Check that we are not setting valid a page that should not be */
798 ASSERT(ProtectionMask & MM_PROTECT_ACCESS);
799 ASSERT((ProtectionMask & MM_GUARDPAGE) == 0);
800
801 /* Start fresh */
802 NewPte->u.Long = 0;
803
804 /* Set the protection and page */
805 NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
806 NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
807
808 /* Make this valid & global */
809 #ifdef _GLOBAL_PAGES_ARE_AWESOME_
810 if (KeFeatureBits & KF_GLOBAL_PAGE)
811 NewPte->u.Hard.Global = 1;
812 #endif
813 NewPte->u.Hard.Valid = 1;
814 }
815
816 //
817 // Creates a valid PTE with the given protection
818 //
819 FORCEINLINE
820 VOID
MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte,IN PMMPTE MappingPte,IN ULONG_PTR ProtectionMask,IN PFN_NUMBER PageFrameNumber)821 MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte,
822 IN PMMPTE MappingPte,
823 IN ULONG_PTR ProtectionMask,
824 IN PFN_NUMBER PageFrameNumber)
825 {
826 /* Check that we are not setting valid a page that should not be */
827 ASSERT(ProtectionMask & MM_PROTECT_ACCESS);
828 ASSERT((ProtectionMask & MM_GUARDPAGE) == 0);
829
830 /* Set the protection and page */
831 NewPte->u.Long = MiDetermineUserGlobalPteMask(MappingPte);
832 NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
833 NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
834 }
835
836 //
837 // Creates a valid user PTE with the given protection
838 //
839 FORCEINLINE
840 VOID
MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte,IN PMMPTE MappingPte,IN ULONG_PTR ProtectionMask,IN PFN_NUMBER PageFrameNumber)841 MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte,
842 IN PMMPTE MappingPte,
843 IN ULONG_PTR ProtectionMask,
844 IN PFN_NUMBER PageFrameNumber)
845 {
846 /* Only valid for kernel, non-session PTEs */
847 ASSERT(MappingPte <= MiHighestUserPte);
848
849 /* Start fresh */
850 NewPte->u.Long = 0;
851
852 /* Check that we are not setting valid a page that should not be */
853 ASSERT(ProtectionMask & MM_PROTECT_ACCESS);
854 ASSERT((ProtectionMask & MM_GUARDPAGE) == 0);
855
856 NewPte->u.Hard.Valid = TRUE;
857 NewPte->u.Hard.Owner = TRUE;
858 NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
859 NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
860 }
861
862 #ifndef _M_AMD64
863 //
864 // Builds a Prototype PTE for the address of the PTE
865 //
866 FORCEINLINE
867 VOID
MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte,IN PMMPTE PointerPte)868 MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte,
869 IN PMMPTE PointerPte)
870 {
871 ULONG_PTR Offset;
872
873 /* Mark this as a prototype */
874 NewPte->u.Long = 0;
875 NewPte->u.Proto.Prototype = 1;
876
877 /*
878 * Prototype PTEs are only valid in paged pool by design, this little trick
879 * lets us only use 30 bits for the address of the PTE, as long as the area
880 * stays 1024MB At most.
881 */
882 Offset = (ULONG_PTR)PointerPte - (ULONG_PTR)MmPagedPoolStart;
883
884 /*
885 * 7 bits go in the "low" (but we assume the bottom 2 are zero)
886 * and the other 21 bits go in the "high"
887 */
888 NewPte->u.Proto.ProtoAddressLow = (Offset & 0x1FC) >> 2;
889 NewPte->u.Proto.ProtoAddressHigh = (Offset & 0x3FFFFE00) >> 9;
890 }
891
892 //
893 // Builds a Subsection PTE for the address of the Segment
894 //
895 FORCEINLINE
896 VOID
MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte,IN PVOID Segment)897 MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte,
898 IN PVOID Segment)
899 {
900 ULONG_PTR Offset;
901
902 /* Mark this as a prototype */
903 NewPte->u.Long = 0;
904 NewPte->u.Subsect.Prototype = 1;
905
906 /*
907 * Segments are only valid either in nonpaged pool. We store the 20 bit
908 * difference either from the top or bottom of nonpaged pool, giving a
909 * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed
910 * 256MB.
911 */
912 if ((ULONG_PTR)Segment < ((ULONG_PTR)MmSubsectionBase + (128 * _1MB)))
913 {
914 Offset = (ULONG_PTR)Segment - (ULONG_PTR)MmSubsectionBase;
915 NewPte->u.Subsect.WhichPool = PagedPool;
916 }
917 else
918 {
919 Offset = (ULONG_PTR)MmNonPagedPoolEnd - (ULONG_PTR)Segment;
920 NewPte->u.Subsect.WhichPool = NonPagedPool;
921 }
922
923 /*
924 * 4 bits go in the "low" (but we assume the bottom 3 are zero)
925 * and the other 20 bits go in the "high"
926 */
927 NewPte->u.Subsect.SubsectionAddressLow = (Offset & 0x78) >> 3;
928 NewPte->u.Subsect.SubsectionAddressHigh = (Offset & 0xFFFFF80) >> 7;
929 }
930
931 FORCEINLINE
932 BOOLEAN
MI_IS_MAPPED_PTE(PMMPTE PointerPte)933 MI_IS_MAPPED_PTE(PMMPTE PointerPte)
934 {
935 /// \todo Make this reasonable code, this is UGLY!
936 return ((PointerPte->u.Long & 0xFFFFFC01) != 0);
937 }
938
939 #endif
940
941 FORCEINLINE
942 VOID
MI_MAKE_TRANSITION_PTE(_Out_ PMMPTE NewPte,_In_ PFN_NUMBER Page,_In_ ULONG Protection)943 MI_MAKE_TRANSITION_PTE(_Out_ PMMPTE NewPte,
944 _In_ PFN_NUMBER Page,
945 _In_ ULONG Protection)
946 {
947 NewPte->u.Long = 0;
948 NewPte->u.Trans.Transition = 1;
949 NewPte->u.Trans.Protection = Protection;
950 NewPte->u.Trans.PageFrameNumber = Page;
951 }
952
953 //
954 // Returns if the page is physically resident (ie: a large page)
955 // FIXFIX: CISC/x86 only?
956 //
957 FORCEINLINE
958 BOOLEAN
MI_IS_PHYSICAL_ADDRESS(IN PVOID Address)959 MI_IS_PHYSICAL_ADDRESS(IN PVOID Address)
960 {
961 PMMPDE PointerPde;
962
963 /* Large pages are never paged out, always physically resident */
964 PointerPde = MiAddressToPde(Address);
965 return ((PointerPde->u.Hard.LargePage) && (PointerPde->u.Hard.Valid));
966 }
967
968 //
969 // Writes a valid PTE
970 //
971 FORCEINLINE
972 VOID
MI_WRITE_VALID_PTE(IN PMMPTE PointerPte,IN MMPTE TempPte)973 MI_WRITE_VALID_PTE(IN PMMPTE PointerPte,
974 IN MMPTE TempPte)
975 {
976 /* Write the valid PTE */
977 ASSERT(PointerPte->u.Hard.Valid == 0);
978 ASSERT(TempPte.u.Hard.Valid == 1);
979 #if _M_AMD64
980 ASSERT(!MI_IS_PAGE_TABLE_ADDRESS(MiPteToAddress(PointerPte)) ||
981 (TempPte.u.Hard.NoExecute == 0));
982 #endif
983 *PointerPte = TempPte;
984 }
985
986 //
987 // Updates a valid PTE
988 //
989 FORCEINLINE
990 VOID
MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte,IN MMPTE TempPte)991 MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte,
992 IN MMPTE TempPte)
993 {
994 /* Write the valid PTE */
995 ASSERT(PointerPte->u.Hard.Valid == 1);
996 ASSERT(TempPte.u.Hard.Valid == 1);
997 ASSERT(PointerPte->u.Hard.PageFrameNumber == TempPte.u.Hard.PageFrameNumber);
998 *PointerPte = TempPte;
999 }
1000
1001 //
1002 // Writes an invalid PTE
1003 //
1004 FORCEINLINE
1005 VOID
MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte,IN MMPTE InvalidPte)1006 MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte,
1007 IN MMPTE InvalidPte)
1008 {
1009 /* Write the invalid PTE */
1010 ASSERT(InvalidPte.u.Hard.Valid == 0);
1011 *PointerPte = InvalidPte;
1012 }
1013
1014 //
1015 // Erase the PTE completely
1016 //
1017 FORCEINLINE
1018 VOID
MI_ERASE_PTE(IN PMMPTE PointerPte)1019 MI_ERASE_PTE(IN PMMPTE PointerPte)
1020 {
1021 /* Zero out the PTE */
1022 ASSERT(PointerPte->u.Long != 0);
1023 PointerPte->u.Long = 0;
1024 }
1025
1026 //
1027 // Writes a valid PDE
1028 //
1029 FORCEINLINE
1030 VOID
MI_WRITE_VALID_PDE(IN PMMPDE PointerPde,IN MMPDE TempPde)1031 MI_WRITE_VALID_PDE(IN PMMPDE PointerPde,
1032 IN MMPDE TempPde)
1033 {
1034 /* Write the valid PDE */
1035 ASSERT(PointerPde->u.Hard.Valid == 0);
1036 #ifdef _M_AMD64
1037 ASSERT(PointerPde->u.Hard.NoExecute == 0);
1038 #endif
1039 ASSERT(TempPde.u.Hard.Valid == 1);
1040 *PointerPde = TempPde;
1041 }
1042
1043 //
1044 // Writes an invalid PDE
1045 //
1046 FORCEINLINE
1047 VOID
MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde,IN MMPDE InvalidPde)1048 MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde,
1049 IN MMPDE InvalidPde)
1050 {
1051 /* Write the invalid PDE */
1052 ASSERT(InvalidPde.u.Hard.Valid == 0);
1053 ASSERT(InvalidPde.u.Long != 0);
1054 #ifdef _M_AMD64
1055 ASSERT(InvalidPde.u.Soft.Protection == MM_EXECUTE_READWRITE);
1056 #endif
1057 *PointerPde = InvalidPde;
1058 }
1059
1060 //
1061 // Checks if the thread already owns a working set
1062 //
1063 FORCEINLINE
1064 BOOLEAN
MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread)1065 MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread)
1066 {
1067 /* If any of these are held, return TRUE */
1068 return ((Thread->OwnsProcessWorkingSetExclusive) ||
1069 (Thread->OwnsProcessWorkingSetShared) ||
1070 (Thread->OwnsSystemWorkingSetExclusive) ||
1071 (Thread->OwnsSystemWorkingSetShared) ||
1072 (Thread->OwnsSessionWorkingSetExclusive) ||
1073 (Thread->OwnsSessionWorkingSetShared));
1074 }
1075
1076 FORCEINLINE
1077 BOOLEAN
MM_ANY_WS_LOCK_HELD_EXCLUSIVE(_In_ PETHREAD Thread)1078 MM_ANY_WS_LOCK_HELD_EXCLUSIVE(_In_ PETHREAD Thread)
1079 {
1080 return ((Thread->OwnsProcessWorkingSetExclusive) ||
1081 (Thread->OwnsSystemWorkingSetExclusive) ||
1082 (Thread->OwnsSessionWorkingSetExclusive));
1083 }
1084
1085 //
1086 // Checks if the process owns the working set lock
1087 //
1088 FORCEINLINE
1089 BOOLEAN
MI_WS_OWNER(IN PEPROCESS Process)1090 MI_WS_OWNER(IN PEPROCESS Process)
1091 {
1092 /* Check if this process is the owner, and that the thread owns the WS */
1093 if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive == 0)
1094 {
1095 DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread());
1096 }
1097 if (KeGetCurrentThread()->ApcState.Process != &Process->Pcb)
1098 {
1099 DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process);
1100 }
1101 return ((KeGetCurrentThread()->ApcState.Process == &Process->Pcb) &&
1102 ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive) ||
1103 (PsGetCurrentThread()->OwnsProcessWorkingSetShared)));
1104 }
1105
1106 //
1107 // New ARM3<->RosMM PAGE Architecture
1108 //
1109 FORCEINLINE
1110 BOOLEAN
MiIsRosSectionObject(IN PSECTION Section)1111 MiIsRosSectionObject(IN PSECTION Section)
1112 {
1113 return Section->u.Flags.filler;
1114 }
1115
1116 #define MI_IS_ROS_PFN(x) ((x)->u4.AweAllocation == TRUE)
1117
1118 VOID
1119 NTAPI
1120 MiDecrementReferenceCount(
1121 IN PMMPFN Pfn1,
1122 IN PFN_NUMBER PageFrameIndex
1123 );
1124
1125 FORCEINLINE
1126 BOOLEAN
MI_IS_WS_UNSAFE(IN PEPROCESS Process)1127 MI_IS_WS_UNSAFE(IN PEPROCESS Process)
1128 {
1129 return (Process->Vm.Flags.AcquiredUnsafe == TRUE);
1130 }
1131
1132 //
1133 // Locks the working set for the given process
1134 //
1135 FORCEINLINE
1136 VOID
MiLockProcessWorkingSet(IN PEPROCESS Process,IN PETHREAD Thread)1137 MiLockProcessWorkingSet(IN PEPROCESS Process,
1138 IN PETHREAD Thread)
1139 {
1140 /* Shouldn't already be owning the process working set */
1141 ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1142 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1143
1144 /* Block APCs, make sure that still nothing is already held */
1145 KeEnterGuardedRegion();
1146 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1147
1148 /* Lock the working set */
1149 ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1150
1151 /* Now claim that we own the lock */
1152 ASSERT(!MI_IS_WS_UNSAFE(Process));
1153 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1154 Thread->OwnsProcessWorkingSetExclusive = TRUE;
1155 }
1156
1157 FORCEINLINE
1158 VOID
MiLockProcessWorkingSetShared(IN PEPROCESS Process,IN PETHREAD Thread)1159 MiLockProcessWorkingSetShared(IN PEPROCESS Process,
1160 IN PETHREAD Thread)
1161 {
1162 /* Shouldn't already be owning the process working set */
1163 ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1164 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1165
1166 /* Block APCs, make sure that still nothing is already held */
1167 KeEnterGuardedRegion();
1168 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1169
1170 /* Lock the working set */
1171 ExAcquirePushLockShared(&Process->Vm.WorkingSetMutex);
1172
1173 /* Now claim that we own the lock */
1174 ASSERT(!MI_IS_WS_UNSAFE(Process));
1175 ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1176 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1177 Thread->OwnsProcessWorkingSetShared = TRUE;
1178 }
1179
1180 FORCEINLINE
1181 VOID
MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process,IN PETHREAD Thread)1182 MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1183 IN PETHREAD Thread)
1184 {
1185 /* Shouldn't already be owning the process working set */
1186 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1187
1188 /* APCs must be blocked, make sure that still nothing is already held */
1189 ASSERT(KeAreAllApcsDisabled() == TRUE);
1190 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1191
1192 /* Lock the working set */
1193 ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1194
1195 /* Now claim that we own the lock */
1196 ASSERT(!MI_IS_WS_UNSAFE(Process));
1197 Process->Vm.Flags.AcquiredUnsafe = 1;
1198 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1199 Thread->OwnsProcessWorkingSetExclusive = TRUE;
1200 }
1201
1202 //
1203 // Unlocks the working set for the given process
1204 //
1205 FORCEINLINE
1206 VOID
MiUnlockProcessWorkingSet(IN PEPROCESS Process,IN PETHREAD Thread)1207 MiUnlockProcessWorkingSet(IN PEPROCESS Process,
1208 IN PETHREAD Thread)
1209 {
1210 /* Make sure we are the owner of a safe acquisition */
1211 ASSERT(MI_WS_OWNER(Process));
1212 ASSERT(!MI_IS_WS_UNSAFE(Process));
1213
1214 /* The thread doesn't own it anymore */
1215 ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1216 Thread->OwnsProcessWorkingSetExclusive = FALSE;
1217
1218 /* Release the lock and re-enable APCs */
1219 ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1220 KeLeaveGuardedRegion();
1221 }
1222
1223 //
1224 // Unlocks the working set for the given process
1225 //
1226 FORCEINLINE
1227 VOID
MiUnlockProcessWorkingSetShared(IN PEPROCESS Process,IN PETHREAD Thread)1228 MiUnlockProcessWorkingSetShared(IN PEPROCESS Process,
1229 IN PETHREAD Thread)
1230 {
1231 /* Make sure we are the owner of a safe acquisition (because shared) */
1232 ASSERT(MI_WS_OWNER(Process));
1233 ASSERT(!MI_IS_WS_UNSAFE(Process));
1234
1235 /* Ensure we are in a shared acquisition */
1236 ASSERT(Thread->OwnsProcessWorkingSetShared == TRUE);
1237 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1238
1239 /* Don't claim the lock anylonger */
1240 Thread->OwnsProcessWorkingSetShared = FALSE;
1241
1242 /* Release the lock and re-enable APCs */
1243 ExReleasePushLockShared(&Process->Vm.WorkingSetMutex);
1244 KeLeaveGuardedRegion();
1245 }
1246
1247 //
1248 // Unlocks the working set for the given process
1249 //
1250 FORCEINLINE
1251 VOID
MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process,IN PETHREAD Thread)1252 MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1253 IN PETHREAD Thread)
1254 {
1255 /* Make sure we are the owner of an unsafe acquisition */
1256 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1257 ASSERT(KeAreAllApcsDisabled() == TRUE);
1258 ASSERT(MI_WS_OWNER(Process));
1259 ASSERT(MI_IS_WS_UNSAFE(Process));
1260
1261 /* No longer unsafe */
1262 Process->Vm.Flags.AcquiredUnsafe = 0;
1263
1264 /* The thread doesn't own it anymore */
1265 ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1266 Thread->OwnsProcessWorkingSetExclusive = FALSE;
1267
1268 /* Release the lock but don't touch APC state */
1269 ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1270 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1271 }
1272
1273 //
1274 // Locks the working set
1275 //
1276 FORCEINLINE
1277 VOID
MiLockWorkingSet(IN PETHREAD Thread,IN PMMSUPPORT WorkingSet)1278 MiLockWorkingSet(IN PETHREAD Thread,
1279 IN PMMSUPPORT WorkingSet)
1280 {
1281 /* Block APCs */
1282 KeEnterGuardedRegion();
1283
1284 /* Working set should be in global memory */
1285 ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1286
1287 /* Thread shouldn't already be owning something */
1288 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1289
1290 /* Lock this working set */
1291 ExAcquirePushLockExclusive(&WorkingSet->WorkingSetMutex);
1292
1293 /* Which working set is this? */
1294 if (WorkingSet == &MmSystemCacheWs)
1295 {
1296 /* Own the system working set */
1297 ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1298 (Thread->OwnsSystemWorkingSetShared == FALSE));
1299 Thread->OwnsSystemWorkingSetExclusive = TRUE;
1300 }
1301 else if (WorkingSet->Flags.SessionSpace)
1302 {
1303 /* Own the session working set */
1304 ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1305 (Thread->OwnsSessionWorkingSetShared == FALSE));
1306 Thread->OwnsSessionWorkingSetExclusive = TRUE;
1307 }
1308 else
1309 {
1310 /* Own the process working set */
1311 ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1312 (Thread->OwnsProcessWorkingSetShared == FALSE));
1313 Thread->OwnsProcessWorkingSetExclusive = TRUE;
1314 }
1315 }
1316
1317 FORCEINLINE
1318 VOID
MiLockWorkingSetShared(_In_ PETHREAD Thread,_In_ PMMSUPPORT WorkingSet)1319 MiLockWorkingSetShared(
1320 _In_ PETHREAD Thread,
1321 _In_ PMMSUPPORT WorkingSet)
1322 {
1323 /* Block APCs */
1324 KeEnterGuardedRegion();
1325
1326 /* Working set should be in global memory */
1327 ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1328
1329 /* Thread shouldn't already be owning something */
1330 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1331
1332 /* Lock this working set */
1333 ExAcquirePushLockShared(&WorkingSet->WorkingSetMutex);
1334
1335 /* Which working set is this? */
1336 if (WorkingSet == &MmSystemCacheWs)
1337 {
1338 /* Own the system working set */
1339 ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1340 (Thread->OwnsSystemWorkingSetShared == FALSE));
1341 Thread->OwnsSystemWorkingSetShared = TRUE;
1342 }
1343 else if (WorkingSet->Flags.SessionSpace)
1344 {
1345 /* Own the session working set */
1346 ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1347 (Thread->OwnsSessionWorkingSetShared == FALSE));
1348 Thread->OwnsSessionWorkingSetShared = TRUE;
1349 }
1350 else
1351 {
1352 /* Own the process working set */
1353 ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1354 (Thread->OwnsProcessWorkingSetShared == FALSE));
1355 Thread->OwnsProcessWorkingSetShared = TRUE;
1356 }
1357 }
1358
1359 //
1360 // Unlocks the working set
1361 //
1362 FORCEINLINE
1363 VOID
MiUnlockWorkingSet(IN PETHREAD Thread,IN PMMSUPPORT WorkingSet)1364 MiUnlockWorkingSet(IN PETHREAD Thread,
1365 IN PMMSUPPORT WorkingSet)
1366 {
1367 /* Working set should be in global memory */
1368 ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1369
1370 /* Which working set is this? */
1371 if (WorkingSet == &MmSystemCacheWs)
1372 {
1373 /* Release the system working set */
1374 ASSERT((Thread->OwnsSystemWorkingSetExclusive == TRUE) &&
1375 (Thread->OwnsSystemWorkingSetShared == FALSE));
1376 Thread->OwnsSystemWorkingSetExclusive = FALSE;
1377 }
1378 else if (WorkingSet->Flags.SessionSpace)
1379 {
1380 /* Release the session working set */
1381 ASSERT((Thread->OwnsSessionWorkingSetExclusive == TRUE) &&
1382 (Thread->OwnsSessionWorkingSetShared == FALSE));
1383 Thread->OwnsSessionWorkingSetExclusive = FALSE;
1384 }
1385 else
1386 {
1387 /* Release the process working set */
1388 ASSERT((Thread->OwnsProcessWorkingSetExclusive == TRUE) &&
1389 (Thread->OwnsProcessWorkingSetShared == FALSE));
1390 Thread->OwnsProcessWorkingSetExclusive = FALSE;
1391 }
1392
1393 /* Release the working set lock */
1394 ExReleasePushLockExclusive(&WorkingSet->WorkingSetMutex);
1395
1396 /* Unblock APCs */
1397 KeLeaveGuardedRegion();
1398 }
1399
1400 FORCEINLINE
1401 VOID
MiUnlockWorkingSetShared(_In_ PETHREAD Thread,_In_ PMMSUPPORT WorkingSet)1402 MiUnlockWorkingSetShared(
1403 _In_ PETHREAD Thread,
1404 _In_ PMMSUPPORT WorkingSet)
1405 {
1406 /* Working set should be in global memory */
1407 ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1408
1409 /* Which working set is this? */
1410 if (WorkingSet == &MmSystemCacheWs)
1411 {
1412 /* Release the system working set */
1413 ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1414 (Thread->OwnsSystemWorkingSetShared == TRUE));
1415 Thread->OwnsSystemWorkingSetShared = FALSE;
1416 }
1417 else if (WorkingSet->Flags.SessionSpace)
1418 {
1419 /* Release the session working set */
1420 ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1421 (Thread->OwnsSessionWorkingSetShared == TRUE));
1422 Thread->OwnsSessionWorkingSetShared = FALSE;
1423 }
1424 else
1425 {
1426 /* Release the process working set */
1427 ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1428 (Thread->OwnsProcessWorkingSetShared == TRUE));
1429 Thread->OwnsProcessWorkingSetShared = FALSE;
1430 }
1431
1432 /* Release the working set lock */
1433 ExReleasePushLockShared(&WorkingSet->WorkingSetMutex);
1434
1435 /* Unblock APCs */
1436 KeLeaveGuardedRegion();
1437 }
1438
1439 FORCEINLINE
1440 BOOLEAN
MiConvertSharedWorkingSetLockToExclusive(_In_ PETHREAD Thread,_In_ PMMSUPPORT Vm)1441 MiConvertSharedWorkingSetLockToExclusive(
1442 _In_ PETHREAD Thread,
1443 _In_ PMMSUPPORT Vm)
1444 {
1445 /* Sanity check: No exclusive lock. */
1446 ASSERT(!Thread->OwnsProcessWorkingSetExclusive);
1447 ASSERT(!Thread->OwnsSessionWorkingSetExclusive);
1448 ASSERT(!Thread->OwnsSystemWorkingSetExclusive);
1449
1450 /* And it should have one and only one shared lock */
1451 ASSERT((Thread->OwnsProcessWorkingSetShared + Thread->OwnsSessionWorkingSetShared + Thread->OwnsSystemWorkingSetShared) == 1);
1452
1453 /* Try. */
1454 if (!ExConvertPushLockSharedToExclusive(&Vm->WorkingSetMutex))
1455 return FALSE;
1456
1457 if (Vm == &MmSystemCacheWs)
1458 {
1459 ASSERT(Thread->OwnsSystemWorkingSetShared);
1460 Thread->OwnsSystemWorkingSetShared = FALSE;
1461 Thread->OwnsSystemWorkingSetExclusive = TRUE;
1462 }
1463 else if (Vm->Flags.SessionSpace)
1464 {
1465 ASSERT(Thread->OwnsSessionWorkingSetShared);
1466 Thread->OwnsSessionWorkingSetShared = FALSE;
1467 Thread->OwnsSessionWorkingSetExclusive = TRUE;
1468 }
1469 else
1470 {
1471 ASSERT(Thread->OwnsProcessWorkingSetShared);
1472 Thread->OwnsProcessWorkingSetShared = FALSE;
1473 Thread->OwnsProcessWorkingSetExclusive = TRUE;
1474 }
1475
1476 return TRUE;
1477 }
1478
1479 FORCEINLINE
1480 VOID
MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process,IN PETHREAD Thread,OUT PBOOLEAN Safe,OUT PBOOLEAN Shared)1481 MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process,
1482 IN PETHREAD Thread,
1483 OUT PBOOLEAN Safe,
1484 OUT PBOOLEAN Shared)
1485 {
1486 ASSERT(MI_WS_OWNER(Process));
1487
1488 /* Check if the current owner is unsafe */
1489 if (MI_IS_WS_UNSAFE(Process))
1490 {
1491 /* Release unsafely */
1492 MiUnlockProcessWorkingSetUnsafe(Process, Thread);
1493 *Safe = FALSE;
1494 *Shared = FALSE;
1495 }
1496 else if (Thread->OwnsProcessWorkingSetExclusive == 1)
1497 {
1498 /* Owner is safe and exclusive, release normally */
1499 MiUnlockProcessWorkingSet(Process, Thread);
1500 *Safe = TRUE;
1501 *Shared = FALSE;
1502 }
1503 else
1504 {
1505 /* Owner is shared (implies safe), release normally */
1506 MiUnlockProcessWorkingSetShared(Process, Thread);
1507 *Safe = TRUE;
1508 *Shared = TRUE;
1509 }
1510 }
1511
1512 FORCEINLINE
1513 VOID
MiLockProcessWorkingSetForFault(IN PEPROCESS Process,IN PETHREAD Thread,IN BOOLEAN Safe,IN BOOLEAN Shared)1514 MiLockProcessWorkingSetForFault(IN PEPROCESS Process,
1515 IN PETHREAD Thread,
1516 IN BOOLEAN Safe,
1517 IN BOOLEAN Shared)
1518 {
1519 /* Check if this was a safe lock or not */
1520 if (Safe)
1521 {
1522 if (Shared)
1523 {
1524 /* Reacquire safely & shared */
1525 MiLockProcessWorkingSetShared(Process, Thread);
1526 }
1527 else
1528 {
1529 /* Reacquire safely */
1530 MiLockProcessWorkingSet(Process, Thread);
1531 }
1532 }
1533 else
1534 {
1535 /* Unsafe lock cannot be shared */
1536 ASSERT(Shared == FALSE);
1537 /* Reacquire unsafely */
1538 MiLockProcessWorkingSetUnsafe(Process, Thread);
1539 }
1540 }
1541
1542 FORCEINLINE
1543 KIRQL
MiAcquireExpansionLock(VOID)1544 MiAcquireExpansionLock(VOID)
1545 {
1546 KIRQL OldIrql;
1547
1548 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1549 KeAcquireSpinLock(&MmExpansionLock, &OldIrql);
1550 ASSERT(MiExpansionLockOwner == NULL);
1551 MiExpansionLockOwner = PsGetCurrentThread();
1552 return OldIrql;
1553 }
1554
1555 FORCEINLINE
1556 VOID
MiReleaseExpansionLock(KIRQL OldIrql)1557 MiReleaseExpansionLock(KIRQL OldIrql)
1558 {
1559 ASSERT(MiExpansionLockOwner == PsGetCurrentThread());
1560 MiExpansionLockOwner = NULL;
1561 KeReleaseSpinLock(&MmExpansionLock, OldIrql);
1562 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1563 }
1564
1565 //
1566 // Returns the ProtoPTE inside a VAD for the given VPN
1567 //
1568 FORCEINLINE
1569 PMMPTE
MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad,IN ULONG_PTR Vpn)1570 MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad,
1571 IN ULONG_PTR Vpn)
1572 {
1573 PMMPTE ProtoPte;
1574
1575 /* Find the offset within the VAD's prototype PTEs */
1576 ProtoPte = Vad->FirstPrototypePte + (Vpn - Vad->StartingVpn);
1577 ASSERT(ProtoPte <= Vad->LastContiguousPte);
1578 return ProtoPte;
1579 }
1580
1581 //
1582 // Returns the PFN Database entry for the given page number
1583 // Warning: This is not necessarily a valid PFN database entry!
1584 //
1585 FORCEINLINE
1586 PMMPFN
MI_PFN_ELEMENT(IN PFN_NUMBER Pfn)1587 MI_PFN_ELEMENT(IN PFN_NUMBER Pfn)
1588 {
1589 /* Get the entry */
1590 return &MmPfnDatabase[Pfn];
1591 };
1592
1593 //
1594 // Drops a locked page without dereferencing it
1595 //
1596 FORCEINLINE
1597 VOID
MiDropLockCount(IN PMMPFN Pfn1)1598 MiDropLockCount(IN PMMPFN Pfn1)
1599 {
1600 /* This page shouldn't be locked, but it should be valid */
1601 ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1602 ASSERT(Pfn1->u2.ShareCount == 0);
1603
1604 /* Is this the last reference to the page */
1605 if (Pfn1->u3.e2.ReferenceCount == 1)
1606 {
1607 /* It better not be valid */
1608 ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1609
1610 /* Is it a prototype PTE? */
1611 if ((Pfn1->u3.e1.PrototypePte == 1) &&
1612 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1613 {
1614 /* FIXME: We should return commit */
1615 DPRINT1("Not returning commit for prototype PTE\n");
1616 }
1617
1618 /* Update the counter */
1619 InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1620 }
1621 }
1622
1623 //
1624 // Drops a locked page and dereferences it
1625 //
1626 FORCEINLINE
1627 VOID
MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1)1628 MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1)
1629 {
1630 USHORT RefCount, OldRefCount;
1631 PFN_NUMBER PageFrameIndex;
1632
1633 /* Loop while we decrement the page successfully */
1634 do
1635 {
1636 /* There should be at least one reference */
1637 OldRefCount = Pfn1->u3.e2.ReferenceCount;
1638 ASSERT(OldRefCount != 0);
1639
1640 /* Are we the last one */
1641 if (OldRefCount == 1)
1642 {
1643 /* The page shoudln't be shared not active at this point */
1644 ASSERT(Pfn1->u3.e2.ReferenceCount == 1);
1645 ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1646 ASSERT(Pfn1->u2.ShareCount == 0);
1647
1648 /* Is it a prototype PTE? */
1649 if ((Pfn1->u3.e1.PrototypePte == 1) &&
1650 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1651 {
1652 /* FIXME: We should return commit */
1653 DPRINT1("Not returning commit for prototype PTE\n");
1654 }
1655
1656 /* Update the counter, and drop a reference the long way */
1657 InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1658 PageFrameIndex = MiGetPfnEntryIndex(Pfn1);
1659 MiDecrementReferenceCount(Pfn1, PageFrameIndex);
1660 return;
1661 }
1662
1663 /* Drop a reference the short way, and that's it */
1664 RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1665 OldRefCount - 1,
1666 OldRefCount);
1667 ASSERT(RefCount != 0);
1668 } while (OldRefCount != RefCount);
1669
1670 /* If we got here, there should be more than one reference */
1671 ASSERT(RefCount > 1);
1672 if (RefCount == 2)
1673 {
1674 /* Is it still being shared? */
1675 if (Pfn1->u2.ShareCount >= 1)
1676 {
1677 /* Then it should be valid */
1678 ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1679
1680 /* Is it a prototype PTE? */
1681 if ((Pfn1->u3.e1.PrototypePte == 1) &&
1682 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1683 {
1684 /* We don't handle ethis */
1685 ASSERT(FALSE);
1686 }
1687
1688 /* Update the counter */
1689 InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1690 }
1691 }
1692 }
1693
1694 //
1695 // References a locked page and updates the counter
1696 // Used in MmProbeAndLockPages to handle different edge cases
1697 //
1698 FORCEINLINE
1699 VOID
MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1)1700 MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1)
1701 {
1702 USHORT RefCount, OldRefCount;
1703
1704 /* Sanity check */
1705 ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1706
1707 /* Does ARM3 own the page? */
1708 if (MI_IS_ROS_PFN(Pfn1))
1709 {
1710 /* ReactOS Mm doesn't track share count */
1711 ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1712 }
1713 else
1714 {
1715 /* On ARM3 pages, we should see a valid share count */
1716 ASSERT((Pfn1->u2.ShareCount != 0) && (Pfn1->u3.e1.PageLocation == ActiveAndValid));
1717
1718 /* Is it a prototype PTE? */
1719 if ((Pfn1->u3.e1.PrototypePte == 1) &&
1720 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1721 {
1722 /* FIXME: We should charge commit */
1723 DPRINT1("Not charging commit for prototype PTE\n");
1724 }
1725 }
1726
1727 /* More locked pages! */
1728 InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1729
1730 /* Loop trying to update the reference count */
1731 do
1732 {
1733 /* Get the current reference count, make sure it's valid */
1734 OldRefCount = Pfn1->u3.e2.ReferenceCount;
1735 ASSERT(OldRefCount != 0);
1736 ASSERT(OldRefCount < 2500);
1737
1738 /* Bump it up by one */
1739 RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1740 OldRefCount + 1,
1741 OldRefCount);
1742 ASSERT(RefCount != 0);
1743 } while (OldRefCount != RefCount);
1744
1745 /* Was this the first lock attempt? If not, undo our bump */
1746 if (OldRefCount != 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1747 }
1748
1749 //
1750 // References a locked page and updates the counter
1751 // Used in all other cases except MmProbeAndLockPages
1752 //
1753 FORCEINLINE
1754 VOID
MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1)1755 MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1)
1756 {
1757 USHORT NewRefCount;
1758
1759 /* Is it a prototype PTE? */
1760 if ((Pfn1->u3.e1.PrototypePte == 1) &&
1761 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1762 {
1763 /* FIXME: We should charge commit */
1764 DPRINT1("Not charging commit for prototype PTE\n");
1765 }
1766
1767 /* More locked pages! */
1768 InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1769
1770 /* Update the reference count */
1771 NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1772 if (NewRefCount == 2)
1773 {
1774 /* Is it locked or shared? */
1775 if (Pfn1->u2.ShareCount)
1776 {
1777 /* It's shared, so make sure it's active */
1778 ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1779 }
1780 else
1781 {
1782 /* It's locked, so we shouldn't lock again */
1783 InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1784 }
1785 }
1786 else
1787 {
1788 /* Someone had already locked the page, so undo our bump */
1789 ASSERT(NewRefCount < 2500);
1790 InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1791 }
1792 }
1793
1794 //
1795 // References a locked page and updates the counter
1796 // Used in all other cases except MmProbeAndLockPages
1797 //
1798 FORCEINLINE
1799 VOID
MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1)1800 MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1)
1801 {
1802 USHORT NewRefCount;
1803
1804 /* Make sure the page isn't used yet */
1805 ASSERT(Pfn1->u2.ShareCount == 0);
1806 ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1807
1808 /* Is it a prototype PTE? */
1809 if ((Pfn1->u3.e1.PrototypePte == 1) &&
1810 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1811 {
1812 /* FIXME: We should charge commit */
1813 DPRINT1("Not charging commit for prototype PTE\n");
1814 }
1815
1816 /* More locked pages! */
1817 InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1818
1819 /* Update the reference count */
1820 NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1821 if (NewRefCount != 1)
1822 {
1823 /* Someone had already locked the page, so undo our bump */
1824 ASSERT(NewRefCount < 2500);
1825 InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1826 }
1827 }
1828
1829
1830
1831 CODE_SEG("INIT")
1832 BOOLEAN
1833 NTAPI
1834 MmArmInitSystem(
1835 IN ULONG Phase,
1836 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1837 );
1838
1839 CODE_SEG("INIT")
1840 VOID
1841 NTAPI
1842 MiInitializeSessionSpaceLayout(VOID);
1843
1844 CODE_SEG("INIT")
1845 NTSTATUS
1846 NTAPI
1847 MiInitMachineDependent(
1848 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1849 );
1850
1851 CODE_SEG("INIT")
1852 VOID
1853 NTAPI
1854 MiComputeColorInformation(
1855 VOID
1856 );
1857
1858 CODE_SEG("INIT")
1859 VOID
1860 NTAPI
1861 MiMapPfnDatabase(
1862 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1863 );
1864
1865 CODE_SEG("INIT")
1866 VOID
1867 NTAPI
1868 MiInitializeColorTables(
1869 VOID
1870 );
1871
1872 CODE_SEG("INIT")
1873 VOID
1874 NTAPI
1875 MiInitializePfnDatabase(
1876 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1877 );
1878
1879 VOID
1880 NTAPI
1881 MiInitializeSessionWsSupport(
1882 VOID
1883 );
1884
1885 VOID
1886 NTAPI
1887 MiInitializeSessionIds(
1888 VOID
1889 );
1890
1891 CODE_SEG("INIT")
1892 BOOLEAN
1893 NTAPI
1894 MiInitializeMemoryEvents(
1895 VOID
1896 );
1897
1898 CODE_SEG("INIT")
1899 PFN_NUMBER
1900 NTAPI
1901 MxGetNextPage(
1902 IN PFN_NUMBER PageCount
1903 );
1904
1905 CODE_SEG("INIT")
1906 PPHYSICAL_MEMORY_DESCRIPTOR
1907 NTAPI
1908 MmInitializeMemoryLimits(
1909 IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1910 IN PBOOLEAN IncludeType
1911 );
1912
1913 PFN_NUMBER
1914 NTAPI
1915 MiPagesInLoaderBlock(
1916 IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1917 IN PBOOLEAN IncludeType
1918 );
1919
1920 VOID
1921 FASTCALL
1922 MiSyncARM3WithROS(
1923 IN PVOID AddressStart,
1924 IN PVOID AddressEnd
1925 );
1926
1927 NTSTATUS
1928 NTAPI
1929 MiRosProtectVirtualMemory(
1930 IN PEPROCESS Process,
1931 IN OUT PVOID *BaseAddress,
1932 IN OUT PSIZE_T NumberOfBytesToProtect,
1933 IN ULONG NewAccessProtection,
1934 OUT PULONG OldAccessProtection OPTIONAL
1935 );
1936
1937 NTSTATUS
1938 NTAPI
1939 MmArmAccessFault(
1940 IN ULONG FaultCode,
1941 IN PVOID Address,
1942 IN KPROCESSOR_MODE Mode,
1943 IN PVOID TrapInformation
1944 );
1945
1946 NTSTATUS
1947 FASTCALL
1948 MiCheckPdeForPagedPool(
1949 IN PVOID Address
1950 );
1951
1952 CODE_SEG("INIT")
1953 VOID
1954 NTAPI
1955 MiInitializeNonPagedPoolThresholds(
1956 VOID
1957 );
1958
1959 CODE_SEG("INIT")
1960 VOID
1961 NTAPI
1962 MiInitializePoolEvents(
1963 VOID
1964 );
1965
1966 CODE_SEG("INIT")
1967 VOID //
1968 NTAPI //
1969 InitializePool( //
1970 IN POOL_TYPE PoolType,// FIXFIX: This should go in ex.h after the pool merge
1971 IN ULONG Threshold //
1972 ); //
1973
1974 // FIXFIX: THIS ONE TOO
1975 CODE_SEG("INIT")
1976 VOID
1977 NTAPI
1978 ExInitializePoolDescriptor(
1979 IN PPOOL_DESCRIPTOR PoolDescriptor,
1980 IN POOL_TYPE PoolType,
1981 IN ULONG PoolIndex,
1982 IN ULONG Threshold,
1983 IN PVOID PoolLock
1984 );
1985
1986 NTSTATUS
1987 NTAPI
1988 MiInitializeSessionPool(
1989 VOID
1990 );
1991
1992 CODE_SEG("INIT")
1993 VOID
1994 NTAPI
1995 MiInitializeSystemPtes(
1996 IN PMMPTE StartingPte,
1997 IN ULONG NumberOfPtes,
1998 IN MMSYSTEM_PTE_POOL_TYPE PoolType
1999 );
2000
2001 PMMPTE
2002 NTAPI
2003 MiReserveSystemPtes(
2004 IN ULONG NumberOfPtes,
2005 IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
2006 );
2007
2008 VOID
2009 NTAPI
2010 MiReleaseSystemPtes(
2011 IN PMMPTE StartingPte,
2012 IN ULONG NumberOfPtes,
2013 IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
2014 );
2015
2016
2017 PFN_NUMBER
2018 NTAPI
2019 MiFindContiguousPages(
2020 IN PFN_NUMBER LowestPfn,
2021 IN PFN_NUMBER HighestPfn,
2022 IN PFN_NUMBER BoundaryPfn,
2023 IN PFN_NUMBER SizeInPages,
2024 IN MEMORY_CACHING_TYPE CacheType
2025 );
2026
2027 PVOID
2028 NTAPI
2029 MiCheckForContiguousMemory(
2030 IN PVOID BaseAddress,
2031 IN PFN_NUMBER BaseAddressPages,
2032 IN PFN_NUMBER SizeInPages,
2033 IN PFN_NUMBER LowestPfn,
2034 IN PFN_NUMBER HighestPfn,
2035 IN PFN_NUMBER BoundaryPfn,
2036 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
2037 );
2038
2039 PMDL
2040 NTAPI
2041 MiAllocatePagesForMdl(
2042 IN PHYSICAL_ADDRESS LowAddress,
2043 IN PHYSICAL_ADDRESS HighAddress,
2044 IN PHYSICAL_ADDRESS SkipBytes,
2045 IN SIZE_T TotalBytes,
2046 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
2047 IN ULONG Flags
2048 );
2049
2050 VOID
2051 NTAPI
2052 MiInsertPageInList(
2053 IN PMMPFNLIST ListHead,
2054 IN PFN_NUMBER PageFrameIndex
2055 );
2056
2057 VOID
2058 NTAPI
2059 MiUnlinkFreeOrZeroedPage(
2060 IN PMMPFN Entry
2061 );
2062
2063 VOID
2064 NTAPI
2065 MiUnlinkPageFromList(
2066 IN PMMPFN Pfn
2067 );
2068
2069 VOID
2070 NTAPI
2071 MiInitializePfn(
2072 IN PFN_NUMBER PageFrameIndex,
2073 IN PMMPTE PointerPte,
2074 IN BOOLEAN Modified
2075 );
2076
2077 NTSTATUS
2078 NTAPI
2079 MiInitializeAndChargePfn(
2080 OUT PPFN_NUMBER PageFrameIndex,
2081 IN PMMPDE PointerPde,
2082 IN PFN_NUMBER ContainingPageFrame,
2083 IN BOOLEAN SessionAllocation
2084 );
2085
2086 VOID
2087 NTAPI
2088 MiInitializePfnAndMakePteValid(
2089 IN PFN_NUMBER PageFrameIndex,
2090 IN PMMPTE PointerPte,
2091 IN MMPTE TempPte
2092 );
2093
2094 VOID
2095 NTAPI
2096 MiInitializePfnForOtherProcess(
2097 IN PFN_NUMBER PageFrameIndex,
2098 IN PVOID PteAddress,
2099 IN PFN_NUMBER PteFrame
2100 );
2101
2102 VOID
2103 NTAPI
2104 MiDecrementShareCount(
2105 IN PMMPFN Pfn1,
2106 IN PFN_NUMBER PageFrameIndex
2107 );
2108
2109 PFN_NUMBER
2110 NTAPI
2111 MiRemoveAnyPage(
2112 IN ULONG Color
2113 );
2114
2115 PFN_NUMBER
2116 NTAPI
2117 MiRemoveZeroPage(
2118 IN ULONG Color
2119 );
2120
2121 VOID
2122 NTAPI
2123 MiZeroPhysicalPage(
2124 IN PFN_NUMBER PageFrameIndex
2125 );
2126
2127 VOID
2128 NTAPI
2129 MiInsertPageInFreeList(
2130 IN PFN_NUMBER PageFrameIndex
2131 );
2132
2133 PFN_COUNT
2134 NTAPI
2135 MiDeleteSystemPageableVm(
2136 IN PMMPTE PointerPte,
2137 IN PFN_NUMBER PageCount,
2138 IN ULONG Flags,
2139 OUT PPFN_NUMBER ValidPages
2140 );
2141
2142 ULONG
2143 NTAPI
2144 MiGetPageProtection(
2145 IN PMMPTE PointerPte
2146 );
2147
2148 PLDR_DATA_TABLE_ENTRY
2149 NTAPI
2150 MiLookupDataTableEntry(
2151 IN PVOID Address
2152 );
2153
2154 CODE_SEG("INIT")
2155 VOID
2156 NTAPI
2157 MiInitializeDriverLargePageList(
2158 VOID
2159 );
2160
2161 CODE_SEG("INIT")
2162 VOID
2163 NTAPI
2164 MiInitializeLargePageSupport(
2165 VOID
2166 );
2167
2168 CODE_SEG("INIT")
2169 VOID
2170 NTAPI
2171 MiSyncCachedRanges(
2172 VOID
2173 );
2174
2175 BOOLEAN
2176 NTAPI
2177 MiIsPfnInUse(
2178 IN PMMPFN Pfn1
2179 );
2180
2181 PMMVAD
2182 NTAPI
2183 MiLocateAddress(
2184 IN PVOID VirtualAddress
2185 );
2186
2187 TABLE_SEARCH_RESULT
2188 NTAPI
2189 MiCheckForConflictingNode(
2190 IN ULONG_PTR StartVpn,
2191 IN ULONG_PTR EndVpn,
2192 IN PMM_AVL_TABLE Table,
2193 OUT PMMADDRESS_NODE *NodeOrParent
2194 );
2195
2196 TABLE_SEARCH_RESULT
2197 NTAPI
2198 MiFindEmptyAddressRangeDownTree(
2199 IN SIZE_T Length,
2200 IN ULONG_PTR BoundaryAddress,
2201 IN ULONG_PTR Alignment,
2202 IN PMM_AVL_TABLE Table,
2203 OUT PULONG_PTR Base,
2204 OUT PMMADDRESS_NODE *Parent
2205 );
2206
2207 NTSTATUS
2208 NTAPI
2209 MiFindEmptyAddressRangeDownBasedTree(
2210 IN SIZE_T Length,
2211 IN ULONG_PTR BoundaryAddress,
2212 IN ULONG_PTR Alignment,
2213 IN PMM_AVL_TABLE Table,
2214 OUT PULONG_PTR Base
2215 );
2216
2217 TABLE_SEARCH_RESULT
2218 NTAPI
2219 MiFindEmptyAddressRangeInTree(
2220 IN SIZE_T Length,
2221 IN ULONG_PTR Alignment,
2222 IN PMM_AVL_TABLE Table,
2223 OUT PMMADDRESS_NODE *PreviousVad,
2224 OUT PULONG_PTR Base
2225 );
2226
2227 NTSTATUS
2228 NTAPI
2229 MiCheckSecuredVad(
2230 IN PMMVAD Vad,
2231 IN PVOID Base,
2232 IN SIZE_T Size,
2233 IN ULONG ProtectionMask
2234 );
2235
2236 VOID
2237 NTAPI
2238 MiInsertVad(
2239 _Inout_ PMMVAD Vad,
2240 _Inout_ PMM_AVL_TABLE VadRoot);
2241
2242 NTSTATUS
2243 NTAPI
2244 MiInsertVadEx(
2245 _Inout_ PMMVAD Vad,
2246 _In_ ULONG_PTR *BaseAddress,
2247 _In_ SIZE_T ViewSize,
2248 _In_ ULONG_PTR HighestAddress,
2249 _In_ ULONG_PTR Alignment,
2250 _In_ ULONG AllocationType);
2251
2252 VOID
2253 NTAPI
2254 MiInsertBasedSection(
2255 IN PSECTION Section
2256 );
2257
2258 NTSTATUS
2259 NTAPI
2260 MiRosUnmapViewOfSection(
2261 IN PEPROCESS Process,
2262 IN PVOID BaseAddress,
2263 IN BOOLEAN SkipDebuggerNotify
2264 );
2265
2266 VOID
2267 NTAPI
2268 MiInsertNode(
2269 IN PMM_AVL_TABLE Table,
2270 IN PMMADDRESS_NODE NewNode,
2271 PMMADDRESS_NODE Parent,
2272 TABLE_SEARCH_RESULT Result
2273 );
2274
2275 VOID
2276 NTAPI
2277 MiRemoveNode(
2278 IN PMMADDRESS_NODE Node,
2279 IN PMM_AVL_TABLE Table
2280 );
2281
2282 PMMADDRESS_NODE
2283 NTAPI
2284 MiGetPreviousNode(
2285 IN PMMADDRESS_NODE Node
2286 );
2287
2288 PMMADDRESS_NODE
2289 NTAPI
2290 MiGetNextNode(
2291 IN PMMADDRESS_NODE Node
2292 );
2293
2294 BOOLEAN
2295 NTAPI
2296 MiInitializeSystemSpaceMap(
2297 IN PMMSESSION InputSession OPTIONAL
2298 );
2299
2300 VOID
2301 NTAPI
2302 MiSessionRemoveProcess(
2303 VOID
2304 );
2305
2306 VOID
2307 NTAPI
2308 MiReleaseProcessReferenceToSessionDataPage(
2309 IN PMM_SESSION_SPACE SessionGlobal
2310 );
2311
2312 VOID
2313 NTAPI
2314 MiSessionAddProcess(
2315 IN PEPROCESS NewProcess
2316 );
2317
2318 ULONG
2319 NTAPI
2320 MiMakeProtectionMask(
2321 IN ULONG Protect
2322 );
2323
2324 VOID
2325 NTAPI
2326 MiDeleteVirtualAddresses(
2327 IN ULONG_PTR Va,
2328 IN ULONG_PTR EndingAddress,
2329 IN PMMVAD Vad
2330 );
2331
2332 VOID
2333 NTAPI
2334 MiDeletePte(
2335 IN PMMPTE PointerPte,
2336 IN PVOID VirtualAddress,
2337 IN PEPROCESS CurrentProcess,
2338 IN PMMPTE PrototypePte
2339 );
2340
2341 ULONG
2342 NTAPI
2343 MiMakeSystemAddressValid(
2344 IN PVOID PageTableVirtualAddress,
2345 IN PEPROCESS CurrentProcess
2346 );
2347
2348 ULONG
2349 NTAPI
2350 MiMakeSystemAddressValidPfn(
2351 IN PVOID VirtualAddress,
2352 IN KIRQL OldIrql
2353 );
2354
2355 VOID
2356 NTAPI
2357 MiRemoveMappedView(
2358 IN PEPROCESS CurrentProcess,
2359 IN PMMVAD Vad
2360 );
2361
2362 PSUBSECTION
2363 NTAPI
2364 MiLocateSubsection(
2365 IN PMMVAD Vad,
2366 IN ULONG_PTR Vpn
2367 );
2368
2369 VOID
2370 NTAPI
2371 MiDeleteARM3Section(
2372 PVOID ObjectBody
2373 );
2374
2375 NTSTATUS
2376 NTAPI
2377 MiQueryMemorySectionName(
2378 IN HANDLE ProcessHandle,
2379 IN PVOID BaseAddress,
2380 OUT PVOID MemoryInformation,
2381 IN SIZE_T MemoryInformationLength,
2382 OUT PSIZE_T ReturnLength
2383 );
2384
2385 NTSTATUS
2386 NTAPI
2387 MiRosUnmapViewInSystemSpace(
2388 IN PVOID MappedBase
2389 );
2390
2391 VOID
2392 NTAPI
2393 MiMakePdeExistAndMakeValid(
2394 IN PMMPDE PointerPde,
2395 IN PEPROCESS TargetProcess,
2396 IN KIRQL OldIrql
2397 );
2398
2399 VOID
2400 NTAPI
2401 MiWriteProtectSystemImage(
2402 _In_ PVOID ImageBase);
2403
2404 //
2405 // MiRemoveZeroPage will use inline code to zero out the page manually if only
2406 // free pages are available. In some scenarios, we don't/can't run that piece of
2407 // code and would rather only have a real zero page. If we can't have a zero page,
2408 // then we'd like to have our own code to grab a free page and zero it out, by
2409 // using MiRemoveAnyPage. This macro implements this.
2410 //
2411 FORCEINLINE
2412 PFN_NUMBER
MiRemoveZeroPageSafe(IN ULONG Color)2413 MiRemoveZeroPageSafe(IN ULONG Color)
2414 {
2415 if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color);
2416 return 0;
2417 }
2418
2419 #if (_MI_PAGING_LEVELS == 2)
2420 FORCEINLINE
2421 BOOLEAN
MiSynchronizeSystemPde(PMMPDE PointerPde)2422 MiSynchronizeSystemPde(PMMPDE PointerPde)
2423 {
2424 ULONG Index;
2425
2426 /* Get the Index from the PDE */
2427 Index = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE);
2428 if (PointerPde->u.Hard.Valid != 0)
2429 {
2430 NT_ASSERT(PointerPde->u.Long == MmSystemPagePtes[Index].u.Long);
2431 return TRUE;
2432 }
2433
2434 if (MmSystemPagePtes[Index].u.Hard.Valid == 0)
2435 {
2436 return FALSE;
2437 }
2438
2439 /* Copy the PDE from the double-mapped system page directory */
2440 MI_WRITE_VALID_PDE(PointerPde, MmSystemPagePtes[Index]);
2441
2442 /* Make sure we re-read the PDE and PTE */
2443 KeMemoryBarrierWithoutFence();
2444
2445 /* Return success */
2446 return TRUE;
2447 }
2448 #endif
2449
2450 #if _MI_PAGING_LEVELS == 2
2451 FORCEINLINE
2452 USHORT
MiIncrementPageTableReferences(IN PVOID Address)2453 MiIncrementPageTableReferences(IN PVOID Address)
2454 {
2455 PUSHORT RefCount;
2456
2457 RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
2458
2459 *RefCount += 1;
2460 ASSERT(*RefCount <= PTE_PER_PAGE);
2461 return *RefCount;
2462 }
2463
2464 FORCEINLINE
2465 USHORT
MiDecrementPageTableReferences(IN PVOID Address)2466 MiDecrementPageTableReferences(IN PVOID Address)
2467 {
2468 PUSHORT RefCount;
2469
2470 RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
2471
2472 *RefCount -= 1;
2473 ASSERT(*RefCount < PTE_PER_PAGE);
2474 return *RefCount;
2475 }
2476 #else
2477 FORCEINLINE
2478 USHORT
MiIncrementPageTableReferences(IN PVOID Address)2479 MiIncrementPageTableReferences(IN PVOID Address)
2480 {
2481 PMMPDE PointerPde = MiAddressToPde(Address);
2482 PMMPFN Pfn;
2483
2484 /* We should not tinker with this one. */
2485 ASSERT(PointerPde != (PMMPDE)PXE_SELFMAP);
2486 DPRINT("Incrementing %p from %p\n", Address, _ReturnAddress());
2487
2488 /* Make sure we're locked */
2489 ASSERT(PsGetCurrentThread()->OwnsProcessWorkingSetExclusive);
2490
2491 /* If we're bumping refcount, then it must be valid! */
2492 ASSERT(PointerPde->u.Hard.Valid == 1);
2493
2494 /* This lies on the PFN */
2495 Pfn = MiGetPfnEntry(PFN_FROM_PDE(PointerPde));
2496 Pfn->OriginalPte.u.Soft.UsedPageTableEntries++;
2497
2498 ASSERT(Pfn->OriginalPte.u.Soft.UsedPageTableEntries <= PTE_PER_PAGE);
2499
2500 return Pfn->OriginalPte.u.Soft.UsedPageTableEntries;
2501 }
2502
2503 FORCEINLINE
2504 USHORT
MiDecrementPageTableReferences(IN PVOID Address)2505 MiDecrementPageTableReferences(IN PVOID Address)
2506 {
2507 PMMPDE PointerPde = MiAddressToPde(Address);
2508 PMMPFN Pfn;
2509
2510 /* We should not tinker with this one. */
2511 ASSERT(PointerPde != (PMMPDE)PXE_SELFMAP);
2512
2513 DPRINT("Decrementing %p from %p\n", PointerPde, _ReturnAddress());
2514
2515 /* Make sure we're locked */
2516 ASSERT(PsGetCurrentThread()->OwnsProcessWorkingSetExclusive);
2517
2518 /* If we're decreasing refcount, then it must be valid! */
2519 ASSERT(PointerPde->u.Hard.Valid == 1);
2520
2521 /* This lies on the PFN */
2522 Pfn = MiGetPfnEntry(PFN_FROM_PDE(PointerPde));
2523
2524 ASSERT(Pfn->OriginalPte.u.Soft.UsedPageTableEntries != 0);
2525 Pfn->OriginalPte.u.Soft.UsedPageTableEntries--;
2526
2527 ASSERT(Pfn->OriginalPte.u.Soft.UsedPageTableEntries < PTE_PER_PAGE);
2528
2529 return Pfn->OriginalPte.u.Soft.UsedPageTableEntries;
2530 }
2531 #endif
2532
2533 #ifdef __cplusplus
2534 } // extern "C"
2535 #endif
2536
2537 FORCEINLINE
2538 VOID
MiDeletePde(_In_ PMMPDE PointerPde,_In_ PEPROCESS CurrentProcess)2539 MiDeletePde(
2540 _In_ PMMPDE PointerPde,
2541 _In_ PEPROCESS CurrentProcess)
2542 {
2543 /* Only for user-mode ones */
2544 ASSERT(MiIsUserPde(PointerPde));
2545
2546 /* Kill this one as a PTE */
2547 MiDeletePte((PMMPTE)PointerPde, MiPdeToPte(PointerPde), CurrentProcess, NULL);
2548 #if _MI_PAGING_LEVELS >= 3
2549 /* Cascade down */
2550 if (MiDecrementPageTableReferences(MiPdeToPte(PointerPde)) == 0)
2551 {
2552 MiDeletePte(MiPdeToPpe(PointerPde), PointerPde, CurrentProcess, NULL);
2553 #if _MI_PAGING_LEVELS == 4
2554 if (MiDecrementPageTableReferences(PointerPde) == 0)
2555 {
2556 MiDeletePte(MiPdeToPxe(PointerPde), MiPdeToPpe(PointerPde), CurrentProcess, NULL);
2557 }
2558 #endif
2559 }
2560 #endif
2561 }
2562
2563 /* EOF */
2564