1 /* 2 * PROJECT: ReactOS Kernel 3 * LICENSE: BSD - See COPYING.ARM in the top level directory 4 * FILE: ntoskrnl/mm/ARM3/miarm.h 5 * PURPOSE: ARM Memory Manager Header 6 * PROGRAMMERS: ReactOS Portable Systems Group 7 */ 8 9 #pragma once 10 11 #ifdef __cplusplus 12 extern "C" { 13 #endif 14 15 #define MI_LOWEST_VAD_ADDRESS (PVOID)MM_LOWEST_USER_ADDRESS 16 17 /* Make the code cleaner with some definitions for size multiples */ 18 #define _1KB (1024u) 19 #define _1MB (1024 * _1KB) 20 #define _1GB (1024 * _1MB) 21 22 /* Everyone loves 64K */ 23 #define _64K (64 * _1KB) 24 25 /* Size of a page table */ 26 #define PT_SIZE (PTE_PER_PAGE * sizeof(MMPTE)) 27 28 /* Size of a page directory */ 29 #define PD_SIZE (PDE_PER_PAGE * sizeof(MMPDE)) 30 31 /* Size of all page directories for a process */ 32 #define SYSTEM_PD_SIZE (PPE_PER_PAGE * PD_SIZE) 33 #ifdef _M_IX86 34 C_ASSERT(SYSTEM_PD_SIZE == PAGE_SIZE); 35 #endif 36 37 // 38 // Protection Bits part of the internal memory manager Protection Mask, from: 39 // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel 40 // https://www.reactos.org/wiki/Techwiki:Memory_Protection_constants 41 // and public assertions. 42 // 43 #define MM_ZERO_ACCESS 0 44 #define MM_READONLY 1 45 #define MM_EXECUTE 2 46 #define MM_EXECUTE_READ 3 47 #define MM_READWRITE 4 48 #define MM_WRITECOPY 5 49 #define MM_EXECUTE_READWRITE 6 50 #define MM_EXECUTE_WRITECOPY 7 51 #define MM_PROTECT_ACCESS 7 52 53 // 54 // These are flags on top of the actual protection mask 55 // 56 #define MM_NOCACHE 0x08 57 #define MM_GUARDPAGE 0x10 58 #define MM_WRITECOMBINE 0x18 59 #define MM_PROTECT_SPECIAL 0x18 60 61 // 62 // These are special cases 63 // 64 #define MM_DECOMMIT (MM_ZERO_ACCESS | MM_GUARDPAGE) 65 #define MM_NOACCESS (MM_ZERO_ACCESS | MM_WRITECOMBINE) 66 #define MM_OUTSWAPPED_KSTACK (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE) 67 #define MM_INVALID_PROTECTION 0xFFFFFFFF 68 69 // 70 // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits 71 // The Memory Manager's definition define the attributes that must be preserved 72 // and these PTE definitions describe the attributes in the hardware sense. This 73 // helps deal with hardware differences between the actual boolean expression of 74 // the argument. 75 // 76 // For example, in the logical attributes, we want to express read-only as a flag 77 // but on x86, it is writability that must be set. On the other hand, on x86, just 78 // like in the kernel, it is disabling the caches that requires a special flag, 79 // while on certain architectures such as ARM, it is enabling the cache which 80 // requires a flag. 81 // 82 #if defined(_M_IX86) 83 // 84 // Access Flags 85 // 86 #define PTE_READONLY 0 // Doesn't exist on x86 87 #define PTE_EXECUTE 0 // Not worrying about NX yet 88 #define PTE_EXECUTE_READ 0 // Not worrying about NX yet 89 #define PTE_READWRITE 0x2 90 #define PTE_WRITECOPY 0x200 91 #define PTE_EXECUTE_READWRITE 0x2 // Not worrying about NX yet 92 #define PTE_EXECUTE_WRITECOPY 0x200 93 #define PTE_PROTOTYPE 0x400 94 95 // 96 // State Flags 97 // 98 #define PTE_VALID 0x1 99 #define PTE_ACCESSED 0x20 100 #define PTE_DIRTY 0x40 101 102 // 103 // Cache flags 104 // 105 #define PTE_ENABLE_CACHE 0 106 #define PTE_DISABLE_CACHE 0x10 107 #define PTE_WRITECOMBINED_CACHE 0x10 108 #define PTE_PROTECT_MASK 0x612 109 #elif defined(_M_AMD64) 110 // 111 // Access Flags 112 // 113 #define PTE_READONLY 0x8000000000000000ULL 114 #define PTE_EXECUTE 0x0000000000000000ULL 115 #define PTE_EXECUTE_READ PTE_EXECUTE /* EXECUTE implies READ on x64 */ 116 #define PTE_READWRITE 0x8000000000000002ULL 117 #define PTE_WRITECOPY 0x8000000000000200ULL 118 #define PTE_EXECUTE_READWRITE 0x0000000000000002ULL 119 #define PTE_EXECUTE_WRITECOPY 0x0000000000000200ULL 120 #define PTE_PROTOTYPE 0x0000000000000400ULL 121 122 // 123 // State Flags 124 // 125 #define PTE_VALID 0x0000000000000001ULL 126 #define PTE_ACCESSED 0x0000000000000020ULL 127 #define PTE_DIRTY 0x0000000000000040ULL 128 129 // 130 // Cache flags 131 // 132 #define PTE_ENABLE_CACHE 0x0000000000000000ULL 133 #define PTE_DISABLE_CACHE 0x0000000000000010ULL 134 #define PTE_WRITECOMBINED_CACHE 0x0000000000000010ULL 135 #define PTE_PROTECT_MASK 0x8000000000000612ULL 136 #elif defined(_M_ARM) 137 #define PTE_READONLY 0x200 138 #define PTE_EXECUTE 0 // Not worrying about NX yet 139 #define PTE_EXECUTE_READ 0 // Not worrying about NX yet 140 #define PTE_READWRITE 0 // Doesn't exist on ARM 141 #define PTE_WRITECOPY 0 // Doesn't exist on ARM 142 #define PTE_EXECUTE_READWRITE 0 // Not worrying about NX yet 143 #define PTE_EXECUTE_WRITECOPY 0 // Not worrying about NX yet 144 #define PTE_PROTOTYPE 0x400 // Using the Shared bit 145 146 // 147 // Cache flags 148 // 149 #define PTE_ENABLE_CACHE 0 150 #define PTE_DISABLE_CACHE 0x10 151 #define PTE_WRITECOMBINED_CACHE 0x10 152 #define PTE_PROTECT_MASK 0x610 153 #else 154 #error Define these please! 155 #endif 156 157 // 158 // Mask for image section page protection 159 // 160 #define IMAGE_SCN_PROTECTION_MASK (IMAGE_SCN_MEM_WRITE | IMAGE_SCN_MEM_READ | IMAGE_SCN_MEM_EXECUTE) 161 162 extern const ULONG_PTR MmProtectToPteMask[32]; 163 extern const ULONG MmProtectToValue[32]; 164 165 // 166 // Assertions for session images, addresses, and PTEs 167 // 168 #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \ 169 (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd)) 170 171 #define MI_IS_SESSION_ADDRESS(Address) \ 172 (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd)) 173 174 #define MI_IS_SESSION_PTE(Pte) \ 175 ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte)) 176 177 #define MI_IS_PAGE_TABLE_ADDRESS(Address) \ 178 (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP)) 179 180 #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \ 181 (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP)) 182 183 #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \ 184 (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd)) 185 186 // 187 // Creates a software PTE with the given protection 188 // 189 #define MI_MAKE_SOFTWARE_PTE(p, x) ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS)) 190 191 // 192 // Marks a PTE as deleted 193 // 194 #define MI_SET_PFN_DELETED(x) ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1)) 195 #define MI_IS_PFN_DELETED(x) ((ULONG_PTR)((x)->PteAddress) & 1) 196 197 // 198 // Special values for LoadedImports 199 // 200 #define MM_SYSLDR_NO_IMPORTS ((PVOID)(ULONG_PTR)-2) 201 #define MM_SYSLDR_BOOT_LOADED ((PVOID)(ULONG_PTR)-1) 202 #define MM_SYSLDR_SINGLE_ENTRY 0x1 203 204 // 205 // Number of initial session IDs 206 // 207 #define MI_INITIAL_SESSION_IDS 64 208 209 #if defined(_M_IX86) || defined(_M_ARM) 210 // 211 // PFN List Sentinel 212 // 213 #define LIST_HEAD 0xFFFFFFFF 214 215 // 216 // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits, 217 // we need a manual definition suited to the number of bits in the PteFrame. 218 // This is used as a LIST_HEAD for the colored list 219 // 220 #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF 221 #elif defined(_M_AMD64) 222 #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL 223 #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL 224 #else 225 #error Define these please! 226 #endif 227 228 // 229 // Returns the color of a page 230 // 231 #define MI_GET_PAGE_COLOR(x) ((x) & MmSecondaryColorMask) 232 #define MI_GET_NEXT_COLOR() (MI_GET_PAGE_COLOR(++MmSystemPageColor)) 233 #define MI_GET_NEXT_PROCESS_COLOR(x) (MI_GET_PAGE_COLOR(++(x)->NextPageColor)) 234 235 // 236 // Prototype PTEs that don't yet have a pagefile association 237 // 238 #ifdef _WIN64 239 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL 240 #else 241 #define MI_PTE_LOOKUP_NEEDED 0xFFFFF 242 #endif 243 244 // 245 // Number of session data and tag pages 246 // 247 #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE) 248 #define MI_SESSION_TAG_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE) 249 250 // 251 // Used by MiCheckSecuredVad 252 // 253 #define MM_READ_WRITE_ALLOWED 11 254 #define MM_READ_ONLY_ALLOWED 10 255 #define MM_NO_ACCESS_ALLOWED 01 256 #define MM_DELETE_CHECK 85 257 258 // 259 // System views are binned into 64K chunks 260 // 261 #define MI_SYSTEM_VIEW_BUCKET_SIZE _64K 262 263 // 264 // FIXFIX: These should go in ex.h after the pool merge 265 // 266 #ifdef _WIN64 267 #define POOL_BLOCK_SIZE 16 268 #else 269 #define POOL_BLOCK_SIZE 8 270 #endif 271 #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE) 272 #define BASE_POOL_TYPE_MASK 1 273 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE)) 274 275 // 276 // Pool debugging/analysis/tracing flags 277 // 278 #define POOL_FLAG_CHECK_TIMERS 0x1 279 #define POOL_FLAG_CHECK_WORKERS 0x2 280 #define POOL_FLAG_CHECK_RESOURCES 0x4 281 #define POOL_FLAG_VERIFIER 0x8 282 #define POOL_FLAG_CHECK_DEADLOCK 0x10 283 #define POOL_FLAG_SPECIAL_POOL 0x20 284 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40 285 #define POOL_FLAG_CRASH_ON_FAILURE 0x80 286 287 // 288 // BAD_POOL_HEADER codes during pool bugcheck 289 // 290 #define POOL_CORRUPTED_LIST 3 291 #define POOL_SIZE_OR_INDEX_MISMATCH 5 292 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6 293 #define POOL_HEADER_NOT_ALIGNED 7 294 #define POOL_HEADER_IS_ZERO 8 295 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9 296 #define POOL_ENTRY_NOT_FOUND 10 297 298 // 299 // BAD_POOL_CALLER codes during pool bugcheck 300 // 301 #define POOL_ENTRY_CORRUPTED 1 302 #define POOL_ENTRY_ALREADY_FREE 6 303 #define POOL_ENTRY_NOT_ALLOCATED 7 304 #define POOL_ALLOC_IRQL_INVALID 8 305 #define POOL_FREE_IRQL_INVALID 9 306 #define POOL_BILLED_PROCESS_INVALID 13 307 #define POOL_HEADER_SIZE_INVALID 32 308 309 typedef struct _POOL_DESCRIPTOR 310 { 311 POOL_TYPE PoolType; 312 ULONG PoolIndex; 313 ULONG RunningAllocs; 314 ULONG RunningDeAllocs; 315 ULONG TotalPages; 316 ULONG TotalBigPages; 317 ULONG Threshold; 318 PVOID LockAddress; 319 PVOID PendingFrees; 320 LONG PendingFreeDepth; 321 SIZE_T TotalBytes; 322 SIZE_T Spare0; 323 LIST_ENTRY ListHeads[POOL_LISTS_PER_PAGE]; 324 } POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR; 325 326 typedef struct _POOL_HEADER 327 { 328 union 329 { 330 struct 331 { 332 #ifdef _WIN64 333 USHORT PreviousSize:8; 334 USHORT PoolIndex:8; 335 USHORT BlockSize:8; 336 USHORT PoolType:8; 337 #else 338 USHORT PreviousSize:9; 339 USHORT PoolIndex:7; 340 USHORT BlockSize:9; 341 USHORT PoolType:7; 342 #endif 343 }; 344 ULONG Ulong1; 345 }; 346 #ifdef _WIN64 347 ULONG PoolTag; 348 #endif 349 union 350 { 351 #ifdef _WIN64 352 PEPROCESS ProcessBilled; 353 #else 354 ULONG PoolTag; 355 #endif 356 struct 357 { 358 USHORT AllocatorBackTraceIndex; 359 USHORT PoolTagHash; 360 }; 361 }; 362 } POOL_HEADER, *PPOOL_HEADER; 363 364 C_ASSERT(sizeof(POOL_HEADER) == POOL_BLOCK_SIZE); 365 C_ASSERT(POOL_BLOCK_SIZE == sizeof(LIST_ENTRY)); 366 367 typedef struct _POOL_TRACKER_TABLE 368 { 369 ULONG Key; 370 LONG NonPagedAllocs; 371 LONG NonPagedFrees; 372 SIZE_T NonPagedBytes; 373 LONG PagedAllocs; 374 LONG PagedFrees; 375 SIZE_T PagedBytes; 376 } POOL_TRACKER_TABLE, *PPOOL_TRACKER_TABLE; 377 378 typedef struct _POOL_TRACKER_BIG_PAGES 379 { 380 PVOID Va; 381 ULONG Key; 382 ULONG NumberOfPages; 383 PVOID QuotaObject; 384 } POOL_TRACKER_BIG_PAGES, *PPOOL_TRACKER_BIG_PAGES; 385 386 extern ULONG ExpNumberOfPagedPools; 387 extern POOL_DESCRIPTOR NonPagedPoolDescriptor; 388 extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16 + 1]; 389 extern PPOOL_TRACKER_TABLE PoolTrackTable; 390 391 // 392 // END FIXFIX 393 // 394 395 typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY 396 { 397 LIST_ENTRY Links; 398 UNICODE_STRING BaseName; 399 } MI_LARGE_PAGE_DRIVER_ENTRY, *PMI_LARGE_PAGE_DRIVER_ENTRY; 400 401 typedef enum _MMSYSTEM_PTE_POOL_TYPE 402 { 403 SystemPteSpace, 404 NonPagedPoolExpansion, 405 MaximumPtePoolTypes 406 } MMSYSTEM_PTE_POOL_TYPE; 407 408 typedef enum _MI_PFN_CACHE_ATTRIBUTE 409 { 410 MiNonCached, 411 MiCached, 412 MiWriteCombined, 413 MiNotMapped 414 } MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE; 415 416 typedef struct _PHYSICAL_MEMORY_RUN 417 { 418 PFN_NUMBER BasePage; 419 PFN_NUMBER PageCount; 420 } PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN; 421 422 typedef struct _PHYSICAL_MEMORY_DESCRIPTOR 423 { 424 ULONG NumberOfRuns; 425 PFN_NUMBER NumberOfPages; 426 PHYSICAL_MEMORY_RUN Run[1]; 427 } PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR; 428 429 typedef struct _MMCOLOR_TABLES 430 { 431 PFN_NUMBER Flink; 432 PVOID Blink; 433 PFN_NUMBER Count; 434 } MMCOLOR_TABLES, *PMMCOLOR_TABLES; 435 436 typedef struct _MI_LARGE_PAGE_RANGES 437 { 438 PFN_NUMBER StartFrame; 439 PFN_NUMBER LastFrame; 440 } MI_LARGE_PAGE_RANGES, *PMI_LARGE_PAGE_RANGES; 441 442 typedef struct _MMVIEW 443 { 444 ULONG_PTR Entry; 445 PCONTROL_AREA ControlArea; 446 } MMVIEW, *PMMVIEW; 447 448 typedef struct _MMSESSION 449 { 450 KGUARDED_MUTEX SystemSpaceViewLock; 451 PKGUARDED_MUTEX SystemSpaceViewLockPointer; 452 PCHAR SystemSpaceViewStart; 453 PMMVIEW SystemSpaceViewTable; 454 ULONG SystemSpaceHashSize; 455 ULONG SystemSpaceHashEntries; 456 ULONG SystemSpaceHashKey; 457 ULONG BitmapFailures; 458 PRTL_BITMAP SystemSpaceBitMap; 459 } MMSESSION, *PMMSESSION; 460 461 typedef struct _MM_SESSION_SPACE_FLAGS 462 { 463 ULONG Initialized:1; 464 ULONG DeletePending:1; 465 ULONG Filler:30; 466 } MM_SESSION_SPACE_FLAGS; 467 468 typedef struct _MM_SESSION_SPACE 469 { 470 struct _MM_SESSION_SPACE *GlobalVirtualAddress; 471 LONG ReferenceCount; 472 union 473 { 474 ULONG LongFlags; 475 MM_SESSION_SPACE_FLAGS Flags; 476 } u; 477 ULONG SessionId; 478 LIST_ENTRY ProcessList; 479 LARGE_INTEGER LastProcessSwappedOutTime; 480 PFN_NUMBER SessionPageDirectoryIndex; 481 SIZE_T NonPageablePages; 482 SIZE_T CommittedPages; 483 PVOID PagedPoolStart; 484 PVOID PagedPoolEnd; 485 PMMPDE PagedPoolBasePde; 486 ULONG Color; 487 LONG ResidentProcessCount; 488 ULONG SessionPoolAllocationFailures[4]; 489 LIST_ENTRY ImageList; 490 LCID LocaleId; 491 ULONG AttachCount; 492 KEVENT AttachEvent; 493 PEPROCESS LastProcess; 494 LONG ProcessReferenceToSession; 495 LIST_ENTRY WsListEntry; 496 GENERAL_LOOKASIDE Lookaside[SESSION_POOL_LOOKASIDES]; 497 MMSESSION Session; 498 KGUARDED_MUTEX PagedPoolMutex; 499 MM_PAGED_POOL_INFO PagedPoolInfo; 500 MMSUPPORT Vm; 501 PMMWSLE Wsle; 502 PDRIVER_UNLOAD Win32KDriverUnload; 503 POOL_DESCRIPTOR PagedPool; 504 #if defined (_M_AMD64) 505 MMPDE PageDirectory; 506 #else 507 PMMPDE PageTables; 508 #endif 509 #if defined (_M_AMD64) 510 PMMPTE SpecialPoolFirstPte; 511 PMMPTE SpecialPoolLastPte; 512 PMMPTE NextPdeForSpecialPoolExpansion; 513 PMMPTE LastPdeForSpecialPoolExpansion; 514 PFN_NUMBER SpecialPagesInUse; 515 #endif 516 LONG ImageLoadingCount; 517 } MM_SESSION_SPACE, *PMM_SESSION_SPACE; 518 519 extern PMM_SESSION_SPACE MmSessionSpace; 520 extern MMPTE HyperTemplatePte; 521 extern MMPDE ValidKernelPde; 522 extern MMPTE ValidKernelPte; 523 extern MMPDE ValidKernelPdeLocal; 524 extern MMPTE ValidKernelPteLocal; 525 extern MMPDE DemandZeroPde; 526 extern MMPTE DemandZeroPte; 527 extern MMPTE PrototypePte; 528 extern MMPTE MmDecommittedPte; 529 extern BOOLEAN MmLargeSystemCache; 530 extern BOOLEAN MmZeroPageFile; 531 extern BOOLEAN MmProtectFreedNonPagedPool; 532 extern BOOLEAN MmTrackLockedPages; 533 extern BOOLEAN MmTrackPtes; 534 extern BOOLEAN MmDynamicPfn; 535 extern BOOLEAN MmMirroring; 536 extern BOOLEAN MmMakeLowMemory; 537 extern BOOLEAN MmEnforceWriteProtection; 538 extern SIZE_T MmAllocationFragment; 539 extern ULONG MmConsumedPoolPercentage; 540 extern ULONG MmVerifyDriverBufferType; 541 extern ULONG MmVerifyDriverLevel; 542 extern WCHAR MmVerifyDriverBuffer[512]; 543 extern WCHAR MmLargePageDriverBuffer[512]; 544 extern LIST_ENTRY MiLargePageDriverList; 545 extern BOOLEAN MiLargePageAllDrivers; 546 extern ULONG MmVerifyDriverBufferLength; 547 extern ULONG MmLargePageDriverBufferLength; 548 extern SIZE_T MmSizeOfNonPagedPoolInBytes; 549 extern SIZE_T MmMaximumNonPagedPoolInBytes; 550 extern PFN_NUMBER MmMaximumNonPagedPoolInPages; 551 extern PFN_NUMBER MmSizeOfPagedPoolInPages; 552 extern PVOID MmNonPagedSystemStart; 553 extern PVOID MmNonPagedPoolStart; 554 extern PVOID MmNonPagedPoolExpansionStart; 555 extern PVOID MmNonPagedPoolEnd; 556 extern SIZE_T MmSizeOfPagedPoolInBytes; 557 extern PVOID MmPagedPoolStart; 558 extern PVOID MmPagedPoolEnd; 559 extern PVOID MmSessionBase; 560 extern SIZE_T MmSessionSize; 561 extern PMMPTE MmFirstReservedMappingPte, MmLastReservedMappingPte; 562 extern PMMPTE MiFirstReservedZeroingPte; 563 extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2][MmMaximumCacheType]; 564 extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock; 565 extern SIZE_T MmBootImageSize; 566 extern PMMPTE MmSystemPtesStart[MaximumPtePoolTypes]; 567 extern PMMPTE MmSystemPtesEnd[MaximumPtePoolTypes]; 568 extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor; 569 extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor; 570 extern ULONG_PTR MxPfnAllocation; 571 extern MM_PAGED_POOL_INFO MmPagedPoolInfo; 572 extern KGUARDED_MUTEX MmPagedPoolMutex; 573 extern KGUARDED_MUTEX MmSectionCommitMutex; 574 extern PVOID MmPagedPoolStart; 575 extern PVOID MmPagedPoolEnd; 576 extern PVOID MmNonPagedSystemStart; 577 extern PVOID MiSystemViewStart; 578 extern SIZE_T MmSystemViewSize; 579 extern PVOID MmSessionBase; 580 extern PVOID MiSessionSpaceEnd; 581 extern PMMPTE MiSessionImagePteStart; 582 extern PMMPTE MiSessionImagePteEnd; 583 extern PMMPTE MiSessionBasePte; 584 extern PMMPTE MiSessionLastPte; 585 extern SIZE_T MmSizeOfPagedPoolInBytes; 586 extern PMMPDE MmSystemPagePtes; 587 extern PVOID MmSystemCacheStart; 588 extern PVOID MmSystemCacheEnd; 589 extern MMSUPPORT MmSystemCacheWs; 590 extern SIZE_T MmAllocatedNonPagedPool; 591 extern ULONG MmSpecialPoolTag; 592 extern PVOID MmHyperSpaceEnd; 593 extern PMMWSL MmSystemCacheWorkingSetList; 594 extern SIZE_T MmMinimumNonPagedPoolSize; 595 extern ULONG MmMinAdditionNonPagedPoolPerMb; 596 extern SIZE_T MmDefaultMaximumNonPagedPool; 597 extern ULONG MmMaxAdditionNonPagedPoolPerMb; 598 extern ULONG MmSecondaryColors; 599 extern ULONG MmSecondaryColorMask; 600 extern ULONG MmNumberOfSystemPtes; 601 extern ULONG MmMaximumNonPagedPoolPercent; 602 extern ULONG MmLargeStackSize; 603 extern PMMCOLOR_TABLES MmFreePagesByColor[FreePageList + 1]; 604 extern MMPFNLIST MmStandbyPageListByPriority[8]; 605 extern ULONG MmProductType; 606 extern MM_SYSTEMSIZE MmSystemSize; 607 extern PKEVENT MiLowMemoryEvent; 608 extern PKEVENT MiHighMemoryEvent; 609 extern PKEVENT MiLowPagedPoolEvent; 610 extern PKEVENT MiHighPagedPoolEvent; 611 extern PKEVENT MiLowNonPagedPoolEvent; 612 extern PKEVENT MiHighNonPagedPoolEvent; 613 extern PFN_NUMBER MmLowMemoryThreshold; 614 extern PFN_NUMBER MmHighMemoryThreshold; 615 extern PFN_NUMBER MiLowPagedPoolThreshold; 616 extern PFN_NUMBER MiHighPagedPoolThreshold; 617 extern PFN_NUMBER MiLowNonPagedPoolThreshold; 618 extern PFN_NUMBER MiHighNonPagedPoolThreshold; 619 extern PFN_NUMBER MmMinimumFreePages; 620 extern PFN_NUMBER MmPlentyFreePages; 621 extern SIZE_T MmMinimumStackCommitInBytes; 622 extern PFN_COUNT MiExpansionPoolPagesInitialCharge; 623 extern PFN_NUMBER MmResidentAvailableAtInit; 624 extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes]; 625 extern PFN_NUMBER MmTotalSystemDriverPages; 626 extern ULONG MmCritsectTimeoutSeconds; 627 extern PVOID MiSessionImageStart; 628 extern PVOID MiSessionImageEnd; 629 extern PMMPTE MiHighestUserPte; 630 extern PMMPDE MiHighestUserPde; 631 extern PFN_NUMBER MmSystemPageDirectory[PPE_PER_PAGE]; 632 extern PMMPTE MmSharedUserDataPte; 633 extern LIST_ENTRY MmProcessList; 634 extern KEVENT MmZeroingPageEvent; 635 extern ULONG MmSystemPageColor; 636 extern ULONG MmProcessColorSeed; 637 extern PMMWSL MmWorkingSetList; 638 extern PFN_NUMBER MiNumberOfFreePages; 639 extern SIZE_T MmSessionViewSize; 640 extern SIZE_T MmSessionPoolSize; 641 extern SIZE_T MmSessionImageSize; 642 extern PVOID MiSystemViewStart; 643 extern PVOID MiSessionPoolEnd; // 0xBE000000 644 extern PVOID MiSessionPoolStart; // 0xBD000000 645 extern PVOID MiSessionViewStart; // 0xBE000000 646 extern PVOID MiSessionSpaceWs; 647 extern ULONG MmMaximumDeadKernelStacks; 648 extern SLIST_HEADER MmDeadStackSListHead; 649 extern MM_AVL_TABLE MmSectionBasedRoot; 650 extern KGUARDED_MUTEX MmSectionBasedMutex; 651 extern PVOID MmHighSectionBase; 652 extern SIZE_T MmSystemLockPagesCount; 653 extern ULONG_PTR MmSubsectionBase; 654 extern LARGE_INTEGER MmCriticalSectionTimeout; 655 extern LIST_ENTRY MmWorkingSetExpansionHead; 656 extern KSPIN_LOCK MmExpansionLock; 657 extern PETHREAD MiExpansionLockOwner; 658 659 FORCEINLINE 660 BOOLEAN 661 MI_IS_PROCESS_WORKING_SET(PMMSUPPORT WorkingSet) 662 { 663 return (WorkingSet != &MmSystemCacheWs) && !WorkingSet->Flags.SessionSpace; 664 } 665 666 FORCEINLINE 667 BOOLEAN 668 MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType) 669 { 670 return ((MemoryType == LoaderFree) || 671 (MemoryType == LoaderLoadedProgram) || 672 (MemoryType == LoaderFirmwareTemporary) || 673 (MemoryType == LoaderOsloaderStack)); 674 } 675 676 FORCEINLINE 677 BOOLEAN 678 MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType) 679 { 680 return ((MemoryType == LoaderFirmwarePermanent) || 681 (MemoryType == LoaderSpecialMemory) || 682 (MemoryType == LoaderHALCachedMemory) || 683 (MemoryType == LoaderBBTMemory)); 684 } 685 686 #ifdef _M_AMD64 687 FORCEINLINE 688 BOOLEAN 689 MiIsUserPxe(PVOID Address) 690 { 691 return ((ULONG_PTR)Address >> 7) == 0x1FFFFEDF6FB7DA0ULL; 692 } 693 694 FORCEINLINE 695 BOOLEAN 696 MiIsUserPpe(PVOID Address) 697 { 698 return ((ULONG_PTR)Address >> 16) == 0xFFFFF6FB7DA0ULL; 699 } 700 701 FORCEINLINE 702 BOOLEAN 703 MiIsUserPde(PVOID Address) 704 { 705 return ((ULONG_PTR)Address >> 25) == 0x7FFFFB7DA0ULL; 706 } 707 708 FORCEINLINE 709 BOOLEAN 710 MiIsUserPte(PVOID Address) 711 { 712 return ((ULONG_PTR)Address >> 34) == 0x3FFFFDA0ULL; 713 } 714 #else 715 FORCEINLINE 716 BOOLEAN 717 MiIsUserPde(PVOID Address) 718 { 719 return ((Address >= (PVOID)MiAddressToPde(NULL)) && 720 (Address <= (PVOID)MiHighestUserPde)); 721 } 722 723 FORCEINLINE 724 BOOLEAN 725 MiIsUserPte(PVOID Address) 726 { 727 return (Address >= (PVOID)PTE_BASE) && (Address <= (PVOID)MiHighestUserPte); 728 } 729 #endif 730 731 // 732 // Figures out the hardware bits for a PTE 733 // 734 FORCEINLINE 735 ULONG_PTR 736 MiDetermineUserGlobalPteMask(IN PVOID PointerPte) 737 { 738 MMPTE TempPte; 739 740 /* Start fresh */ 741 TempPte.u.Long = 0; 742 743 /* Make it valid and accessed */ 744 TempPte.u.Hard.Valid = TRUE; 745 MI_MAKE_ACCESSED_PAGE(&TempPte); 746 747 /* Is this for user-mode? */ 748 if ( 749 #if (_MI_PAGING_LEVELS == 4) 750 MiIsUserPxe(PointerPte) || 751 #endif 752 #if (_MI_PAGING_LEVELS >= 3) 753 MiIsUserPpe(PointerPte) || 754 #endif 755 MiIsUserPde(PointerPte) || 756 MiIsUserPte(PointerPte)) 757 { 758 /* Set the owner bit */ 759 MI_MAKE_OWNER_PAGE(&TempPte); 760 } 761 762 /* FIXME: We should also set the global bit */ 763 764 /* Return the protection */ 765 return TempPte.u.Long; 766 } 767 768 // 769 // Creates a valid kernel PTE with the given protection 770 // 771 FORCEINLINE 772 VOID 773 MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte, 774 IN PMMPTE MappingPte, 775 IN ULONG_PTR ProtectionMask, 776 IN PFN_NUMBER PageFrameNumber) 777 { 778 /* Only valid for kernel, non-session PTEs */ 779 ASSERT(MappingPte > MiHighestUserPte); 780 ASSERT(!MI_IS_SESSION_PTE(MappingPte)); 781 ASSERT((MappingPte < (PMMPTE)PDE_BASE) || (MappingPte > (PMMPTE)PDE_TOP)); 782 783 /* Check that we are not setting valid a page that should not be */ 784 ASSERT(ProtectionMask & MM_PROTECT_ACCESS); 785 ASSERT((ProtectionMask & MM_GUARDPAGE) == 0); 786 787 /* Start fresh */ 788 NewPte->u.Long = 0; 789 790 /* Set the protection and page */ 791 NewPte->u.Hard.PageFrameNumber = PageFrameNumber; 792 NewPte->u.Long |= MmProtectToPteMask[ProtectionMask]; 793 794 /* Make this valid & global */ 795 #ifdef _GLOBAL_PAGES_ARE_AWESOME_ 796 if (KeFeatureBits & KF_GLOBAL_PAGE) 797 NewPte->u.Hard.Global = 1; 798 #endif 799 NewPte->u.Hard.Valid = 1; 800 } 801 802 // 803 // Creates a valid PTE with the given protection 804 // 805 FORCEINLINE 806 VOID 807 MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte, 808 IN PMMPTE MappingPte, 809 IN ULONG_PTR ProtectionMask, 810 IN PFN_NUMBER PageFrameNumber) 811 { 812 /* Check that we are not setting valid a page that should not be */ 813 ASSERT(ProtectionMask & MM_PROTECT_ACCESS); 814 ASSERT((ProtectionMask & MM_GUARDPAGE) == 0); 815 816 /* Set the protection and page */ 817 NewPte->u.Long = MiDetermineUserGlobalPteMask(MappingPte); 818 NewPte->u.Long |= MmProtectToPteMask[ProtectionMask]; 819 NewPte->u.Hard.PageFrameNumber = PageFrameNumber; 820 } 821 822 // 823 // Creates a valid user PTE with the given protection 824 // 825 FORCEINLINE 826 VOID 827 MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte, 828 IN PMMPTE MappingPte, 829 IN ULONG_PTR ProtectionMask, 830 IN PFN_NUMBER PageFrameNumber) 831 { 832 /* Only valid for kernel, non-session PTEs */ 833 ASSERT(MappingPte <= MiHighestUserPte); 834 835 /* Start fresh */ 836 NewPte->u.Long = 0; 837 838 /* Check that we are not setting valid a page that should not be */ 839 ASSERT(ProtectionMask & MM_PROTECT_ACCESS); 840 ASSERT((ProtectionMask & MM_GUARDPAGE) == 0); 841 842 NewPte->u.Hard.Valid = TRUE; 843 NewPte->u.Hard.Owner = TRUE; 844 NewPte->u.Hard.PageFrameNumber = PageFrameNumber; 845 NewPte->u.Long |= MmProtectToPteMask[ProtectionMask]; 846 } 847 848 #ifndef _M_AMD64 849 // 850 // Builds a Prototype PTE for the address of the PTE 851 // 852 FORCEINLINE 853 VOID 854 MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte, 855 IN PMMPTE PointerPte) 856 { 857 ULONG_PTR Offset; 858 859 /* Mark this as a prototype */ 860 NewPte->u.Long = 0; 861 NewPte->u.Proto.Prototype = 1; 862 863 /* 864 * Prototype PTEs are only valid in paged pool by design, this little trick 865 * lets us only use 30 bits for the address of the PTE, as long as the area 866 * stays 1024MB At most. 867 */ 868 Offset = (ULONG_PTR)PointerPte - (ULONG_PTR)MmPagedPoolStart; 869 870 /* 871 * 7 bits go in the "low" (but we assume the bottom 2 are zero) 872 * and the other 21 bits go in the "high" 873 */ 874 NewPte->u.Proto.ProtoAddressLow = (Offset & 0x1FC) >> 2; 875 NewPte->u.Proto.ProtoAddressHigh = (Offset & 0x3FFFFE00) >> 9; 876 } 877 878 // 879 // Builds a Subsection PTE for the address of the Segment 880 // 881 FORCEINLINE 882 VOID 883 MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte, 884 IN PVOID Segment) 885 { 886 ULONG_PTR Offset; 887 888 /* Mark this as a prototype */ 889 NewPte->u.Long = 0; 890 NewPte->u.Subsect.Prototype = 1; 891 892 /* 893 * Segments are only valid either in nonpaged pool. We store the 20 bit 894 * difference either from the top or bottom of nonpaged pool, giving a 895 * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed 896 * 256MB. 897 */ 898 if ((ULONG_PTR)Segment < ((ULONG_PTR)MmSubsectionBase + (128 * _1MB))) 899 { 900 Offset = (ULONG_PTR)Segment - (ULONG_PTR)MmSubsectionBase; 901 NewPte->u.Subsect.WhichPool = PagedPool; 902 } 903 else 904 { 905 Offset = (ULONG_PTR)MmNonPagedPoolEnd - (ULONG_PTR)Segment; 906 NewPte->u.Subsect.WhichPool = NonPagedPool; 907 } 908 909 /* 910 * 4 bits go in the "low" (but we assume the bottom 3 are zero) 911 * and the other 20 bits go in the "high" 912 */ 913 NewPte->u.Subsect.SubsectionAddressLow = (Offset & 0x78) >> 3; 914 NewPte->u.Subsect.SubsectionAddressHigh = (Offset & 0xFFFFF80) >> 7; 915 } 916 917 FORCEINLINE 918 BOOLEAN 919 MI_IS_MAPPED_PTE(PMMPTE PointerPte) 920 { 921 /// \todo Make this reasonable code, this is UGLY! 922 return ((PointerPte->u.Long & 0xFFFFFC01) != 0); 923 } 924 925 #endif 926 927 FORCEINLINE 928 VOID 929 MI_MAKE_TRANSITION_PTE(_Out_ PMMPTE NewPte, 930 _In_ PFN_NUMBER Page, 931 _In_ ULONG Protection) 932 { 933 NewPte->u.Long = 0; 934 NewPte->u.Trans.Transition = 1; 935 NewPte->u.Trans.Protection = Protection; 936 NewPte->u.Trans.PageFrameNumber = Page; 937 } 938 939 // 940 // Returns if the page is physically resident (ie: a large page) 941 // FIXFIX: CISC/x86 only? 942 // 943 FORCEINLINE 944 BOOLEAN 945 MI_IS_PHYSICAL_ADDRESS(IN PVOID Address) 946 { 947 PMMPDE PointerPde; 948 949 /* Large pages are never paged out, always physically resident */ 950 PointerPde = MiAddressToPde(Address); 951 return ((PointerPde->u.Hard.LargePage) && (PointerPde->u.Hard.Valid)); 952 } 953 954 // 955 // Writes a valid PTE 956 // 957 FORCEINLINE 958 VOID 959 MI_WRITE_VALID_PTE(IN PMMPTE PointerPte, 960 IN MMPTE TempPte) 961 { 962 /* Write the valid PTE */ 963 ASSERT(PointerPte->u.Hard.Valid == 0); 964 ASSERT(TempPte.u.Hard.Valid == 1); 965 #if _M_AMD64 966 ASSERT(!MI_IS_PAGE_TABLE_ADDRESS(MiPteToAddress(PointerPte)) || 967 (TempPte.u.Hard.NoExecute == 0)); 968 #endif 969 *PointerPte = TempPte; 970 } 971 972 // 973 // Updates a valid PTE 974 // 975 FORCEINLINE 976 VOID 977 MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte, 978 IN MMPTE TempPte) 979 { 980 /* Write the valid PTE */ 981 ASSERT(PointerPte->u.Hard.Valid == 1); 982 ASSERT(TempPte.u.Hard.Valid == 1); 983 ASSERT(PointerPte->u.Hard.PageFrameNumber == TempPte.u.Hard.PageFrameNumber); 984 *PointerPte = TempPte; 985 } 986 987 // 988 // Writes an invalid PTE 989 // 990 FORCEINLINE 991 VOID 992 MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte, 993 IN MMPTE InvalidPte) 994 { 995 /* Write the invalid PTE */ 996 ASSERT(InvalidPte.u.Hard.Valid == 0); 997 ASSERT(InvalidPte.u.Long != 0); 998 *PointerPte = InvalidPte; 999 } 1000 1001 // 1002 // Erase the PTE completely 1003 // 1004 FORCEINLINE 1005 VOID 1006 MI_ERASE_PTE(IN PMMPTE PointerPte) 1007 { 1008 /* Zero out the PTE */ 1009 ASSERT(PointerPte->u.Long != 0); 1010 PointerPte->u.Long = 0; 1011 } 1012 1013 // 1014 // Writes a valid PDE 1015 // 1016 FORCEINLINE 1017 VOID 1018 MI_WRITE_VALID_PDE(IN PMMPDE PointerPde, 1019 IN MMPDE TempPde) 1020 { 1021 /* Write the valid PDE */ 1022 ASSERT(PointerPde->u.Hard.Valid == 0); 1023 #ifdef _M_AMD64 1024 ASSERT(PointerPde->u.Hard.NoExecute == 0); 1025 #endif 1026 ASSERT(TempPde.u.Hard.Valid == 1); 1027 *PointerPde = TempPde; 1028 } 1029 1030 // 1031 // Writes an invalid PDE 1032 // 1033 FORCEINLINE 1034 VOID 1035 MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde, 1036 IN MMPDE InvalidPde) 1037 { 1038 /* Write the invalid PDE */ 1039 ASSERT(InvalidPde.u.Hard.Valid == 0); 1040 ASSERT(InvalidPde.u.Long != 0); 1041 #ifdef _M_AMD64 1042 ASSERT(InvalidPde.u.Soft.Protection == MM_EXECUTE_READWRITE); 1043 #endif 1044 *PointerPde = InvalidPde; 1045 } 1046 1047 // 1048 // Checks if the thread already owns a working set 1049 // 1050 FORCEINLINE 1051 BOOLEAN 1052 MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread) 1053 { 1054 /* If any of these are held, return TRUE */ 1055 return ((Thread->OwnsProcessWorkingSetExclusive) || 1056 (Thread->OwnsProcessWorkingSetShared) || 1057 (Thread->OwnsSystemWorkingSetExclusive) || 1058 (Thread->OwnsSystemWorkingSetShared) || 1059 (Thread->OwnsSessionWorkingSetExclusive) || 1060 (Thread->OwnsSessionWorkingSetShared)); 1061 } 1062 1063 FORCEINLINE 1064 BOOLEAN 1065 MM_ANY_WS_LOCK_HELD_EXCLUSIVE(_In_ PETHREAD Thread) 1066 { 1067 return ((Thread->OwnsProcessWorkingSetExclusive) || 1068 (Thread->OwnsSystemWorkingSetExclusive) || 1069 (Thread->OwnsSessionWorkingSetExclusive)); 1070 } 1071 1072 // 1073 // Checks if the process owns the working set lock 1074 // 1075 FORCEINLINE 1076 BOOLEAN 1077 MI_WS_OWNER(IN PEPROCESS Process) 1078 { 1079 /* Check if this process is the owner, and that the thread owns the WS */ 1080 if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive == 0) 1081 { 1082 DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread()); 1083 } 1084 if (KeGetCurrentThread()->ApcState.Process != &Process->Pcb) 1085 { 1086 DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process); 1087 } 1088 return ((KeGetCurrentThread()->ApcState.Process == &Process->Pcb) && 1089 ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive) || 1090 (PsGetCurrentThread()->OwnsProcessWorkingSetShared))); 1091 } 1092 1093 // 1094 // New ARM3<->RosMM PAGE Architecture 1095 // 1096 FORCEINLINE 1097 BOOLEAN 1098 MiIsRosSectionObject(IN PSECTION Section) 1099 { 1100 return Section->u.Flags.filler; 1101 } 1102 1103 #define MI_IS_ROS_PFN(x) ((x)->u4.AweAllocation == TRUE) 1104 1105 VOID 1106 NTAPI 1107 MiDecrementReferenceCount( 1108 IN PMMPFN Pfn1, 1109 IN PFN_NUMBER PageFrameIndex 1110 ); 1111 1112 FORCEINLINE 1113 BOOLEAN 1114 MI_IS_WS_UNSAFE(IN PEPROCESS Process) 1115 { 1116 return (Process->Vm.Flags.AcquiredUnsafe == TRUE); 1117 } 1118 1119 // 1120 // Locks the working set for the given process 1121 // 1122 FORCEINLINE 1123 VOID 1124 MiLockProcessWorkingSet(IN PEPROCESS Process, 1125 IN PETHREAD Thread) 1126 { 1127 /* Shouldn't already be owning the process working set */ 1128 ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE); 1129 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); 1130 1131 /* Block APCs, make sure that still nothing is already held */ 1132 KeEnterGuardedRegion(); 1133 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread)); 1134 1135 /* Lock the working set */ 1136 ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex); 1137 1138 /* Now claim that we own the lock */ 1139 ASSERT(!MI_IS_WS_UNSAFE(Process)); 1140 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); 1141 Thread->OwnsProcessWorkingSetExclusive = TRUE; 1142 } 1143 1144 FORCEINLINE 1145 VOID 1146 MiLockProcessWorkingSetShared(IN PEPROCESS Process, 1147 IN PETHREAD Thread) 1148 { 1149 /* Shouldn't already be owning the process working set */ 1150 ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE); 1151 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); 1152 1153 /* Block APCs, make sure that still nothing is already held */ 1154 KeEnterGuardedRegion(); 1155 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread)); 1156 1157 /* Lock the working set */ 1158 ExAcquirePushLockShared(&Process->Vm.WorkingSetMutex); 1159 1160 /* Now claim that we own the lock */ 1161 ASSERT(!MI_IS_WS_UNSAFE(Process)); 1162 ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE); 1163 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); 1164 Thread->OwnsProcessWorkingSetShared = TRUE; 1165 } 1166 1167 FORCEINLINE 1168 VOID 1169 MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process, 1170 IN PETHREAD Thread) 1171 { 1172 /* Shouldn't already be owning the process working set */ 1173 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); 1174 1175 /* APCs must be blocked, make sure that still nothing is already held */ 1176 ASSERT(KeAreAllApcsDisabled() == TRUE); 1177 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread)); 1178 1179 /* Lock the working set */ 1180 ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex); 1181 1182 /* Now claim that we own the lock */ 1183 ASSERT(!MI_IS_WS_UNSAFE(Process)); 1184 Process->Vm.Flags.AcquiredUnsafe = 1; 1185 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); 1186 Thread->OwnsProcessWorkingSetExclusive = TRUE; 1187 } 1188 1189 // 1190 // Unlocks the working set for the given process 1191 // 1192 FORCEINLINE 1193 VOID 1194 MiUnlockProcessWorkingSet(IN PEPROCESS Process, 1195 IN PETHREAD Thread) 1196 { 1197 /* Make sure we are the owner of a safe acquisition */ 1198 ASSERT(MI_WS_OWNER(Process)); 1199 ASSERT(!MI_IS_WS_UNSAFE(Process)); 1200 1201 /* The thread doesn't own it anymore */ 1202 ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE); 1203 Thread->OwnsProcessWorkingSetExclusive = FALSE; 1204 1205 /* Release the lock and re-enable APCs */ 1206 ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex); 1207 KeLeaveGuardedRegion(); 1208 } 1209 1210 // 1211 // Unlocks the working set for the given process 1212 // 1213 FORCEINLINE 1214 VOID 1215 MiUnlockProcessWorkingSetShared(IN PEPROCESS Process, 1216 IN PETHREAD Thread) 1217 { 1218 /* Make sure we are the owner of a safe acquisition (because shared) */ 1219 ASSERT(MI_WS_OWNER(Process)); 1220 ASSERT(!MI_IS_WS_UNSAFE(Process)); 1221 1222 /* Ensure we are in a shared acquisition */ 1223 ASSERT(Thread->OwnsProcessWorkingSetShared == TRUE); 1224 ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); 1225 1226 /* Don't claim the lock anylonger */ 1227 Thread->OwnsProcessWorkingSetShared = FALSE; 1228 1229 /* Release the lock and re-enable APCs */ 1230 ExReleasePushLockShared(&Process->Vm.WorkingSetMutex); 1231 KeLeaveGuardedRegion(); 1232 } 1233 1234 // 1235 // Unlocks the working set for the given process 1236 // 1237 FORCEINLINE 1238 VOID 1239 MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process, 1240 IN PETHREAD Thread) 1241 { 1242 /* Make sure we are the owner of an unsafe acquisition */ 1243 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1244 ASSERT(KeAreAllApcsDisabled() == TRUE); 1245 ASSERT(MI_WS_OWNER(Process)); 1246 ASSERT(MI_IS_WS_UNSAFE(Process)); 1247 1248 /* No longer unsafe */ 1249 Process->Vm.Flags.AcquiredUnsafe = 0; 1250 1251 /* The thread doesn't own it anymore */ 1252 ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE); 1253 Thread->OwnsProcessWorkingSetExclusive = FALSE; 1254 1255 /* Release the lock but don't touch APC state */ 1256 ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex); 1257 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1258 } 1259 1260 // 1261 // Locks the working set 1262 // 1263 FORCEINLINE 1264 VOID 1265 MiLockWorkingSet(IN PETHREAD Thread, 1266 IN PMMSUPPORT WorkingSet) 1267 { 1268 /* Block APCs */ 1269 KeEnterGuardedRegion(); 1270 1271 /* Working set should be in global memory */ 1272 ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE); 1273 1274 /* Thread shouldn't already be owning something */ 1275 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread)); 1276 1277 /* Lock this working set */ 1278 ExAcquirePushLockExclusive(&WorkingSet->WorkingSetMutex); 1279 1280 /* Which working set is this? */ 1281 if (WorkingSet == &MmSystemCacheWs) 1282 { 1283 /* Own the system working set */ 1284 ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) && 1285 (Thread->OwnsSystemWorkingSetShared == FALSE)); 1286 Thread->OwnsSystemWorkingSetExclusive = TRUE; 1287 } 1288 else if (WorkingSet->Flags.SessionSpace) 1289 { 1290 /* Own the session working set */ 1291 ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) && 1292 (Thread->OwnsSessionWorkingSetShared == FALSE)); 1293 Thread->OwnsSessionWorkingSetExclusive = TRUE; 1294 } 1295 else 1296 { 1297 /* Own the process working set */ 1298 ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) && 1299 (Thread->OwnsProcessWorkingSetShared == FALSE)); 1300 Thread->OwnsProcessWorkingSetExclusive = TRUE; 1301 } 1302 } 1303 1304 FORCEINLINE 1305 VOID 1306 MiLockWorkingSetShared( 1307 _In_ PETHREAD Thread, 1308 _In_ PMMSUPPORT WorkingSet) 1309 { 1310 /* Block APCs */ 1311 KeEnterGuardedRegion(); 1312 1313 /* Working set should be in global memory */ 1314 ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE); 1315 1316 /* Thread shouldn't already be owning something */ 1317 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread)); 1318 1319 /* Lock this working set */ 1320 ExAcquirePushLockShared(&WorkingSet->WorkingSetMutex); 1321 1322 /* Which working set is this? */ 1323 if (WorkingSet == &MmSystemCacheWs) 1324 { 1325 /* Own the system working set */ 1326 ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) && 1327 (Thread->OwnsSystemWorkingSetShared == FALSE)); 1328 Thread->OwnsSystemWorkingSetShared = TRUE; 1329 } 1330 else if (WorkingSet->Flags.SessionSpace) 1331 { 1332 /* Own the session working set */ 1333 ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) && 1334 (Thread->OwnsSessionWorkingSetShared == FALSE)); 1335 Thread->OwnsSessionWorkingSetShared = TRUE; 1336 } 1337 else 1338 { 1339 /* Own the process working set */ 1340 ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) && 1341 (Thread->OwnsProcessWorkingSetShared == FALSE)); 1342 Thread->OwnsProcessWorkingSetShared = TRUE; 1343 } 1344 } 1345 1346 // 1347 // Unlocks the working set 1348 // 1349 FORCEINLINE 1350 VOID 1351 MiUnlockWorkingSet(IN PETHREAD Thread, 1352 IN PMMSUPPORT WorkingSet) 1353 { 1354 /* Working set should be in global memory */ 1355 ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE); 1356 1357 /* Which working set is this? */ 1358 if (WorkingSet == &MmSystemCacheWs) 1359 { 1360 /* Release the system working set */ 1361 ASSERT((Thread->OwnsSystemWorkingSetExclusive == TRUE) && 1362 (Thread->OwnsSystemWorkingSetShared == FALSE)); 1363 Thread->OwnsSystemWorkingSetExclusive = FALSE; 1364 } 1365 else if (WorkingSet->Flags.SessionSpace) 1366 { 1367 /* Release the session working set */ 1368 ASSERT((Thread->OwnsSessionWorkingSetExclusive == TRUE) && 1369 (Thread->OwnsSessionWorkingSetShared == FALSE)); 1370 Thread->OwnsSessionWorkingSetExclusive = FALSE; 1371 } 1372 else 1373 { 1374 /* Release the process working set */ 1375 ASSERT((Thread->OwnsProcessWorkingSetExclusive == TRUE) && 1376 (Thread->OwnsProcessWorkingSetShared == FALSE)); 1377 Thread->OwnsProcessWorkingSetExclusive = FALSE; 1378 } 1379 1380 /* Release the working set lock */ 1381 ExReleasePushLockExclusive(&WorkingSet->WorkingSetMutex); 1382 1383 /* Unblock APCs */ 1384 KeLeaveGuardedRegion(); 1385 } 1386 1387 FORCEINLINE 1388 VOID 1389 MiUnlockWorkingSetShared( 1390 _In_ PETHREAD Thread, 1391 _In_ PMMSUPPORT WorkingSet) 1392 { 1393 /* Working set should be in global memory */ 1394 ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE); 1395 1396 /* Which working set is this? */ 1397 if (WorkingSet == &MmSystemCacheWs) 1398 { 1399 /* Release the system working set */ 1400 ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) && 1401 (Thread->OwnsSystemWorkingSetShared == TRUE)); 1402 Thread->OwnsSystemWorkingSetShared = FALSE; 1403 } 1404 else if (WorkingSet->Flags.SessionSpace) 1405 { 1406 /* Release the session working set */ 1407 ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) && 1408 (Thread->OwnsSessionWorkingSetShared == TRUE)); 1409 Thread->OwnsSessionWorkingSetShared = FALSE; 1410 } 1411 else 1412 { 1413 /* Release the process working set */ 1414 ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) && 1415 (Thread->OwnsProcessWorkingSetShared == TRUE)); 1416 Thread->OwnsProcessWorkingSetShared = FALSE; 1417 } 1418 1419 /* Release the working set lock */ 1420 ExReleasePushLockShared(&WorkingSet->WorkingSetMutex); 1421 1422 /* Unblock APCs */ 1423 KeLeaveGuardedRegion(); 1424 } 1425 1426 FORCEINLINE 1427 BOOLEAN 1428 MiConvertSharedWorkingSetLockToExclusive( 1429 _In_ PETHREAD Thread, 1430 _In_ PMMSUPPORT Vm) 1431 { 1432 /* Sanity check: No exclusive lock. */ 1433 ASSERT(!Thread->OwnsProcessWorkingSetExclusive); 1434 ASSERT(!Thread->OwnsSessionWorkingSetExclusive); 1435 ASSERT(!Thread->OwnsSystemWorkingSetExclusive); 1436 1437 /* And it should have one and only one shared lock */ 1438 ASSERT((Thread->OwnsProcessWorkingSetShared + Thread->OwnsSessionWorkingSetShared + Thread->OwnsSystemWorkingSetShared) == 1); 1439 1440 /* Try. */ 1441 if (!ExConvertPushLockSharedToExclusive(&Vm->WorkingSetMutex)) 1442 return FALSE; 1443 1444 if (Vm == &MmSystemCacheWs) 1445 { 1446 ASSERT(Thread->OwnsSystemWorkingSetShared); 1447 Thread->OwnsSystemWorkingSetShared = FALSE; 1448 Thread->OwnsSystemWorkingSetExclusive = TRUE; 1449 } 1450 else if (Vm->Flags.SessionSpace) 1451 { 1452 ASSERT(Thread->OwnsSessionWorkingSetShared); 1453 Thread->OwnsSessionWorkingSetShared = FALSE; 1454 Thread->OwnsSessionWorkingSetExclusive = TRUE; 1455 } 1456 else 1457 { 1458 ASSERT(Thread->OwnsProcessWorkingSetShared); 1459 Thread->OwnsProcessWorkingSetShared = FALSE; 1460 Thread->OwnsProcessWorkingSetExclusive = TRUE; 1461 } 1462 1463 return TRUE; 1464 } 1465 1466 FORCEINLINE 1467 VOID 1468 MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process, 1469 IN PETHREAD Thread, 1470 OUT PBOOLEAN Safe, 1471 OUT PBOOLEAN Shared) 1472 { 1473 ASSERT(MI_WS_OWNER(Process)); 1474 1475 /* Check if the current owner is unsafe */ 1476 if (MI_IS_WS_UNSAFE(Process)) 1477 { 1478 /* Release unsafely */ 1479 MiUnlockProcessWorkingSetUnsafe(Process, Thread); 1480 *Safe = FALSE; 1481 *Shared = FALSE; 1482 } 1483 else if (Thread->OwnsProcessWorkingSetExclusive == 1) 1484 { 1485 /* Owner is safe and exclusive, release normally */ 1486 MiUnlockProcessWorkingSet(Process, Thread); 1487 *Safe = TRUE; 1488 *Shared = FALSE; 1489 } 1490 else 1491 { 1492 /* Owner is shared (implies safe), release normally */ 1493 MiUnlockProcessWorkingSetShared(Process, Thread); 1494 *Safe = TRUE; 1495 *Shared = TRUE; 1496 } 1497 } 1498 1499 FORCEINLINE 1500 VOID 1501 MiLockProcessWorkingSetForFault(IN PEPROCESS Process, 1502 IN PETHREAD Thread, 1503 IN BOOLEAN Safe, 1504 IN BOOLEAN Shared) 1505 { 1506 /* Check if this was a safe lock or not */ 1507 if (Safe) 1508 { 1509 if (Shared) 1510 { 1511 /* Reacquire safely & shared */ 1512 MiLockProcessWorkingSetShared(Process, Thread); 1513 } 1514 else 1515 { 1516 /* Reacquire safely */ 1517 MiLockProcessWorkingSet(Process, Thread); 1518 } 1519 } 1520 else 1521 { 1522 /* Unsafe lock cannot be shared */ 1523 ASSERT(Shared == FALSE); 1524 /* Reacquire unsafely */ 1525 MiLockProcessWorkingSetUnsafe(Process, Thread); 1526 } 1527 } 1528 1529 FORCEINLINE 1530 KIRQL 1531 MiAcquireExpansionLock(VOID) 1532 { 1533 KIRQL OldIrql; 1534 1535 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1536 KeAcquireSpinLock(&MmExpansionLock, &OldIrql); 1537 ASSERT(MiExpansionLockOwner == NULL); 1538 MiExpansionLockOwner = PsGetCurrentThread(); 1539 return OldIrql; 1540 } 1541 1542 FORCEINLINE 1543 VOID 1544 MiReleaseExpansionLock(KIRQL OldIrql) 1545 { 1546 ASSERT(MiExpansionLockOwner == PsGetCurrentThread()); 1547 MiExpansionLockOwner = NULL; 1548 KeReleaseSpinLock(&MmExpansionLock, OldIrql); 1549 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); 1550 } 1551 1552 // 1553 // Returns the ProtoPTE inside a VAD for the given VPN 1554 // 1555 FORCEINLINE 1556 PMMPTE 1557 MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad, 1558 IN ULONG_PTR Vpn) 1559 { 1560 PMMPTE ProtoPte; 1561 1562 /* Find the offset within the VAD's prototype PTEs */ 1563 ProtoPte = Vad->FirstPrototypePte + (Vpn - Vad->StartingVpn); 1564 ASSERT(ProtoPte <= Vad->LastContiguousPte); 1565 return ProtoPte; 1566 } 1567 1568 // 1569 // Returns the PFN Database entry for the given page number 1570 // Warning: This is not necessarily a valid PFN database entry! 1571 // 1572 FORCEINLINE 1573 PMMPFN 1574 MI_PFN_ELEMENT(IN PFN_NUMBER Pfn) 1575 { 1576 /* Get the entry */ 1577 return &MmPfnDatabase[Pfn]; 1578 }; 1579 1580 // 1581 // Drops a locked page without dereferencing it 1582 // 1583 FORCEINLINE 1584 VOID 1585 MiDropLockCount(IN PMMPFN Pfn1) 1586 { 1587 /* This page shouldn't be locked, but it should be valid */ 1588 ASSERT(Pfn1->u3.e2.ReferenceCount != 0); 1589 ASSERT(Pfn1->u2.ShareCount == 0); 1590 1591 /* Is this the last reference to the page */ 1592 if (Pfn1->u3.e2.ReferenceCount == 1) 1593 { 1594 /* It better not be valid */ 1595 ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid); 1596 1597 /* Is it a prototype PTE? */ 1598 if ((Pfn1->u3.e1.PrototypePte == 1) && 1599 (Pfn1->OriginalPte.u.Soft.Prototype == 1)) 1600 { 1601 /* FIXME: We should return commit */ 1602 DPRINT1("Not returning commit for prototype PTE\n"); 1603 } 1604 1605 /* Update the counter */ 1606 InterlockedDecrementSizeT(&MmSystemLockPagesCount); 1607 } 1608 } 1609 1610 // 1611 // Drops a locked page and dereferences it 1612 // 1613 FORCEINLINE 1614 VOID 1615 MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1) 1616 { 1617 USHORT RefCount, OldRefCount; 1618 PFN_NUMBER PageFrameIndex; 1619 1620 /* Loop while we decrement the page successfully */ 1621 do 1622 { 1623 /* There should be at least one reference */ 1624 OldRefCount = Pfn1->u3.e2.ReferenceCount; 1625 ASSERT(OldRefCount != 0); 1626 1627 /* Are we the last one */ 1628 if (OldRefCount == 1) 1629 { 1630 /* The page shoudln't be shared not active at this point */ 1631 ASSERT(Pfn1->u3.e2.ReferenceCount == 1); 1632 ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid); 1633 ASSERT(Pfn1->u2.ShareCount == 0); 1634 1635 /* Is it a prototype PTE? */ 1636 if ((Pfn1->u3.e1.PrototypePte == 1) && 1637 (Pfn1->OriginalPte.u.Soft.Prototype == 1)) 1638 { 1639 /* FIXME: We should return commit */ 1640 DPRINT1("Not returning commit for prototype PTE\n"); 1641 } 1642 1643 /* Update the counter, and drop a reference the long way */ 1644 InterlockedDecrementSizeT(&MmSystemLockPagesCount); 1645 PageFrameIndex = MiGetPfnEntryIndex(Pfn1); 1646 MiDecrementReferenceCount(Pfn1, PageFrameIndex); 1647 return; 1648 } 1649 1650 /* Drop a reference the short way, and that's it */ 1651 RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount, 1652 OldRefCount - 1, 1653 OldRefCount); 1654 ASSERT(RefCount != 0); 1655 } while (OldRefCount != RefCount); 1656 1657 /* If we got here, there should be more than one reference */ 1658 ASSERT(RefCount > 1); 1659 if (RefCount == 2) 1660 { 1661 /* Is it still being shared? */ 1662 if (Pfn1->u2.ShareCount >= 1) 1663 { 1664 /* Then it should be valid */ 1665 ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid); 1666 1667 /* Is it a prototype PTE? */ 1668 if ((Pfn1->u3.e1.PrototypePte == 1) && 1669 (Pfn1->OriginalPte.u.Soft.Prototype == 1)) 1670 { 1671 /* We don't handle ethis */ 1672 ASSERT(FALSE); 1673 } 1674 1675 /* Update the counter */ 1676 InterlockedDecrementSizeT(&MmSystemLockPagesCount); 1677 } 1678 } 1679 } 1680 1681 // 1682 // References a locked page and updates the counter 1683 // Used in MmProbeAndLockPages to handle different edge cases 1684 // 1685 FORCEINLINE 1686 VOID 1687 MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1) 1688 { 1689 USHORT RefCount, OldRefCount; 1690 1691 /* Sanity check */ 1692 ASSERT(Pfn1->u3.e2.ReferenceCount != 0); 1693 1694 /* Does ARM3 own the page? */ 1695 if (MI_IS_ROS_PFN(Pfn1)) 1696 { 1697 /* ReactOS Mm doesn't track share count */ 1698 ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid); 1699 } 1700 else 1701 { 1702 /* On ARM3 pages, we should see a valid share count */ 1703 ASSERT((Pfn1->u2.ShareCount != 0) && (Pfn1->u3.e1.PageLocation == ActiveAndValid)); 1704 1705 /* Is it a prototype PTE? */ 1706 if ((Pfn1->u3.e1.PrototypePte == 1) && 1707 (Pfn1->OriginalPte.u.Soft.Prototype == 1)) 1708 { 1709 /* FIXME: We should charge commit */ 1710 DPRINT1("Not charging commit for prototype PTE\n"); 1711 } 1712 } 1713 1714 /* More locked pages! */ 1715 InterlockedIncrementSizeT(&MmSystemLockPagesCount); 1716 1717 /* Loop trying to update the reference count */ 1718 do 1719 { 1720 /* Get the current reference count, make sure it's valid */ 1721 OldRefCount = Pfn1->u3.e2.ReferenceCount; 1722 ASSERT(OldRefCount != 0); 1723 ASSERT(OldRefCount < 2500); 1724 1725 /* Bump it up by one */ 1726 RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount, 1727 OldRefCount + 1, 1728 OldRefCount); 1729 ASSERT(RefCount != 0); 1730 } while (OldRefCount != RefCount); 1731 1732 /* Was this the first lock attempt? If not, undo our bump */ 1733 if (OldRefCount != 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount); 1734 } 1735 1736 // 1737 // References a locked page and updates the counter 1738 // Used in all other cases except MmProbeAndLockPages 1739 // 1740 FORCEINLINE 1741 VOID 1742 MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1) 1743 { 1744 USHORT NewRefCount; 1745 1746 /* Is it a prototype PTE? */ 1747 if ((Pfn1->u3.e1.PrototypePte == 1) && 1748 (Pfn1->OriginalPte.u.Soft.Prototype == 1)) 1749 { 1750 /* FIXME: We should charge commit */ 1751 DPRINT1("Not charging commit for prototype PTE\n"); 1752 } 1753 1754 /* More locked pages! */ 1755 InterlockedIncrementSizeT(&MmSystemLockPagesCount); 1756 1757 /* Update the reference count */ 1758 NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount); 1759 if (NewRefCount == 2) 1760 { 1761 /* Is it locked or shared? */ 1762 if (Pfn1->u2.ShareCount) 1763 { 1764 /* It's shared, so make sure it's active */ 1765 ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid); 1766 } 1767 else 1768 { 1769 /* It's locked, so we shouldn't lock again */ 1770 InterlockedDecrementSizeT(&MmSystemLockPagesCount); 1771 } 1772 } 1773 else 1774 { 1775 /* Someone had already locked the page, so undo our bump */ 1776 ASSERT(NewRefCount < 2500); 1777 InterlockedDecrementSizeT(&MmSystemLockPagesCount); 1778 } 1779 } 1780 1781 // 1782 // References a locked page and updates the counter 1783 // Used in all other cases except MmProbeAndLockPages 1784 // 1785 FORCEINLINE 1786 VOID 1787 MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1) 1788 { 1789 USHORT NewRefCount; 1790 1791 /* Make sure the page isn't used yet */ 1792 ASSERT(Pfn1->u2.ShareCount == 0); 1793 ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid); 1794 1795 /* Is it a prototype PTE? */ 1796 if ((Pfn1->u3.e1.PrototypePte == 1) && 1797 (Pfn1->OriginalPte.u.Soft.Prototype == 1)) 1798 { 1799 /* FIXME: We should charge commit */ 1800 DPRINT1("Not charging commit for prototype PTE\n"); 1801 } 1802 1803 /* More locked pages! */ 1804 InterlockedIncrementSizeT(&MmSystemLockPagesCount); 1805 1806 /* Update the reference count */ 1807 NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount); 1808 if (NewRefCount != 1) 1809 { 1810 /* Someone had already locked the page, so undo our bump */ 1811 ASSERT(NewRefCount < 2500); 1812 InterlockedDecrementSizeT(&MmSystemLockPagesCount); 1813 } 1814 } 1815 1816 1817 1818 CODE_SEG("INIT") 1819 BOOLEAN 1820 NTAPI 1821 MmArmInitSystem( 1822 IN ULONG Phase, 1823 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1824 ); 1825 1826 CODE_SEG("INIT") 1827 VOID 1828 NTAPI 1829 MiInitializeSessionSpaceLayout(VOID); 1830 1831 CODE_SEG("INIT") 1832 NTSTATUS 1833 NTAPI 1834 MiInitMachineDependent( 1835 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1836 ); 1837 1838 CODE_SEG("INIT") 1839 VOID 1840 NTAPI 1841 MiComputeColorInformation( 1842 VOID 1843 ); 1844 1845 CODE_SEG("INIT") 1846 VOID 1847 NTAPI 1848 MiMapPfnDatabase( 1849 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1850 ); 1851 1852 CODE_SEG("INIT") 1853 VOID 1854 NTAPI 1855 MiInitializeColorTables( 1856 VOID 1857 ); 1858 1859 CODE_SEG("INIT") 1860 VOID 1861 NTAPI 1862 MiInitializePfnDatabase( 1863 IN PLOADER_PARAMETER_BLOCK LoaderBlock 1864 ); 1865 1866 VOID 1867 NTAPI 1868 MiInitializeSessionWsSupport( 1869 VOID 1870 ); 1871 1872 VOID 1873 NTAPI 1874 MiInitializeSessionIds( 1875 VOID 1876 ); 1877 1878 CODE_SEG("INIT") 1879 BOOLEAN 1880 NTAPI 1881 MiInitializeMemoryEvents( 1882 VOID 1883 ); 1884 1885 CODE_SEG("INIT") 1886 PFN_NUMBER 1887 NTAPI 1888 MxGetNextPage( 1889 IN PFN_NUMBER PageCount 1890 ); 1891 1892 CODE_SEG("INIT") 1893 PPHYSICAL_MEMORY_DESCRIPTOR 1894 NTAPI 1895 MmInitializeMemoryLimits( 1896 IN PLOADER_PARAMETER_BLOCK LoaderBlock, 1897 IN PBOOLEAN IncludeType 1898 ); 1899 1900 PFN_NUMBER 1901 NTAPI 1902 MiPagesInLoaderBlock( 1903 IN PLOADER_PARAMETER_BLOCK LoaderBlock, 1904 IN PBOOLEAN IncludeType 1905 ); 1906 1907 VOID 1908 FASTCALL 1909 MiSyncARM3WithROS( 1910 IN PVOID AddressStart, 1911 IN PVOID AddressEnd 1912 ); 1913 1914 NTSTATUS 1915 NTAPI 1916 MiRosProtectVirtualMemory( 1917 IN PEPROCESS Process, 1918 IN OUT PVOID *BaseAddress, 1919 IN OUT PSIZE_T NumberOfBytesToProtect, 1920 IN ULONG NewAccessProtection, 1921 OUT PULONG OldAccessProtection OPTIONAL 1922 ); 1923 1924 NTSTATUS 1925 NTAPI 1926 MmArmAccessFault( 1927 IN ULONG FaultCode, 1928 IN PVOID Address, 1929 IN KPROCESSOR_MODE Mode, 1930 IN PVOID TrapInformation 1931 ); 1932 1933 NTSTATUS 1934 FASTCALL 1935 MiCheckPdeForPagedPool( 1936 IN PVOID Address 1937 ); 1938 1939 CODE_SEG("INIT") 1940 VOID 1941 NTAPI 1942 MiInitializeNonPagedPoolThresholds( 1943 VOID 1944 ); 1945 1946 CODE_SEG("INIT") 1947 VOID 1948 NTAPI 1949 MiInitializePoolEvents( 1950 VOID 1951 ); 1952 1953 CODE_SEG("INIT") 1954 VOID // 1955 NTAPI // 1956 InitializePool( // 1957 IN POOL_TYPE PoolType,// FIXFIX: This should go in ex.h after the pool merge 1958 IN ULONG Threshold // 1959 ); // 1960 1961 // FIXFIX: THIS ONE TOO 1962 CODE_SEG("INIT") 1963 VOID 1964 NTAPI 1965 ExInitializePoolDescriptor( 1966 IN PPOOL_DESCRIPTOR PoolDescriptor, 1967 IN POOL_TYPE PoolType, 1968 IN ULONG PoolIndex, 1969 IN ULONG Threshold, 1970 IN PVOID PoolLock 1971 ); 1972 1973 NTSTATUS 1974 NTAPI 1975 MiInitializeSessionPool( 1976 VOID 1977 ); 1978 1979 CODE_SEG("INIT") 1980 VOID 1981 NTAPI 1982 MiInitializeSystemPtes( 1983 IN PMMPTE StartingPte, 1984 IN ULONG NumberOfPtes, 1985 IN MMSYSTEM_PTE_POOL_TYPE PoolType 1986 ); 1987 1988 PMMPTE 1989 NTAPI 1990 MiReserveSystemPtes( 1991 IN ULONG NumberOfPtes, 1992 IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType 1993 ); 1994 1995 VOID 1996 NTAPI 1997 MiReleaseSystemPtes( 1998 IN PMMPTE StartingPte, 1999 IN ULONG NumberOfPtes, 2000 IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType 2001 ); 2002 2003 2004 PFN_NUMBER 2005 NTAPI 2006 MiFindContiguousPages( 2007 IN PFN_NUMBER LowestPfn, 2008 IN PFN_NUMBER HighestPfn, 2009 IN PFN_NUMBER BoundaryPfn, 2010 IN PFN_NUMBER SizeInPages, 2011 IN MEMORY_CACHING_TYPE CacheType 2012 ); 2013 2014 PVOID 2015 NTAPI 2016 MiCheckForContiguousMemory( 2017 IN PVOID BaseAddress, 2018 IN PFN_NUMBER BaseAddressPages, 2019 IN PFN_NUMBER SizeInPages, 2020 IN PFN_NUMBER LowestPfn, 2021 IN PFN_NUMBER HighestPfn, 2022 IN PFN_NUMBER BoundaryPfn, 2023 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute 2024 ); 2025 2026 PMDL 2027 NTAPI 2028 MiAllocatePagesForMdl( 2029 IN PHYSICAL_ADDRESS LowAddress, 2030 IN PHYSICAL_ADDRESS HighAddress, 2031 IN PHYSICAL_ADDRESS SkipBytes, 2032 IN SIZE_T TotalBytes, 2033 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute, 2034 IN ULONG Flags 2035 ); 2036 2037 VOID 2038 NTAPI 2039 MiInsertPageInList( 2040 IN PMMPFNLIST ListHead, 2041 IN PFN_NUMBER PageFrameIndex 2042 ); 2043 2044 VOID 2045 NTAPI 2046 MiUnlinkFreeOrZeroedPage( 2047 IN PMMPFN Entry 2048 ); 2049 2050 VOID 2051 NTAPI 2052 MiUnlinkPageFromList( 2053 IN PMMPFN Pfn 2054 ); 2055 2056 VOID 2057 NTAPI 2058 MiInitializePfn( 2059 IN PFN_NUMBER PageFrameIndex, 2060 IN PMMPTE PointerPte, 2061 IN BOOLEAN Modified 2062 ); 2063 2064 NTSTATUS 2065 NTAPI 2066 MiInitializeAndChargePfn( 2067 OUT PPFN_NUMBER PageFrameIndex, 2068 IN PMMPDE PointerPde, 2069 IN PFN_NUMBER ContainingPageFrame, 2070 IN BOOLEAN SessionAllocation 2071 ); 2072 2073 VOID 2074 NTAPI 2075 MiInitializePfnAndMakePteValid( 2076 IN PFN_NUMBER PageFrameIndex, 2077 IN PMMPTE PointerPte, 2078 IN MMPTE TempPte 2079 ); 2080 2081 VOID 2082 NTAPI 2083 MiInitializePfnForOtherProcess( 2084 IN PFN_NUMBER PageFrameIndex, 2085 IN PVOID PteAddress, 2086 IN PFN_NUMBER PteFrame 2087 ); 2088 2089 VOID 2090 NTAPI 2091 MiDecrementShareCount( 2092 IN PMMPFN Pfn1, 2093 IN PFN_NUMBER PageFrameIndex 2094 ); 2095 2096 PFN_NUMBER 2097 NTAPI 2098 MiRemoveAnyPage( 2099 IN ULONG Color 2100 ); 2101 2102 PFN_NUMBER 2103 NTAPI 2104 MiRemoveZeroPage( 2105 IN ULONG Color 2106 ); 2107 2108 VOID 2109 NTAPI 2110 MiZeroPhysicalPage( 2111 IN PFN_NUMBER PageFrameIndex 2112 ); 2113 2114 VOID 2115 NTAPI 2116 MiInsertPageInFreeList( 2117 IN PFN_NUMBER PageFrameIndex 2118 ); 2119 2120 PFN_COUNT 2121 NTAPI 2122 MiDeleteSystemPageableVm( 2123 IN PMMPTE PointerPte, 2124 IN PFN_NUMBER PageCount, 2125 IN ULONG Flags, 2126 OUT PPFN_NUMBER ValidPages 2127 ); 2128 2129 ULONG 2130 NTAPI 2131 MiGetPageProtection( 2132 IN PMMPTE PointerPte 2133 ); 2134 2135 PLDR_DATA_TABLE_ENTRY 2136 NTAPI 2137 MiLookupDataTableEntry( 2138 IN PVOID Address 2139 ); 2140 2141 CODE_SEG("INIT") 2142 VOID 2143 NTAPI 2144 MiInitializeDriverLargePageList( 2145 VOID 2146 ); 2147 2148 CODE_SEG("INIT") 2149 VOID 2150 NTAPI 2151 MiInitializeLargePageSupport( 2152 VOID 2153 ); 2154 2155 CODE_SEG("INIT") 2156 VOID 2157 NTAPI 2158 MiSyncCachedRanges( 2159 VOID 2160 ); 2161 2162 BOOLEAN 2163 NTAPI 2164 MiIsPfnInUse( 2165 IN PMMPFN Pfn1 2166 ); 2167 2168 PMMVAD 2169 NTAPI 2170 MiLocateAddress( 2171 IN PVOID VirtualAddress 2172 ); 2173 2174 TABLE_SEARCH_RESULT 2175 NTAPI 2176 MiCheckForConflictingNode( 2177 IN ULONG_PTR StartVpn, 2178 IN ULONG_PTR EndVpn, 2179 IN PMM_AVL_TABLE Table, 2180 OUT PMMADDRESS_NODE *NodeOrParent 2181 ); 2182 2183 TABLE_SEARCH_RESULT 2184 NTAPI 2185 MiFindEmptyAddressRangeDownTree( 2186 IN SIZE_T Length, 2187 IN ULONG_PTR BoundaryAddress, 2188 IN ULONG_PTR Alignment, 2189 IN PMM_AVL_TABLE Table, 2190 OUT PULONG_PTR Base, 2191 OUT PMMADDRESS_NODE *Parent 2192 ); 2193 2194 NTSTATUS 2195 NTAPI 2196 MiFindEmptyAddressRangeDownBasedTree( 2197 IN SIZE_T Length, 2198 IN ULONG_PTR BoundaryAddress, 2199 IN ULONG_PTR Alignment, 2200 IN PMM_AVL_TABLE Table, 2201 OUT PULONG_PTR Base 2202 ); 2203 2204 TABLE_SEARCH_RESULT 2205 NTAPI 2206 MiFindEmptyAddressRangeInTree( 2207 IN SIZE_T Length, 2208 IN ULONG_PTR Alignment, 2209 IN PMM_AVL_TABLE Table, 2210 OUT PMMADDRESS_NODE *PreviousVad, 2211 OUT PULONG_PTR Base 2212 ); 2213 2214 NTSTATUS 2215 NTAPI 2216 MiCheckSecuredVad( 2217 IN PMMVAD Vad, 2218 IN PVOID Base, 2219 IN SIZE_T Size, 2220 IN ULONG ProtectionMask 2221 ); 2222 2223 VOID 2224 NTAPI 2225 MiInsertVad( 2226 _Inout_ PMMVAD Vad, 2227 _Inout_ PMM_AVL_TABLE VadRoot); 2228 2229 NTSTATUS 2230 NTAPI 2231 MiInsertVadEx( 2232 _Inout_ PMMVAD Vad, 2233 _In_ ULONG_PTR *BaseAddress, 2234 _In_ SIZE_T ViewSize, 2235 _In_ ULONG_PTR HighestAddress, 2236 _In_ ULONG_PTR Alignment, 2237 _In_ ULONG AllocationType); 2238 2239 VOID 2240 NTAPI 2241 MiInsertBasedSection( 2242 IN PSECTION Section 2243 ); 2244 2245 NTSTATUS 2246 NTAPI 2247 MiRosUnmapViewOfSection( 2248 IN PEPROCESS Process, 2249 IN PVOID BaseAddress, 2250 IN BOOLEAN SkipDebuggerNotify 2251 ); 2252 2253 VOID 2254 NTAPI 2255 MiInsertNode( 2256 IN PMM_AVL_TABLE Table, 2257 IN PMMADDRESS_NODE NewNode, 2258 PMMADDRESS_NODE Parent, 2259 TABLE_SEARCH_RESULT Result 2260 ); 2261 2262 VOID 2263 NTAPI 2264 MiRemoveNode( 2265 IN PMMADDRESS_NODE Node, 2266 IN PMM_AVL_TABLE Table 2267 ); 2268 2269 PMMADDRESS_NODE 2270 NTAPI 2271 MiGetPreviousNode( 2272 IN PMMADDRESS_NODE Node 2273 ); 2274 2275 PMMADDRESS_NODE 2276 NTAPI 2277 MiGetNextNode( 2278 IN PMMADDRESS_NODE Node 2279 ); 2280 2281 BOOLEAN 2282 NTAPI 2283 MiInitializeSystemSpaceMap( 2284 IN PMMSESSION InputSession OPTIONAL 2285 ); 2286 2287 VOID 2288 NTAPI 2289 MiSessionRemoveProcess( 2290 VOID 2291 ); 2292 2293 VOID 2294 NTAPI 2295 MiReleaseProcessReferenceToSessionDataPage( 2296 IN PMM_SESSION_SPACE SessionGlobal 2297 ); 2298 2299 VOID 2300 NTAPI 2301 MiSessionAddProcess( 2302 IN PEPROCESS NewProcess 2303 ); 2304 2305 ULONG 2306 NTAPI 2307 MiMakeProtectionMask( 2308 IN ULONG Protect 2309 ); 2310 2311 VOID 2312 NTAPI 2313 MiDeleteVirtualAddresses( 2314 IN ULONG_PTR Va, 2315 IN ULONG_PTR EndingAddress, 2316 IN PMMVAD Vad 2317 ); 2318 2319 VOID 2320 NTAPI 2321 MiDeletePte( 2322 IN PMMPTE PointerPte, 2323 IN PVOID VirtualAddress, 2324 IN PEPROCESS CurrentProcess, 2325 IN PMMPTE PrototypePte 2326 ); 2327 2328 ULONG 2329 NTAPI 2330 MiMakeSystemAddressValid( 2331 IN PVOID PageTableVirtualAddress, 2332 IN PEPROCESS CurrentProcess 2333 ); 2334 2335 ULONG 2336 NTAPI 2337 MiMakeSystemAddressValidPfn( 2338 IN PVOID VirtualAddress, 2339 IN KIRQL OldIrql 2340 ); 2341 2342 VOID 2343 NTAPI 2344 MiRemoveMappedView( 2345 IN PEPROCESS CurrentProcess, 2346 IN PMMVAD Vad 2347 ); 2348 2349 PSUBSECTION 2350 NTAPI 2351 MiLocateSubsection( 2352 IN PMMVAD Vad, 2353 IN ULONG_PTR Vpn 2354 ); 2355 2356 VOID 2357 NTAPI 2358 MiDeleteARM3Section( 2359 PVOID ObjectBody 2360 ); 2361 2362 NTSTATUS 2363 NTAPI 2364 MiQueryMemorySectionName( 2365 IN HANDLE ProcessHandle, 2366 IN PVOID BaseAddress, 2367 OUT PVOID MemoryInformation, 2368 IN SIZE_T MemoryInformationLength, 2369 OUT PSIZE_T ReturnLength 2370 ); 2371 2372 NTSTATUS 2373 NTAPI 2374 MiRosUnmapViewInSystemSpace( 2375 IN PVOID MappedBase 2376 ); 2377 2378 VOID 2379 NTAPI 2380 MiMakePdeExistAndMakeValid( 2381 IN PMMPDE PointerPde, 2382 IN PEPROCESS TargetProcess, 2383 IN KIRQL OldIrql 2384 ); 2385 2386 VOID 2387 NTAPI 2388 MiWriteProtectSystemImage( 2389 _In_ PVOID ImageBase); 2390 2391 // 2392 // MiRemoveZeroPage will use inline code to zero out the page manually if only 2393 // free pages are available. In some scenarios, we don't/can't run that piece of 2394 // code and would rather only have a real zero page. If we can't have a zero page, 2395 // then we'd like to have our own code to grab a free page and zero it out, by 2396 // using MiRemoveAnyPage. This macro implements this. 2397 // 2398 FORCEINLINE 2399 PFN_NUMBER 2400 MiRemoveZeroPageSafe(IN ULONG Color) 2401 { 2402 if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color); 2403 return 0; 2404 } 2405 2406 #if (_MI_PAGING_LEVELS == 2) 2407 FORCEINLINE 2408 BOOLEAN 2409 MiSynchronizeSystemPde(PMMPDE PointerPde) 2410 { 2411 ULONG Index; 2412 2413 /* Get the Index from the PDE */ 2414 Index = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE); 2415 if (PointerPde->u.Hard.Valid != 0) 2416 { 2417 NT_ASSERT(PointerPde->u.Long == MmSystemPagePtes[Index].u.Long); 2418 return TRUE; 2419 } 2420 2421 if (MmSystemPagePtes[Index].u.Hard.Valid == 0) 2422 { 2423 return FALSE; 2424 } 2425 2426 /* Copy the PDE from the double-mapped system page directory */ 2427 MI_WRITE_VALID_PDE(PointerPde, MmSystemPagePtes[Index]); 2428 2429 /* Make sure we re-read the PDE and PTE */ 2430 KeMemoryBarrierWithoutFence(); 2431 2432 /* Return success */ 2433 return TRUE; 2434 } 2435 #endif 2436 2437 #if _MI_PAGING_LEVELS == 2 2438 FORCEINLINE 2439 USHORT 2440 MiIncrementPageTableReferences(IN PVOID Address) 2441 { 2442 PUSHORT RefCount; 2443 2444 RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]; 2445 2446 *RefCount += 1; 2447 ASSERT(*RefCount <= PTE_PER_PAGE); 2448 return *RefCount; 2449 } 2450 2451 FORCEINLINE 2452 USHORT 2453 MiDecrementPageTableReferences(IN PVOID Address) 2454 { 2455 PUSHORT RefCount; 2456 2457 RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]; 2458 2459 *RefCount -= 1; 2460 ASSERT(*RefCount < PTE_PER_PAGE); 2461 return *RefCount; 2462 } 2463 #else 2464 FORCEINLINE 2465 USHORT 2466 MiIncrementPageTableReferences(IN PVOID Address) 2467 { 2468 PMMPDE PointerPde = MiAddressToPde(Address); 2469 PMMPFN Pfn; 2470 2471 /* We should not tinker with this one. */ 2472 ASSERT(PointerPde != (PMMPDE)PXE_SELFMAP); 2473 DPRINT("Incrementing %p from %p\n", Address, _ReturnAddress()); 2474 2475 /* Make sure we're locked */ 2476 ASSERT(PsGetCurrentThread()->OwnsProcessWorkingSetExclusive); 2477 2478 /* If we're bumping refcount, then it must be valid! */ 2479 ASSERT(PointerPde->u.Hard.Valid == 1); 2480 2481 /* This lies on the PFN */ 2482 Pfn = MiGetPfnEntry(PFN_FROM_PDE(PointerPde)); 2483 Pfn->OriginalPte.u.Soft.UsedPageTableEntries++; 2484 2485 ASSERT(Pfn->OriginalPte.u.Soft.UsedPageTableEntries <= PTE_PER_PAGE); 2486 2487 return Pfn->OriginalPte.u.Soft.UsedPageTableEntries; 2488 } 2489 2490 FORCEINLINE 2491 USHORT 2492 MiDecrementPageTableReferences(IN PVOID Address) 2493 { 2494 PMMPDE PointerPde = MiAddressToPde(Address); 2495 PMMPFN Pfn; 2496 2497 /* We should not tinker with this one. */ 2498 ASSERT(PointerPde != (PMMPDE)PXE_SELFMAP); 2499 2500 DPRINT("Decrementing %p from %p\n", PointerPde, _ReturnAddress()); 2501 2502 /* Make sure we're locked */ 2503 ASSERT(PsGetCurrentThread()->OwnsProcessWorkingSetExclusive); 2504 2505 /* If we're decreasing refcount, then it must be valid! */ 2506 ASSERT(PointerPde->u.Hard.Valid == 1); 2507 2508 /* This lies on the PFN */ 2509 Pfn = MiGetPfnEntry(PFN_FROM_PDE(PointerPde)); 2510 2511 ASSERT(Pfn->OriginalPte.u.Soft.UsedPageTableEntries != 0); 2512 Pfn->OriginalPte.u.Soft.UsedPageTableEntries--; 2513 2514 ASSERT(Pfn->OriginalPte.u.Soft.UsedPageTableEntries < PTE_PER_PAGE); 2515 2516 return Pfn->OriginalPte.u.Soft.UsedPageTableEntries; 2517 } 2518 #endif 2519 2520 #ifdef __cplusplus 2521 } // extern "C" 2522 #endif 2523 2524 FORCEINLINE 2525 VOID 2526 MiDeletePde( 2527 _In_ PMMPDE PointerPde, 2528 _In_ PEPROCESS CurrentProcess) 2529 { 2530 /* Only for user-mode ones */ 2531 ASSERT(MiIsUserPde(PointerPde)); 2532 2533 /* Kill this one as a PTE */ 2534 MiDeletePte((PMMPTE)PointerPde, MiPdeToPte(PointerPde), CurrentProcess, NULL); 2535 #if _MI_PAGING_LEVELS >= 3 2536 /* Cascade down */ 2537 if (MiDecrementPageTableReferences(MiPdeToPte(PointerPde)) == 0) 2538 { 2539 MiDeletePte(MiPdeToPpe(PointerPde), PointerPde, CurrentProcess, NULL); 2540 #if _MI_PAGING_LEVELS == 4 2541 if (MiDecrementPageTableReferences(PointerPde) == 0) 2542 { 2543 MiDeletePte(MiPdeToPxe(PointerPde), MiPdeToPpe(PointerPde), CurrentProcess, NULL); 2544 } 2545 #endif 2546 } 2547 #endif 2548 } 2549 2550 /* EOF */ 2551