xref: /reactos/ntoskrnl/mm/rmap.c (revision 299e4305) 
1 /*
2  * COPYRIGHT:       See COPYING in the top directory
3  * PROJECT:         ReactOS kernel
4  * FILE:            ntoskrnl/mm/rmap.c
5  * PURPOSE:         Kernel memory management functions
6  *
7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
8  */
9 
10 /* INCLUDES *****************************************************************/
11 
12 #include <ntoskrnl.h>
13 #include <cache/section/newmm.h>
14 #define NDEBUG
15 #include <debug.h>
16 
17 /* TYPES ********************************************************************/
18 
19 /* GLOBALS ******************************************************************/
20 
21 static NPAGED_LOOKASIDE_LIST RmapLookasideList;
22 
23 /* FUNCTIONS ****************************************************************/
24 
25 _IRQL_requires_max_(DISPATCH_LEVEL)
26 static
27 VOID
28 NTAPI
29 RmapListFree(
30     _In_ __drv_freesMem(Mem) PVOID P)
31 {
32     ExFreePoolWithTag(P, TAG_RMAP);
33 }
34 
35 CODE_SEG("INIT")
36 VOID
37 NTAPI
38 MmInitializeRmapList(VOID)
39 {
40     ExInitializeNPagedLookasideList (&RmapLookasideList,
41                                      NULL,
42                                      RmapListFree,
43                                      0,
44                                      sizeof(MM_RMAP_ENTRY),
45                                      TAG_RMAP,
46                                      50);
47 }
48 
49 NTSTATUS
50 NTAPI
51 MmPageOutPhysicalAddress(PFN_NUMBER Page)
52 {
53     PMM_RMAP_ENTRY entry;
54     PMEMORY_AREA MemoryArea;
55     PMMSUPPORT AddressSpace;
56     PVOID Address = NULL;
57     PEPROCESS Process = NULL;
58     NTSTATUS Status = STATUS_SUCCESS;
59     PMM_SECTION_SEGMENT Segment;
60     LARGE_INTEGER SegmentOffset;
61     KIRQL OldIrql;
62 
63 GetEntry:
64     OldIrql = MiAcquirePfnLock();
65 
66     entry = MmGetRmapListHeadPage(Page);
67 
68     while (entry && RMAP_IS_SEGMENT(entry->Address))
69         entry = entry->Next;
70 
71     if (entry == NULL)
72     {
73         MiReleasePfnLock(OldIrql);
74         goto WriteSegment;
75     }
76 
77     Process = entry->Process;
78     Address = entry->Address;
79 
80     if ((((ULONG_PTR)Address) & 0xFFF) != 0)
81     {
82         KeBugCheck(MEMORY_MANAGEMENT);
83     }
84 
85     /* This is for user-mode address only */
86     ASSERT(Address < MmSystemRangeStart);
87 
88     if (!ExAcquireRundownProtection(&Process->RundownProtect))
89     {
90         MiReleasePfnLock(OldIrql);
91         return STATUS_PROCESS_IS_TERMINATING;
92     }
93 
94     Status = ObReferenceObjectByPointer(Process, PROCESS_ALL_ACCESS, NULL, KernelMode);
95     MiReleasePfnLock(OldIrql);
96     if (!NT_SUCCESS(Status))
97     {
98         ExReleaseRundownProtection(&Process->RundownProtect);
99         return Status;
100     }
101     AddressSpace = &Process->Vm;
102 
103     MmLockAddressSpace(AddressSpace);
104 
105     MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address);
106     if (MemoryArea == NULL || MemoryArea->DeleteInProgress)
107     {
108         MmUnlockAddressSpace(AddressSpace);
109         ExReleaseRundownProtection(&Process->RundownProtect);
110         ObDereferenceObject(Process);
111         goto GetEntry;
112     }
113 
114 
115     /* Attach to it, if needed */
116     ASSERT(PsGetCurrentProcess() == PsInitialSystemProcess);
117     if (Process != PsInitialSystemProcess)
118         KeAttachProcess(&Process->Pcb);
119 
120     if (MmGetPfnForProcess(Process, Address) != Page)
121     {
122         /* This changed in the short window where we didn't have any locks */
123         if (Process != PsInitialSystemProcess)
124             KeDetachProcess();
125         MmUnlockAddressSpace(AddressSpace);
126         ExReleaseRundownProtection(&Process->RundownProtect);
127         ObDereferenceObject(Process);
128         goto GetEntry;
129     }
130 
131     if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW)
132     {
133         ULONG_PTR Entry;
134         BOOLEAN Dirty;
135         PFN_NUMBER MapPage;
136         LARGE_INTEGER Offset;
137         BOOLEAN Released;
138         BOOLEAN Unmapped;
139 
140         Offset.QuadPart = MemoryArea->SectionData.ViewOffset +
141                  ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea));
142 
143         Segment = MemoryArea->SectionData.Segment;
144 
145         MmLockSectionSegment(Segment);
146 
147         Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
148         if (Entry && MM_IS_WAIT_PTE(Entry))
149         {
150             /* The segment is being read or something. Give up */
151             MmUnlockSectionSegment(Segment);
152             if (Process != PsInitialSystemProcess)
153                 KeDetachProcess();
154             MmUnlockAddressSpace(AddressSpace);
155             ExReleaseRundownProtection(&Process->RundownProtect);
156             ObDereferenceObject(Process);
157             return(STATUS_UNSUCCESSFUL);
158         }
159 
160         /* Delete this virtual mapping in the process */
161         MmDeleteRmap(Page, Process, Address);
162         Unmapped = MmDeleteVirtualMapping(Process, Address, &Dirty, &MapPage);
163         if (!Unmapped || (MapPage != Page))
164         {
165             /*
166              * Something's corrupted, we got there because this process is
167              * supposed to be mapping this page there.
168              */
169             KeBugCheckEx(MEMORY_MANAGEMENT,
170                          (ULONG_PTR)Process,
171                          (ULONG_PTR)Address,
172                          (ULONG_PTR)__FILE__,
173                          __LINE__);
174         }
175 
176         if (Page != PFN_FROM_SSE(Entry))
177         {
178             SWAPENTRY SwapEntry;
179 
180             /* This page is private to the process */
181             MmUnlockSectionSegment(Segment);
182 
183             /* Check if we should write it back to the page file */
184             SwapEntry = MmGetSavedSwapEntryPage(Page);
185 
186             if ((SwapEntry == 0) && Dirty)
187             {
188                 /* We don't have a Swap entry, yet the page is dirty. Get one */
189                 SwapEntry = MmAllocSwapPage();
190                 if (!SwapEntry)
191                 {
192                     PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea),
193                             &MemoryArea->SectionData.RegionListHead,
194                             Address, NULL);
195 
196                     /* We can't, so let this page in the Process VM */
197                     MmCreateVirtualMapping(Process, Address, Region->Protect, Page);
198                     MmInsertRmap(Page, Process, Address);
199                     MmSetDirtyPage(Process, Address);
200 
201                     MmUnlockAddressSpace(AddressSpace);
202                     if (Process != PsInitialSystemProcess)
203                         KeDetachProcess();
204                     ExReleaseRundownProtection(&Process->RundownProtect);
205                     ObDereferenceObject(Process);
206 
207                     return STATUS_UNSUCCESSFUL;
208                 }
209             }
210 
211             if (Dirty)
212             {
213                 SWAPENTRY Dummy;
214 
215                 /* Put a wait entry into the process and unlock */
216                 MmCreatePageFileMapping(Process, Address, MM_WAIT_ENTRY);
217                 MmUnlockAddressSpace(AddressSpace);
218 
219                 Status = MmWriteToSwapPage(SwapEntry, Page);
220 
221                 MmLockAddressSpace(AddressSpace);
222                 MmDeletePageFileMapping(Process, Address, &Dummy);
223                 ASSERT(Dummy == MM_WAIT_ENTRY);
224 
225                 if (!NT_SUCCESS(Status))
226                 {
227                     /* We failed at saving the content of this page. Keep it in */
228                     PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea),
229                             &MemoryArea->SectionData.RegionListHead,
230                             Address, NULL);
231 
232                     /* This Swap Entry is useless to us */
233                     MmSetSavedSwapEntryPage(Page, 0);
234                     MmFreeSwapPage(SwapEntry);
235 
236                     /* We can't, so let this page in the Process VM */
237                     MmCreateVirtualMapping(Process, Address, Region->Protect, Page);
238                     MmInsertRmap(Page, Process, Address);
239                     MmSetDirtyPage(Process, Address);
240 
241                     MmUnlockAddressSpace(AddressSpace);
242                     if (Process != PsInitialSystemProcess)
243                         KeDetachProcess();
244                     ExReleaseRundownProtection(&Process->RundownProtect);
245                     ObDereferenceObject(Process);
246 
247                     return STATUS_UNSUCCESSFUL;
248                 }
249             }
250 
251             if (SwapEntry)
252             {
253                 /* Keep this in the process VM */
254                 MmCreatePageFileMapping(Process, Address, SwapEntry);
255                 MmSetSavedSwapEntryPage(Page, 0);
256             }
257 
258             /* We can finally let this page go */
259             MmUnlockAddressSpace(AddressSpace);
260             if (Process != PsInitialSystemProcess)
261                 KeDetachProcess();
262 #if DBG
263             OldIrql = MiAcquirePfnLock();
264             ASSERT(MmGetRmapListHeadPage(Page) == NULL);
265             MiReleasePfnLock(OldIrql);
266 #endif
267             MmReleasePageMemoryConsumer(MC_USER, Page);
268 
269             ExReleaseRundownProtection(&Process->RundownProtect);
270             ObDereferenceObject(Process);
271 
272             return STATUS_SUCCESS;
273         }
274 
275         /* One less mapping referencing this segment */
276         Released = MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, TRUE, NULL);
277 
278         MmUnlockSectionSegment(Segment);
279         if (Process != PsInitialSystemProcess)
280             KeDetachProcess();
281         MmUnlockAddressSpace(AddressSpace);
282 
283         ExReleaseRundownProtection(&Process->RundownProtect);
284         ObDereferenceObject(Process);
285 
286         if (Released) return STATUS_SUCCESS;
287     }
288 #ifdef NEWCC
289     else if (Type == MEMORY_AREA_CACHE)
290     {
291         /* NEWCC does locking itself */
292         MmUnlockAddressSpace(AddressSpace);
293         Status = MmpPageOutPhysicalAddress(Page);
294     }
295 #endif
296     else
297     {
298         KeBugCheck(MEMORY_MANAGEMENT);
299     }
300 
301 WriteSegment:
302     /* Now write this page to file, if needed */
303     Segment = MmGetSectionAssociation(Page, &SegmentOffset);
304     if (Segment)
305     {
306         BOOLEAN Released;
307 
308         MmLockSectionSegment(Segment);
309 
310         Released = MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE);
311 
312         MmUnlockSectionSegment(Segment);
313         MmDereferenceSegment(Segment);
314 
315         if (Released)
316         {
317             return STATUS_SUCCESS;
318         }
319     }
320 
321     /* If we are here, then we didn't release the page */
322     return STATUS_UNSUCCESSFUL;
323 }
324 
325 VOID
326 NTAPI
327 MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process,
328              PVOID Address)
329 {
330     PMM_RMAP_ENTRY current_entry;
331     PMM_RMAP_ENTRY new_entry;
332     ULONG PrevSize;
333     KIRQL OldIrql;
334 
335     if (!RMAP_IS_SEGMENT(Address))
336         Address = (PVOID)PAGE_ROUND_DOWN(Address);
337 
338     new_entry = ExAllocateFromNPagedLookasideList(&RmapLookasideList);
339     if (new_entry == NULL)
340     {
341         KeBugCheck(MEMORY_MANAGEMENT);
342     }
343     new_entry->Address = Address;
344     new_entry->Process = (PEPROCESS)Process;
345 #if DBG
346     new_entry->Caller = _ReturnAddress();
347 #endif
348 
349     if (
350         !RMAP_IS_SEGMENT(Address) &&
351         MmGetPfnForProcess(Process, Address) != Page)
352     {
353         DPRINT1("Insert rmap (%d, 0x%.8X) 0x%.8X which doesn't match physical "
354                 "address 0x%.8X\n", Process ? Process->UniqueProcessId : 0,
355                 Address,
356                 MmGetPfnForProcess(Process, Address) << PAGE_SHIFT,
357                 Page << PAGE_SHIFT);
358         KeBugCheck(MEMORY_MANAGEMENT);
359     }
360 
361     OldIrql = MiAcquirePfnLock();
362     current_entry = MmGetRmapListHeadPage(Page);
363 
364     PMM_RMAP_ENTRY previous_entry = NULL;
365     /* Keep the list sorted */
366     while (current_entry && (current_entry->Address < Address))
367     {
368         previous_entry = current_entry;
369         current_entry = current_entry->Next;
370     }
371 
372     /* In case of clash in the address, sort by process */
373     if (current_entry && (current_entry->Address == Address))
374     {
375         while (current_entry && (current_entry->Process < Process))
376         {
377             previous_entry = current_entry;
378             current_entry = current_entry->Next;
379         }
380     }
381 
382     if (current_entry && (current_entry->Address == Address) && (current_entry->Process == Process))
383     {
384 #if DBG
385         DbgPrint("MmInsertRmap tries to add a second rmap entry for address %p\n", current_entry->Address);
386         DbgPrint("    current caller  %p\n", new_entry->Caller);
387         DbgPrint("    previous caller %p\n", current_entry->Caller);
388 #endif
389         KeBugCheck(MEMORY_MANAGEMENT);
390     }
391 
392     new_entry->Next = current_entry;
393     if (previous_entry)
394         previous_entry->Next = new_entry;
395     else
396         MmSetRmapListHeadPage(Page, new_entry);
397 
398     MiReleasePfnLock(OldIrql);
399 
400     if (!RMAP_IS_SEGMENT(Address))
401     {
402         ASSERT(Process != NULL);
403         PrevSize = InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, PAGE_SIZE);
404         if (PrevSize >= Process->Vm.PeakWorkingSetSize)
405         {
406             Process->Vm.PeakWorkingSetSize = PrevSize + PAGE_SIZE;
407         }
408     }
409 }
410 
411 VOID
412 NTAPI
413 MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process,
414              PVOID Address)
415 {
416     PMM_RMAP_ENTRY current_entry, previous_entry;
417     KIRQL OldIrql;
418 
419     OldIrql = MiAcquirePfnLock();
420     previous_entry = NULL;
421     current_entry = MmGetRmapListHeadPage(Page);
422 
423     while (current_entry != NULL)
424     {
425         if (current_entry->Process == (PEPROCESS)Process &&
426                 current_entry->Address == Address)
427         {
428             if (previous_entry == NULL)
429             {
430                 MmSetRmapListHeadPage(Page, current_entry->Next);
431             }
432             else
433             {
434                 previous_entry->Next = current_entry->Next;
435             }
436             MiReleasePfnLock(OldIrql);
437 
438             ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry);
439             if (!RMAP_IS_SEGMENT(Address))
440             {
441                 ASSERT(Process != NULL);
442                 (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE);
443             }
444             return;
445         }
446         previous_entry = current_entry;
447         current_entry = current_entry->Next;
448     }
449     KeBugCheck(MEMORY_MANAGEMENT);
450 }
451 
452 /*
453 
454 Return the process pointer given when a previous call to MmInsertRmap was
455 called with a process and address pointer that conform to the segment rmap
456 schema.  In short, this requires the address part to be 0xffffff00 + n
457 where n is between 0 and 255.  When such an rmap exists, it specifies a
458 segment rmap in which the process part is a pointer to a slice of a section
459 page table, and the low 8 bits of the address represent a page index in the
460 page table slice.  Together, this information is used by
461 MmGetSectionAssociation to determine which page entry points to this page in
462 the segment page table.
463 
464 */
465 
466 PVOID
467 NTAPI
468 MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset)
469 {
470     PCACHE_SECTION_PAGE_TABLE Result = NULL;
471     PMM_RMAP_ENTRY current_entry;//, previous_entry;
472 
473     /* Must hold the PFN lock */
474     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
475 
476     //previous_entry = NULL;
477     current_entry = MmGetRmapListHeadPage(Page);
478     while (current_entry != NULL)
479     {
480         if (RMAP_IS_SEGMENT(current_entry->Address))
481         {
482             Result = (PCACHE_SECTION_PAGE_TABLE)current_entry->Process;
483             *RawOffset = (ULONG_PTR)current_entry->Address & ~RMAP_SEGMENT_MASK;
484             if (*Result->Segment->Flags & MM_SEGMENT_INDELETE)
485             {
486                 return NULL;
487             }
488 
489             return Result;
490         }
491         //previous_entry = current_entry;
492         current_entry = current_entry->Next;
493     }
494 
495     return NULL;
496 }
497 
498 /*
499 
500 Remove the section rmap associated with the indicated page, if it exists.
501 
502 */
503 
504 VOID
505 NTAPI
506 MmDeleteSectionAssociation(PFN_NUMBER Page)
507 {
508     PMM_RMAP_ENTRY current_entry, previous_entry;
509     KIRQL OldIrql = MiAcquirePfnLock();
510 
511     previous_entry = NULL;
512     current_entry = MmGetRmapListHeadPage(Page);
513     while (current_entry != NULL)
514     {
515         if (RMAP_IS_SEGMENT(current_entry->Address))
516         {
517             if (previous_entry == NULL)
518             {
519                 MmSetRmapListHeadPage(Page, current_entry->Next);
520             }
521             else
522             {
523                 previous_entry->Next = current_entry->Next;
524             }
525             MiReleasePfnLock(OldIrql);
526             ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry);
527             return;
528         }
529         previous_entry = current_entry;
530         current_entry = current_entry->Next;
531     }
532     MiReleasePfnLock(OldIrql);
533 }
534