xref: /reactos/ntoskrnl/mm/rmap.c (revision 6a31fe6c) 
1 /*
2  * COPYRIGHT:       See COPYING in the top directory
3  * PROJECT:         ReactOS kernel
4  * FILE:            ntoskrnl/mm/rmap.c
5  * PURPOSE:         Kernel memory managment functions
6  *
7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
8  */
9 
10 /* INCLUDES *****************************************************************/
11 
12 #include <ntoskrnl.h>
13 #include <cache/section/newmm.h>
14 #define NDEBUG
15 #include <debug.h>
16 
17 /* TYPES ********************************************************************/
18 
19 /* GLOBALS ******************************************************************/
20 
21 static NPAGED_LOOKASIDE_LIST RmapLookasideList;
22 
23 /* FUNCTIONS ****************************************************************/
24 
25 _IRQL_requires_max_(DISPATCH_LEVEL)
26 static
27 VOID
28 NTAPI
29 RmapListFree(
30     _In_ __drv_freesMem(Mem) PVOID P)
31 {
32     ExFreePoolWithTag(P, TAG_RMAP);
33 }
34 
35 CODE_SEG("INIT")
36 VOID
37 NTAPI
38 MmInitializeRmapList(VOID)
39 {
40     ExInitializeNPagedLookasideList (&RmapLookasideList,
41                                      NULL,
42                                      RmapListFree,
43                                      0,
44                                      sizeof(MM_RMAP_ENTRY),
45                                      TAG_RMAP,
46                                      50);
47 }
48 
49 NTSTATUS
50 NTAPI
51 MmPageOutPhysicalAddress(PFN_NUMBER Page)
52 {
53     PMM_RMAP_ENTRY entry;
54     PMEMORY_AREA MemoryArea;
55     PMMSUPPORT AddressSpace;
56     PVOID Address = NULL;
57     PEPROCESS Process = NULL;
58     NTSTATUS Status = STATUS_SUCCESS;
59     PMM_SECTION_SEGMENT Segment;
60     LARGE_INTEGER SegmentOffset;
61     KIRQL OldIrql;
62 
63 GetEntry:
64     OldIrql = MiAcquirePfnLock();
65 
66     entry = MmGetRmapListHeadPage(Page);
67 
68     while (entry && RMAP_IS_SEGMENT(entry->Address))
69         entry = entry->Next;
70 
71     if (entry == NULL)
72     {
73         MiReleasePfnLock(OldIrql);
74         goto WriteSegment;
75     }
76 
77     Process = entry->Process;
78     Address = entry->Address;
79 
80     if ((((ULONG_PTR)Address) & 0xFFF) != 0)
81     {
82         KeBugCheck(MEMORY_MANAGEMENT);
83     }
84 
85     /* This is for user-mode address only */
86     ASSERT(Address < MmSystemRangeStart);
87 
88     if (!ExAcquireRundownProtection(&Process->RundownProtect))
89     {
90         MiReleasePfnLock(OldIrql);
91         return STATUS_PROCESS_IS_TERMINATING;
92     }
93 
94     Status = ObReferenceObjectByPointer(Process, PROCESS_ALL_ACCESS, NULL, KernelMode);
95     MiReleasePfnLock(OldIrql);
96     if (!NT_SUCCESS(Status))
97     {
98         ExReleaseRundownProtection(&Process->RundownProtect);
99         return Status;
100     }
101     AddressSpace = &Process->Vm;
102 
103     MmLockAddressSpace(AddressSpace);
104 
105     if (MmGetPfnForProcess(Process, Address) != Page)
106     {
107         /* This changed in the short window where we didn't have any locks */
108         MmUnlockAddressSpace(AddressSpace);
109         ExReleaseRundownProtection(&Process->RundownProtect);
110         ObDereferenceObject(Process);
111         goto GetEntry;
112     }
113 
114     MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address);
115     if (MemoryArea == NULL || MemoryArea->DeleteInProgress)
116     {
117         MmUnlockAddressSpace(AddressSpace);
118         ExReleaseRundownProtection(&Process->RundownProtect);
119         ObDereferenceObject(Process);
120         goto GetEntry;
121     }
122 
123     if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW)
124     {
125         ULONG_PTR Entry;
126         BOOLEAN Dirty;
127         PFN_NUMBER MapPage;
128         LARGE_INTEGER Offset;
129         BOOLEAN Released;
130 
131         Offset.QuadPart = MemoryArea->SectionData.ViewOffset +
132                  ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea));
133 
134         Segment = MemoryArea->SectionData.Segment;
135 
136         MmLockSectionSegment(Segment);
137 
138         Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
139         if (Entry && MM_IS_WAIT_PTE(Entry))
140         {
141             /* The segment is being read or something. Give up */
142             MmUnlockSectionSegment(Segment);
143             MmUnlockAddressSpace(AddressSpace);
144             if (Address < MmSystemRangeStart)
145             {
146                 ExReleaseRundownProtection(&Process->RundownProtect);
147                 ObDereferenceObject(Process);
148             }
149             return(STATUS_UNSUCCESSFUL);
150         }
151 
152         /* Delete this virtual mapping in the process */
153         MmDeleteVirtualMapping(Process, Address, &Dirty, &MapPage);
154 
155         /* We checked this earlier */
156         ASSERT(MapPage == Page);
157 
158         if (Page != PFN_FROM_SSE(Entry))
159         {
160             SWAPENTRY SwapEntry;
161 
162             /* This page is private to the process */
163             MmUnlockSectionSegment(Segment);
164 
165             /* Check if we should write it back to the page file */
166             SwapEntry = MmGetSavedSwapEntryPage(Page);
167 
168             if ((SwapEntry == 0) && Dirty)
169             {
170                 /* We don't have a Swap entry, yet the page is dirty. Get one */
171                 SwapEntry = MmAllocSwapPage();
172                 if (!SwapEntry)
173                 {
174                     PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea),
175                             &MemoryArea->SectionData.RegionListHead,
176                             Address, NULL);
177 
178                     /* We can't, so let this page in the Process VM */
179                     MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1);
180                     MmSetDirtyPage(Process, Address);
181 
182                     MmUnlockAddressSpace(AddressSpace);
183                     if (Address < MmSystemRangeStart)
184                     {
185                         ExReleaseRundownProtection(&Process->RundownProtect);
186                         ObDereferenceObject(Process);
187                     }
188 
189                     return STATUS_UNSUCCESSFUL;
190                 }
191             }
192 
193             if (Dirty)
194             {
195                 Status = MmWriteToSwapPage(SwapEntry, Page);
196                 if (!NT_SUCCESS(Status))
197                 {
198                     /* We failed at saving the content of this page. Keep it in */
199                     PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea),
200                             &MemoryArea->SectionData.RegionListHead,
201                             Address, NULL);
202 
203                     /* This Swap Entry is useless to us */
204                     MmSetSavedSwapEntryPage(Page, 0);
205                     MmFreeSwapPage(SwapEntry);
206 
207                     /* We can't, so let this page in the Process VM */
208                     MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1);
209                     MmSetDirtyPage(Process, Address);
210 
211                     MmUnlockAddressSpace(AddressSpace);
212                     ExReleaseRundownProtection(&Process->RundownProtect);
213                     ObDereferenceObject(Process);
214 
215                     return STATUS_UNSUCCESSFUL;
216                 }
217             }
218 
219             if (SwapEntry)
220             {
221                 /* Keep this in the process VM */
222                 MmCreatePageFileMapping(Process, Address, SwapEntry);
223                 MmSetSavedSwapEntryPage(Page, 0);
224             }
225 
226             MmUnlockAddressSpace(AddressSpace);
227 
228             /* We can finally let this page go */
229             MmDeleteRmap(Page, Process, Address);
230 #if DBG
231             OldIrql = MiAcquirePfnLock();
232             ASSERT(MmGetRmapListHeadPage(Page) == NULL);
233             MiReleasePfnLock(OldIrql);
234 #endif
235             MmReleasePageMemoryConsumer(MC_USER, Page);
236 
237             ExReleaseRundownProtection(&Process->RundownProtect);
238             ObDereferenceObject(Process);
239 
240             return STATUS_SUCCESS;
241         }
242 
243         /* Delete this RMAP */
244         MmDeleteRmap(Page, Process, Address);
245 
246         /* One less mapping referencing this segment */
247         Released = MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, TRUE, NULL);
248 
249         MmUnlockSectionSegment(Segment);
250         MmUnlockAddressSpace(AddressSpace);
251 
252         ExReleaseRundownProtection(&Process->RundownProtect);
253         ObDereferenceObject(Process);
254 
255         if (Released) return STATUS_SUCCESS;
256     }
257 #ifdef NEWCC
258     else if (Type == MEMORY_AREA_CACHE)
259     {
260         /* NEWCC does locking itself */
261         MmUnlockAddressSpace(AddressSpace);
262         Status = MmpPageOutPhysicalAddress(Page);
263     }
264 #endif
265     else
266     {
267         KeBugCheck(MEMORY_MANAGEMENT);
268     }
269 
270 WriteSegment:
271     /* Now write this page to file, if needed */
272     Segment = MmGetSectionAssociation(Page, &SegmentOffset);
273     if (Segment)
274     {
275         BOOLEAN Released;
276 
277         MmLockSectionSegment(Segment);
278 
279         Released = MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE);
280 
281         MmUnlockSectionSegment(Segment);
282 
283         MmDereferenceSegment(Segment);
284 
285         if (Released)
286         {
287             return STATUS_SUCCESS;
288         }
289     }
290 
291     /* If we are here, then we didn't release the page */
292     return STATUS_UNSUCCESSFUL;
293 }
294 
295 VOID
296 NTAPI
297 MmSetCleanAllRmaps(PFN_NUMBER Page)
298 {
299     PMM_RMAP_ENTRY current_entry;
300     KIRQL OldIrql;
301 
302     OldIrql = MiAcquirePfnLock();
303     current_entry = MmGetRmapListHeadPage(Page);
304     if (current_entry == NULL)
305     {
306         DPRINT1("MmSetCleanAllRmaps: No rmaps.\n");
307         KeBugCheck(MEMORY_MANAGEMENT);
308     }
309     while (current_entry != NULL)
310     {
311         if (!RMAP_IS_SEGMENT(current_entry->Address))
312             MmSetCleanPage(current_entry->Process, current_entry->Address);
313         current_entry = current_entry->Next;
314     }
315     MiReleasePfnLock(OldIrql);
316 }
317 
318 BOOLEAN
319 NTAPI
320 MmIsDirtyPageRmap(PFN_NUMBER Page)
321 {
322     PMM_RMAP_ENTRY current_entry;
323     KIRQL OldIrql;
324     BOOLEAN Dirty = FALSE;
325 
326     OldIrql = MiAcquirePfnLock();
327     current_entry = MmGetRmapListHeadPage(Page);
328     if (current_entry == NULL)
329     {
330         DPRINT1("MmIsDirtyPageRmap: No rmaps.\n");
331         KeBugCheck(MEMORY_MANAGEMENT);
332     }
333     while (current_entry != NULL)
334     {
335         if (!RMAP_IS_SEGMENT(current_entry->Address))
336         {
337             if (MmIsDirtyPage(current_entry->Process, current_entry->Address))
338             {
339                 Dirty = TRUE;
340                 break;
341             }
342         }
343         current_entry = current_entry->Next;
344     }
345     MiReleasePfnLock(OldIrql);
346 
347     return Dirty;
348 }
349 
350 VOID
351 NTAPI
352 MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process,
353              PVOID Address)
354 {
355     PMM_RMAP_ENTRY current_entry;
356     PMM_RMAP_ENTRY new_entry;
357     ULONG PrevSize;
358     KIRQL OldIrql;
359 
360     if (!RMAP_IS_SEGMENT(Address))
361         Address = (PVOID)PAGE_ROUND_DOWN(Address);
362 
363     new_entry = ExAllocateFromNPagedLookasideList(&RmapLookasideList);
364     if (new_entry == NULL)
365     {
366         KeBugCheck(MEMORY_MANAGEMENT);
367     }
368     new_entry->Address = Address;
369     new_entry->Process = (PEPROCESS)Process;
370 #if DBG
371     new_entry->Caller = _ReturnAddress();
372 #endif
373 
374     if (
375         !RMAP_IS_SEGMENT(Address) &&
376         MmGetPfnForProcess(Process, Address) != Page)
377     {
378         DPRINT1("Insert rmap (%d, 0x%.8X) 0x%.8X which doesn't match physical "
379                 "address 0x%.8X\n", Process ? Process->UniqueProcessId : 0,
380                 Address,
381                 MmGetPfnForProcess(Process, Address) << PAGE_SHIFT,
382                 Page << PAGE_SHIFT);
383         KeBugCheck(MEMORY_MANAGEMENT);
384     }
385 
386     OldIrql = MiAcquirePfnLock();
387     current_entry = MmGetRmapListHeadPage(Page);
388 
389     PMM_RMAP_ENTRY previous_entry = NULL;
390     /* Keep the list sorted */
391     while (current_entry && (current_entry->Address < Address))
392     {
393         previous_entry = current_entry;
394         current_entry = current_entry->Next;
395     }
396 
397     /* In case of clash in the address, sort by process */
398     if (current_entry && (current_entry->Address == Address))
399     {
400         while (current_entry && (current_entry->Process < Process))
401         {
402             previous_entry = current_entry;
403             current_entry = current_entry->Next;
404         }
405     }
406 
407     if (current_entry && (current_entry->Address == Address) && (current_entry->Process == Process))
408     {
409 #if DBG
410         DbgPrint("MmInsertRmap tries to add a second rmap entry for address %p\n", current_entry->Address);
411         DbgPrint("    current caller  %p\n", new_entry->Caller);
412         DbgPrint("    previous caller %p\n", current_entry->Caller);
413 #endif
414         KeBugCheck(MEMORY_MANAGEMENT);
415     }
416 
417     new_entry->Next = current_entry;
418     if (previous_entry)
419         previous_entry->Next = new_entry;
420     else
421         MmSetRmapListHeadPage(Page, new_entry);
422 
423     MiReleasePfnLock(OldIrql);
424 
425     if (!RMAP_IS_SEGMENT(Address))
426     {
427         ASSERT(Process != NULL);
428         PrevSize = InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, PAGE_SIZE);
429         if (PrevSize >= Process->Vm.PeakWorkingSetSize)
430         {
431             Process->Vm.PeakWorkingSetSize = PrevSize + PAGE_SIZE;
432         }
433     }
434 }
435 
436 VOID
437 NTAPI
438 MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process,
439              PVOID Address)
440 {
441     PMM_RMAP_ENTRY current_entry, previous_entry;
442     KIRQL OldIrql;
443 
444     OldIrql = MiAcquirePfnLock();
445     previous_entry = NULL;
446     current_entry = MmGetRmapListHeadPage(Page);
447 
448     while (current_entry != NULL)
449     {
450         if (current_entry->Process == (PEPROCESS)Process &&
451                 current_entry->Address == Address)
452         {
453             if (previous_entry == NULL)
454             {
455                 MmSetRmapListHeadPage(Page, current_entry->Next);
456             }
457             else
458             {
459                 previous_entry->Next = current_entry->Next;
460             }
461             MiReleasePfnLock(OldIrql);
462 
463             ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry);
464             if (!RMAP_IS_SEGMENT(Address))
465             {
466                 ASSERT(Process != NULL);
467                 (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE);
468             }
469             return;
470         }
471         previous_entry = current_entry;
472         current_entry = current_entry->Next;
473     }
474     KeBugCheck(MEMORY_MANAGEMENT);
475 }
476 
477 /*
478 
479 Return the process pointer given when a previous call to MmInsertRmap was
480 called with a process and address pointer that conform to the segment rmap
481 schema.  In short, this requires the address part to be 0xffffff00 + n
482 where n is between 0 and 255.  When such an rmap exists, it specifies a
483 segment rmap in which the process part is a pointer to a slice of a section
484 page table, and the low 8 bits of the address represent a page index in the
485 page table slice.  Together, this information is used by
486 MmGetSectionAssociation to determine which page entry points to this page in
487 the segment page table.
488 
489 */
490 
491 PVOID
492 NTAPI
493 MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset)
494 {
495     PCACHE_SECTION_PAGE_TABLE Result = NULL;
496     PMM_RMAP_ENTRY current_entry;//, previous_entry;
497     KIRQL OldIrql = MiAcquirePfnLock();
498 
499     //previous_entry = NULL;
500     current_entry = MmGetRmapListHeadPage(Page);
501     while (current_entry != NULL)
502     {
503         if (RMAP_IS_SEGMENT(current_entry->Address))
504         {
505             Result = (PCACHE_SECTION_PAGE_TABLE)current_entry->Process;
506             *RawOffset = (ULONG_PTR)current_entry->Address & ~RMAP_SEGMENT_MASK;
507             if (*Result->Segment->Flags & MM_SEGMENT_INDELETE)
508             {
509                 MiReleasePfnLock(OldIrql);
510                 return NULL;
511             }
512 
513             InterlockedIncrement64(Result->Segment->ReferenceCount);
514             MiReleasePfnLock(OldIrql);
515             return Result;
516         }
517         //previous_entry = current_entry;
518         current_entry = current_entry->Next;
519     }
520     MiReleasePfnLock(OldIrql);
521     return NULL;
522 }
523 
524 /*
525 
526 Remove the section rmap associated with the indicated page, if it exists.
527 
528 */
529 
530 VOID
531 NTAPI
532 MmDeleteSectionAssociation(PFN_NUMBER Page)
533 {
534     PMM_RMAP_ENTRY current_entry, previous_entry;
535     KIRQL OldIrql = MiAcquirePfnLock();
536 
537     previous_entry = NULL;
538     current_entry = MmGetRmapListHeadPage(Page);
539     while (current_entry != NULL)
540     {
541         if (RMAP_IS_SEGMENT(current_entry->Address))
542         {
543             if (previous_entry == NULL)
544             {
545                 MmSetRmapListHeadPage(Page, current_entry->Next);
546             }
547             else
548             {
549                 previous_entry->Next = current_entry->Next;
550             }
551             MiReleasePfnLock(OldIrql);
552             ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry);
553             return;
554         }
555         previous_entry = current_entry;
556         current_entry = current_entry->Next;
557     }
558     MiReleasePfnLock(OldIrql);
559 }
560