1 /*
2 * COPYRIGHT: See COPYING in the top directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/rmap.c
5 * PURPOSE: Kernel memory management functions
6 *
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
8 */
9
10 /* INCLUDES *****************************************************************/
11
12 #include <ntoskrnl.h>
13 #include <cache/section/newmm.h>
14 #define NDEBUG
15 #include <debug.h>
16
17 /* TYPES ********************************************************************/
18
19 /* GLOBALS ******************************************************************/
20
21 static NPAGED_LOOKASIDE_LIST RmapLookasideList;
22
23 /* FUNCTIONS ****************************************************************/
24
_IRQL_requires_max_(DISPATCH_LEVEL)25 _IRQL_requires_max_(DISPATCH_LEVEL)
26 static
27 VOID
28 NTAPI
29 RmapListFree(
30 _In_ __drv_freesMem(Mem) PVOID P)
31 {
32 ExFreePoolWithTag(P, TAG_RMAP);
33 }
34
35 CODE_SEG("INIT")
36 VOID
37 NTAPI
MmInitializeRmapList(VOID)38 MmInitializeRmapList(VOID)
39 {
40 ExInitializeNPagedLookasideList (&RmapLookasideList,
41 NULL,
42 RmapListFree,
43 0,
44 sizeof(MM_RMAP_ENTRY),
45 TAG_RMAP,
46 50);
47 }
48
49 NTSTATUS
50 NTAPI
MmPageOutPhysicalAddress(PFN_NUMBER Page)51 MmPageOutPhysicalAddress(PFN_NUMBER Page)
52 {
53 PMM_RMAP_ENTRY entry;
54 PMEMORY_AREA MemoryArea;
55 PMMSUPPORT AddressSpace;
56 PVOID Address = NULL;
57 PEPROCESS Process = NULL;
58 NTSTATUS Status = STATUS_SUCCESS;
59 PMM_SECTION_SEGMENT Segment;
60 LARGE_INTEGER SegmentOffset;
61 KIRQL OldIrql;
62
63 GetEntry:
64 OldIrql = MiAcquirePfnLock();
65
66 entry = MmGetRmapListHeadPage(Page);
67
68 while (entry && RMAP_IS_SEGMENT(entry->Address))
69 entry = entry->Next;
70
71 if (entry == NULL)
72 {
73 MiReleasePfnLock(OldIrql);
74 goto WriteSegment;
75 }
76
77 Process = entry->Process;
78 Address = entry->Address;
79
80 if ((((ULONG_PTR)Address) & 0xFFF) != 0)
81 {
82 KeBugCheck(MEMORY_MANAGEMENT);
83 }
84
85 /* This is for user-mode address only */
86 ASSERT(Address < MmSystemRangeStart);
87
88 if (!ExAcquireRundownProtection(&Process->RundownProtect))
89 {
90 MiReleasePfnLock(OldIrql);
91 return STATUS_PROCESS_IS_TERMINATING;
92 }
93
94 Status = ObReferenceObjectByPointer(Process, PROCESS_ALL_ACCESS, NULL, KernelMode);
95 MiReleasePfnLock(OldIrql);
96 if (!NT_SUCCESS(Status))
97 {
98 ExReleaseRundownProtection(&Process->RundownProtect);
99 return Status;
100 }
101 AddressSpace = &Process->Vm;
102
103 MmLockAddressSpace(AddressSpace);
104
105 MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address);
106 if (MemoryArea == NULL || MemoryArea->DeleteInProgress)
107 {
108 MmUnlockAddressSpace(AddressSpace);
109 ExReleaseRundownProtection(&Process->RundownProtect);
110 ObDereferenceObject(Process);
111 goto GetEntry;
112 }
113
114
115 /* Attach to it, if needed */
116 ASSERT(PsGetCurrentProcess() == PsInitialSystemProcess);
117 if (Process != PsInitialSystemProcess)
118 KeAttachProcess(&Process->Pcb);
119
120 if (MmGetPfnForProcess(Process, Address) != Page)
121 {
122 /* This changed in the short window where we didn't have any locks */
123 if (Process != PsInitialSystemProcess)
124 KeDetachProcess();
125 MmUnlockAddressSpace(AddressSpace);
126 ExReleaseRundownProtection(&Process->RundownProtect);
127 ObDereferenceObject(Process);
128 goto GetEntry;
129 }
130
131 if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW)
132 {
133 ULONG_PTR Entry;
134 BOOLEAN Dirty;
135 PFN_NUMBER MapPage;
136 LARGE_INTEGER Offset;
137 BOOLEAN Released;
138 BOOLEAN Unmapped;
139
140 Offset.QuadPart = MemoryArea->SectionData.ViewOffset +
141 ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea));
142
143 Segment = MemoryArea->SectionData.Segment;
144
145 MmLockSectionSegment(Segment);
146
147 Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
148 if (Entry && MM_IS_WAIT_PTE(Entry))
149 {
150 /* The segment is being read or something. Give up */
151 MmUnlockSectionSegment(Segment);
152 if (Process != PsInitialSystemProcess)
153 KeDetachProcess();
154 MmUnlockAddressSpace(AddressSpace);
155 ExReleaseRundownProtection(&Process->RundownProtect);
156 ObDereferenceObject(Process);
157 return(STATUS_UNSUCCESSFUL);
158 }
159
160 /* Delete this virtual mapping in the process */
161 MmDeleteRmap(Page, Process, Address);
162 Unmapped = MmDeleteVirtualMapping(Process, Address, &Dirty, &MapPage);
163 if (!Unmapped || (MapPage != Page))
164 {
165 /*
166 * Something's corrupted, we got there because this process is
167 * supposed to be mapping this page there.
168 */
169 KeBugCheckEx(MEMORY_MANAGEMENT,
170 (ULONG_PTR)Process,
171 (ULONG_PTR)Address,
172 (ULONG_PTR)__FILE__,
173 __LINE__);
174 }
175
176 if (Page != PFN_FROM_SSE(Entry))
177 {
178 SWAPENTRY SwapEntry;
179
180 /* This page is private to the process */
181 MmUnlockSectionSegment(Segment);
182
183 /* Check if we should write it back to the page file */
184 SwapEntry = MmGetSavedSwapEntryPage(Page);
185
186 if ((SwapEntry == 0) && Dirty)
187 {
188 /* We don't have a Swap entry, yet the page is dirty. Get one */
189 SwapEntry = MmAllocSwapPage();
190 if (!SwapEntry)
191 {
192 PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea),
193 &MemoryArea->SectionData.RegionListHead,
194 Address, NULL);
195
196 /* We can't, so let this page in the Process VM */
197 MmCreateVirtualMapping(Process, Address, Region->Protect, Page);
198 MmInsertRmap(Page, Process, Address);
199 MmSetDirtyPage(Process, Address);
200
201 MmUnlockAddressSpace(AddressSpace);
202 if (Process != PsInitialSystemProcess)
203 KeDetachProcess();
204 ExReleaseRundownProtection(&Process->RundownProtect);
205 ObDereferenceObject(Process);
206
207 return STATUS_UNSUCCESSFUL;
208 }
209 }
210
211 if (Dirty)
212 {
213 SWAPENTRY Dummy;
214
215 /* Put a wait entry into the process and unlock */
216 MmCreatePageFileMapping(Process, Address, MM_WAIT_ENTRY);
217 MmUnlockAddressSpace(AddressSpace);
218
219 Status = MmWriteToSwapPage(SwapEntry, Page);
220
221 MmLockAddressSpace(AddressSpace);
222 MmDeletePageFileMapping(Process, Address, &Dummy);
223 ASSERT(Dummy == MM_WAIT_ENTRY);
224
225 if (!NT_SUCCESS(Status))
226 {
227 /* We failed at saving the content of this page. Keep it in */
228 PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea),
229 &MemoryArea->SectionData.RegionListHead,
230 Address, NULL);
231
232 /* This Swap Entry is useless to us */
233 MmSetSavedSwapEntryPage(Page, 0);
234 MmFreeSwapPage(SwapEntry);
235
236 /* We can't, so let this page in the Process VM */
237 MmCreateVirtualMapping(Process, Address, Region->Protect, Page);
238 MmInsertRmap(Page, Process, Address);
239 MmSetDirtyPage(Process, Address);
240
241 MmUnlockAddressSpace(AddressSpace);
242 if (Process != PsInitialSystemProcess)
243 KeDetachProcess();
244 ExReleaseRundownProtection(&Process->RundownProtect);
245 ObDereferenceObject(Process);
246
247 return STATUS_UNSUCCESSFUL;
248 }
249 }
250
251 if (SwapEntry)
252 {
253 /* Keep this in the process VM */
254 MmCreatePageFileMapping(Process, Address, SwapEntry);
255 MmSetSavedSwapEntryPage(Page, 0);
256 }
257
258 /* We can finally let this page go */
259 MmUnlockAddressSpace(AddressSpace);
260 if (Process != PsInitialSystemProcess)
261 KeDetachProcess();
262 #if DBG
263 OldIrql = MiAcquirePfnLock();
264 ASSERT(MmGetRmapListHeadPage(Page) == NULL);
265 MiReleasePfnLock(OldIrql);
266 #endif
267 MmReleasePageMemoryConsumer(MC_USER, Page);
268
269 ExReleaseRundownProtection(&Process->RundownProtect);
270 ObDereferenceObject(Process);
271
272 return STATUS_SUCCESS;
273 }
274
275 /* One less mapping referencing this segment */
276 Released = MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, TRUE, NULL);
277
278 MmUnlockSectionSegment(Segment);
279 if (Process != PsInitialSystemProcess)
280 KeDetachProcess();
281 MmUnlockAddressSpace(AddressSpace);
282
283 ExReleaseRundownProtection(&Process->RundownProtect);
284 ObDereferenceObject(Process);
285
286 if (Released) return STATUS_SUCCESS;
287 }
288 #ifdef NEWCC
289 else if (Type == MEMORY_AREA_CACHE)
290 {
291 /* NEWCC does locking itself */
292 MmUnlockAddressSpace(AddressSpace);
293 Status = MmpPageOutPhysicalAddress(Page);
294 }
295 #endif
296 else
297 {
298 KeBugCheck(MEMORY_MANAGEMENT);
299 }
300
301 WriteSegment:
302 /* Now write this page to file, if needed */
303 Segment = MmGetSectionAssociation(Page, &SegmentOffset);
304 if (Segment)
305 {
306 BOOLEAN Released;
307
308 MmLockSectionSegment(Segment);
309
310 Released = MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE);
311
312 MmUnlockSectionSegment(Segment);
313 MmDereferenceSegment(Segment);
314
315 if (Released)
316 {
317 return STATUS_SUCCESS;
318 }
319 }
320
321 /* If we are here, then we didn't release the page */
322 return STATUS_UNSUCCESSFUL;
323 }
324
325 VOID
326 NTAPI
MmInsertRmap(PFN_NUMBER Page,PEPROCESS Process,PVOID Address)327 MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process,
328 PVOID Address)
329 {
330 PMM_RMAP_ENTRY current_entry;
331 PMM_RMAP_ENTRY new_entry;
332 ULONG PrevSize;
333 KIRQL OldIrql;
334
335 if (!RMAP_IS_SEGMENT(Address))
336 Address = (PVOID)PAGE_ROUND_DOWN(Address);
337
338 new_entry = ExAllocateFromNPagedLookasideList(&RmapLookasideList);
339 if (new_entry == NULL)
340 {
341 KeBugCheck(MEMORY_MANAGEMENT);
342 }
343 new_entry->Address = Address;
344 new_entry->Process = (PEPROCESS)Process;
345 #if DBG
346 new_entry->Caller = _ReturnAddress();
347 #endif
348
349 if (
350 !RMAP_IS_SEGMENT(Address) &&
351 MmGetPfnForProcess(Process, Address) != Page)
352 {
353 DPRINT1("Insert rmap (%d, 0x%.8X) 0x%.8X which doesn't match physical "
354 "address 0x%.8X\n", Process ? Process->UniqueProcessId : 0,
355 Address,
356 MmGetPfnForProcess(Process, Address) << PAGE_SHIFT,
357 Page << PAGE_SHIFT);
358 KeBugCheck(MEMORY_MANAGEMENT);
359 }
360
361 OldIrql = MiAcquirePfnLock();
362 current_entry = MmGetRmapListHeadPage(Page);
363
364 PMM_RMAP_ENTRY previous_entry = NULL;
365 /* Keep the list sorted */
366 while (current_entry && (current_entry->Address < Address))
367 {
368 previous_entry = current_entry;
369 current_entry = current_entry->Next;
370 }
371
372 /* In case of clash in the address, sort by process */
373 if (current_entry && (current_entry->Address == Address))
374 {
375 while (current_entry && (current_entry->Process < Process))
376 {
377 previous_entry = current_entry;
378 current_entry = current_entry->Next;
379 }
380 }
381
382 if (current_entry && (current_entry->Address == Address) && (current_entry->Process == Process))
383 {
384 #if DBG
385 DbgPrint("MmInsertRmap tries to add a second rmap entry for address %p\n", current_entry->Address);
386 DbgPrint(" current caller %p\n", new_entry->Caller);
387 DbgPrint(" previous caller %p\n", current_entry->Caller);
388 #endif
389 KeBugCheck(MEMORY_MANAGEMENT);
390 }
391
392 new_entry->Next = current_entry;
393 if (previous_entry)
394 previous_entry->Next = new_entry;
395 else
396 MmSetRmapListHeadPage(Page, new_entry);
397
398 MiReleasePfnLock(OldIrql);
399
400 if (!RMAP_IS_SEGMENT(Address))
401 {
402 ASSERT(Process != NULL);
403 PrevSize = InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, PAGE_SIZE);
404 if (PrevSize >= Process->Vm.PeakWorkingSetSize)
405 {
406 Process->Vm.PeakWorkingSetSize = PrevSize + PAGE_SIZE;
407 }
408 }
409 }
410
411 VOID
412 NTAPI
MmDeleteRmap(PFN_NUMBER Page,PEPROCESS Process,PVOID Address)413 MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process,
414 PVOID Address)
415 {
416 PMM_RMAP_ENTRY current_entry, previous_entry;
417 KIRQL OldIrql;
418
419 OldIrql = MiAcquirePfnLock();
420 previous_entry = NULL;
421 current_entry = MmGetRmapListHeadPage(Page);
422
423 while (current_entry != NULL)
424 {
425 if (current_entry->Process == (PEPROCESS)Process &&
426 current_entry->Address == Address)
427 {
428 if (previous_entry == NULL)
429 {
430 MmSetRmapListHeadPage(Page, current_entry->Next);
431 }
432 else
433 {
434 previous_entry->Next = current_entry->Next;
435 }
436 MiReleasePfnLock(OldIrql);
437
438 ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry);
439 if (!RMAP_IS_SEGMENT(Address))
440 {
441 ASSERT(Process != NULL);
442 (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE);
443 }
444 return;
445 }
446 previous_entry = current_entry;
447 current_entry = current_entry->Next;
448 }
449 KeBugCheck(MEMORY_MANAGEMENT);
450 }
451
452 /*
453
454 Return the process pointer given when a previous call to MmInsertRmap was
455 called with a process and address pointer that conform to the segment rmap
456 schema. In short, this requires the address part to be 0xffffff00 + n
457 where n is between 0 and 255. When such an rmap exists, it specifies a
458 segment rmap in which the process part is a pointer to a slice of a section
459 page table, and the low 8 bits of the address represent a page index in the
460 page table slice. Together, this information is used by
461 MmGetSectionAssociation to determine which page entry points to this page in
462 the segment page table.
463
464 */
465
466 PVOID
467 NTAPI
MmGetSegmentRmap(PFN_NUMBER Page,PULONG RawOffset)468 MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset)
469 {
470 PCACHE_SECTION_PAGE_TABLE Result = NULL;
471 PMM_RMAP_ENTRY current_entry;//, previous_entry;
472
473 /* Must hold the PFN lock */
474 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
475
476 //previous_entry = NULL;
477 current_entry = MmGetRmapListHeadPage(Page);
478 while (current_entry != NULL)
479 {
480 if (RMAP_IS_SEGMENT(current_entry->Address))
481 {
482 Result = (PCACHE_SECTION_PAGE_TABLE)current_entry->Process;
483 *RawOffset = (ULONG_PTR)current_entry->Address & ~RMAP_SEGMENT_MASK;
484 if (*Result->Segment->Flags & MM_SEGMENT_INDELETE)
485 {
486 return NULL;
487 }
488
489 return Result;
490 }
491 //previous_entry = current_entry;
492 current_entry = current_entry->Next;
493 }
494
495 return NULL;
496 }
497
498 /*
499
500 Remove the section rmap associated with the indicated page, if it exists.
501
502 */
503
504 VOID
505 NTAPI
MmDeleteSectionAssociation(PFN_NUMBER Page)506 MmDeleteSectionAssociation(PFN_NUMBER Page)
507 {
508 PMM_RMAP_ENTRY current_entry, previous_entry;
509 KIRQL OldIrql = MiAcquirePfnLock();
510
511 previous_entry = NULL;
512 current_entry = MmGetRmapListHeadPage(Page);
513 while (current_entry != NULL)
514 {
515 if (RMAP_IS_SEGMENT(current_entry->Address))
516 {
517 if (previous_entry == NULL)
518 {
519 MmSetRmapListHeadPage(Page, current_entry->Next);
520 }
521 else
522 {
523 previous_entry->Next = current_entry->Next;
524 }
525 MiReleasePfnLock(OldIrql);
526 ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry);
527 return;
528 }
529 previous_entry = current_entry;
530 current_entry = current_entry->Next;
531 }
532 MiReleasePfnLock(OldIrql);
533 }
534