1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/freelist.c
5 * PURPOSE: Handle the list of free physical pages
6 *
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
8 * Robert Bergkvist
9 */
10
11 /* INCLUDES ****************************************************************/
12
13 #include <ntoskrnl.h>
14 #define NDEBUG
15 #include <debug.h>
16
17 #define MODULE_INVOLVED_IN_ARM3
18 #include "ARM3/miarm.h"
19
20 #define ASSERT_IS_ROS_PFN(x) ASSERT(MI_IS_ROS_PFN(x) == TRUE);
21
22 /* GLOBALS ****************************************************************/
23
24 PMMPFN MmPfnDatabase;
25
26 PFN_NUMBER MmAvailablePages;
27 PFN_NUMBER MmResidentAvailablePages;
28 PFN_NUMBER MmResidentAvailableAtInit;
29
30 SIZE_T MmTotalCommittedPages;
31 SIZE_T MmSharedCommit;
32 SIZE_T MmDriverCommit;
33 SIZE_T MmProcessCommit;
34 SIZE_T MmPagedPoolCommit;
35 SIZE_T MmPeakCommitment;
36 SIZE_T MmtotalCommitLimitMaximum;
37
38 PMMPFN FirstUserLRUPfn;
39 PMMPFN LastUserLRUPfn;
40
41 /* FUNCTIONS *************************************************************/
42
43 PFN_NUMBER
44 NTAPI
MmGetLRUFirstUserPage(VOID)45 MmGetLRUFirstUserPage(VOID)
46 {
47 PFN_NUMBER Page;
48 KIRQL OldIrql;
49
50 /* Find the first user page */
51 OldIrql = MiAcquirePfnLock();
52
53 if (FirstUserLRUPfn == NULL)
54 {
55 MiReleasePfnLock(OldIrql);
56 return 0;
57 }
58
59 Page = MiGetPfnEntryIndex(FirstUserLRUPfn);
60 MmReferencePage(Page);
61
62 MiReleasePfnLock(OldIrql);
63
64 return Page;
65 }
66
67 static
68 VOID
MmInsertLRULastUserPage(PFN_NUMBER Page)69 MmInsertLRULastUserPage(PFN_NUMBER Page)
70 {
71 MI_ASSERT_PFN_LOCK_HELD();
72
73 PMMPFN Pfn = MiGetPfnEntry(Page);
74
75 if (FirstUserLRUPfn == NULL)
76 FirstUserLRUPfn = Pfn;
77
78 Pfn->PreviousLRU = LastUserLRUPfn;
79
80 if (LastUserLRUPfn != NULL)
81 LastUserLRUPfn->NextLRU = Pfn;
82 LastUserLRUPfn = Pfn;
83 }
84
85 static
86 VOID
MmRemoveLRUUserPage(PFN_NUMBER Page)87 MmRemoveLRUUserPage(PFN_NUMBER Page)
88 {
89 MI_ASSERT_PFN_LOCK_HELD();
90
91 /* Unset the page as a user page */
92 ASSERT(Page != 0);
93
94 PMMPFN Pfn = MiGetPfnEntry(Page);
95
96 ASSERT_IS_ROS_PFN(Pfn);
97
98 if (Pfn->PreviousLRU)
99 {
100 ASSERT(Pfn->PreviousLRU->NextLRU == Pfn);
101 Pfn->PreviousLRU->NextLRU = Pfn->NextLRU;
102 }
103 else
104 {
105 ASSERT(FirstUserLRUPfn == Pfn);
106 FirstUserLRUPfn = Pfn->NextLRU;
107 }
108
109 if (Pfn->NextLRU)
110 {
111 ASSERT(Pfn->NextLRU->PreviousLRU == Pfn);
112 Pfn->NextLRU->PreviousLRU = Pfn->PreviousLRU;
113 }
114 else
115 {
116 ASSERT(Pfn == LastUserLRUPfn);
117 LastUserLRUPfn = Pfn->PreviousLRU;
118 }
119
120 Pfn->PreviousLRU = Pfn->NextLRU = NULL;
121 }
122
123 PFN_NUMBER
124 NTAPI
MmGetLRUNextUserPage(PFN_NUMBER PreviousPage,BOOLEAN MoveToLast)125 MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast)
126 {
127 PFN_NUMBER Page = 0;
128 KIRQL OldIrql;
129
130 /* Find the next user page */
131 OldIrql = MiAcquirePfnLock();
132
133 PMMPFN PreviousPfn = MiGetPfnEntry(PreviousPage);
134 PMMPFN NextPfn = PreviousPfn->NextLRU;
135
136 /*
137 * Move this one at the end of the list.
138 * It may be freed by MmDereferencePage below.
139 * If it's not, then it means it is still hanging in some process address space.
140 * This avoids paging-out e.g. ntdll early just because it's mapped first time.
141 */
142 if ((MoveToLast) && (MmGetReferenceCountPage(PreviousPage) > 1))
143 {
144 MmRemoveLRUUserPage(PreviousPage);
145 MmInsertLRULastUserPage(PreviousPage);
146 }
147
148 if (NextPfn)
149 {
150 Page = MiGetPfnEntryIndex(NextPfn);
151 MmReferencePage(Page);
152 }
153
154 MmDereferencePage(PreviousPage);
155
156 MiReleasePfnLock(OldIrql);
157
158 return Page;
159 }
160
161 BOOLEAN
162 NTAPI
MiIsPfnFree(IN PMMPFN Pfn1)163 MiIsPfnFree(IN PMMPFN Pfn1)
164 {
165 /* Must be a free or zero page, with no references, linked */
166 return ((Pfn1->u3.e1.PageLocation <= StandbyPageList) &&
167 (Pfn1->u1.Flink) &&
168 (Pfn1->u2.Blink) &&
169 !(Pfn1->u3.e2.ReferenceCount));
170 }
171
172 BOOLEAN
173 NTAPI
MiIsPfnInUse(IN PMMPFN Pfn1)174 MiIsPfnInUse(IN PMMPFN Pfn1)
175 {
176 /* Standby list or higher, unlinked, and with references */
177 return !MiIsPfnFree(Pfn1);
178 }
179
180 PMDL
181 NTAPI
MiAllocatePagesForMdl(IN PHYSICAL_ADDRESS LowAddress,IN PHYSICAL_ADDRESS HighAddress,IN PHYSICAL_ADDRESS SkipBytes,IN SIZE_T TotalBytes,IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,IN ULONG MdlFlags)182 MiAllocatePagesForMdl(IN PHYSICAL_ADDRESS LowAddress,
183 IN PHYSICAL_ADDRESS HighAddress,
184 IN PHYSICAL_ADDRESS SkipBytes,
185 IN SIZE_T TotalBytes,
186 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
187 IN ULONG MdlFlags)
188 {
189 PMDL Mdl;
190 PFN_NUMBER PageCount, LowPage, HighPage, SkipPages, PagesFound = 0, Page;
191 PPFN_NUMBER MdlPage, LastMdlPage;
192 KIRQL OldIrql;
193 PMMPFN Pfn1;
194 INT LookForZeroedPages;
195
196 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
197 DPRINT("ARM3-DEBUG: Being called with %I64x %I64x %I64x %lx %d %lu\n", LowAddress, HighAddress, SkipBytes, TotalBytes, CacheAttribute, MdlFlags);
198
199 //
200 // Convert the low address into a PFN
201 //
202 LowPage = (PFN_NUMBER)(LowAddress.QuadPart >> PAGE_SHIFT);
203
204 //
205 // Convert, and normalize, the high address into a PFN
206 //
207 HighPage = (PFN_NUMBER)(HighAddress.QuadPart >> PAGE_SHIFT);
208 if (HighPage > MmHighestPhysicalPage) HighPage = MmHighestPhysicalPage;
209
210 //
211 // Validate skipbytes and convert them into pages
212 //
213 if (BYTE_OFFSET(SkipBytes.LowPart)) return NULL;
214 SkipPages = (PFN_NUMBER)(SkipBytes.QuadPart >> PAGE_SHIFT);
215
216 /* This isn't supported at all */
217 if (SkipPages) DPRINT1("WARNING: Caller requesting SkipBytes, MDL might be mismatched\n");
218
219 //
220 // Now compute the number of pages the MDL will cover
221 //
222 PageCount = (PFN_NUMBER)ADDRESS_AND_SIZE_TO_SPAN_PAGES(0, TotalBytes);
223 do
224 {
225 //
226 // Try creating an MDL for these many pages
227 //
228 Mdl = MmCreateMdl(NULL, NULL, PageCount << PAGE_SHIFT);
229 if (Mdl) break;
230
231 //
232 // This function is not required to return the amount of pages requested
233 // In fact, it can return as little as 1 page, and callers are supposed
234 // to deal with this scenario. So re-attempt the allocation with less
235 // pages than before, and see if it worked this time.
236 //
237 PageCount -= (PageCount >> 4);
238 } while (PageCount);
239
240 //
241 // Wow, not even a single page was around!
242 //
243 if (!Mdl) return NULL;
244
245 //
246 // This is where the page array starts....
247 //
248 MdlPage = (PPFN_NUMBER)(Mdl + 1);
249
250 //
251 // Lock the PFN database
252 //
253 OldIrql = MiAcquirePfnLock();
254
255 //
256 // Are we looking for any pages, without discriminating?
257 //
258 if ((LowPage == 0) && (HighPage == MmHighestPhysicalPage))
259 {
260 //
261 // Well then, let's go shopping
262 //
263 while (PagesFound < PageCount)
264 {
265 /* Grab a page */
266 MI_SET_USAGE(MI_USAGE_MDL);
267 MI_SET_PROCESS2("Kernel");
268
269 /* FIXME: This check should be smarter */
270 Page = 0;
271 if (MmAvailablePages != 0)
272 Page = MiRemoveAnyPage(0);
273
274 if (Page == 0)
275 {
276 /* This is not good... hopefully we have at least SOME pages */
277 ASSERT(PagesFound);
278 break;
279 }
280
281 /* Grab the page entry for it */
282 Pfn1 = MiGetPfnEntry(Page);
283
284 //
285 // Make sure it's really free
286 //
287 ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
288
289 /* Now setup the page and mark it */
290 Pfn1->u3.e2.ReferenceCount = 1;
291 Pfn1->u2.ShareCount = 1;
292 MI_SET_PFN_DELETED(Pfn1);
293 Pfn1->u4.PteFrame = 0x1FFEDCB;
294 Pfn1->u3.e1.StartOfAllocation = 1;
295 Pfn1->u3.e1.EndOfAllocation = 1;
296 Pfn1->u4.VerifierAllocation = 0;
297
298 //
299 // Save it into the MDL
300 //
301 *MdlPage++ = MiGetPfnEntryIndex(Pfn1);
302 PagesFound++;
303 }
304 }
305 else
306 {
307 //
308 // You want specific range of pages. We'll do this in two runs
309 //
310 for (LookForZeroedPages = 1; LookForZeroedPages >= 0; LookForZeroedPages--)
311 {
312 //
313 // Scan the range you specified
314 //
315 for (Page = LowPage; Page < HighPage; Page++)
316 {
317 //
318 // Get the PFN entry for this page
319 //
320 Pfn1 = MiGetPfnEntry(Page);
321 ASSERT(Pfn1);
322
323 //
324 // Make sure it's free and if this is our first pass, zeroed
325 //
326 if (MiIsPfnInUse(Pfn1)) continue;
327 if ((Pfn1->u3.e1.PageLocation == ZeroedPageList) != LookForZeroedPages) continue;
328
329 /* Remove the page from the free or zero list */
330 ASSERT(Pfn1->u3.e1.ReadInProgress == 0);
331 MI_SET_USAGE(MI_USAGE_MDL);
332 MI_SET_PROCESS2("Kernel");
333 MiUnlinkFreeOrZeroedPage(Pfn1);
334
335 //
336 // Sanity checks
337 //
338 ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
339
340 //
341 // Now setup the page and mark it
342 //
343 Pfn1->u3.e2.ReferenceCount = 1;
344 Pfn1->u2.ShareCount = 1;
345 MI_SET_PFN_DELETED(Pfn1);
346 Pfn1->u4.PteFrame = 0x1FFEDCB;
347 Pfn1->u3.e1.StartOfAllocation = 1;
348 Pfn1->u3.e1.EndOfAllocation = 1;
349 Pfn1->u4.VerifierAllocation = 0;
350
351 //
352 // Save this page into the MDL
353 //
354 *MdlPage++ = Page;
355 if (++PagesFound == PageCount) break;
356 }
357
358 //
359 // If the first pass was enough, don't keep going, otherwise, go again
360 //
361 if (PagesFound == PageCount) break;
362 }
363 }
364
365 //
366 // Now release the PFN count
367 //
368 MiReleasePfnLock(OldIrql);
369
370 //
371 // We might've found less pages, but not more ;-)
372 //
373 if (PagesFound != PageCount) ASSERT(PagesFound < PageCount);
374 if (!PagesFound)
375 {
376 //
377 // If we didn' tfind any pages at all, fail
378 //
379 DPRINT1("NO MDL PAGES!\n");
380 ExFreePoolWithTag(Mdl, TAG_MDL);
381 return NULL;
382 }
383
384 //
385 // Write out how many pages we found
386 //
387 Mdl->ByteCount = (ULONG)(PagesFound << PAGE_SHIFT);
388
389 //
390 // Terminate the MDL array if there's certain missing pages
391 //
392 if (PagesFound != PageCount) *MdlPage = LIST_HEAD;
393
394 //
395 // Now go back and loop over all the MDL pages
396 //
397 MdlPage = (PPFN_NUMBER)(Mdl + 1);
398 LastMdlPage = MdlPage + PagesFound;
399 while (MdlPage < LastMdlPage)
400 {
401 //
402 // Check if we've reached the end
403 //
404 Page = *MdlPage++;
405 if (Page == LIST_HEAD) break;
406
407 //
408 // Get the PFN entry for the page and check if we should zero it out
409 //
410 Pfn1 = MiGetPfnEntry(Page);
411 ASSERT(Pfn1);
412 if (Pfn1->u3.e1.PageLocation != ZeroedPageList) MiZeroPhysicalPage(Page);
413 Pfn1->u3.e1.PageLocation = ActiveAndValid;
414 }
415
416 //
417 // We're done, mark the pages as locked
418 //
419 Mdl->Process = NULL;
420 Mdl->MdlFlags |= MDL_PAGES_LOCKED;
421 return Mdl;
422 }
423
424 VOID
425 NTAPI
MmSetRmapListHeadPage(PFN_NUMBER Pfn,PMM_RMAP_ENTRY ListHead)426 MmSetRmapListHeadPage(PFN_NUMBER Pfn, PMM_RMAP_ENTRY ListHead)
427 {
428 PMMPFN Pfn1;
429
430 /* PFN database must be locked */
431 MI_ASSERT_PFN_LOCK_HELD();
432
433 Pfn1 = MiGetPfnEntry(Pfn);
434 ASSERT(Pfn1);
435 ASSERT_IS_ROS_PFN(Pfn1);
436
437 if (ListHead)
438 {
439 /* Should not be trying to insert an RMAP for a non-active page */
440 ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
441
442 /* Set the list head address */
443 Pfn1->RmapListHead = ListHead;
444 }
445 else
446 {
447 /* ReactOS semantics dictate the page is STILL active right now */
448 ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
449
450 /* In this case, the RMAP is actually being removed, so clear field */
451 Pfn1->RmapListHead = NULL;
452
453 /* ReactOS semantics will now release the page, which will make it free and enter a colored list */
454 }
455 }
456
457 PMM_RMAP_ENTRY
458 NTAPI
MmGetRmapListHeadPage(PFN_NUMBER Pfn)459 MmGetRmapListHeadPage(PFN_NUMBER Pfn)
460 {
461 PMMPFN Pfn1;
462
463 /* PFN database must be locked */
464 MI_ASSERT_PFN_LOCK_HELD();
465
466 /* Get the entry */
467 Pfn1 = MiGetPfnEntry(Pfn);
468 ASSERT(Pfn1);
469
470 if (!MI_IS_ROS_PFN(Pfn1))
471 {
472 return NULL;
473 }
474
475 /* Should not have an RMAP for a non-active page */
476 ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
477
478 /* Get the list head */
479 return Pfn1->RmapListHead;
480 }
481
482 VOID
483 NTAPI
MmSetSavedSwapEntryPage(PFN_NUMBER Pfn,SWAPENTRY SwapEntry)484 MmSetSavedSwapEntryPage(PFN_NUMBER Pfn, SWAPENTRY SwapEntry)
485 {
486 KIRQL oldIrql;
487 PMMPFN Pfn1;
488
489 Pfn1 = MiGetPfnEntry(Pfn);
490 ASSERT(Pfn1);
491 ASSERT_IS_ROS_PFN(Pfn1);
492
493 oldIrql = MiAcquirePfnLock();
494 Pfn1->u1.SwapEntry = SwapEntry;
495 MiReleasePfnLock(oldIrql);
496 }
497
498 SWAPENTRY
499 NTAPI
MmGetSavedSwapEntryPage(PFN_NUMBER Pfn)500 MmGetSavedSwapEntryPage(PFN_NUMBER Pfn)
501 {
502 SWAPENTRY SwapEntry;
503 KIRQL oldIrql;
504 PMMPFN Pfn1;
505
506 Pfn1 = MiGetPfnEntry(Pfn);
507 ASSERT(Pfn1);
508 ASSERT_IS_ROS_PFN(Pfn1);
509
510 oldIrql = MiAcquirePfnLock();
511 SwapEntry = Pfn1->u1.SwapEntry;
512 MiReleasePfnLock(oldIrql);
513
514 return(SwapEntry);
515 }
516
517 VOID
518 NTAPI
MmReferencePage(PFN_NUMBER Pfn)519 MmReferencePage(PFN_NUMBER Pfn)
520 {
521 PMMPFN Pfn1;
522
523 DPRINT("MmReferencePage(PysicalAddress %x)\n", Pfn << PAGE_SHIFT);
524
525 MI_ASSERT_PFN_LOCK_HELD();
526 ASSERT(Pfn != 0);
527 ASSERT(Pfn <= MmHighestPhysicalPage);
528
529 Pfn1 = MiGetPfnEntry(Pfn);
530 ASSERT(Pfn1);
531 ASSERT_IS_ROS_PFN(Pfn1);
532
533 ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
534 Pfn1->u3.e2.ReferenceCount++;
535 }
536
537 ULONG
538 NTAPI
MmGetReferenceCountPage(PFN_NUMBER Pfn)539 MmGetReferenceCountPage(PFN_NUMBER Pfn)
540 {
541 ULONG RCount;
542 PMMPFN Pfn1;
543
544 MI_ASSERT_PFN_LOCK_HELD();
545
546 DPRINT("MmGetReferenceCountPage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
547
548 Pfn1 = MiGetPfnEntry(Pfn);
549 ASSERT(Pfn1);
550 ASSERT_IS_ROS_PFN(Pfn1);
551
552 RCount = Pfn1->u3.e2.ReferenceCount;
553
554 return(RCount);
555 }
556
557 BOOLEAN
558 NTAPI
MmIsPageInUse(PFN_NUMBER Pfn)559 MmIsPageInUse(PFN_NUMBER Pfn)
560 {
561 return MiIsPfnInUse(MiGetPfnEntry(Pfn));
562 }
563
564 VOID
565 NTAPI
MmDereferencePage(PFN_NUMBER Pfn)566 MmDereferencePage(PFN_NUMBER Pfn)
567 {
568 PMMPFN Pfn1;
569 DPRINT("MmDereferencePage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
570
571 MI_ASSERT_PFN_LOCK_HELD();
572
573 Pfn1 = MiGetPfnEntry(Pfn);
574 ASSERT(Pfn1);
575 ASSERT_IS_ROS_PFN(Pfn1);
576
577 ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
578 Pfn1->u3.e2.ReferenceCount--;
579 if (Pfn1->u3.e2.ReferenceCount == 0)
580 {
581 /* Apply LRU hack */
582 if (Pfn1->u4.MustBeCached)
583 {
584 MmRemoveLRUUserPage(Pfn);
585 Pfn1->u4.MustBeCached = 0;
586 }
587
588 /* Mark the page temporarily as valid, we're going to make it free soon */
589 Pfn1->u3.e1.PageLocation = ActiveAndValid;
590
591 /* It's not a ROS PFN anymore */
592 Pfn1->u4.AweAllocation = FALSE;
593
594 /* Bring it back into the free list */
595 DPRINT("Legacy free: %lx\n", Pfn);
596 MiInsertPageInFreeList(Pfn);
597 }
598 }
599
600 PFN_NUMBER
601 NTAPI
MmAllocPage(ULONG Type)602 MmAllocPage(ULONG Type)
603 {
604 PFN_NUMBER PfnOffset;
605 PMMPFN Pfn1;
606 KIRQL OldIrql;
607
608 OldIrql = MiAcquirePfnLock();
609
610 #if MI_TRACE_PFNS
611 switch(Type)
612 {
613 case MC_SYSTEM:
614 MI_SET_USAGE(MI_USAGE_CACHE);
615 break;
616 case MC_USER:
617 MI_SET_USAGE(MI_USAGE_SECTION);
618 break;
619 default:
620 ASSERT(FALSE);
621 }
622 #endif
623
624 PfnOffset = MiRemoveZeroPage(MI_GET_NEXT_COLOR());
625 if (!PfnOffset)
626 {
627 MiReleasePfnLock(OldIrql);
628 return 0;
629 }
630
631 DPRINT("Legacy allocate: %lx\n", PfnOffset);
632 Pfn1 = MiGetPfnEntry(PfnOffset);
633 Pfn1->u3.e2.ReferenceCount = 1;
634 Pfn1->u3.e1.PageLocation = ActiveAndValid;
635
636 /* This marks the PFN as a ReactOS PFN */
637 Pfn1->u4.AweAllocation = TRUE;
638
639 /* Allocate the extra ReactOS Data and zero it out */
640 Pfn1->u1.SwapEntry = 0;
641 Pfn1->RmapListHead = NULL;
642
643 Pfn1->NextLRU = NULL;
644 Pfn1->PreviousLRU = NULL;
645
646 if (Type == MC_USER)
647 {
648 Pfn1->u4.MustBeCached = 1; /* HACK again */
649 MmInsertLRULastUserPage(PfnOffset);
650 }
651
652 MiReleasePfnLock(OldIrql);
653 return PfnOffset;
654 }
655
656 /* EOF */
657