1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/freelist.c
5 * PURPOSE: Handle the list of free physical pages
6 *
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
8 * Robert Bergkvist
9 */
10
11 /* INCLUDES ****************************************************************/
12
13 #include <ntoskrnl.h>
14 #define NDEBUG
15 #include <debug.h>
16
17 #define MODULE_INVOLVED_IN_ARM3
18 #include "ARM3/miarm.h"
19
20 #define ASSERT_IS_ROS_PFN(x) ASSERT(MI_IS_ROS_PFN(x) == TRUE);
21
22 /* GLOBALS ****************************************************************/
23
24 PMMPFN MmPfnDatabase;
25
26 PFN_NUMBER MmAvailablePages;
27 PFN_NUMBER MmResidentAvailablePages;
28 PFN_NUMBER MmResidentAvailableAtInit;
29
30 SIZE_T MmTotalCommittedPages;
31 SIZE_T MmSharedCommit;
32 SIZE_T MmDriverCommit;
33 SIZE_T MmProcessCommit;
34 SIZE_T MmPagedPoolCommit;
35 SIZE_T MmPeakCommitment;
36 SIZE_T MmtotalCommitLimitMaximum;
37
38 PMMPFN FirstUserLRUPfn;
39 PMMPFN LastUserLRUPfn;
40
41 /* FUNCTIONS *************************************************************/
42
43 PFN_NUMBER
44 NTAPI
MmGetLRUFirstUserPage(VOID)45 MmGetLRUFirstUserPage(VOID)
46 {
47 PFN_NUMBER Page;
48 KIRQL OldIrql;
49
50 /* Find the first user page */
51 OldIrql = MiAcquirePfnLock();
52
53 if (FirstUserLRUPfn == NULL)
54 {
55 MiReleasePfnLock(OldIrql);
56 return 0;
57 }
58
59 Page = MiGetPfnEntryIndex(FirstUserLRUPfn);
60 MmReferencePage(Page);
61
62 MiReleasePfnLock(OldIrql);
63
64 return Page;
65 }
66
67 static
68 VOID
MmInsertLRULastUserPage(PFN_NUMBER Page)69 MmInsertLRULastUserPage(PFN_NUMBER Page)
70 {
71 MI_ASSERT_PFN_LOCK_HELD();
72
73 PMMPFN Pfn = MiGetPfnEntry(Page);
74
75 if (FirstUserLRUPfn == NULL)
76 FirstUserLRUPfn = Pfn;
77
78 Pfn->PreviousLRU = LastUserLRUPfn;
79
80 if (LastUserLRUPfn != NULL)
81 LastUserLRUPfn->NextLRU = Pfn;
82 LastUserLRUPfn = Pfn;
83 }
84
85 static
86 VOID
MmRemoveLRUUserPage(PFN_NUMBER Page)87 MmRemoveLRUUserPage(PFN_NUMBER Page)
88 {
89 MI_ASSERT_PFN_LOCK_HELD();
90
91 /* Unset the page as a user page */
92 ASSERT(Page != 0);
93
94 PMMPFN Pfn = MiGetPfnEntry(Page);
95
96 ASSERT_IS_ROS_PFN(Pfn);
97
98 if (Pfn->PreviousLRU)
99 {
100 ASSERT(Pfn->PreviousLRU->NextLRU == Pfn);
101 Pfn->PreviousLRU->NextLRU = Pfn->NextLRU;
102 }
103 else
104 {
105 ASSERT(FirstUserLRUPfn == Pfn);
106 FirstUserLRUPfn = Pfn->NextLRU;
107 }
108
109 if (Pfn->NextLRU)
110 {
111 ASSERT(Pfn->NextLRU->PreviousLRU == Pfn);
112 Pfn->NextLRU->PreviousLRU = Pfn->PreviousLRU;
113 }
114 else
115 {
116 ASSERT(Pfn == LastUserLRUPfn);
117 LastUserLRUPfn = Pfn->PreviousLRU;
118 }
119
120 Pfn->PreviousLRU = Pfn->NextLRU = NULL;
121 }
122
123 PFN_NUMBER
124 NTAPI
MmGetLRUNextUserPage(PFN_NUMBER PreviousPage,BOOLEAN MoveToLast)125 MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast)
126 {
127 PFN_NUMBER Page = 0;
128 KIRQL OldIrql;
129
130 /* Find the next user page */
131 OldIrql = MiAcquirePfnLock();
132
133 PMMPFN PreviousPfn = MiGetPfnEntry(PreviousPage);
134 PMMPFN NextPfn = PreviousPfn->NextLRU;
135
136 /*
137 * Move this one at the end of the list.
138 * It may be freed by MmDereferencePage below.
139 * If it's not, then it means it is still hanging in some process address space.
140 * This avoids paging-out e.g. ntdll early just because it's mapped first time.
141 */
142 if ((MoveToLast) && (MmGetReferenceCountPage(PreviousPage) > 1))
143 {
144 MmRemoveLRUUserPage(PreviousPage);
145 MmInsertLRULastUserPage(PreviousPage);
146 }
147
148 if (NextPfn)
149 {
150 Page = MiGetPfnEntryIndex(NextPfn);
151 MmReferencePage(Page);
152 }
153
154 MmDereferencePage(PreviousPage);
155
156 MiReleasePfnLock(OldIrql);
157
158 return Page;
159 }
160
161 BOOLEAN
162 NTAPI
MiIsPfnFree(IN PMMPFN Pfn1)163 MiIsPfnFree(IN PMMPFN Pfn1)
164 {
165 /* Must be a free or zero page, with no references, linked */
166 return ((Pfn1->u3.e1.PageLocation <= StandbyPageList) &&
167 (Pfn1->u1.Flink) &&
168 (Pfn1->u2.Blink) &&
169 !(Pfn1->u3.e2.ReferenceCount));
170 }
171
172 BOOLEAN
173 NTAPI
MiIsPfnInUse(IN PMMPFN Pfn1)174 MiIsPfnInUse(IN PMMPFN Pfn1)
175 {
176 /* Standby list or higher, unlinked, and with references */
177 return !MiIsPfnFree(Pfn1);
178 }
179
180 PMDL
181 NTAPI
MiAllocatePagesForMdl(IN PHYSICAL_ADDRESS LowAddress,IN PHYSICAL_ADDRESS HighAddress,IN PHYSICAL_ADDRESS SkipBytes,IN SIZE_T TotalBytes,IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,IN ULONG MdlFlags)182 MiAllocatePagesForMdl(IN PHYSICAL_ADDRESS LowAddress,
183 IN PHYSICAL_ADDRESS HighAddress,
184 IN PHYSICAL_ADDRESS SkipBytes,
185 IN SIZE_T TotalBytes,
186 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
187 IN ULONG MdlFlags)
188 {
189 PMDL Mdl;
190 PFN_NUMBER PageCount, LowPage, HighPage, SkipPages, PagesFound = 0, Page;
191 PPFN_NUMBER MdlPage, LastMdlPage;
192 KIRQL OldIrql;
193 PMMPFN Pfn1;
194 INT LookForZeroedPages;
195 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
196 DPRINT1("ARM3-DEBUG: Being called with %I64x %I64x %I64x %lx %d %lu\n", LowAddress, HighAddress, SkipBytes, TotalBytes, CacheAttribute, MdlFlags);
197
198 //
199 // Convert the low address into a PFN
200 //
201 LowPage = (PFN_NUMBER)(LowAddress.QuadPart >> PAGE_SHIFT);
202
203 //
204 // Convert, and normalize, the high address into a PFN
205 //
206 HighPage = (PFN_NUMBER)(HighAddress.QuadPart >> PAGE_SHIFT);
207 if (HighPage > MmHighestPhysicalPage) HighPage = MmHighestPhysicalPage;
208
209 //
210 // Validate skipbytes and convert them into pages
211 //
212 if (BYTE_OFFSET(SkipBytes.LowPart)) return NULL;
213 SkipPages = (PFN_NUMBER)(SkipBytes.QuadPart >> PAGE_SHIFT);
214
215 /* This isn't supported at all */
216 if (SkipPages) DPRINT1("WARNING: Caller requesting SkipBytes, MDL might be mismatched\n");
217
218 //
219 // Now compute the number of pages the MDL will cover
220 //
221 PageCount = (PFN_NUMBER)ADDRESS_AND_SIZE_TO_SPAN_PAGES(0, TotalBytes);
222 do
223 {
224 //
225 // Try creating an MDL for these many pages
226 //
227 Mdl = MmCreateMdl(NULL, NULL, PageCount << PAGE_SHIFT);
228 if (Mdl) break;
229
230 //
231 // This function is not required to return the amount of pages requested
232 // In fact, it can return as little as 1 page, and callers are supposed
233 // to deal with this scenario. So re-attempt the allocation with less
234 // pages than before, and see if it worked this time.
235 //
236 PageCount -= (PageCount >> 4);
237 } while (PageCount);
238
239 //
240 // Wow, not even a single page was around!
241 //
242 if (!Mdl) return NULL;
243
244 //
245 // This is where the page array starts....
246 //
247 MdlPage = (PPFN_NUMBER)(Mdl + 1);
248
249 //
250 // Lock the PFN database
251 //
252 OldIrql = MiAcquirePfnLock();
253
254 //
255 // Are we looking for any pages, without discriminating?
256 //
257 if ((LowPage == 0) && (HighPage == MmHighestPhysicalPage))
258 {
259 //
260 // Well then, let's go shopping
261 //
262 while (PagesFound < PageCount)
263 {
264 /* Grab a page */
265 MI_SET_USAGE(MI_USAGE_MDL);
266 MI_SET_PROCESS2("Kernel");
267
268 /* FIXME: This check should be smarter */
269 Page = 0;
270 if (MmAvailablePages != 0)
271 Page = MiRemoveAnyPage(0);
272
273 if (Page == 0)
274 {
275 /* This is not good... hopefully we have at least SOME pages */
276 ASSERT(PagesFound);
277 break;
278 }
279
280 /* Grab the page entry for it */
281 Pfn1 = MiGetPfnEntry(Page);
282
283 //
284 // Make sure it's really free
285 //
286 ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
287
288 /* Now setup the page and mark it */
289 Pfn1->u3.e2.ReferenceCount = 1;
290 Pfn1->u2.ShareCount = 1;
291 MI_SET_PFN_DELETED(Pfn1);
292 Pfn1->u4.PteFrame = 0x1FFEDCB;
293 Pfn1->u3.e1.StartOfAllocation = 1;
294 Pfn1->u3.e1.EndOfAllocation = 1;
295 Pfn1->u4.VerifierAllocation = 0;
296
297 //
298 // Save it into the MDL
299 //
300 *MdlPage++ = MiGetPfnEntryIndex(Pfn1);
301 PagesFound++;
302 }
303 }
304 else
305 {
306 //
307 // You want specific range of pages. We'll do this in two runs
308 //
309 for (LookForZeroedPages = 1; LookForZeroedPages >= 0; LookForZeroedPages--)
310 {
311 //
312 // Scan the range you specified
313 //
314 for (Page = LowPage; Page < HighPage; Page++)
315 {
316 //
317 // Get the PFN entry for this page
318 //
319 Pfn1 = MiGetPfnEntry(Page);
320 ASSERT(Pfn1);
321
322 //
323 // Make sure it's free and if this is our first pass, zeroed
324 //
325 if (MiIsPfnInUse(Pfn1)) continue;
326 if ((Pfn1->u3.e1.PageLocation == ZeroedPageList) != LookForZeroedPages) continue;
327
328 /* Remove the page from the free or zero list */
329 ASSERT(Pfn1->u3.e1.ReadInProgress == 0);
330 MI_SET_USAGE(MI_USAGE_MDL);
331 MI_SET_PROCESS2("Kernel");
332 MiUnlinkFreeOrZeroedPage(Pfn1);
333
334 //
335 // Sanity checks
336 //
337 ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
338
339 //
340 // Now setup the page and mark it
341 //
342 Pfn1->u3.e2.ReferenceCount = 1;
343 Pfn1->u2.ShareCount = 1;
344 MI_SET_PFN_DELETED(Pfn1);
345 Pfn1->u4.PteFrame = 0x1FFEDCB;
346 Pfn1->u3.e1.StartOfAllocation = 1;
347 Pfn1->u3.e1.EndOfAllocation = 1;
348 Pfn1->u4.VerifierAllocation = 0;
349
350 //
351 // Save this page into the MDL
352 //
353 *MdlPage++ = Page;
354 if (++PagesFound == PageCount) break;
355 }
356
357 //
358 // If the first pass was enough, don't keep going, otherwise, go again
359 //
360 if (PagesFound == PageCount) break;
361 }
362 }
363
364 //
365 // Now release the PFN count
366 //
367 MiReleasePfnLock(OldIrql);
368
369 //
370 // We might've found less pages, but not more ;-)
371 //
372 if (PagesFound != PageCount) ASSERT(PagesFound < PageCount);
373 if (!PagesFound)
374 {
375 //
376 // If we didn' tfind any pages at all, fail
377 //
378 DPRINT1("NO MDL PAGES!\n");
379 ExFreePoolWithTag(Mdl, TAG_MDL);
380 return NULL;
381 }
382
383 //
384 // Write out how many pages we found
385 //
386 Mdl->ByteCount = (ULONG)(PagesFound << PAGE_SHIFT);
387
388 //
389 // Terminate the MDL array if there's certain missing pages
390 //
391 if (PagesFound != PageCount) *MdlPage = LIST_HEAD;
392
393 //
394 // Now go back and loop over all the MDL pages
395 //
396 MdlPage = (PPFN_NUMBER)(Mdl + 1);
397 LastMdlPage = MdlPage + PagesFound;
398 while (MdlPage < LastMdlPage)
399 {
400 //
401 // Check if we've reached the end
402 //
403 Page = *MdlPage++;
404 if (Page == LIST_HEAD) break;
405
406 //
407 // Get the PFN entry for the page and check if we should zero it out
408 //
409 Pfn1 = MiGetPfnEntry(Page);
410 ASSERT(Pfn1);
411 if (Pfn1->u3.e1.PageLocation != ZeroedPageList) MiZeroPhysicalPage(Page);
412 Pfn1->u3.e1.PageLocation = ActiveAndValid;
413 }
414
415 //
416 // We're done, mark the pages as locked
417 //
418 Mdl->Process = NULL;
419 Mdl->MdlFlags |= MDL_PAGES_LOCKED;
420 return Mdl;
421 }
422
423 VOID
424 NTAPI
MmSetRmapListHeadPage(PFN_NUMBER Pfn,PMM_RMAP_ENTRY ListHead)425 MmSetRmapListHeadPage(PFN_NUMBER Pfn, PMM_RMAP_ENTRY ListHead)
426 {
427 PMMPFN Pfn1;
428
429 /* PFN database must be locked */
430 MI_ASSERT_PFN_LOCK_HELD();
431
432 Pfn1 = MiGetPfnEntry(Pfn);
433 ASSERT(Pfn1);
434 ASSERT_IS_ROS_PFN(Pfn1);
435
436 if (ListHead)
437 {
438 /* Should not be trying to insert an RMAP for a non-active page */
439 ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
440
441 /* Set the list head address */
442 Pfn1->RmapListHead = ListHead;
443 }
444 else
445 {
446 /* ReactOS semantics dictate the page is STILL active right now */
447 ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
448
449 /* In this case, the RMAP is actually being removed, so clear field */
450 Pfn1->RmapListHead = NULL;
451
452 /* ReactOS semantics will now release the page, which will make it free and enter a colored list */
453 }
454 }
455
456 PMM_RMAP_ENTRY
457 NTAPI
MmGetRmapListHeadPage(PFN_NUMBER Pfn)458 MmGetRmapListHeadPage(PFN_NUMBER Pfn)
459 {
460 PMMPFN Pfn1;
461
462 /* PFN database must be locked */
463 MI_ASSERT_PFN_LOCK_HELD();
464
465 /* Get the entry */
466 Pfn1 = MiGetPfnEntry(Pfn);
467 ASSERT(Pfn1);
468
469 if (!MI_IS_ROS_PFN(Pfn1))
470 {
471 return NULL;
472 }
473
474 /* Should not have an RMAP for a non-active page */
475 ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
476
477 /* Get the list head */
478 return Pfn1->RmapListHead;
479 }
480
481 VOID
482 NTAPI
MmSetSavedSwapEntryPage(PFN_NUMBER Pfn,SWAPENTRY SwapEntry)483 MmSetSavedSwapEntryPage(PFN_NUMBER Pfn, SWAPENTRY SwapEntry)
484 {
485 KIRQL oldIrql;
486 PMMPFN Pfn1;
487
488 Pfn1 = MiGetPfnEntry(Pfn);
489 ASSERT(Pfn1);
490 ASSERT_IS_ROS_PFN(Pfn1);
491
492 oldIrql = MiAcquirePfnLock();
493 Pfn1->u1.SwapEntry = SwapEntry;
494 MiReleasePfnLock(oldIrql);
495 }
496
497 SWAPENTRY
498 NTAPI
MmGetSavedSwapEntryPage(PFN_NUMBER Pfn)499 MmGetSavedSwapEntryPage(PFN_NUMBER Pfn)
500 {
501 SWAPENTRY SwapEntry;
502 KIRQL oldIrql;
503 PMMPFN Pfn1;
504
505 Pfn1 = MiGetPfnEntry(Pfn);
506 ASSERT(Pfn1);
507 ASSERT_IS_ROS_PFN(Pfn1);
508
509 oldIrql = MiAcquirePfnLock();
510 SwapEntry = Pfn1->u1.SwapEntry;
511 MiReleasePfnLock(oldIrql);
512
513 return(SwapEntry);
514 }
515
516 VOID
517 NTAPI
MmReferencePage(PFN_NUMBER Pfn)518 MmReferencePage(PFN_NUMBER Pfn)
519 {
520 PMMPFN Pfn1;
521
522 DPRINT("MmReferencePage(PysicalAddress %x)\n", Pfn << PAGE_SHIFT);
523
524 MI_ASSERT_PFN_LOCK_HELD();
525 ASSERT(Pfn != 0);
526 ASSERT(Pfn <= MmHighestPhysicalPage);
527
528 Pfn1 = MiGetPfnEntry(Pfn);
529 ASSERT(Pfn1);
530 ASSERT_IS_ROS_PFN(Pfn1);
531
532 ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
533 Pfn1->u3.e2.ReferenceCount++;
534 }
535
536 ULONG
537 NTAPI
MmGetReferenceCountPage(PFN_NUMBER Pfn)538 MmGetReferenceCountPage(PFN_NUMBER Pfn)
539 {
540 ULONG RCount;
541 PMMPFN Pfn1;
542
543 MI_ASSERT_PFN_LOCK_HELD();
544
545 DPRINT("MmGetReferenceCountPage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
546
547 Pfn1 = MiGetPfnEntry(Pfn);
548 ASSERT(Pfn1);
549 ASSERT_IS_ROS_PFN(Pfn1);
550
551 RCount = Pfn1->u3.e2.ReferenceCount;
552
553 return(RCount);
554 }
555
556 BOOLEAN
557 NTAPI
MmIsPageInUse(PFN_NUMBER Pfn)558 MmIsPageInUse(PFN_NUMBER Pfn)
559 {
560 return MiIsPfnInUse(MiGetPfnEntry(Pfn));
561 }
562
563 VOID
564 NTAPI
MmDereferencePage(PFN_NUMBER Pfn)565 MmDereferencePage(PFN_NUMBER Pfn)
566 {
567 PMMPFN Pfn1;
568 DPRINT("MmDereferencePage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
569
570 MI_ASSERT_PFN_LOCK_HELD();
571
572 Pfn1 = MiGetPfnEntry(Pfn);
573 ASSERT(Pfn1);
574 ASSERT_IS_ROS_PFN(Pfn1);
575
576 ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
577 Pfn1->u3.e2.ReferenceCount--;
578 if (Pfn1->u3.e2.ReferenceCount == 0)
579 {
580 /* Apply LRU hack */
581 if (Pfn1->u4.MustBeCached)
582 {
583 MmRemoveLRUUserPage(Pfn);
584 Pfn1->u4.MustBeCached = 0;
585 }
586
587 /* Mark the page temporarily as valid, we're going to make it free soon */
588 Pfn1->u3.e1.PageLocation = ActiveAndValid;
589
590 /* It's not a ROS PFN anymore */
591 Pfn1->u4.AweAllocation = FALSE;
592
593 /* Bring it back into the free list */
594 DPRINT("Legacy free: %lx\n", Pfn);
595 MiInsertPageInFreeList(Pfn);
596 }
597 }
598
599 PFN_NUMBER
600 NTAPI
MmAllocPage(ULONG Type)601 MmAllocPage(ULONG Type)
602 {
603 PFN_NUMBER PfnOffset;
604 PMMPFN Pfn1;
605 KIRQL OldIrql;
606
607 OldIrql = MiAcquirePfnLock();
608
609 #if MI_TRACE_PFNS
610 switch(Type)
611 {
612 case MC_SYSTEM:
613 MI_SET_USAGE(MI_USAGE_CACHE);
614 break;
615 case MC_USER:
616 MI_SET_USAGE(MI_USAGE_SECTION);
617 break;
618 default:
619 ASSERT(FALSE);
620 }
621 #endif
622
623 PfnOffset = MiRemoveZeroPage(MI_GET_NEXT_COLOR());
624 if (!PfnOffset)
625 {
626 MiReleasePfnLock(OldIrql);
627 return 0;
628 }
629
630 DPRINT("Legacy allocate: %lx\n", PfnOffset);
631 Pfn1 = MiGetPfnEntry(PfnOffset);
632 Pfn1->u3.e2.ReferenceCount = 1;
633 Pfn1->u3.e1.PageLocation = ActiveAndValid;
634
635 /* This marks the PFN as a ReactOS PFN */
636 Pfn1->u4.AweAllocation = TRUE;
637
638 /* Allocate the extra ReactOS Data and zero it out */
639 Pfn1->u1.SwapEntry = 0;
640 Pfn1->RmapListHead = NULL;
641
642 Pfn1->NextLRU = NULL;
643 Pfn1->PreviousLRU = NULL;
644
645 if (Type == MC_USER)
646 {
647 Pfn1->u4.MustBeCached = 1; /* HACK again */
648 MmInsertLRULastUserPage(PfnOffset);
649 }
650
651 MiReleasePfnLock(OldIrql);
652 return PfnOffset;
653 }
654
655 /* EOF */
656