xref: /reactos/ntoskrnl/mm/balance.c (revision 003b19dc) 
1 /*
2  * COPYRIGHT:       See COPYING in the top level directory
3  * PROJECT:         ReactOS kernel
4  * FILE:            ntoskrnl/mm/balance.c
5  * PURPOSE:         kernel memory managment functions
6  *
7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
8  *                  Cameron Gutman (cameron.gutman@reactos.org)
9  */
10 
11 /* INCLUDES *****************************************************************/
12 
13 #include <ntoskrnl.h>
14 #define NDEBUG
15 #include <debug.h>
16 
17 #include "ARM3/miarm.h"
18 
19 /* TYPES ********************************************************************/
20 typedef struct _MM_ALLOCATION_REQUEST
21 {
22     PFN_NUMBER Page;
23     LIST_ENTRY ListEntry;
24     KEVENT Event;
25 }
26 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
27 /* GLOBALS ******************************************************************/
28 
29 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
30 static ULONG MiMinimumAvailablePages;
31 static LIST_ENTRY AllocationListHead;
32 static KSPIN_LOCK AllocationListLock;
33 static ULONG MiMinimumPagesPerRun;
34 
35 static CLIENT_ID MiBalancerThreadId;
36 static HANDLE MiBalancerThreadHandle = NULL;
37 static KEVENT MiBalancerEvent;
38 static KTIMER MiBalancerTimer;
39 
40 static LONG PageOutThreadActive;
41 
42 /* FUNCTIONS ****************************************************************/
43 
44 CODE_SEG("INIT")
45 VOID
46 NTAPI
47 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
48 {
49     memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
50     InitializeListHead(&AllocationListHead);
51     KeInitializeSpinLock(&AllocationListLock);
52 
53     /* Set up targets. */
54     MiMinimumAvailablePages = 256;
55     MiMinimumPagesPerRun = 256;
56     MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages / 2;
57 }
58 
59 CODE_SEG("INIT")
60 VOID
61 NTAPI
62 MmInitializeMemoryConsumer(
63     ULONG Consumer,
64     NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
65 {
66     MiMemoryConsumers[Consumer].Trim = Trim;
67 }
68 
69 VOID
70 NTAPI
71 MiZeroPhysicalPage(
72     IN PFN_NUMBER PageFrameIndex
73 );
74 
75 NTSTATUS
76 NTAPI
77 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
78 {
79     if (Page == 0)
80     {
81         DPRINT1("Tried to release page zero.\n");
82         KeBugCheck(MEMORY_MANAGEMENT);
83     }
84 
85     (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
86 
87     MmDereferencePage(Page);
88 
89     return(STATUS_SUCCESS);
90 }
91 
92 ULONG
93 NTAPI
94 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
95 {
96     ULONG Target = InitialTarget;
97     ULONG NrFreedPages = 0;
98     NTSTATUS Status;
99 
100     /* Make sure we can trim this consumer */
101     if (!MiMemoryConsumers[Consumer].Trim)
102     {
103         /* Return the unmodified initial target */
104         return InitialTarget;
105     }
106 
107     if (MmAvailablePages < MiMinimumAvailablePages)
108     {
109         /* Global page limit exceeded */
110         Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
111     }
112     else if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
113     {
114         /* Consumer page limit exceeded */
115         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
116     }
117 
118     if (Target)
119     {
120         /* Now swap the pages out */
121         Status = MiMemoryConsumers[Consumer].Trim(Target, MmAvailablePages < MiMinimumAvailablePages, &NrFreedPages);
122 
123         DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
124 
125         if (!NT_SUCCESS(Status))
126         {
127             KeBugCheck(MEMORY_MANAGEMENT);
128         }
129     }
130 
131     /* Return the page count needed to be freed to meet the initial target */
132     return (InitialTarget > NrFreedPages) ? (InitialTarget - NrFreedPages) : 0;
133 }
134 
135 NTSTATUS
136 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
137 {
138     PFN_NUMBER CurrentPage;
139     NTSTATUS Status;
140 
141     (*NrFreedPages) = 0;
142 
143     DPRINT1("MM BALANCER: %s\n", Priority ? "Paging out!" : "Removing access bit!");
144 
145     CurrentPage = MmGetLRUFirstUserPage();
146     while (CurrentPage != 0 && Target > 0)
147     {
148         if (Priority)
149         {
150             Status = MmPageOutPhysicalAddress(CurrentPage);
151             if (NT_SUCCESS(Status))
152             {
153                 DPRINT("Succeeded\n");
154                 Target--;
155                 (*NrFreedPages)++;
156             }
157         }
158         else
159         {
160             /* When not paging-out agressively, just reset the accessed bit */
161             PEPROCESS Process = NULL;
162             PVOID Address = NULL;
163             BOOLEAN Accessed = FALSE;
164 
165             /*
166              * We have a lock-ordering problem here. We cant lock the PFN DB before the Process address space.
167              * So we must use circonvoluted loops.
168              * Well...
169              */
170             while (TRUE)
171             {
172                 KAPC_STATE ApcState;
173                 KIRQL OldIrql = MiAcquirePfnLock();
174                 PMM_RMAP_ENTRY Entry = MmGetRmapListHeadPage(CurrentPage);
175                 while (Entry)
176                 {
177                     if (RMAP_IS_SEGMENT(Entry->Address))
178                     {
179                         Entry = Entry->Next;
180                         continue;
181                     }
182 
183                     /* Check that we didn't treat this entry before */
184                     if (Entry->Address < Address)
185                     {
186                         Entry = Entry->Next;
187                         continue;
188                     }
189 
190                     if ((Entry->Address == Address) && (Entry->Process <= Process))
191                     {
192                         Entry = Entry->Next;
193                         continue;
194                     }
195 
196                     break;
197                 }
198 
199                 if (!Entry)
200                 {
201                     MiReleasePfnLock(OldIrql);
202                     break;
203                 }
204 
205                 Process = Entry->Process;
206                 Address = Entry->Address;
207 
208                 MiReleasePfnLock(OldIrql);
209 
210                 KeStackAttachProcess(&Process->Pcb, &ApcState);
211 
212                 MmLockAddressSpace(&Process->Vm);
213 
214                 /* Be sure this is still valid. */
215                 PMMPTE Pte = MiAddressToPte(Address);
216                 if (Pte->u.Hard.Valid)
217                 {
218                     Accessed = Accessed || Pte->u.Hard.Accessed;
219                     Pte->u.Hard.Accessed = 0;
220 
221                     /* There is no need to invalidate, the balancer thread is never on a user process */
222                     //KeInvalidateTlbEntry(Address);
223                 }
224 
225                 MmUnlockAddressSpace(&Process->Vm);
226 
227                 KeUnstackDetachProcess(&ApcState);
228             }
229 
230             if (!Accessed)
231             {
232                 /* Nobody accessed this page since the last time we check. Time to clean up */
233 
234                 Status = MmPageOutPhysicalAddress(CurrentPage);
235                 // DPRINT1("Paged-out one page: %s\n", NT_SUCCESS(Status) ? "Yes" : "No");
236                 (void)Status;
237             }
238 
239             /* Done for this page. */
240             Target--;
241         }
242 
243         CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE);
244     }
245 
246     if (CurrentPage)
247     {
248         KIRQL OldIrql = MiAcquirePfnLock();
249         MmDereferencePage(CurrentPage);
250         MiReleasePfnLock(OldIrql);
251     }
252 
253     return STATUS_SUCCESS;
254 }
255 
256 static BOOLEAN
257 MiIsBalancerThread(VOID)
258 {
259     return (MiBalancerThreadHandle != NULL) &&
260            (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
261 }
262 
263 VOID
264 NTAPI
265 MmRebalanceMemoryConsumers(VOID)
266 {
267     // if (InterlockedCompareExchange(&PageOutThreadActive, 0, 1) == 0)
268     {
269         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
270     }
271 }
272 
273 NTSTATUS
274 NTAPI
275 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
276                             PPFN_NUMBER AllocatedPage)
277 {
278     PFN_NUMBER Page;
279 
280     /* Update the target */
281     InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
282 
283     /*
284      * Actually allocate the page.
285      */
286     Page = MmAllocPage(Consumer);
287     if (Page == 0)
288     {
289         KeBugCheck(NO_PAGES_AVAILABLE);
290     }
291     *AllocatedPage = Page;
292 
293     return(STATUS_SUCCESS);
294 }
295 
296 
297 VOID NTAPI
298 MiBalancerThread(PVOID Unused)
299 {
300     PVOID WaitObjects[2];
301     NTSTATUS Status;
302     ULONG i;
303 
304     WaitObjects[0] = &MiBalancerEvent;
305     WaitObjects[1] = &MiBalancerTimer;
306 
307     while (1)
308     {
309         Status = KeWaitForMultipleObjects(2,
310                                           WaitObjects,
311                                           WaitAny,
312                                           Executive,
313                                           KernelMode,
314                                           FALSE,
315                                           NULL,
316                                           NULL);
317 
318         if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
319         {
320             ULONG InitialTarget = 0;
321 
322 #if (_MI_PAGING_LEVELS == 2)
323             if (!MiIsBalancerThread())
324             {
325                 /* Clean up the unused PDEs */
326                 ULONG_PTR Address;
327                 PEPROCESS Process = PsGetCurrentProcess();
328 
329                 /* Acquire PFN lock */
330                 KIRQL OldIrql = MiAcquirePfnLock();
331                 PMMPDE pointerPde;
332                 for (Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS;
333                      Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS;
334                      Address += PTE_PER_PAGE * PAGE_SIZE)
335                 {
336                     if (MiQueryPageTableReferences((PVOID)Address) == 0)
337                     {
338                         pointerPde = MiAddressToPde(Address);
339                         if (pointerPde->u.Hard.Valid)
340                             MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL);
341                         ASSERT(pointerPde->u.Hard.Valid == 0);
342                     }
343                 }
344                 /* Release lock */
345                 MiReleasePfnLock(OldIrql);
346             }
347 #endif
348             do
349             {
350                 ULONG OldTarget = InitialTarget;
351 
352                 /* Trim each consumer */
353                 for (i = 0; i < MC_MAXIMUM; i++)
354                 {
355                     InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
356                 }
357 
358                 /* No pages left to swap! */
359                 if (InitialTarget != 0 &&
360                         InitialTarget == OldTarget)
361                 {
362                     /* Game over */
363                     KeBugCheck(NO_PAGES_AVAILABLE);
364                 }
365             }
366             while (InitialTarget != 0);
367 
368             if (Status == STATUS_WAIT_0)
369                 InterlockedDecrement(&PageOutThreadActive);
370         }
371         else
372         {
373             DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
374             KeBugCheck(MEMORY_MANAGEMENT);
375         }
376     }
377 }
378 
379 BOOLEAN MmRosNotifyAvailablePage(PFN_NUMBER Page)
380 {
381     PLIST_ENTRY Entry;
382     PMM_ALLOCATION_REQUEST Request;
383     PMMPFN Pfn1;
384 
385     /* Make sure the PFN lock is held */
386     MI_ASSERT_PFN_LOCK_HELD();
387 
388     if (!MiMinimumAvailablePages)
389     {
390         /* Dirty way to know if we were initialized. */
391         return FALSE;
392     }
393 
394     Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock);
395     if (!Entry)
396         return FALSE;
397 
398     Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
399     MiZeroPhysicalPage(Page);
400     Request->Page = Page;
401 
402     Pfn1 = MiGetPfnEntry(Page);
403     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
404     Pfn1->u3.e2.ReferenceCount = 1;
405     Pfn1->u3.e1.PageLocation = ActiveAndValid;
406 
407     /* This marks the PFN as a ReactOS PFN */
408     Pfn1->u4.AweAllocation = TRUE;
409 
410     /* Allocate the extra ReactOS Data and zero it out */
411     Pfn1->u1.SwapEntry = 0;
412     Pfn1->RmapListHead = NULL;
413 
414     KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
415 
416     return TRUE;
417 }
418 
419 CODE_SEG("INIT")
420 VOID
421 NTAPI
422 MiInitBalancerThread(VOID)
423 {
424     KPRIORITY Priority;
425     NTSTATUS Status;
426     LARGE_INTEGER Timeout;
427 
428     KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
429     KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
430 
431     Timeout.QuadPart = -20000000; /* 2 sec */
432     KeSetTimerEx(&MiBalancerTimer,
433                  Timeout,
434                  2000,         /* 2 sec */
435                  NULL);
436 
437     Status = PsCreateSystemThread(&MiBalancerThreadHandle,
438                                   THREAD_ALL_ACCESS,
439                                   NULL,
440                                   NULL,
441                                   &MiBalancerThreadId,
442                                   MiBalancerThread,
443                                   NULL);
444     if (!NT_SUCCESS(Status))
445     {
446         KeBugCheck(MEMORY_MANAGEMENT);
447     }
448 
449     Priority = LOW_REALTIME_PRIORITY + 1;
450     NtSetInformationThread(MiBalancerThreadHandle,
451                            ThreadPriority,
452                            &Priority,
453                            sizeof(Priority));
454 
455 }
456 
457 
458 /* EOF */
459