xref: /reactos/ntoskrnl/mm/balance.c (revision e5993f13) 
1 /*
2  * COPYRIGHT:       See COPYING in the top level directory
3  * PROJECT:         ReactOS kernel
4  * FILE:            ntoskrnl/mm/balance.c
5  * PURPOSE:         kernel memory management functions
6  *
7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
8  *                  Cameron Gutman (cameron.gutman@reactos.org)
9  */
10 
11 /* INCLUDES *****************************************************************/
12 
13 #include <ntoskrnl.h>
14 #define NDEBUG
15 #include <debug.h>
16 
17 #include "ARM3/miarm.h"
18 
19 
20 /* TYPES ********************************************************************/
21 typedef struct _MM_ALLOCATION_REQUEST
22 {
23     PFN_NUMBER Page;
24     LIST_ENTRY ListEntry;
25     KEVENT Event;
26 }
27 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
28 /* GLOBALS ******************************************************************/
29 
30 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
31 static ULONG MiMinimumAvailablePages;
32 static ULONG MiMinimumPagesPerRun;
33 static CLIENT_ID MiBalancerThreadId;
34 static HANDLE MiBalancerThreadHandle = NULL;
35 static KEVENT MiBalancerEvent;
36 static KEVENT MiBalancerDoneEvent;
37 static KTIMER MiBalancerTimer;
38 
39 static LONG PageOutThreadActive;
40 
41 /* FUNCTIONS ****************************************************************/
42 
43 CODE_SEG("INIT")
44 VOID
45 NTAPI
46 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
47 {
48     memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
49 
50     /* Set up targets. */
51     MiMinimumAvailablePages = 256;
52     MiMinimumPagesPerRun = 256;
53     MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages / 2;
54 }
55 
56 CODE_SEG("INIT")
57 VOID
58 NTAPI
59 MmInitializeMemoryConsumer(
60     ULONG Consumer,
61     NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
62 {
63     MiMemoryConsumers[Consumer].Trim = Trim;
64 }
65 
66 VOID
67 NTAPI
68 MiZeroPhysicalPage(
69     IN PFN_NUMBER PageFrameIndex
70 );
71 
72 NTSTATUS
73 NTAPI
74 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
75 {
76     KIRQL OldIrql;
77 
78     if (Page == 0)
79     {
80         DPRINT1("Tried to release page zero.\n");
81         KeBugCheck(MEMORY_MANAGEMENT);
82     }
83 
84     (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
85     UpdateTotalCommittedPages(-1);
86 
87     OldIrql = MiAcquirePfnLock();
88 
89     MmDereferencePage(Page);
90 
91     MiReleasePfnLock(OldIrql);
92 
93     return(STATUS_SUCCESS);
94 }
95 
96 ULONG
97 NTAPI
98 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
99 {
100     ULONG Target = InitialTarget;
101     ULONG NrFreedPages = 0;
102     NTSTATUS Status;
103 
104     /* Make sure we can trim this consumer */
105     if (!MiMemoryConsumers[Consumer].Trim)
106     {
107         /* Return the unmodified initial target */
108         return InitialTarget;
109     }
110 
111     if (MmAvailablePages < MiMinimumAvailablePages)
112     {
113         /* Global page limit exceeded */
114         Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
115     }
116     else if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
117     {
118         /* Consumer page limit exceeded */
119         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
120     }
121 
122     if (Target)
123     {
124         /* Now swap the pages out */
125         Status = MiMemoryConsumers[Consumer].Trim(Target, MmAvailablePages < MiMinimumAvailablePages, &NrFreedPages);
126 
127         DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
128 
129         if (!NT_SUCCESS(Status))
130         {
131             KeBugCheck(MEMORY_MANAGEMENT);
132         }
133     }
134 
135     /* Return the page count needed to be freed to meet the initial target */
136     return (InitialTarget > NrFreedPages) ? (InitialTarget - NrFreedPages) : 0;
137 }
138 
139 NTSTATUS
140 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
141 {
142     PFN_NUMBER FirstPage, CurrentPage;
143     NTSTATUS Status;
144 
145     (*NrFreedPages) = 0;
146 
147     DPRINT1("MM BALANCER: %s\n", Priority ? "Paging out!" : "Removing access bit!");
148 
149     FirstPage = MmGetLRUFirstUserPage();
150     CurrentPage = FirstPage;
151     while (CurrentPage != 0 && Target > 0)
152     {
153         if (Priority)
154         {
155             Status = MmPageOutPhysicalAddress(CurrentPage);
156             if (NT_SUCCESS(Status))
157             {
158                 DPRINT("Succeeded\n");
159                 Target--;
160                 (*NrFreedPages)++;
161                 if (CurrentPage == FirstPage)
162                 {
163                     FirstPage = 0;
164                 }
165             }
166         }
167         else
168         {
169             /* When not paging-out agressively, just reset the accessed bit */
170             PEPROCESS Process = NULL;
171             PVOID Address = NULL;
172             BOOLEAN Accessed = FALSE;
173 
174             /*
175              * We have a lock-ordering problem here. We cant lock the PFN DB before the Process address space.
176              * So we must use circonvoluted loops.
177              * Well...
178              */
179             while (TRUE)
180             {
181                 KAPC_STATE ApcState;
182                 KIRQL OldIrql = MiAcquirePfnLock();
183                 PMM_RMAP_ENTRY Entry = MmGetRmapListHeadPage(CurrentPage);
184                 while (Entry)
185                 {
186                     if (RMAP_IS_SEGMENT(Entry->Address))
187                     {
188                         Entry = Entry->Next;
189                         continue;
190                     }
191 
192                     /* Check that we didn't treat this entry before */
193                     if (Entry->Address < Address)
194                     {
195                         Entry = Entry->Next;
196                         continue;
197                     }
198 
199                     if ((Entry->Address == Address) && (Entry->Process <= Process))
200                     {
201                         Entry = Entry->Next;
202                         continue;
203                     }
204 
205                     break;
206                 }
207 
208                 if (!Entry)
209                 {
210                     MiReleasePfnLock(OldIrql);
211                     break;
212                 }
213 
214                 Process = Entry->Process;
215                 Address = Entry->Address;
216 
217                 ObReferenceObject(Process);
218 
219                 if (!ExAcquireRundownProtection(&Process->RundownProtect))
220                 {
221                     ObDereferenceObject(Process);
222                     MiReleasePfnLock(OldIrql);
223                     continue;
224                 }
225 
226                 MiReleasePfnLock(OldIrql);
227 
228                 KeStackAttachProcess(&Process->Pcb, &ApcState);
229                 MiLockProcessWorkingSet(Process, PsGetCurrentThread());
230 
231                 /* Be sure this is still valid. */
232                 if (MmIsAddressValid(Address))
233                 {
234                     PMMPTE Pte = MiAddressToPte(Address);
235                     Accessed = Accessed || Pte->u.Hard.Accessed;
236                     Pte->u.Hard.Accessed = 0;
237 
238                     /* There is no need to invalidate, the balancer thread is never on a user process */
239                     //KeInvalidateTlbEntry(Address);
240                 }
241 
242                 MiUnlockProcessWorkingSet(Process, PsGetCurrentThread());
243 
244                 KeUnstackDetachProcess(&ApcState);
245                 ExReleaseRundownProtection(&Process->RundownProtect);
246                 ObDereferenceObject(Process);
247             }
248 
249             if (!Accessed)
250             {
251                 /* Nobody accessed this page since the last time we check. Time to clean up */
252 
253                 Status = MmPageOutPhysicalAddress(CurrentPage);
254                 if (NT_SUCCESS(Status))
255                 {
256                     if (CurrentPage == FirstPage)
257                     {
258                         FirstPage = 0;
259                     }
260                 }
261                 // DPRINT1("Paged-out one page: %s\n", NT_SUCCESS(Status) ? "Yes" : "No");
262             }
263 
264             /* Done for this page. */
265             Target--;
266         }
267 
268         CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE);
269         if (FirstPage == 0)
270         {
271             FirstPage = CurrentPage;
272         }
273         else if (CurrentPage == FirstPage)
274         {
275             DPRINT1("We are back at the start, abort!\n");
276             return STATUS_SUCCESS;
277         }
278     }
279 
280     if (CurrentPage)
281     {
282         KIRQL OldIrql = MiAcquirePfnLock();
283         MmDereferencePage(CurrentPage);
284         MiReleasePfnLock(OldIrql);
285     }
286 
287     return STATUS_SUCCESS;
288 }
289 
290 VOID
291 NTAPI
292 MmRebalanceMemoryConsumers(VOID)
293 {
294     // if (InterlockedCompareExchange(&PageOutThreadActive, 0, 1) == 0)
295     {
296         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
297     }
298 }
299 
300 VOID
301 NTAPI
302 MmRebalanceMemoryConsumersAndWait(VOID)
303 {
304     ASSERT(PsGetCurrentProcess()->AddressCreationLock.Owner != KeGetCurrentThread());
305     ASSERT(!MM_ANY_WS_LOCK_HELD(PsGetCurrentThread()));
306     ASSERT(KeGetCurrentIrql() < DISPATCH_LEVEL);
307 
308     KeResetEvent(&MiBalancerDoneEvent);
309     MmRebalanceMemoryConsumers();
310     KeWaitForSingleObject(&MiBalancerDoneEvent, Executive, KernelMode, FALSE, NULL);
311 }
312 
313 NTSTATUS
314 NTAPI
315 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
316                             PPFN_NUMBER AllocatedPage)
317 {
318     PFN_NUMBER Page;
319 
320     /*
321      * Actually allocate the page.
322      */
323     Page = MmAllocPage(Consumer);
324     if (Page == 0)
325     {
326         *AllocatedPage = 0;
327         return STATUS_NO_MEMORY;
328     }
329     *AllocatedPage = Page;
330 
331     /* Update the target */
332     InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
333     UpdateTotalCommittedPages(1);
334 
335     return(STATUS_SUCCESS);
336 }
337 
338 
339 VOID NTAPI
340 MiBalancerThread(PVOID Unused)
341 {
342     PVOID WaitObjects[2];
343     NTSTATUS Status;
344     ULONG i;
345 
346     WaitObjects[0] = &MiBalancerEvent;
347     WaitObjects[1] = &MiBalancerTimer;
348 
349     while (1)
350     {
351         KeSetEvent(&MiBalancerDoneEvent, IO_NO_INCREMENT, FALSE);
352         Status = KeWaitForMultipleObjects(2,
353                                           WaitObjects,
354                                           WaitAny,
355                                           Executive,
356                                           KernelMode,
357                                           FALSE,
358                                           NULL,
359                                           NULL);
360 
361         if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
362         {
363             ULONG InitialTarget = 0;
364 
365             do
366             {
367                 ULONG OldTarget = InitialTarget;
368 
369                 /* Trim each consumer */
370                 for (i = 0; i < MC_MAXIMUM; i++)
371                 {
372                     InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
373                 }
374 
375                 /* No pages left to swap! */
376                 if (InitialTarget != 0 &&
377                         InitialTarget == OldTarget)
378                 {
379                     /* Game over */
380                     KeBugCheck(NO_PAGES_AVAILABLE);
381                 }
382             }
383             while (InitialTarget != 0);
384 
385             if (Status == STATUS_WAIT_0)
386                 InterlockedDecrement(&PageOutThreadActive);
387         }
388         else
389         {
390             DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
391             KeBugCheck(MEMORY_MANAGEMENT);
392         }
393     }
394 }
395 
396 CODE_SEG("INIT")
397 VOID
398 NTAPI
399 MiInitBalancerThread(VOID)
400 {
401     KPRIORITY Priority;
402     NTSTATUS Status;
403     LARGE_INTEGER Timeout;
404 
405     KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
406     KeInitializeEvent(&MiBalancerDoneEvent, SynchronizationEvent, FALSE);
407     KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
408 
409     Timeout.QuadPart = -20000000; /* 2 sec */
410     KeSetTimerEx(&MiBalancerTimer,
411                  Timeout,
412                  2000,         /* 2 sec */
413                  NULL);
414 
415     Status = PsCreateSystemThread(&MiBalancerThreadHandle,
416                                   THREAD_ALL_ACCESS,
417                                   NULL,
418                                   NULL,
419                                   &MiBalancerThreadId,
420                                   MiBalancerThread,
421                                   NULL);
422     if (!NT_SUCCESS(Status))
423     {
424         KeBugCheck(MEMORY_MANAGEMENT);
425     }
426 
427     Priority = LOW_REALTIME_PRIORITY + 1;
428     NtSetInformationThread(MiBalancerThreadHandle,
429                            ThreadPriority,
430                            &Priority,
431                            sizeof(Priority));
432 
433 }
434 
435 
436 /* EOF */
437