xref: /reactos/ntoskrnl/mm/balance.c (revision 565bf9e3) 
1 /*
2  * COPYRIGHT:   See COPYING in the top level directory
3  * PROJECT:     ReactOS kernel
4  * PURPOSE:     kernel memory management functions
5  * PROGRAMMERS: David Welch <welch@cwcom.net>
6  *              Cameron Gutman <cameron.gutman@reactos.org>
7  */
8 
9 /* INCLUDES *****************************************************************/
10 
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14 
15 #include "ARM3/miarm.h"
16 
17 
18 /* TYPES ********************************************************************/
19 typedef struct _MM_ALLOCATION_REQUEST
20 {
21     PFN_NUMBER Page;
22     LIST_ENTRY ListEntry;
23     KEVENT Event;
24 }
25 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
26 /* GLOBALS ******************************************************************/
27 
28 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
29 static ULONG MiMinimumAvailablePages;
30 static ULONG MiMinimumPagesPerRun;
31 static CLIENT_ID MiBalancerThreadId;
32 static HANDLE MiBalancerThreadHandle = NULL;
33 static KEVENT MiBalancerEvent;
34 static KEVENT MiBalancerDoneEvent;
35 static KTIMER MiBalancerTimer;
36 
37 static LONG PageOutThreadActive;
38 
39 /* FUNCTIONS ****************************************************************/
40 
41 CODE_SEG("INIT")
42 VOID
43 NTAPI
44 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
45 {
46     memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
47 
48     /* Set up targets. */
49     MiMinimumAvailablePages = 256;
50     MiMinimumPagesPerRun = 256;
51     MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages / 2;
52 }
53 
54 CODE_SEG("INIT")
55 VOID
56 NTAPI
57 MmInitializeMemoryConsumer(
58     ULONG Consumer,
59     NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
60 {
61     MiMemoryConsumers[Consumer].Trim = Trim;
62 }
63 
64 VOID
65 NTAPI
66 MiZeroPhysicalPage(
67     IN PFN_NUMBER PageFrameIndex
68 );
69 
70 NTSTATUS
71 NTAPI
72 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
73 {
74     KIRQL OldIrql;
75 
76     if (Page == 0)
77     {
78         DPRINT1("Tried to release page zero.\n");
79         KeBugCheck(MEMORY_MANAGEMENT);
80     }
81 
82     (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
83     UpdateTotalCommittedPages(-1);
84 
85     OldIrql = MiAcquirePfnLock();
86 
87     MmDereferencePage(Page);
88 
89     MiReleasePfnLock(OldIrql);
90 
91     return(STATUS_SUCCESS);
92 }
93 
94 ULONG
95 NTAPI
96 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
97 {
98     ULONG Target = InitialTarget;
99     ULONG NrFreedPages = 0;
100     NTSTATUS Status;
101 
102     /* Make sure we can trim this consumer */
103     if (!MiMemoryConsumers[Consumer].Trim)
104     {
105         /* Return the unmodified initial target */
106         return InitialTarget;
107     }
108 
109     if (MmAvailablePages < MiMinimumAvailablePages)
110     {
111         /* Global page limit exceeded */
112         Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
113     }
114     else if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
115     {
116         /* Consumer page limit exceeded */
117         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
118     }
119 
120     if (Target)
121     {
122         /* Now swap the pages out */
123         Status = MiMemoryConsumers[Consumer].Trim(Target, MmAvailablePages < MiMinimumAvailablePages, &NrFreedPages);
124 
125         DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
126 
127         if (!NT_SUCCESS(Status))
128         {
129             KeBugCheck(MEMORY_MANAGEMENT);
130         }
131     }
132 
133     /* Return the page count needed to be freed to meet the initial target */
134     return (InitialTarget > NrFreedPages) ? (InitialTarget - NrFreedPages) : 0;
135 }
136 
137 NTSTATUS
138 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
139 {
140     PFN_NUMBER FirstPage, CurrentPage;
141     NTSTATUS Status;
142 
143     (*NrFreedPages) = 0;
144 
145     DPRINT("MM BALANCER: %s\n", Priority ? "Paging out!" : "Removing access bit!");
146 
147     FirstPage = MmGetLRUFirstUserPage();
148     CurrentPage = FirstPage;
149     while (CurrentPage != 0 && Target > 0)
150     {
151         if (Priority)
152         {
153             Status = MmPageOutPhysicalAddress(CurrentPage);
154             if (NT_SUCCESS(Status))
155             {
156                 DPRINT("Succeeded\n");
157                 Target--;
158                 (*NrFreedPages)++;
159                 if (CurrentPage == FirstPage)
160                 {
161                     FirstPage = 0;
162                 }
163             }
164         }
165         else
166         {
167             /* When not paging-out agressively, just reset the accessed bit */
168             PEPROCESS Process = NULL;
169             PVOID Address = NULL;
170             BOOLEAN Accessed = FALSE;
171 
172             /*
173              * We have a lock-ordering problem here. We cant lock the PFN DB before the Process address space.
174              * So we must use circonvoluted loops.
175              * Well...
176              */
177             while (TRUE)
178             {
179                 KAPC_STATE ApcState;
180                 KIRQL OldIrql = MiAcquirePfnLock();
181                 PMM_RMAP_ENTRY Entry = MmGetRmapListHeadPage(CurrentPage);
182                 while (Entry)
183                 {
184                     if (RMAP_IS_SEGMENT(Entry->Address))
185                     {
186                         Entry = Entry->Next;
187                         continue;
188                     }
189 
190                     /* Check that we didn't treat this entry before */
191                     if (Entry->Address < Address)
192                     {
193                         Entry = Entry->Next;
194                         continue;
195                     }
196 
197                     if ((Entry->Address == Address) && (Entry->Process <= Process))
198                     {
199                         Entry = Entry->Next;
200                         continue;
201                     }
202 
203                     break;
204                 }
205 
206                 if (!Entry)
207                 {
208                     MiReleasePfnLock(OldIrql);
209                     break;
210                 }
211 
212                 Process = Entry->Process;
213                 Address = Entry->Address;
214 
215                 ObReferenceObject(Process);
216 
217                 if (!ExAcquireRundownProtection(&Process->RundownProtect))
218                 {
219                     ObDereferenceObject(Process);
220                     MiReleasePfnLock(OldIrql);
221                     continue;
222                 }
223 
224                 MiReleasePfnLock(OldIrql);
225 
226                 KeStackAttachProcess(&Process->Pcb, &ApcState);
227                 MiLockProcessWorkingSet(Process, PsGetCurrentThread());
228 
229                 /* Be sure this is still valid. */
230                 if (MmIsAddressValid(Address))
231                 {
232                     PMMPTE Pte = MiAddressToPte(Address);
233                     Accessed = Accessed || Pte->u.Hard.Accessed;
234                     Pte->u.Hard.Accessed = 0;
235 
236                     /* There is no need to invalidate, the balancer thread is never on a user process */
237                     //KeInvalidateTlbEntry(Address);
238                 }
239 
240                 MiUnlockProcessWorkingSet(Process, PsGetCurrentThread());
241 
242                 KeUnstackDetachProcess(&ApcState);
243                 ExReleaseRundownProtection(&Process->RundownProtect);
244                 ObDereferenceObject(Process);
245             }
246 
247             if (!Accessed)
248             {
249                 /* Nobody accessed this page since the last time we check. Time to clean up */
250 
251                 Status = MmPageOutPhysicalAddress(CurrentPage);
252                 if (NT_SUCCESS(Status))
253                 {
254                     if (CurrentPage == FirstPage)
255                     {
256                         FirstPage = 0;
257                     }
258                 }
259                 // DPRINT1("Paged-out one page: %s\n", NT_SUCCESS(Status) ? "Yes" : "No");
260             }
261 
262             /* Done for this page. */
263             Target--;
264         }
265 
266         CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE);
267         if (FirstPage == 0)
268         {
269             FirstPage = CurrentPage;
270         }
271         else if (CurrentPage == FirstPage)
272         {
273             DPRINT1("We are back at the start, abort!\n");
274             return STATUS_SUCCESS;
275         }
276     }
277 
278     if (CurrentPage)
279     {
280         KIRQL OldIrql = MiAcquirePfnLock();
281         MmDereferencePage(CurrentPage);
282         MiReleasePfnLock(OldIrql);
283     }
284 
285     return STATUS_SUCCESS;
286 }
287 
288 VOID
289 NTAPI
290 MmRebalanceMemoryConsumers(VOID)
291 {
292     // if (InterlockedCompareExchange(&PageOutThreadActive, 0, 1) == 0)
293     {
294         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
295     }
296 }
297 
298 VOID
299 NTAPI
300 MmRebalanceMemoryConsumersAndWait(VOID)
301 {
302     ASSERT(PsGetCurrentProcess()->AddressCreationLock.Owner != KeGetCurrentThread());
303     ASSERT(!MM_ANY_WS_LOCK_HELD(PsGetCurrentThread()));
304     ASSERT(KeGetCurrentIrql() < DISPATCH_LEVEL);
305 
306     KeResetEvent(&MiBalancerDoneEvent);
307     MmRebalanceMemoryConsumers();
308     KeWaitForSingleObject(&MiBalancerDoneEvent, Executive, KernelMode, FALSE, NULL);
309 }
310 
311 NTSTATUS
312 NTAPI
313 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
314                             PPFN_NUMBER AllocatedPage)
315 {
316     PFN_NUMBER Page;
317     static INT i = 0;
318     static LARGE_INTEGER TinyTime = {{-1L, -1L}};
319 
320     /* Delay some requests for the Memory Manager to recover pages */
321     if (i++ >= 100)
322     {
323         KeDelayExecutionThread(KernelMode, FALSE, &TinyTime);
324         i = 0;
325     }
326 
327     /*
328      * Actually allocate the page.
329      */
330     Page = MmAllocPage(Consumer);
331     if (Page == 0)
332     {
333         *AllocatedPage = 0;
334         return STATUS_NO_MEMORY;
335     }
336     *AllocatedPage = Page;
337 
338     /* Update the target */
339     InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
340     UpdateTotalCommittedPages(1);
341 
342     return(STATUS_SUCCESS);
343 }
344 
345 VOID
346 CcRosTrimCache(
347     _In_ ULONG Target,
348     _Out_ PULONG NrFreed);
349 
350 VOID NTAPI
351 MiBalancerThread(PVOID Unused)
352 {
353     PVOID WaitObjects[2];
354     NTSTATUS Status;
355     ULONG i;
356 
357     WaitObjects[0] = &MiBalancerEvent;
358     WaitObjects[1] = &MiBalancerTimer;
359 
360     while (1)
361     {
362         KeSetEvent(&MiBalancerDoneEvent, IO_NO_INCREMENT, FALSE);
363         Status = KeWaitForMultipleObjects(2,
364                                           WaitObjects,
365                                           WaitAny,
366                                           Executive,
367                                           KernelMode,
368                                           FALSE,
369                                           NULL,
370                                           NULL);
371 
372         if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
373         {
374             ULONG InitialTarget = 0;
375             ULONG Target;
376             ULONG NrFreedPages;
377 
378             do
379             {
380                 ULONG OldTarget = InitialTarget;
381 
382                 /* Trim each consumer */
383                 for (i = 0; i < MC_MAXIMUM; i++)
384                 {
385                     InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
386                 }
387 
388                 /* Trim cache */
389                 Target = max(InitialTarget, abs(MiMinimumAvailablePages - MmAvailablePages));
390                 if (Target)
391                 {
392                     CcRosTrimCache(Target, &NrFreedPages);
393                     InitialTarget -= min(NrFreedPages, InitialTarget);
394                 }
395 
396                 /* No pages left to swap! */
397                 if (InitialTarget != 0 &&
398                         InitialTarget == OldTarget)
399                 {
400                     /* Game over */
401                     KeBugCheck(NO_PAGES_AVAILABLE);
402                 }
403             }
404             while (InitialTarget != 0);
405 
406             if (Status == STATUS_WAIT_0)
407                 InterlockedDecrement(&PageOutThreadActive);
408         }
409         else
410         {
411             DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
412             KeBugCheck(MEMORY_MANAGEMENT);
413         }
414     }
415 }
416 
417 CODE_SEG("INIT")
418 VOID
419 NTAPI
420 MiInitBalancerThread(VOID)
421 {
422     KPRIORITY Priority;
423     NTSTATUS Status;
424     LARGE_INTEGER Timeout;
425 
426     KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
427     KeInitializeEvent(&MiBalancerDoneEvent, SynchronizationEvent, FALSE);
428     KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
429 
430     Timeout.QuadPart = -20000000; /* 2 sec */
431     KeSetTimerEx(&MiBalancerTimer,
432                  Timeout,
433                  2000,         /* 2 sec */
434                  NULL);
435 
436     Status = PsCreateSystemThread(&MiBalancerThreadHandle,
437                                   THREAD_ALL_ACCESS,
438                                   NULL,
439                                   NULL,
440                                   &MiBalancerThreadId,
441                                   MiBalancerThread,
442                                   NULL);
443     if (!NT_SUCCESS(Status))
444     {
445         KeBugCheck(MEMORY_MANAGEMENT);
446     }
447 
448     Priority = LOW_REALTIME_PRIORITY + 1;
449     NtSetInformationThread(MiBalancerThreadHandle,
450                            ThreadPriority,
451                            &Priority,
452                            sizeof(Priority));
453 
454 }
455 
456 
457 /* EOF */
458