xref: /reactos/ntoskrnl/mm/balance.c (revision 40462c92) 
1 /*
2  * COPYRIGHT:       See COPYING in the top level directory
3  * PROJECT:         ReactOS kernel
4  * FILE:            ntoskrnl/mm/balance.c
5  * PURPOSE:         kernel memory managment functions
6  *
7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
8  *                  Cameron Gutman (cameron.gutman@reactos.org)
9  */
10 
11 /* INCLUDES *****************************************************************/
12 
13 #include <ntoskrnl.h>
14 #define NDEBUG
15 #include <debug.h>
16 
17 #include "ARM3/miarm.h"
18 
19 /* TYPES ********************************************************************/
20 typedef struct _MM_ALLOCATION_REQUEST
21 {
22     PFN_NUMBER Page;
23     LIST_ENTRY ListEntry;
24     KEVENT Event;
25 }
26 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
27 /* GLOBALS ******************************************************************/
28 
29 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
30 static ULONG MiMinimumAvailablePages;
31 static ULONG MiNrTotalPages;
32 static LIST_ENTRY AllocationListHead;
33 static KSPIN_LOCK AllocationListLock;
34 static ULONG MiMinimumPagesPerRun;
35 
36 static CLIENT_ID MiBalancerThreadId;
37 static HANDLE MiBalancerThreadHandle = NULL;
38 static KEVENT MiBalancerEvent;
39 static KTIMER MiBalancerTimer;
40 
41 /* FUNCTIONS ****************************************************************/
42 
43 CODE_SEG("INIT")
44 VOID
45 NTAPI
46 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
47 {
48     memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
49     InitializeListHead(&AllocationListHead);
50     KeInitializeSpinLock(&AllocationListLock);
51 
52     MiNrTotalPages = NrAvailablePages;
53 
54     /* Set up targets. */
55     MiMinimumAvailablePages = 256;
56     MiMinimumPagesPerRun = 256;
57     if ((NrAvailablePages + NrSystemPages) >= 8192)
58     {
59         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
60     }
61     else if ((NrAvailablePages + NrSystemPages) >= 4096)
62     {
63         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
64     }
65     else
66     {
67         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
68     }
69     MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
70 }
71 
72 CODE_SEG("INIT")
73 VOID
74 NTAPI
75 MmInitializeMemoryConsumer(
76     ULONG Consumer,
77     NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
78 {
79     MiMemoryConsumers[Consumer].Trim = Trim;
80 }
81 
82 VOID
83 NTAPI
84 MiZeroPhysicalPage(
85     IN PFN_NUMBER PageFrameIndex
86 );
87 
88 NTSTATUS
89 NTAPI
90 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
91 {
92     if (Page == 0)
93     {
94         DPRINT1("Tried to release page zero.\n");
95         KeBugCheck(MEMORY_MANAGEMENT);
96     }
97 
98     if (MmGetReferenceCountPage(Page) == 1)
99     {
100         if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
101         (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
102     }
103 
104     MmDereferencePage(Page);
105 
106     return(STATUS_SUCCESS);
107 }
108 
109 ULONG
110 NTAPI
111 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
112 {
113     ULONG Target = InitialTarget;
114     ULONG NrFreedPages = 0;
115     NTSTATUS Status;
116 
117     /* Make sure we can trim this consumer */
118     if (!MiMemoryConsumers[Consumer].Trim)
119     {
120         /* Return the unmodified initial target */
121         return InitialTarget;
122     }
123 
124     if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
125     {
126         /* Consumer page limit exceeded */
127         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
128     }
129     if (MmAvailablePages < MiMinimumAvailablePages)
130     {
131         /* Global page limit exceeded */
132         Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
133     }
134 
135     if (Target)
136     {
137         if (!InitialTarget)
138         {
139             /* If there was no initial target,
140              * swap at least MiMinimumPagesPerRun */
141             Target = max(Target, MiMinimumPagesPerRun);
142         }
143 
144         /* Now swap the pages out */
145         Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
146 
147         DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
148 
149         if (!NT_SUCCESS(Status))
150         {
151             KeBugCheck(MEMORY_MANAGEMENT);
152         }
153 
154         /* Update the target */
155         if (NrFreedPages < Target)
156             Target -= NrFreedPages;
157         else
158             Target = 0;
159 
160         /* Return the remaining pages needed to meet the target */
161         return Target;
162     }
163     else
164     {
165         /* Initial target is zero and we don't have anything else to add */
166         return 0;
167     }
168 }
169 
170 NTSTATUS
171 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
172 {
173     PFN_NUMBER CurrentPage;
174     PFN_NUMBER NextPage;
175     NTSTATUS Status;
176 
177     (*NrFreedPages) = 0;
178 
179     CurrentPage = MmGetLRUFirstUserPage();
180     while (CurrentPage != 0 && Target > 0)
181     {
182         Status = MmPageOutPhysicalAddress(CurrentPage);
183         if (NT_SUCCESS(Status))
184         {
185             DPRINT("Succeeded\n");
186             Target--;
187             (*NrFreedPages)++;
188         }
189 
190         NextPage = MmGetLRUNextUserPage(CurrentPage);
191         if (NextPage <= CurrentPage)
192         {
193             /* We wrapped around, so we're done */
194             break;
195         }
196         CurrentPage = NextPage;
197     }
198 
199     return STATUS_SUCCESS;
200 }
201 
202 static BOOLEAN
203 MiIsBalancerThread(VOID)
204 {
205     return (MiBalancerThreadHandle != NULL) &&
206            (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
207 }
208 
209 VOID
210 NTAPI
211 MmRebalanceMemoryConsumers(VOID)
212 {
213     if (MiBalancerThreadHandle != NULL &&
214         !MiIsBalancerThread())
215     {
216         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
217     }
218 }
219 
220 NTSTATUS
221 NTAPI
222 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
223                             PPFN_NUMBER AllocatedPage)
224 {
225     ULONG PagesUsed;
226     PFN_NUMBER Page;
227 
228     /*
229      * Make sure we don't exceed our individual target.
230      */
231     PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
232     if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget &&
233             !MiIsBalancerThread())
234     {
235         MmRebalanceMemoryConsumers();
236     }
237 
238     /*
239      * Allocate always memory for the non paged pool and for the pager thread.
240      */
241     if ((Consumer == MC_SYSTEM) /* || MiIsBalancerThread() */)
242     {
243         Page = MmAllocPage(Consumer);
244         if (Page == 0)
245         {
246             KeBugCheck(NO_PAGES_AVAILABLE);
247         }
248         if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
249         *AllocatedPage = Page;
250         if (MmAvailablePages < MiMinimumAvailablePages)
251             MmRebalanceMemoryConsumers();
252         return(STATUS_SUCCESS);
253     }
254 
255     /*
256      * Make sure we don't exceed global targets.
257      */
258     if (((MmAvailablePages < MiMinimumAvailablePages) && !MiIsBalancerThread())
259             || (MmAvailablePages < (MiMinimumAvailablePages / 2)))
260     {
261         MM_ALLOCATION_REQUEST Request;
262 
263         if (!CanWait)
264         {
265             (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
266             MmRebalanceMemoryConsumers();
267             return(STATUS_NO_MEMORY);
268         }
269 
270         /* Insert an allocation request. */
271         Request.Page = 0;
272         KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
273 
274         ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock);
275         MmRebalanceMemoryConsumers();
276 
277         KeWaitForSingleObject(&Request.Event,
278                               0,
279                               KernelMode,
280                               FALSE,
281                               NULL);
282 
283         Page = Request.Page;
284         if (Page == 0)
285         {
286             KeBugCheck(NO_PAGES_AVAILABLE);
287         }
288 
289         if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
290         *AllocatedPage = Page;
291 
292         if (MmAvailablePages < MiMinimumAvailablePages)
293         {
294             MmRebalanceMemoryConsumers();
295         }
296 
297         return(STATUS_SUCCESS);
298     }
299 
300     /*
301      * Actually allocate the page.
302      */
303     Page = MmAllocPage(Consumer);
304     if (Page == 0)
305     {
306         KeBugCheck(NO_PAGES_AVAILABLE);
307     }
308     if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
309     *AllocatedPage = Page;
310 
311     if (MmAvailablePages < MiMinimumAvailablePages)
312     {
313         MmRebalanceMemoryConsumers();
314     }
315 
316     return(STATUS_SUCCESS);
317 }
318 
319 
320 VOID NTAPI
321 MiBalancerThread(PVOID Unused)
322 {
323     PVOID WaitObjects[2];
324     NTSTATUS Status;
325     ULONG i;
326 
327     WaitObjects[0] = &MiBalancerEvent;
328     WaitObjects[1] = &MiBalancerTimer;
329 
330     while (1)
331     {
332         Status = KeWaitForMultipleObjects(2,
333                                           WaitObjects,
334                                           WaitAny,
335                                           Executive,
336                                           KernelMode,
337                                           FALSE,
338                                           NULL,
339                                           NULL);
340 
341         if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
342         {
343             ULONG InitialTarget = 0;
344 
345 #if (_MI_PAGING_LEVELS == 2)
346             if (!MiIsBalancerThread())
347             {
348                 /* Clean up the unused PDEs */
349                 ULONG_PTR Address;
350                 PEPROCESS Process = PsGetCurrentProcess();
351 
352                 /* Acquire PFN lock */
353                 KIRQL OldIrql = MiAcquirePfnLock();
354                 PMMPDE pointerPde;
355                 for (Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS;
356                      Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS;
357                      Address += PTE_PER_PAGE * PAGE_SIZE)
358                 {
359                     if (MiQueryPageTableReferences((PVOID)Address) == 0)
360                     {
361                         pointerPde = MiAddressToPde(Address);
362                         if (pointerPde->u.Hard.Valid)
363                             MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL);
364                         ASSERT(pointerPde->u.Hard.Valid == 0);
365                     }
366                 }
367                 /* Release lock */
368                 MiReleasePfnLock(OldIrql);
369             }
370 #endif
371             do
372             {
373                 ULONG OldTarget = InitialTarget;
374 
375                 /* Trim each consumer */
376                 for (i = 0; i < MC_MAXIMUM; i++)
377                 {
378                     InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
379                 }
380 
381                 /* No pages left to swap! */
382                 if (InitialTarget != 0 &&
383                         InitialTarget == OldTarget)
384                 {
385                     /* Game over */
386                     KeBugCheck(NO_PAGES_AVAILABLE);
387                 }
388             }
389             while (InitialTarget != 0);
390         }
391         else
392         {
393             DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
394             KeBugCheck(MEMORY_MANAGEMENT);
395         }
396     }
397 }
398 
399 BOOLEAN MmRosNotifyAvailablePage(PFN_NUMBER Page)
400 {
401     PLIST_ENTRY Entry;
402     PMM_ALLOCATION_REQUEST Request;
403     PMMPFN Pfn1;
404 
405     /* Make sure the PFN lock is held */
406     MI_ASSERT_PFN_LOCK_HELD();
407 
408     if (!MiMinimumAvailablePages)
409     {
410         /* Dirty way to know if we were initialized. */
411         return FALSE;
412     }
413 
414     Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock);
415     if (!Entry)
416         return FALSE;
417 
418     Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
419     MiZeroPhysicalPage(Page);
420     Request->Page = Page;
421 
422     Pfn1 = MiGetPfnEntry(Page);
423     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
424     Pfn1->u3.e2.ReferenceCount = 1;
425     Pfn1->u3.e1.PageLocation = ActiveAndValid;
426 
427     /* This marks the PFN as a ReactOS PFN */
428     Pfn1->u4.AweAllocation = TRUE;
429 
430     /* Allocate the extra ReactOS Data and zero it out */
431     Pfn1->u1.SwapEntry = 0;
432     Pfn1->RmapListHead = NULL;
433 
434     KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
435 
436     return TRUE;
437 }
438 
439 CODE_SEG("INIT")
440 VOID
441 NTAPI
442 MiInitBalancerThread(VOID)
443 {
444     KPRIORITY Priority;
445     NTSTATUS Status;
446 #if !defined(__GNUC__)
447 
448     LARGE_INTEGER dummyJunkNeeded;
449     dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
450     ;
451 #endif
452 
453 
454     KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
455     KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
456     KeSetTimerEx(&MiBalancerTimer,
457 #if defined(__GNUC__)
458                  (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
459 #else
460                  dummyJunkNeeded,
461 #endif
462                  2000,         /* 2 sec */
463                  NULL);
464 
465     Status = PsCreateSystemThread(&MiBalancerThreadHandle,
466                                   THREAD_ALL_ACCESS,
467                                   NULL,
468                                   NULL,
469                                   &MiBalancerThreadId,
470                                   MiBalancerThread,
471                                   NULL);
472     if (!NT_SUCCESS(Status))
473     {
474         KeBugCheck(MEMORY_MANAGEMENT);
475     }
476 
477     Priority = LOW_REALTIME_PRIORITY + 1;
478     NtSetInformationThread(MiBalancerThreadHandle,
479                            ThreadPriority,
480                            &Priority,
481                            sizeof(Priority));
482 
483 }
484 
485 
486 /* EOF */
487