xref: /reactos/ntoskrnl/mm/balance.c (revision e5813c46) 
1 /*
2  * COPYRIGHT:       See COPYING in the top level directory
3  * PROJECT:         ReactOS kernel
4  * FILE:            ntoskrnl/mm/balance.c
5  * PURPOSE:         kernel memory managment functions
6  *
7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
8  *                  Cameron Gutman (cameron.gutman@reactos.org)
9  */
10 
11 /* INCLUDES *****************************************************************/
12 
13 #include <ntoskrnl.h>
14 #define NDEBUG
15 #include <debug.h>
16 
17 #include "ARM3/miarm.h"
18 
19 #if defined (ALLOC_PRAGMA)
20 #pragma alloc_text(INIT, MmInitializeBalancer)
21 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
22 #pragma alloc_text(INIT, MiInitBalancerThread)
23 #endif
24 
25 
26 /* TYPES ********************************************************************/
27 typedef struct _MM_ALLOCATION_REQUEST
28 {
29     PFN_NUMBER Page;
30     LIST_ENTRY ListEntry;
31     KEVENT Event;
32 }
33 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
34 /* GLOBALS ******************************************************************/
35 
36 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
37 static ULONG MiMinimumAvailablePages;
38 static ULONG MiNrTotalPages;
39 static LIST_ENTRY AllocationListHead;
40 static KSPIN_LOCK AllocationListLock;
41 static ULONG MiMinimumPagesPerRun;
42 
43 static CLIENT_ID MiBalancerThreadId;
44 static HANDLE MiBalancerThreadHandle = NULL;
45 static KEVENT MiBalancerEvent;
46 static KTIMER MiBalancerTimer;
47 
48 /* FUNCTIONS ****************************************************************/
49 
50 VOID
51 INIT_FUNCTION
52 NTAPI
53 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
54 {
55     memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
56     InitializeListHead(&AllocationListHead);
57     KeInitializeSpinLock(&AllocationListLock);
58 
59     MiNrTotalPages = NrAvailablePages;
60 
61     /* Set up targets. */
62     MiMinimumAvailablePages = 256;
63     MiMinimumPagesPerRun = 256;
64     if ((NrAvailablePages + NrSystemPages) >= 8192)
65     {
66         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
67     }
68     else if ((NrAvailablePages + NrSystemPages) >= 4096)
69     {
70         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
71     }
72     else
73     {
74         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
75     }
76     MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
77 }
78 
79 VOID
80 INIT_FUNCTION
81 NTAPI
82 MmInitializeMemoryConsumer(
83     ULONG Consumer,
84     NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
85 {
86     MiMemoryConsumers[Consumer].Trim = Trim;
87 }
88 
89 VOID
90 NTAPI
91 MiZeroPhysicalPage(
92     IN PFN_NUMBER PageFrameIndex
93 );
94 
95 NTSTATUS
96 NTAPI
97 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
98 {
99     if (Page == 0)
100     {
101         DPRINT1("Tried to release page zero.\n");
102         KeBugCheck(MEMORY_MANAGEMENT);
103     }
104 
105     if (MmGetReferenceCountPage(Page) == 1)
106     {
107         if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
108         (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
109     }
110 
111     MmDereferencePage(Page);
112 
113     return(STATUS_SUCCESS);
114 }
115 
116 ULONG
117 NTAPI
118 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
119 {
120     ULONG Target = InitialTarget;
121     ULONG NrFreedPages = 0;
122     NTSTATUS Status;
123 
124     /* Make sure we can trim this consumer */
125     if (!MiMemoryConsumers[Consumer].Trim)
126     {
127         /* Return the unmodified initial target */
128         return InitialTarget;
129     }
130 
131     if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
132     {
133         /* Consumer page limit exceeded */
134         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
135     }
136     if (MmAvailablePages < MiMinimumAvailablePages)
137     {
138         /* Global page limit exceeded */
139         Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
140     }
141 
142     if (Target)
143     {
144         if (!InitialTarget)
145         {
146             /* If there was no initial target,
147              * swap at least MiMinimumPagesPerRun */
148             Target = max(Target, MiMinimumPagesPerRun);
149         }
150 
151         /* Now swap the pages out */
152         Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
153 
154         DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
155 
156         if (!NT_SUCCESS(Status))
157         {
158             KeBugCheck(MEMORY_MANAGEMENT);
159         }
160 
161         /* Update the target */
162         if (NrFreedPages < Target)
163             Target -= NrFreedPages;
164         else
165             Target = 0;
166 
167         /* Return the remaining pages needed to meet the target */
168         return Target;
169     }
170     else
171     {
172         /* Initial target is zero and we don't have anything else to add */
173         return 0;
174     }
175 }
176 
177 NTSTATUS
178 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
179 {
180     PFN_NUMBER CurrentPage;
181     PFN_NUMBER NextPage;
182     NTSTATUS Status;
183 
184     (*NrFreedPages) = 0;
185 
186     CurrentPage = MmGetLRUFirstUserPage();
187     while (CurrentPage != 0 && Target > 0)
188     {
189         Status = MmPageOutPhysicalAddress(CurrentPage);
190         if (NT_SUCCESS(Status))
191         {
192             DPRINT("Succeeded\n");
193             Target--;
194             (*NrFreedPages)++;
195         }
196 
197         NextPage = MmGetLRUNextUserPage(CurrentPage);
198         if (NextPage <= CurrentPage)
199         {
200             /* We wrapped around, so we're done */
201             break;
202         }
203         CurrentPage = NextPage;
204     }
205 
206     return STATUS_SUCCESS;
207 }
208 
209 static BOOLEAN
210 MiIsBalancerThread(VOID)
211 {
212     return (MiBalancerThreadHandle != NULL) &&
213            (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
214 }
215 
216 VOID
217 NTAPI
218 MmRebalanceMemoryConsumers(VOID)
219 {
220     if (MiBalancerThreadHandle != NULL &&
221         !MiIsBalancerThread())
222     {
223         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
224     }
225 }
226 
227 NTSTATUS
228 NTAPI
229 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
230                             PPFN_NUMBER AllocatedPage)
231 {
232     ULONG PagesUsed;
233     PFN_NUMBER Page;
234 
235     /*
236      * Make sure we don't exceed our individual target.
237      */
238     PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
239     if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget &&
240             !MiIsBalancerThread())
241     {
242         MmRebalanceMemoryConsumers();
243     }
244 
245     /*
246      * Allocate always memory for the non paged pool and for the pager thread.
247      */
248     if ((Consumer == MC_SYSTEM) /* || MiIsBalancerThread() */)
249     {
250         Page = MmAllocPage(Consumer);
251         if (Page == 0)
252         {
253             KeBugCheck(NO_PAGES_AVAILABLE);
254         }
255         if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
256         *AllocatedPage = Page;
257         if (MmAvailablePages < MiMinimumAvailablePages)
258             MmRebalanceMemoryConsumers();
259         return(STATUS_SUCCESS);
260     }
261 
262     /*
263      * Make sure we don't exceed global targets.
264      */
265     if (((MmAvailablePages < MiMinimumAvailablePages) && !MiIsBalancerThread())
266             || (MmAvailablePages < (MiMinimumAvailablePages / 2)))
267     {
268         MM_ALLOCATION_REQUEST Request;
269 
270         if (!CanWait)
271         {
272             (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
273             MmRebalanceMemoryConsumers();
274             return(STATUS_NO_MEMORY);
275         }
276 
277         /* Insert an allocation request. */
278         Request.Page = 0;
279         KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
280 
281         ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock);
282         MmRebalanceMemoryConsumers();
283 
284         KeWaitForSingleObject(&Request.Event,
285                               0,
286                               KernelMode,
287                               FALSE,
288                               NULL);
289 
290         Page = Request.Page;
291         if (Page == 0)
292         {
293             KeBugCheck(NO_PAGES_AVAILABLE);
294         }
295 
296         if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
297         *AllocatedPage = Page;
298 
299         if (MmAvailablePages < MiMinimumAvailablePages)
300         {
301             MmRebalanceMemoryConsumers();
302         }
303 
304         return(STATUS_SUCCESS);
305     }
306 
307     /*
308      * Actually allocate the page.
309      */
310     Page = MmAllocPage(Consumer);
311     if (Page == 0)
312     {
313         KeBugCheck(NO_PAGES_AVAILABLE);
314     }
315     if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
316     *AllocatedPage = Page;
317 
318     if (MmAvailablePages < MiMinimumAvailablePages)
319     {
320         MmRebalanceMemoryConsumers();
321     }
322 
323     return(STATUS_SUCCESS);
324 }
325 
326 
327 VOID NTAPI
328 MiBalancerThread(PVOID Unused)
329 {
330     PVOID WaitObjects[2];
331     NTSTATUS Status;
332     ULONG i;
333 
334     WaitObjects[0] = &MiBalancerEvent;
335     WaitObjects[1] = &MiBalancerTimer;
336 
337     while (1)
338     {
339         Status = KeWaitForMultipleObjects(2,
340                                           WaitObjects,
341                                           WaitAny,
342                                           Executive,
343                                           KernelMode,
344                                           FALSE,
345                                           NULL,
346                                           NULL);
347 
348         if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
349         {
350             ULONG InitialTarget = 0;
351 
352 #if (_MI_PAGING_LEVELS == 2)
353             if (!MiIsBalancerThread())
354             {
355                 /* Clean up the unused PDEs */
356                 ULONG_PTR Address;
357                 PEPROCESS Process = PsGetCurrentProcess();
358 
359                 /* Acquire PFN lock */
360                 KIRQL OldIrql = MiAcquirePfnLock();
361                 PMMPDE pointerPde;
362                 for (Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS;
363                      Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS;
364                      Address += PTE_PER_PAGE * PAGE_SIZE)
365                 {
366                     if (MiQueryPageTableReferences((PVOID)Address) == 0)
367                     {
368                         pointerPde = MiAddressToPde(Address);
369                         if (pointerPde->u.Hard.Valid)
370                             MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL);
371                         ASSERT(pointerPde->u.Hard.Valid == 0);
372                     }
373                 }
374                 /* Release lock */
375                 MiReleasePfnLock(OldIrql);
376             }
377 #endif
378             do
379             {
380                 ULONG OldTarget = InitialTarget;
381 
382                 /* Trim each consumer */
383                 for (i = 0; i < MC_MAXIMUM; i++)
384                 {
385                     InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
386                 }
387 
388                 /* No pages left to swap! */
389                 if (InitialTarget != 0 &&
390                         InitialTarget == OldTarget)
391                 {
392                     /* Game over */
393                     KeBugCheck(NO_PAGES_AVAILABLE);
394                 }
395             }
396             while (InitialTarget != 0);
397         }
398         else
399         {
400             DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
401             KeBugCheck(MEMORY_MANAGEMENT);
402         }
403     }
404 }
405 
406 BOOLEAN MmRosNotifyAvailablePage(PFN_NUMBER Page)
407 {
408     PLIST_ENTRY Entry;
409     PMM_ALLOCATION_REQUEST Request;
410     PMMPFN Pfn1;
411 
412     /* Make sure the PFN lock is held */
413     MI_ASSERT_PFN_LOCK_HELD();
414 
415     if (!MiMinimumAvailablePages)
416     {
417         /* Dirty way to know if we were initialized. */
418         return FALSE;
419     }
420 
421     Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock);
422     if (!Entry)
423         return FALSE;
424 
425     Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
426     MiZeroPhysicalPage(Page);
427     Request->Page = Page;
428 
429     Pfn1 = MiGetPfnEntry(Page);
430     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
431     Pfn1->u3.e2.ReferenceCount = 1;
432     Pfn1->u3.e1.PageLocation = ActiveAndValid;
433 
434     /* This marks the PFN as a ReactOS PFN */
435     Pfn1->u4.AweAllocation = TRUE;
436 
437     /* Allocate the extra ReactOS Data and zero it out */
438     Pfn1->u1.SwapEntry = 0;
439     Pfn1->RmapListHead = NULL;
440 
441     KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
442 
443     return TRUE;
444 }
445 
446 VOID
447 INIT_FUNCTION
448 NTAPI
449 MiInitBalancerThread(VOID)
450 {
451     KPRIORITY Priority;
452     NTSTATUS Status;
453 #if !defined(__GNUC__)
454 
455     LARGE_INTEGER dummyJunkNeeded;
456     dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
457     ;
458 #endif
459 
460 
461     KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
462     KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
463     KeSetTimerEx(&MiBalancerTimer,
464 #if defined(__GNUC__)
465                  (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
466 #else
467                  dummyJunkNeeded,
468 #endif
469                  2000,         /* 2 sec */
470                  NULL);
471 
472     Status = PsCreateSystemThread(&MiBalancerThreadHandle,
473                                   THREAD_ALL_ACCESS,
474                                   NULL,
475                                   NULL,
476                                   &MiBalancerThreadId,
477                                   MiBalancerThread,
478                                   NULL);
479     if (!NT_SUCCESS(Status))
480     {
481         KeBugCheck(MEMORY_MANAGEMENT);
482     }
483 
484     Priority = LOW_REALTIME_PRIORITY + 1;
485     NtSetInformationThread(MiBalancerThreadHandle,
486                            ThreadPriority,
487                            &Priority,
488                            sizeof(Priority));
489 
490 }
491 
492 
493 /* EOF */
494