1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * PURPOSE: kernel memory management functions
5 * PROGRAMMERS: David Welch <welch@cwcom.net>
6 * Cameron Gutman <cameron.gutman@reactos.org>
7 */
8
9 /* INCLUDES *****************************************************************/
10
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14
15 #include "ARM3/miarm.h"
16
17
18 /* TYPES ********************************************************************/
19 typedef struct _MM_ALLOCATION_REQUEST
20 {
21 PFN_NUMBER Page;
22 LIST_ENTRY ListEntry;
23 KEVENT Event;
24 }
25 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
26 /* GLOBALS ******************************************************************/
27
28 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
29 static ULONG MiMinimumAvailablePages;
30 static ULONG MiMinimumPagesPerRun;
31 static CLIENT_ID MiBalancerThreadId;
32 static HANDLE MiBalancerThreadHandle = NULL;
33 static KEVENT MiBalancerEvent;
34 static KEVENT MiBalancerDoneEvent;
35 static KTIMER MiBalancerTimer;
36
37 static LONG PageOutThreadActive;
38
39 /* FUNCTIONS ****************************************************************/
40
41 CODE_SEG("INIT")
42 VOID
43 NTAPI
MmInitializeBalancer(ULONG NrAvailablePages,ULONG NrSystemPages)44 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
45 {
46 memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
47
48 /* Set up targets. */
49 MiMinimumAvailablePages = 256;
50 MiMinimumPagesPerRun = 256;
51 MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages / 2;
52 }
53
54 CODE_SEG("INIT")
55 VOID
56 NTAPI
MmInitializeMemoryConsumer(ULONG Consumer,NTSTATUS (* Trim)(ULONG Target,ULONG Priority,PULONG NrFreed))57 MmInitializeMemoryConsumer(
58 ULONG Consumer,
59 NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
60 {
61 MiMemoryConsumers[Consumer].Trim = Trim;
62 }
63
64 VOID
65 NTAPI
66 MiZeroPhysicalPage(
67 IN PFN_NUMBER PageFrameIndex
68 );
69
70 NTSTATUS
71 NTAPI
MmReleasePageMemoryConsumer(ULONG Consumer,PFN_NUMBER Page)72 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
73 {
74 KIRQL OldIrql;
75
76 if (Page == 0)
77 {
78 DPRINT1("Tried to release page zero.\n");
79 KeBugCheck(MEMORY_MANAGEMENT);
80 }
81
82 (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
83 UpdateTotalCommittedPages(-1);
84
85 OldIrql = MiAcquirePfnLock();
86
87 MmDereferencePage(Page);
88
89 MiReleasePfnLock(OldIrql);
90
91 return(STATUS_SUCCESS);
92 }
93
94 ULONG
95 NTAPI
MiTrimMemoryConsumer(ULONG Consumer,ULONG InitialTarget)96 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
97 {
98 ULONG Target = InitialTarget;
99 ULONG NrFreedPages = 0;
100 NTSTATUS Status;
101
102 /* Make sure we can trim this consumer */
103 if (!MiMemoryConsumers[Consumer].Trim)
104 {
105 /* Return the unmodified initial target */
106 return InitialTarget;
107 }
108
109 if (MmAvailablePages < MiMinimumAvailablePages)
110 {
111 /* Global page limit exceeded */
112 Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
113 }
114 else if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
115 {
116 /* Consumer page limit exceeded */
117 Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
118 }
119
120 if (Target)
121 {
122 /* Now swap the pages out */
123 Status = MiMemoryConsumers[Consumer].Trim(Target, MmAvailablePages < MiMinimumAvailablePages, &NrFreedPages);
124
125 DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
126
127 if (!NT_SUCCESS(Status))
128 {
129 KeBugCheck(MEMORY_MANAGEMENT);
130 }
131 }
132
133 /* Return the page count needed to be freed to meet the initial target */
134 return (InitialTarget > NrFreedPages) ? (InitialTarget - NrFreedPages) : 0;
135 }
136
137 NTSTATUS
MmTrimUserMemory(ULONG Target,ULONG Priority,PULONG NrFreedPages)138 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
139 {
140 PFN_NUMBER FirstPage, CurrentPage;
141 NTSTATUS Status;
142
143 (*NrFreedPages) = 0;
144
145 DPRINT("MM BALANCER: %s\n", Priority ? "Paging out!" : "Removing access bit!");
146
147 FirstPage = MmGetLRUFirstUserPage();
148 CurrentPage = FirstPage;
149 while (CurrentPage != 0 && Target > 0)
150 {
151 if (Priority)
152 {
153 Status = MmPageOutPhysicalAddress(CurrentPage);
154 if (NT_SUCCESS(Status))
155 {
156 DPRINT("Succeeded\n");
157 Target--;
158 (*NrFreedPages)++;
159 if (CurrentPage == FirstPage)
160 {
161 FirstPage = 0;
162 }
163 }
164 }
165 else
166 {
167 /* When not paging-out agressively, just reset the accessed bit */
168 PEPROCESS Process = NULL;
169 PVOID Address = NULL;
170 BOOLEAN Accessed = FALSE;
171
172 /*
173 * We have a lock-ordering problem here. We cant lock the PFN DB before the Process address space.
174 * So we must use circonvoluted loops.
175 * Well...
176 */
177 while (TRUE)
178 {
179 KAPC_STATE ApcState;
180 KIRQL OldIrql = MiAcquirePfnLock();
181 PMM_RMAP_ENTRY Entry = MmGetRmapListHeadPage(CurrentPage);
182 while (Entry)
183 {
184 if (RMAP_IS_SEGMENT(Entry->Address))
185 {
186 Entry = Entry->Next;
187 continue;
188 }
189
190 /* Check that we didn't treat this entry before */
191 if (Entry->Address < Address)
192 {
193 Entry = Entry->Next;
194 continue;
195 }
196
197 if ((Entry->Address == Address) && (Entry->Process <= Process))
198 {
199 Entry = Entry->Next;
200 continue;
201 }
202
203 break;
204 }
205
206 if (!Entry)
207 {
208 MiReleasePfnLock(OldIrql);
209 break;
210 }
211
212 Process = Entry->Process;
213 Address = Entry->Address;
214
215 ObReferenceObject(Process);
216
217 if (!ExAcquireRundownProtection(&Process->RundownProtect))
218 {
219 ObDereferenceObject(Process);
220 MiReleasePfnLock(OldIrql);
221 continue;
222 }
223
224 MiReleasePfnLock(OldIrql);
225
226 KeStackAttachProcess(&Process->Pcb, &ApcState);
227 MiLockProcessWorkingSet(Process, PsGetCurrentThread());
228
229 /* Be sure this is still valid. */
230 if (MmIsAddressValid(Address))
231 {
232 PMMPTE Pte = MiAddressToPte(Address);
233 Accessed = Accessed || Pte->u.Hard.Accessed;
234 Pte->u.Hard.Accessed = 0;
235
236 /* There is no need to invalidate, the balancer thread is never on a user process */
237 //KeInvalidateTlbEntry(Address);
238 }
239
240 MiUnlockProcessWorkingSet(Process, PsGetCurrentThread());
241
242 KeUnstackDetachProcess(&ApcState);
243 ExReleaseRundownProtection(&Process->RundownProtect);
244 ObDereferenceObject(Process);
245 }
246
247 if (!Accessed)
248 {
249 /* Nobody accessed this page since the last time we check. Time to clean up */
250
251 Status = MmPageOutPhysicalAddress(CurrentPage);
252 if (NT_SUCCESS(Status))
253 {
254 if (CurrentPage == FirstPage)
255 {
256 FirstPage = 0;
257 }
258 }
259 // DPRINT1("Paged-out one page: %s\n", NT_SUCCESS(Status) ? "Yes" : "No");
260 }
261
262 /* Done for this page. */
263 Target--;
264 }
265
266 CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE);
267 if (FirstPage == 0)
268 {
269 FirstPage = CurrentPage;
270 }
271 else if (CurrentPage == FirstPage)
272 {
273 DPRINT1("We are back at the start, abort!\n");
274 return STATUS_SUCCESS;
275 }
276 }
277
278 if (CurrentPage)
279 {
280 KIRQL OldIrql = MiAcquirePfnLock();
281 MmDereferencePage(CurrentPage);
282 MiReleasePfnLock(OldIrql);
283 }
284
285 return STATUS_SUCCESS;
286 }
287
288 VOID
289 NTAPI
MmRebalanceMemoryConsumers(VOID)290 MmRebalanceMemoryConsumers(VOID)
291 {
292 if (InterlockedCompareExchange(&PageOutThreadActive, 1, 0) == 0)
293 {
294 KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
295 }
296 }
297
298 VOID
299 NTAPI
MmRebalanceMemoryConsumersAndWait(VOID)300 MmRebalanceMemoryConsumersAndWait(VOID)
301 {
302 ASSERT(PsGetCurrentProcess()->AddressCreationLock.Owner != KeGetCurrentThread());
303 ASSERT(!MM_ANY_WS_LOCK_HELD(PsGetCurrentThread()));
304 ASSERT(KeGetCurrentIrql() < DISPATCH_LEVEL);
305
306 KeResetEvent(&MiBalancerDoneEvent);
307 MmRebalanceMemoryConsumers();
308 KeWaitForSingleObject(&MiBalancerDoneEvent, Executive, KernelMode, FALSE, NULL);
309 }
310
311 NTSTATUS
312 NTAPI
MmRequestPageMemoryConsumer(ULONG Consumer,BOOLEAN CanWait,PPFN_NUMBER AllocatedPage)313 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
314 PPFN_NUMBER AllocatedPage)
315 {
316 PFN_NUMBER Page;
317
318 /* Delay some requests for the Memory Manager to recover pages (CORE-17624).
319 * FIXME: This is suboptimal.
320 */
321 static INT i = 0;
322 static LARGE_INTEGER TinyTime = {{-1L, -1L}};
323 if (i++ >= 100)
324 {
325 KeDelayExecutionThread(KernelMode, FALSE, &TinyTime);
326 i = 0;
327 }
328
329 /*
330 * Actually allocate the page.
331 */
332 Page = MmAllocPage(Consumer);
333 if (Page == 0)
334 {
335 *AllocatedPage = 0;
336 return STATUS_NO_MEMORY;
337 }
338 *AllocatedPage = Page;
339
340 /* Update the target */
341 InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
342 UpdateTotalCommittedPages(1);
343
344 return(STATUS_SUCCESS);
345 }
346
347 VOID
348 CcRosTrimCache(
349 _In_ ULONG Target,
350 _Out_ PULONG NrFreed);
351
352 VOID
353 NTAPI
MiBalancerThread(PVOID Unused)354 MiBalancerThread(PVOID Unused)
355 {
356 PVOID WaitObjects[2];
357 NTSTATUS Status;
358
359 WaitObjects[0] = &MiBalancerEvent;
360 WaitObjects[1] = &MiBalancerTimer;
361
362 while (TRUE)
363 {
364 KeSetEvent(&MiBalancerDoneEvent, IO_NO_INCREMENT, FALSE);
365 Status = KeWaitForMultipleObjects(_countof(WaitObjects),
366 WaitObjects,
367 WaitAny,
368 Executive,
369 KernelMode,
370 FALSE,
371 NULL,
372 NULL);
373
374 if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
375 {
376 ULONG InitialTarget = 0;
377 ULONG Target;
378 ULONG NrFreedPages;
379
380 do
381 {
382 ULONG OldTarget = InitialTarget;
383
384 /* Trim each consumer */
385 for (ULONG i = 0; i < MC_MAXIMUM; i++)
386 {
387 InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
388 }
389
390 /* Trim cache */
391 Target = max(InitialTarget, abs(MiMinimumAvailablePages - MmAvailablePages));
392 if (Target)
393 {
394 CcRosTrimCache(Target, &NrFreedPages);
395 InitialTarget -= min(NrFreedPages, InitialTarget);
396 }
397
398 /* No pages left to swap! */
399 if (InitialTarget != 0 &&
400 InitialTarget == OldTarget)
401 {
402 /* Game over */
403 KeBugCheck(NO_PAGES_AVAILABLE);
404 }
405 }
406 while (InitialTarget != 0);
407
408 if (Status == STATUS_WAIT_0)
409 {
410 LONG Active = InterlockedExchange(&PageOutThreadActive, 0);
411 ASSERT(Active == 1);
412 DBG_UNREFERENCED_LOCAL_VARIABLE(Active);
413 }
414 }
415 else
416 {
417 DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
418 KeBugCheck(MEMORY_MANAGEMENT);
419 }
420 }
421 }
422
423 CODE_SEG("INIT")
424 VOID
425 NTAPI
MiInitBalancerThread(VOID)426 MiInitBalancerThread(VOID)
427 {
428 KPRIORITY Priority;
429 NTSTATUS Status;
430 LARGE_INTEGER Timeout;
431
432 KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
433 KeInitializeEvent(&MiBalancerDoneEvent, SynchronizationEvent, FALSE);
434 KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
435
436 Timeout.QuadPart = -20000000; /* 2 sec */
437 KeSetTimerEx(&MiBalancerTimer,
438 Timeout,
439 2000, /* 2 sec */
440 NULL);
441
442 Status = PsCreateSystemThread(&MiBalancerThreadHandle,
443 THREAD_ALL_ACCESS,
444 NULL,
445 NULL,
446 &MiBalancerThreadId,
447 MiBalancerThread,
448 NULL);
449 if (!NT_SUCCESS(Status))
450 {
451 KeBugCheck(MEMORY_MANAGEMENT);
452 }
453
454 Priority = LOW_REALTIME_PRIORITY + 1;
455 NtSetInformationThread(MiBalancerThreadHandle,
456 ThreadPriority,
457 &Priority,
458 sizeof(Priority));
459
460 }
461
462
463 /* EOF */
464