1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * PURPOSE: kernel memory management functions
5 * PROGRAMMERS: David Welch <welch@cwcom.net>
6 * Cameron Gutman <cameron.gutman@reactos.org>
7 */
8
9 /* INCLUDES *****************************************************************/
10
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14
15 #include "ARM3/miarm.h"
16
17
18 /* TYPES ********************************************************************/
19 typedef struct _MM_ALLOCATION_REQUEST
20 {
21 PFN_NUMBER Page;
22 LIST_ENTRY ListEntry;
23 KEVENT Event;
24 }
25 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
26 /* GLOBALS ******************************************************************/
27
28 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
29 static ULONG MiMinimumAvailablePages;
30 static ULONG MiMinimumPagesPerRun;
31 static CLIENT_ID MiBalancerThreadId;
32 static HANDLE MiBalancerThreadHandle = NULL;
33 static KEVENT MiBalancerEvent;
34 static KEVENT MiBalancerDoneEvent;
35 static KTIMER MiBalancerTimer;
36
37 static LONG PageOutThreadActive;
38
39 /* FUNCTIONS ****************************************************************/
40
41 CODE_SEG("INIT")
42 VOID
43 NTAPI
MmInitializeBalancer(ULONG NrAvailablePages,ULONG NrSystemPages)44 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
45 {
46 memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
47
48 /* Set up targets. */
49 MiMinimumAvailablePages = 256;
50 MiMinimumPagesPerRun = 256;
51 MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages / 2;
52 }
53
54 CODE_SEG("INIT")
55 VOID
56 NTAPI
MmInitializeMemoryConsumer(ULONG Consumer,NTSTATUS (* Trim)(ULONG Target,ULONG Priority,PULONG NrFreed))57 MmInitializeMemoryConsumer(
58 ULONG Consumer,
59 NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
60 {
61 MiMemoryConsumers[Consumer].Trim = Trim;
62 }
63
64 VOID
65 NTAPI
66 MiZeroPhysicalPage(
67 IN PFN_NUMBER PageFrameIndex
68 );
69
70 NTSTATUS
71 NTAPI
MmReleasePageMemoryConsumer(ULONG Consumer,PFN_NUMBER Page)72 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
73 {
74 KIRQL OldIrql;
75
76 if (Page == 0)
77 {
78 DPRINT1("Tried to release page zero.\n");
79 KeBugCheck(MEMORY_MANAGEMENT);
80 }
81
82 (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
83 UpdateTotalCommittedPages(-1);
84
85 OldIrql = MiAcquirePfnLock();
86
87 MmDereferencePage(Page);
88
89 MiReleasePfnLock(OldIrql);
90
91 return(STATUS_SUCCESS);
92 }
93
94 ULONG
95 NTAPI
MiTrimMemoryConsumer(ULONG Consumer,ULONG InitialTarget)96 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
97 {
98 ULONG Target = InitialTarget;
99 ULONG NrFreedPages = 0;
100 NTSTATUS Status;
101
102 /* Make sure we can trim this consumer */
103 if (!MiMemoryConsumers[Consumer].Trim)
104 {
105 /* Return the unmodified initial target */
106 return InitialTarget;
107 }
108
109 if (MmAvailablePages < MiMinimumAvailablePages)
110 {
111 /* Global page limit exceeded */
112 Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
113 }
114 else if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
115 {
116 /* Consumer page limit exceeded */
117 Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
118 }
119
120 if (Target)
121 {
122 /* Now swap the pages out */
123 Status = MiMemoryConsumers[Consumer].Trim(Target, MmAvailablePages < MiMinimumAvailablePages, &NrFreedPages);
124
125 DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
126
127 if (!NT_SUCCESS(Status))
128 {
129 KeBugCheck(MEMORY_MANAGEMENT);
130 }
131 }
132
133 /* Return the page count needed to be freed to meet the initial target */
134 return (InitialTarget > NrFreedPages) ? (InitialTarget - NrFreedPages) : 0;
135 }
136
137 NTSTATUS
MmTrimUserMemory(ULONG Target,ULONG Priority,PULONG NrFreedPages)138 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
139 {
140 PFN_NUMBER FirstPage, CurrentPage;
141 NTSTATUS Status;
142
143 (*NrFreedPages) = 0;
144
145 DPRINT("MM BALANCER: %s\n", Priority ? "Paging out!" : "Removing access bit!");
146
147 FirstPage = MmGetLRUFirstUserPage();
148 CurrentPage = FirstPage;
149 while (CurrentPage != 0 && Target > 0)
150 {
151 if (Priority)
152 {
153 Status = MmPageOutPhysicalAddress(CurrentPage);
154 if (NT_SUCCESS(Status))
155 {
156 DPRINT("Succeeded\n");
157 Target--;
158 (*NrFreedPages)++;
159 if (CurrentPage == FirstPage)
160 {
161 FirstPage = 0;
162 }
163 }
164 }
165 else
166 {
167 /* When not paging-out agressively, just reset the accessed bit */
168 PEPROCESS Process = NULL;
169 PVOID Address = NULL;
170 BOOLEAN Accessed = FALSE;
171
172 /*
173 * We have a lock-ordering problem here. We cant lock the PFN DB before the Process address space.
174 * So we must use circonvoluted loops.
175 * Well...
176 */
177 while (TRUE)
178 {
179 KAPC_STATE ApcState;
180 KIRQL OldIrql = MiAcquirePfnLock();
181 PMM_RMAP_ENTRY Entry = MmGetRmapListHeadPage(CurrentPage);
182 while (Entry)
183 {
184 if (RMAP_IS_SEGMENT(Entry->Address))
185 {
186 Entry = Entry->Next;
187 continue;
188 }
189
190 /* Check that we didn't treat this entry before */
191 if (Entry->Address < Address)
192 {
193 Entry = Entry->Next;
194 continue;
195 }
196
197 if ((Entry->Address == Address) && (Entry->Process <= Process))
198 {
199 Entry = Entry->Next;
200 continue;
201 }
202
203 break;
204 }
205
206 if (!Entry)
207 {
208 MiReleasePfnLock(OldIrql);
209 break;
210 }
211
212 Process = Entry->Process;
213 Address = Entry->Address;
214
215 ObReferenceObject(Process);
216
217 if (!ExAcquireRundownProtection(&Process->RundownProtect))
218 {
219 ObDereferenceObject(Process);
220 MiReleasePfnLock(OldIrql);
221 continue;
222 }
223
224 MiReleasePfnLock(OldIrql);
225
226 KeStackAttachProcess(&Process->Pcb, &ApcState);
227 MiLockProcessWorkingSet(Process, PsGetCurrentThread());
228
229 /* Be sure this is still valid. */
230 if (MmIsAddressValid(Address))
231 {
232 PMMPTE Pte = MiAddressToPte(Address);
233 Accessed = Accessed || Pte->u.Hard.Accessed;
234 Pte->u.Hard.Accessed = 0;
235
236 /* There is no need to invalidate, the balancer thread is never on a user process */
237 //KeInvalidateTlbEntry(Address);
238 }
239
240 MiUnlockProcessWorkingSet(Process, PsGetCurrentThread());
241
242 KeUnstackDetachProcess(&ApcState);
243 ExReleaseRundownProtection(&Process->RundownProtect);
244 ObDereferenceObject(Process);
245 }
246
247 if (!Accessed)
248 {
249 /* Nobody accessed this page since the last time we check. Time to clean up */
250
251 Status = MmPageOutPhysicalAddress(CurrentPage);
252 if (NT_SUCCESS(Status))
253 {
254 if (CurrentPage == FirstPage)
255 {
256 FirstPage = 0;
257 }
258 }
259 // DPRINT1("Paged-out one page: %s\n", NT_SUCCESS(Status) ? "Yes" : "No");
260 }
261
262 /* Done for this page. */
263 Target--;
264 }
265
266 CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE);
267 if (FirstPage == 0)
268 {
269 FirstPage = CurrentPage;
270 }
271 else if (CurrentPage == FirstPage)
272 {
273 DPRINT1("We are back at the start, abort!\n");
274 return STATUS_SUCCESS;
275 }
276 }
277
278 if (CurrentPage)
279 {
280 KIRQL OldIrql = MiAcquirePfnLock();
281 MmDereferencePage(CurrentPage);
282 MiReleasePfnLock(OldIrql);
283 }
284
285 return STATUS_SUCCESS;
286 }
287
288 VOID
289 NTAPI
MmRebalanceMemoryConsumers(VOID)290 MmRebalanceMemoryConsumers(VOID)
291 {
292 // if (InterlockedCompareExchange(&PageOutThreadActive, 0, 1) == 0)
293 {
294 KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
295 }
296 }
297
298 VOID
299 NTAPI
MmRebalanceMemoryConsumersAndWait(VOID)300 MmRebalanceMemoryConsumersAndWait(VOID)
301 {
302 ASSERT(PsGetCurrentProcess()->AddressCreationLock.Owner != KeGetCurrentThread());
303 ASSERT(!MM_ANY_WS_LOCK_HELD(PsGetCurrentThread()));
304 ASSERT(KeGetCurrentIrql() < DISPATCH_LEVEL);
305
306 KeResetEvent(&MiBalancerDoneEvent);
307 MmRebalanceMemoryConsumers();
308 KeWaitForSingleObject(&MiBalancerDoneEvent, Executive, KernelMode, FALSE, NULL);
309 }
310
311 NTSTATUS
312 NTAPI
MmRequestPageMemoryConsumer(ULONG Consumer,BOOLEAN CanWait,PPFN_NUMBER AllocatedPage)313 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
314 PPFN_NUMBER AllocatedPage)
315 {
316 PFN_NUMBER Page;
317 static INT i = 0;
318 static LARGE_INTEGER TinyTime = {{-1L, -1L}};
319
320 /* Delay some requests for the Memory Manager to recover pages */
321 if (i++ >= 100)
322 {
323 KeDelayExecutionThread(KernelMode, FALSE, &TinyTime);
324 i = 0;
325 }
326
327 /*
328 * Actually allocate the page.
329 */
330 Page = MmAllocPage(Consumer);
331 if (Page == 0)
332 {
333 *AllocatedPage = 0;
334 return STATUS_NO_MEMORY;
335 }
336 *AllocatedPage = Page;
337
338 /* Update the target */
339 InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
340 UpdateTotalCommittedPages(1);
341
342 return(STATUS_SUCCESS);
343 }
344
345 VOID
346 CcRosTrimCache(
347 _In_ ULONG Target,
348 _Out_ PULONG NrFreed);
349
350 VOID NTAPI
MiBalancerThread(PVOID Unused)351 MiBalancerThread(PVOID Unused)
352 {
353 PVOID WaitObjects[2];
354 NTSTATUS Status;
355 ULONG i;
356
357 WaitObjects[0] = &MiBalancerEvent;
358 WaitObjects[1] = &MiBalancerTimer;
359
360 while (1)
361 {
362 KeSetEvent(&MiBalancerDoneEvent, IO_NO_INCREMENT, FALSE);
363 Status = KeWaitForMultipleObjects(2,
364 WaitObjects,
365 WaitAny,
366 Executive,
367 KernelMode,
368 FALSE,
369 NULL,
370 NULL);
371
372 if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
373 {
374 ULONG InitialTarget = 0;
375 ULONG Target;
376 ULONG NrFreedPages;
377
378 do
379 {
380 ULONG OldTarget = InitialTarget;
381
382 /* Trim each consumer */
383 for (i = 0; i < MC_MAXIMUM; i++)
384 {
385 InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
386 }
387
388 /* Trim cache */
389 Target = max(InitialTarget, abs(MiMinimumAvailablePages - MmAvailablePages));
390 if (Target)
391 {
392 CcRosTrimCache(Target, &NrFreedPages);
393 InitialTarget -= min(NrFreedPages, InitialTarget);
394 }
395
396 /* No pages left to swap! */
397 if (InitialTarget != 0 &&
398 InitialTarget == OldTarget)
399 {
400 /* Game over */
401 KeBugCheck(NO_PAGES_AVAILABLE);
402 }
403 }
404 while (InitialTarget != 0);
405
406 if (Status == STATUS_WAIT_0)
407 InterlockedDecrement(&PageOutThreadActive);
408 }
409 else
410 {
411 DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
412 KeBugCheck(MEMORY_MANAGEMENT);
413 }
414 }
415 }
416
417 CODE_SEG("INIT")
418 VOID
419 NTAPI
MiInitBalancerThread(VOID)420 MiInitBalancerThread(VOID)
421 {
422 KPRIORITY Priority;
423 NTSTATUS Status;
424 LARGE_INTEGER Timeout;
425
426 KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
427 KeInitializeEvent(&MiBalancerDoneEvent, SynchronizationEvent, FALSE);
428 KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
429
430 Timeout.QuadPart = -20000000; /* 2 sec */
431 KeSetTimerEx(&MiBalancerTimer,
432 Timeout,
433 2000, /* 2 sec */
434 NULL);
435
436 Status = PsCreateSystemThread(&MiBalancerThreadHandle,
437 THREAD_ALL_ACCESS,
438 NULL,
439 NULL,
440 &MiBalancerThreadId,
441 MiBalancerThread,
442 NULL);
443 if (!NT_SUCCESS(Status))
444 {
445 KeBugCheck(MEMORY_MANAGEMENT);
446 }
447
448 Priority = LOW_REALTIME_PRIORITY + 1;
449 NtSetInformationThread(MiBalancerThreadHandle,
450 ThreadPriority,
451 &Priority,
452 sizeof(Priority));
453
454 }
455
456
457 /* EOF */
458