1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/cc/lazywrite.c
5 * PURPOSE: Cache manager
6 *
7 * PROGRAMMERS: Pierre Schweitzer (pierre@reactos.org)
8 */
9
10 /* INCLUDES *****************************************************************/
11
12 #include <ntoskrnl.h>
13 #define NDEBUG
14 #include <debug.h>
15
16 /* Counters:
17 * - Amount of pages flushed by lazy writer
18 * - Number of times lazy writer ran
19 */
20 ULONG CcLazyWritePages = 0;
21 ULONG CcLazyWriteIos = 0;
22
23 /* Internal vars (MS):
24 * - Lazy writer status structure
25 * - Lookaside list where to allocate work items
26 * - Queue for high priority work items (read ahead)
27 * - Queue for regular work items
28 * - Available worker threads
29 * - Queue for stuff to be queued after lazy writer is done
30 * - Marker for throttling queues
31 * - Number of ongoing workers
32 * - Three seconds delay for lazy writer
33 * - One second delay for lazy writer
34 * - Zero delay for lazy writer
35 * - Number of worker threads
36 */
37 LAZY_WRITER LazyWriter;
38 NPAGED_LOOKASIDE_LIST CcTwilightLookasideList;
39 LIST_ENTRY CcExpressWorkQueue;
40 LIST_ENTRY CcRegularWorkQueue;
41 LIST_ENTRY CcIdleWorkerThreadList;
42 LIST_ENTRY CcPostTickWorkQueue;
43 BOOLEAN CcQueueThrottle = FALSE;
44 ULONG CcNumberActiveWorkerThreads = 0;
45 LARGE_INTEGER CcFirstDelay = RTL_CONSTANT_LARGE_INTEGER((LONGLONG)-1*3000*1000*10);
46 LARGE_INTEGER CcIdleDelay = RTL_CONSTANT_LARGE_INTEGER((LONGLONG)-1*1000*1000*10);
47 LARGE_INTEGER CcNoDelay = RTL_CONSTANT_LARGE_INTEGER((LONGLONG)0);
48 ULONG CcNumberWorkerThreads;
49
50 /* FUNCTIONS *****************************************************************/
51
52 VOID
CcPostWorkQueue(IN PWORK_QUEUE_ENTRY WorkItem,IN PLIST_ENTRY WorkQueue)53 CcPostWorkQueue(
54 IN PWORK_QUEUE_ENTRY WorkItem,
55 IN PLIST_ENTRY WorkQueue)
56 {
57 KIRQL OldIrql;
58 PWORK_QUEUE_ITEM ThreadToSpawn;
59
60 /* First of all, insert the item in the queue */
61 OldIrql = KeAcquireQueuedSpinLock(LockQueueWorkQueueLock);
62 InsertTailList(WorkQueue, &WorkItem->WorkQueueLinks);
63
64 /* Now, define whether we have to spawn a new work thread
65 * We will spawn a new one if:
66 * - There's no throttle in action
67 * - There's still at least one idle thread
68 */
69 ThreadToSpawn = NULL;
70 if (!CcQueueThrottle && !IsListEmpty(&CcIdleWorkerThreadList))
71 {
72 PLIST_ENTRY ListEntry;
73
74 /* Get the idle thread */
75 ListEntry = RemoveHeadList(&CcIdleWorkerThreadList);
76 ThreadToSpawn = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ITEM, List);
77
78 /* We're going to have one more! */
79 CcNumberActiveWorkerThreads += 1;
80 }
81
82 KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql);
83
84 /* If we have a thread to spawn, do it! */
85 if (ThreadToSpawn != NULL)
86 {
87 /* We NULLify it to be consistent with initialization */
88 ThreadToSpawn->List.Flink = NULL;
89 ExQueueWorkItem(ThreadToSpawn, CriticalWorkQueue);
90 }
91 }
92
93 VOID
94 NTAPI
CcScanDpc(IN PKDPC Dpc,IN PVOID DeferredContext,IN PVOID SystemArgument1,IN PVOID SystemArgument2)95 CcScanDpc(
96 IN PKDPC Dpc,
97 IN PVOID DeferredContext,
98 IN PVOID SystemArgument1,
99 IN PVOID SystemArgument2)
100 {
101 PWORK_QUEUE_ENTRY WorkItem;
102
103 /* Allocate a work item */
104 WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList);
105 if (WorkItem == NULL)
106 {
107 LazyWriter.ScanActive = FALSE;
108 return;
109 }
110
111 /* And post it, it will be for lazy write */
112 WorkItem->Function = LazyScan;
113 CcPostWorkQueue(WorkItem, &CcRegularWorkQueue);
114 }
115
116 VOID
CcWriteBehind(VOID)117 CcWriteBehind(VOID)
118 {
119 ULONG Target, Count;
120
121 Target = CcTotalDirtyPages / 8;
122 if (Target != 0)
123 {
124 /* Flush! */
125 DPRINT("Lazy writer starting (%d)\n", Target);
126 CcRosFlushDirtyPages(Target, &Count, FALSE, TRUE);
127
128 /* And update stats */
129 CcLazyWritePages += Count;
130 ++CcLazyWriteIos;
131 DPRINT("Lazy writer done (%d)\n", Count);
132 }
133
134 /* Make sure we're not throttling writes after this */
135 while (MmAvailablePages < MmThrottleTop)
136 {
137 /* Break if we can't even find one to free */
138 if (!CcRosFreeOneUnusedVacb())
139 {
140 break;
141 }
142 }
143 }
144
145 VOID
CcLazyWriteScan(VOID)146 CcLazyWriteScan(VOID)
147 {
148 ULONG Target;
149 KIRQL OldIrql;
150 PLIST_ENTRY ListEntry;
151 LIST_ENTRY ToPost;
152 PWORK_QUEUE_ENTRY WorkItem;
153
154 /* Do we have entries to queue after we're done? */
155 InitializeListHead(&ToPost);
156 OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
157 if (LazyWriter.OtherWork)
158 {
159 while (!IsListEmpty(&CcPostTickWorkQueue))
160 {
161 ListEntry = RemoveHeadList(&CcPostTickWorkQueue);
162 WorkItem = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ENTRY, WorkQueueLinks);
163 InsertTailList(&ToPost, &WorkItem->WorkQueueLinks);
164 }
165 LazyWriter.OtherWork = FALSE;
166 }
167 KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql);
168
169 /* Our target is one-eighth of the dirty pages */
170 Target = CcTotalDirtyPages / 8;
171 if (Target != 0)
172 {
173 /* There is stuff to flush, schedule a write-behind operation */
174
175 /* Allocate a work item */
176 WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList);
177 if (WorkItem != NULL)
178 {
179 WorkItem->Function = WriteBehind;
180 CcPostWorkQueue(WorkItem, &CcRegularWorkQueue);
181 }
182 }
183
184 /* Post items that were due for end of run */
185 while (!IsListEmpty(&ToPost))
186 {
187 ListEntry = RemoveHeadList(&ToPost);
188 WorkItem = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ENTRY, WorkQueueLinks);
189 CcPostWorkQueue(WorkItem, &CcRegularWorkQueue);
190 }
191
192 /* If we have deferred writes, try them now! */
193 if (!IsListEmpty(&CcDeferredWrites))
194 {
195 CcPostDeferredWrites();
196 /* Reschedule immediately a lazy writer run
197 * Keep us active to have short idle delay
198 */
199 CcScheduleLazyWriteScan(FALSE);
200 }
201 else
202 {
203 /* We're no longer active */
204 OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
205 LazyWriter.ScanActive = FALSE;
206 KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql);
207 }
208 }
209
CcScheduleLazyWriteScan(IN BOOLEAN NoDelay)210 VOID CcScheduleLazyWriteScan(
211 IN BOOLEAN NoDelay)
212 {
213 /* If no delay, immediately start lazy writer,
214 * no matter it was already started
215 */
216 if (NoDelay)
217 {
218 LazyWriter.ScanActive = TRUE;
219 KeSetTimer(&LazyWriter.ScanTimer, CcNoDelay, &LazyWriter.ScanDpc);
220 }
221 /* Otherwise, if it's not running, just wait three seconds to start it */
222 else if (!LazyWriter.ScanActive)
223 {
224 LazyWriter.ScanActive = TRUE;
225 KeSetTimer(&LazyWriter.ScanTimer, CcFirstDelay, &LazyWriter.ScanDpc);
226 }
227 /* Finally, already running, so queue for the next second */
228 else
229 {
230 KeSetTimer(&LazyWriter.ScanTimer, CcIdleDelay, &LazyWriter.ScanDpc);
231 }
232 }
233
234 VOID
235 NTAPI
CcWorkerThread(IN PVOID Parameter)236 CcWorkerThread(
237 IN PVOID Parameter)
238 {
239 KIRQL OldIrql;
240 BOOLEAN DropThrottle, WritePerformed;
241 PWORK_QUEUE_ITEM Item;
242 #if DBG
243 PIRP TopLevel;
244 #endif
245
246 /* Get back our thread item */
247 Item = Parameter;
248 /* And by default, don't touch throttle */
249 DropThrottle = FALSE;
250 /* No write performed */
251 WritePerformed = FALSE;
252
253 #if DBG
254 /* Top level IRP should be clean when started
255 * Save it to catch buggy drivers (or bugs!)
256 */
257 TopLevel = IoGetTopLevelIrp();
258 if (TopLevel != NULL)
259 {
260 DPRINT1("(%p) TopLevel IRP for this thread: %p\n", PsGetCurrentThread(), TopLevel);
261 }
262 #endif
263
264 /* Loop till we have jobs */
265 while (TRUE)
266 {
267 PWORK_QUEUE_ENTRY WorkItem;
268
269 /* Lock queues */
270 OldIrql = KeAcquireQueuedSpinLock(LockQueueWorkQueueLock);
271
272 /* If we have to touch throttle, reset it now! */
273 if (DropThrottle)
274 {
275 CcQueueThrottle = FALSE;
276 DropThrottle = FALSE;
277 }
278
279 /* Check first if we have read ahead to do */
280 if (IsListEmpty(&CcExpressWorkQueue))
281 {
282 /* If not, check regular queue */
283 if (IsListEmpty(&CcRegularWorkQueue))
284 {
285 break;
286 }
287 else
288 {
289 WorkItem = CONTAINING_RECORD(CcRegularWorkQueue.Flink, WORK_QUEUE_ENTRY, WorkQueueLinks);
290 }
291 }
292 else
293 {
294 WorkItem = CONTAINING_RECORD(CcExpressWorkQueue.Flink, WORK_QUEUE_ENTRY, WorkQueueLinks);
295 }
296
297 /* Get our work item, if someone is waiting for us to finish
298 * and we're not the only thread in queue
299 * then, quit running to let the others do
300 * and throttle so that noone starts till current activity is over
301 */
302 if (WorkItem->Function == SetDone && CcNumberActiveWorkerThreads > 1)
303 {
304 CcQueueThrottle = TRUE;
305 break;
306 }
307
308 /* Otherwise, remove current entry */
309 RemoveEntryList(&WorkItem->WorkQueueLinks);
310 KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql);
311
312 /* And handle it */
313 switch (WorkItem->Function)
314 {
315 case ReadAhead:
316 CcPerformReadAhead(WorkItem->Parameters.Read.FileObject);
317 break;
318
319 case WriteBehind:
320 PsGetCurrentThread()->MemoryMaker = 1;
321 CcWriteBehind();
322 PsGetCurrentThread()->MemoryMaker = 0;
323 WritePerformed = TRUE;
324 break;
325
326 case LazyScan:
327 CcLazyWriteScan();
328 break;
329
330 case SetDone:
331 KeSetEvent(WorkItem->Parameters.Event.Event, IO_NO_INCREMENT, FALSE);
332 DropThrottle = TRUE;
333 break;
334
335 default:
336 DPRINT1("Ignored item: %p (%d)\n", WorkItem, WorkItem->Function);
337 break;
338 }
339
340 /* And release the item */
341 ExFreeToNPagedLookasideList(&CcTwilightLookasideList, WorkItem);
342 }
343
344 /* Our thread is available again */
345 InsertTailList(&CcIdleWorkerThreadList, &Item->List);
346 /* One less worker */
347 --CcNumberActiveWorkerThreads;
348 KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql);
349
350 /* If there are pending write openations and we have at least 20 dirty pages */
351 if (!IsListEmpty(&CcDeferredWrites) && CcTotalDirtyPages >= 20)
352 {
353 /* And if we performed a write operation previously, then
354 * stress the system a bit and reschedule a scan to find
355 * stuff to write
356 */
357 if (WritePerformed)
358 {
359 CcLazyWriteScan();
360 }
361 }
362
363 #if DBG
364 /* Top level shouldn't have changed */
365 if (TopLevel != IoGetTopLevelIrp())
366 {
367 DPRINT1("(%p) Mismatching TopLevel: %p, %p\n", PsGetCurrentThread(), TopLevel, IoGetTopLevelIrp());
368 }
369 #endif
370 }
371
372 /*
373 * @implemented
374 */
375 NTSTATUS
376 NTAPI
CcWaitForCurrentLazyWriterActivity(VOID)377 CcWaitForCurrentLazyWriterActivity (
378 VOID)
379 {
380 KIRQL OldIrql;
381 KEVENT WaitEvent;
382 PWORK_QUEUE_ENTRY WorkItem;
383
384 /* Allocate a work item */
385 WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList);
386 if (WorkItem == NULL)
387 {
388 return STATUS_INSUFFICIENT_RESOURCES;
389 }
390
391 /* We want lazy writer to set our event */
392 WorkItem->Function = SetDone;
393 KeInitializeEvent(&WaitEvent, NotificationEvent, FALSE);
394 WorkItem->Parameters.Event.Event = &WaitEvent;
395
396 /* Use the post tick queue */
397 OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
398 InsertTailList(&CcPostTickWorkQueue, &WorkItem->WorkQueueLinks);
399
400 /* Inform the lazy writer it will have to handle the post tick queue */
401 LazyWriter.OtherWork = TRUE;
402 /* And if it's not running, queue a lazy writer run
403 * And start it NOW, we want the response now
404 */
405 if (!LazyWriter.ScanActive)
406 {
407 CcScheduleLazyWriteScan(TRUE);
408 }
409
410 KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql);
411
412 /* And now, wait until lazy writer replies */
413 return KeWaitForSingleObject(&WaitEvent, Executive, KernelMode, FALSE, NULL);
414 }
415