1 /* 2 * COPYRIGHT: See COPYING in the top level directory 3 * PROJECT: ReactOS kernel 4 * FILE: ntoskrnl/cc/lazywrite.c 5 * PURPOSE: Cache manager 6 * 7 * PROGRAMMERS: Pierre Schweitzer (pierre@reactos.org) 8 */ 9 10 /* INCLUDES *****************************************************************/ 11 12 #include <ntoskrnl.h> 13 #define NDEBUG 14 #include <debug.h> 15 16 /* Counters: 17 * - Amount of pages flushed by lazy writer 18 * - Number of times lazy writer ran 19 */ 20 ULONG CcLazyWritePages = 0; 21 ULONG CcLazyWriteIos = 0; 22 23 /* Internal vars (MS): 24 * - Lazy writer status structure 25 * - Lookaside list where to allocate work items 26 * - Queue for high priority work items (read ahead) 27 * - Queue for regular work items 28 * - Available worker threads 29 * - Queue for stuff to be queued after lazy writer is done 30 * - Marker for throttling queues 31 * - Number of ongoing workers 32 * - Three seconds delay for lazy writer 33 * - One second delay for lazy writer 34 * - Zero delay for lazy writer 35 * - Number of worker threads 36 */ 37 LAZY_WRITER LazyWriter; 38 NPAGED_LOOKASIDE_LIST CcTwilightLookasideList; 39 LIST_ENTRY CcExpressWorkQueue; 40 LIST_ENTRY CcRegularWorkQueue; 41 LIST_ENTRY CcIdleWorkerThreadList; 42 LIST_ENTRY CcPostTickWorkQueue; 43 BOOLEAN CcQueueThrottle = FALSE; 44 ULONG CcNumberActiveWorkerThreads = 0; 45 LARGE_INTEGER CcFirstDelay = RTL_CONSTANT_LARGE_INTEGER((LONGLONG)-1*3000*1000*10); 46 LARGE_INTEGER CcIdleDelay = RTL_CONSTANT_LARGE_INTEGER((LONGLONG)-1*1000*1000*10); 47 LARGE_INTEGER CcNoDelay = RTL_CONSTANT_LARGE_INTEGER((LONGLONG)0); 48 ULONG CcNumberWorkerThreads; 49 50 /* FUNCTIONS *****************************************************************/ 51 52 VOID 53 CcPostWorkQueue( 54 IN PWORK_QUEUE_ENTRY WorkItem, 55 IN PLIST_ENTRY WorkQueue) 56 { 57 KIRQL OldIrql; 58 PWORK_QUEUE_ITEM ThreadToSpawn; 59 60 /* First of all, insert the item in the queue */ 61 OldIrql = KeAcquireQueuedSpinLock(LockQueueWorkQueueLock); 62 InsertTailList(WorkQueue, &WorkItem->WorkQueueLinks); 63 64 /* Now, define whether we have to spawn a new work thread 65 * We will spawn a new one if: 66 * - There's no throttle in action 67 * - There's still at least one idle thread 68 */ 69 ThreadToSpawn = NULL; 70 if (!CcQueueThrottle && !IsListEmpty(&CcIdleWorkerThreadList)) 71 { 72 PLIST_ENTRY ListEntry; 73 74 /* Get the idle thread */ 75 ListEntry = RemoveHeadList(&CcIdleWorkerThreadList); 76 ThreadToSpawn = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ITEM, List); 77 78 /* We're going to have one more! */ 79 CcNumberActiveWorkerThreads += 1; 80 } 81 82 KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql); 83 84 /* If we have a thread to spawn, do it! */ 85 if (ThreadToSpawn != NULL) 86 { 87 /* We NULLify it to be consistent with initialization */ 88 ThreadToSpawn->List.Flink = NULL; 89 ExQueueWorkItem(ThreadToSpawn, CriticalWorkQueue); 90 } 91 } 92 93 VOID 94 NTAPI 95 CcScanDpc( 96 IN PKDPC Dpc, 97 IN PVOID DeferredContext, 98 IN PVOID SystemArgument1, 99 IN PVOID SystemArgument2) 100 { 101 PWORK_QUEUE_ENTRY WorkItem; 102 103 /* Allocate a work item */ 104 WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList); 105 if (WorkItem == NULL) 106 { 107 LazyWriter.ScanActive = FALSE; 108 return; 109 } 110 111 /* And post it, it will be for lazy write */ 112 WorkItem->Function = LazyScan; 113 CcPostWorkQueue(WorkItem, &CcRegularWorkQueue); 114 } 115 116 VOID 117 CcWriteBehind(VOID) 118 { 119 ULONG Target, Count; 120 121 Target = CcTotalDirtyPages / 8; 122 if (Target != 0) 123 { 124 /* Flush! */ 125 DPRINT("Lazy writer starting (%d)\n", Target); 126 CcRosFlushDirtyPages(Target, &Count, FALSE, TRUE); 127 128 /* And update stats */ 129 CcLazyWritePages += Count; 130 ++CcLazyWriteIos; 131 DPRINT("Lazy writer done (%d)\n", Count); 132 } 133 134 /* Make sure we're not throttling writes after this */ 135 while (MmAvailablePages < MmThrottleTop) 136 { 137 /* Break if we can't even find one to free */ 138 if (!CcRosFreeOneUnusedVacb()) 139 { 140 break; 141 } 142 } 143 } 144 145 VOID 146 CcLazyWriteScan(VOID) 147 { 148 ULONG Target; 149 KIRQL OldIrql; 150 PLIST_ENTRY ListEntry; 151 LIST_ENTRY ToPost; 152 PWORK_QUEUE_ENTRY WorkItem; 153 154 /* Do we have entries to queue after we're done? */ 155 InitializeListHead(&ToPost); 156 OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); 157 if (LazyWriter.OtherWork) 158 { 159 while (!IsListEmpty(&CcPostTickWorkQueue)) 160 { 161 ListEntry = RemoveHeadList(&CcPostTickWorkQueue); 162 WorkItem = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ENTRY, WorkQueueLinks); 163 InsertTailList(&ToPost, &WorkItem->WorkQueueLinks); 164 } 165 LazyWriter.OtherWork = FALSE; 166 } 167 KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); 168 169 /* Our target is one-eighth of the dirty pages */ 170 Target = CcTotalDirtyPages / 8; 171 if (Target != 0) 172 { 173 /* There is stuff to flush, schedule a write-behind operation */ 174 175 /* Allocate a work item */ 176 WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList); 177 if (WorkItem != NULL) 178 { 179 WorkItem->Function = WriteBehind; 180 CcPostWorkQueue(WorkItem, &CcRegularWorkQueue); 181 } 182 } 183 184 /* Post items that were due for end of run */ 185 while (!IsListEmpty(&ToPost)) 186 { 187 ListEntry = RemoveHeadList(&ToPost); 188 WorkItem = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ENTRY, WorkQueueLinks); 189 CcPostWorkQueue(WorkItem, &CcRegularWorkQueue); 190 } 191 192 /* If we have deferred writes, try them now! */ 193 if (!IsListEmpty(&CcDeferredWrites)) 194 { 195 CcPostDeferredWrites(); 196 /* Reschedule immediately a lazy writer run 197 * Keep us active to have short idle delay 198 */ 199 CcScheduleLazyWriteScan(FALSE); 200 } 201 else 202 { 203 /* We're no longer active */ 204 OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); 205 LazyWriter.ScanActive = FALSE; 206 KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); 207 } 208 } 209 210 VOID CcScheduleLazyWriteScan( 211 IN BOOLEAN NoDelay) 212 { 213 /* If no delay, immediately start lazy writer, 214 * no matter it was already started 215 */ 216 if (NoDelay) 217 { 218 LazyWriter.ScanActive = TRUE; 219 KeSetTimer(&LazyWriter.ScanTimer, CcNoDelay, &LazyWriter.ScanDpc); 220 } 221 /* Otherwise, if it's not running, just wait three seconds to start it */ 222 else if (!LazyWriter.ScanActive) 223 { 224 LazyWriter.ScanActive = TRUE; 225 KeSetTimer(&LazyWriter.ScanTimer, CcFirstDelay, &LazyWriter.ScanDpc); 226 } 227 /* Finally, already running, so queue for the next second */ 228 else 229 { 230 KeSetTimer(&LazyWriter.ScanTimer, CcIdleDelay, &LazyWriter.ScanDpc); 231 } 232 } 233 234 VOID 235 NTAPI 236 CcWorkerThread( 237 IN PVOID Parameter) 238 { 239 KIRQL OldIrql; 240 BOOLEAN DropThrottle, WritePerformed; 241 PWORK_QUEUE_ITEM Item; 242 #if DBG 243 PIRP TopLevel; 244 #endif 245 246 /* Get back our thread item */ 247 Item = Parameter; 248 /* And by default, don't touch throttle */ 249 DropThrottle = FALSE; 250 /* No write performed */ 251 WritePerformed = FALSE; 252 253 #if DBG 254 /* Top level IRP should be clean when started 255 * Save it to catch buggy drivers (or bugs!) 256 */ 257 TopLevel = IoGetTopLevelIrp(); 258 if (TopLevel != NULL) 259 { 260 DPRINT1("(%p) TopLevel IRP for this thread: %p\n", PsGetCurrentThread(), TopLevel); 261 } 262 #endif 263 264 /* Loop till we have jobs */ 265 while (TRUE) 266 { 267 PWORK_QUEUE_ENTRY WorkItem; 268 269 /* Lock queues */ 270 OldIrql = KeAcquireQueuedSpinLock(LockQueueWorkQueueLock); 271 272 /* If we have to touch throttle, reset it now! */ 273 if (DropThrottle) 274 { 275 CcQueueThrottle = FALSE; 276 DropThrottle = FALSE; 277 } 278 279 /* Check first if we have read ahead to do */ 280 if (IsListEmpty(&CcExpressWorkQueue)) 281 { 282 /* If not, check regular queue */ 283 if (IsListEmpty(&CcRegularWorkQueue)) 284 { 285 break; 286 } 287 else 288 { 289 WorkItem = CONTAINING_RECORD(CcRegularWorkQueue.Flink, WORK_QUEUE_ENTRY, WorkQueueLinks); 290 } 291 } 292 else 293 { 294 WorkItem = CONTAINING_RECORD(CcExpressWorkQueue.Flink, WORK_QUEUE_ENTRY, WorkQueueLinks); 295 } 296 297 /* Get our work item, if someone is waiting for us to finish 298 * and we're not the only thread in queue 299 * then, quit running to let the others do 300 * and throttle so that noone starts till current activity is over 301 */ 302 if (WorkItem->Function == SetDone && CcNumberActiveWorkerThreads > 1) 303 { 304 CcQueueThrottle = TRUE; 305 break; 306 } 307 308 /* Otherwise, remove current entry */ 309 RemoveEntryList(&WorkItem->WorkQueueLinks); 310 KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql); 311 312 /* And handle it */ 313 switch (WorkItem->Function) 314 { 315 case ReadAhead: 316 CcPerformReadAhead(WorkItem->Parameters.Read.FileObject); 317 break; 318 319 case WriteBehind: 320 PsGetCurrentThread()->MemoryMaker = 1; 321 CcWriteBehind(); 322 PsGetCurrentThread()->MemoryMaker = 0; 323 WritePerformed = TRUE; 324 break; 325 326 case LazyScan: 327 CcLazyWriteScan(); 328 break; 329 330 case SetDone: 331 KeSetEvent(WorkItem->Parameters.Event.Event, IO_NO_INCREMENT, FALSE); 332 DropThrottle = TRUE; 333 break; 334 335 default: 336 DPRINT1("Ignored item: %p (%d)\n", WorkItem, WorkItem->Function); 337 break; 338 } 339 340 /* And release the item */ 341 ExFreeToNPagedLookasideList(&CcTwilightLookasideList, WorkItem); 342 } 343 344 /* Our thread is available again */ 345 InsertTailList(&CcIdleWorkerThreadList, &Item->List); 346 /* One less worker */ 347 --CcNumberActiveWorkerThreads; 348 KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql); 349 350 /* If there are pending write openations and we have at least 20 dirty pages */ 351 if (!IsListEmpty(&CcDeferredWrites) && CcTotalDirtyPages >= 20) 352 { 353 /* And if we performed a write operation previously, then 354 * stress the system a bit and reschedule a scan to find 355 * stuff to write 356 */ 357 if (WritePerformed) 358 { 359 CcLazyWriteScan(); 360 } 361 } 362 363 #if DBG 364 /* Top level shouldn't have changed */ 365 if (TopLevel != IoGetTopLevelIrp()) 366 { 367 DPRINT1("(%p) Mismatching TopLevel: %p, %p\n", PsGetCurrentThread(), TopLevel, IoGetTopLevelIrp()); 368 } 369 #endif 370 } 371 372 /* 373 * @implemented 374 */ 375 NTSTATUS 376 NTAPI 377 CcWaitForCurrentLazyWriterActivity ( 378 VOID) 379 { 380 KIRQL OldIrql; 381 KEVENT WaitEvent; 382 PWORK_QUEUE_ENTRY WorkItem; 383 384 /* Allocate a work item */ 385 WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList); 386 if (WorkItem == NULL) 387 { 388 return STATUS_INSUFFICIENT_RESOURCES; 389 } 390 391 /* We want lazy writer to set our event */ 392 WorkItem->Function = SetDone; 393 KeInitializeEvent(&WaitEvent, NotificationEvent, FALSE); 394 WorkItem->Parameters.Event.Event = &WaitEvent; 395 396 /* Use the post tick queue */ 397 OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); 398 InsertTailList(&CcPostTickWorkQueue, &WorkItem->WorkQueueLinks); 399 400 /* Inform the lazy writer it will have to handle the post tick queue */ 401 LazyWriter.OtherWork = TRUE; 402 /* And if it's not running, queue a lazy writer run 403 * And start it NOW, we want the response now 404 */ 405 if (!LazyWriter.ScanActive) 406 { 407 CcScheduleLazyWriteScan(TRUE); 408 } 409 410 KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); 411 412 /* And now, wait until lazy writer replies */ 413 return KeWaitForSingleObject(&WaitEvent, Executive, KernelMode, FALSE, NULL); 414 } 415