1 //
2 // Copyright (C) Microsoft. All rights reserved.
3 //
4 #include "../ioprivshared.hpp"
5
6
7 extern "C" {
8 #if defined(EVENT_TRACING)
9 #include "FxIoQueueKm.tmh"
10 #endif
11 }
12
13 _Must_inspect_result_
14 NTSTATUS
QueueForwardProgressIrpLocked(__in PIRP Irp)15 FxIoQueue::QueueForwardProgressIrpLocked(
16 __in PIRP Irp
17 )
18 {
19 InsertTailList(&m_FwdProgContext->m_PendedIrpList,
20 &Irp->Tail.Overlay.ListEntry);
21
22 Irp->Tail.Overlay.DriverContext[FX_IRP_QUEUE_CSQ_CONTEXT_ENTRY] =
23 m_FwdProgContext;
24
25 IoSetCancelRoutine(Irp, _WdmCancelRoutineForReservedIrp);
26
27 if (Irp->Cancel) {
28 if (IoSetCancelRoutine(Irp, NULL) != NULL){
29
30 Irp->Tail.Overlay.DriverContext[FX_IRP_QUEUE_CSQ_CONTEXT_ENTRY] = NULL;
31
32 RemoveEntryList(&Irp->Tail.Overlay.ListEntry);
33 InitializeListHead(&Irp->Tail.Overlay.ListEntry);
34
35 return STATUS_CANCELLED;
36 }
37 else {
38 //
39 // CancelRoutine will complete the IRP
40 //
41 DO_NOTHING();
42 }
43 }
44
45 IoMarkIrpPending(Irp);
46
47 return STATUS_PENDING;
48 }
49
50
51 _Must_inspect_result_
52 PIRP
GetForwardProgressIrpLocked(__in_opt PFILE_OBJECT FileObject)53 FxIoQueue::GetForwardProgressIrpLocked(
54 __in_opt PFILE_OBJECT FileObject
55 )
56 /*++
57
58 Routine Description:
59 Remove an IRP from the pending irp list if it matches with the input
60 fileobject. If the fileobject value is NULL, return the first one from
61 the pending list.
62
63 --*/
64 {
65 PIRP pIrp;
66 PLIST_ENTRY thisEntry, nextEntry, listHead;
67 PIO_STACK_LOCATION pIrpStack;
68
69 pIrp = NULL;
70 listHead = &m_FwdProgContext->m_PendedIrpList;
71
72 for(thisEntry = listHead->Flink;
73 thisEntry != listHead;
74 thisEntry = nextEntry) {
75 nextEntry = thisEntry->Flink;
76
77 pIrp = CONTAINING_RECORD(thisEntry, IRP, Tail.Overlay.ListEntry);
78 pIrpStack = IoGetCurrentIrpStackLocation(pIrp);
79
80 if (FileObject == NULL || FileObject == pIrpStack->FileObject) {
81
82 RemoveEntryList(thisEntry);
83 InitializeListHead(thisEntry);
84
85 if (NULL != IoSetCancelRoutine (pIrp, NULL)) {
86 pIrp->Tail.Overlay.DriverContext[FX_IRP_QUEUE_CSQ_CONTEXT_ENTRY] = NULL;
87 break;
88 }
89 else {
90 //
91 // Irp is canceled and the cancel routines is waiting to run.
92 // Continue to get the next the irp in the list.
93 //
94 DO_NOTHING();
95 }
96 }
97
98 pIrp = NULL;
99 }
100
101 return pIrp;
102 }
103
104 VOID
FreeAllReservedRequests(__in BOOLEAN Verify)105 FxIoQueue::FreeAllReservedRequests(
106 __in BOOLEAN Verify
107 )
108 /*++
109
110 Routine Description:
111 Called from dispose to Free all the reserved requests.
112
113 Verify -
114 TRUE - Make sure the number of request freed matches with the count of
115 request created.
116 FALSE - Called when we fail to allocate all the reserved requests
117 during config at init time. So we don't verify because the
118 count of request freed wouldn't match with the configured value.
119 --*/
120 {
121 ULONG count;
122 KIRQL oldIrql;
123
124 count = 0;
125
126 //
127 // Since forward progress request are allocated only for top level
128 // queues which cant be deleted so we dont need to add a reference on the
129 // queue for each reserved request since the Queue is guaranteed to be
130 // around when the request is being freed even if the Request was forwarded
131 // to another Queue.
132 //
133 m_FwdProgContext->m_PendedReserveLock.Acquire(&oldIrql);
134
135 while (!IsListEmpty(&m_FwdProgContext->m_ReservedRequestList)) {
136 PLIST_ENTRY pEntry;
137 FxRequest * pRequest;
138
139 pEntry = RemoveHeadList(&m_FwdProgContext->m_ReservedRequestList);
140 pRequest = FxRequest::_FromOwnerListEntry(FxListEntryForwardProgress,
141 pEntry);
142 pRequest->FreeRequest();
143
144 count++;
145 }
146
147 if (Verify) {
148 ASSERT(count == m_FwdProgContext->m_NumberOfReservedRequests);
149 }
150
151 ASSERT(IsListEmpty(&m_FwdProgContext->m_ReservedRequestInUseList));
152 ASSERT(IsListEmpty(&m_FwdProgContext->m_PendedIrpList));
153
154 m_FwdProgContext->m_PendedReserveLock.Release(oldIrql);
155
156 return;
157 }
158
159 VOID
ReturnReservedRequest(__in FxRequest * ReservedRequest)160 FxIoQueue::ReturnReservedRequest(
161 __in FxRequest *ReservedRequest
162 )
163 /*++
164
165 Routine Description:
166 Reuse the ReservedRequest if there are pended IRPs otherwise
167 add it back to the reserve list.
168
169 --*/
170 {
171 KIRQL oldIrql;
172 PIRP pIrp;
173 PFX_DRIVER_GLOBALS pFxDriverGlobals;
174
175 pFxDriverGlobals = GetDriverGlobals();
176 pIrp= NULL;
177
178 ASSERT(ReservedRequest->GetRefCnt() == 1);
179
180 if (pFxDriverGlobals->FxVerifierOn) {
181 ReservedRequest->ClearVerifierFlags(
182 FXREQUEST_FLAG_RESERVED_REQUEST_ASSOCIATED_WITH_IRP);
183 }
184
185 //
186 // Reuse the reserved request for dispatching the next pending IRP.
187 //
188 m_FwdProgContext->m_PendedReserveLock.Acquire(&oldIrql);
189
190 pIrp = GetForwardProgressIrpLocked(NULL);
191
192 m_FwdProgContext->m_PendedReserveLock.Release(oldIrql);
193
194 ReservedRequest->ClearFieldsForReuse();
195
196 if (pIrp != NULL) {
197 //
198 // Associate the reserved request with the Pended IRP
199 //
200 ReservedRequest->m_Irp.SetIrp(pIrp);
201 ReservedRequest->AssignMemoryBuffers(m_Device->GetIoType());
202
203 if (pFxDriverGlobals->FxVerifierOn) {
204 ReservedRequest->SetVerifierFlags(
205 FXREQUEST_FLAG_RESERVED_REQUEST_ASSOCIATED_WITH_IRP);
206 }
207
208 //
209 // Ignore the return status because QueueRequest will complete the
210 // request on it own if it fails to queue request.
211 //
212 (VOID) QueueRequest(ReservedRequest);
213 }
214 else {
215 PutBackReservedRequest(ReservedRequest);
216 }
217 }
218
219 VOID
GetForwardProgressIrps(__in PLIST_ENTRY IrpListHead,__in_opt MdFileObject FileObject)220 FxIoQueue::GetForwardProgressIrps(
221 __in PLIST_ENTRY IrpListHead,
222 __in_opt MdFileObject FileObject
223 )
224 /*++
225
226 Routine Description:
227
228 This function is called to retrieve the list of reserved queued IRPs.
229 The IRP's Tail.Overlay.ListEntry field is used to link these structs together.
230
231 --*/
232 {
233 PIRP irp;
234 KIRQL irql;
235
236 m_FwdProgContext->m_PendedReserveLock.Acquire(&irql);
237
238 do {
239 irp = GetForwardProgressIrpLocked(FileObject);
240 if (irp != NULL) {
241 //
242 // Add it to the cleanupList. We will complete the IRP after
243 // releasing the lock.
244 //
245 InsertTailList(IrpListHead, &irp->Tail.Overlay.ListEntry);
246 }
247 } while (irp != NULL);
248
249 m_FwdProgContext->m_PendedReserveLock.Release(irql);
250 }
251
252 VOID
_WdmCancelRoutineForReservedIrp(__inout struct _DEVICE_OBJECT * DeviceObject,__in __drv_useCancelIRQL PIRP Irp)253 FxIoQueue::_WdmCancelRoutineForReservedIrp(
254 __inout struct _DEVICE_OBJECT * DeviceObject,
255 __in __drv_useCancelIRQL PIRP Irp
256 )
257 /*++
258
259 Routine Description:
260
261 Cancel routine for forward progress irp.
262
263 --*/
264 {
265 KIRQL irql;
266 PFXIO_FORWARD_PROGRESS_CONTEXT m_FwdPrgContext;
267
268 UNREFERENCED_PARAMETER (DeviceObject);
269
270 IoReleaseCancelSpinLock(Irp->CancelIrql);
271
272 m_FwdPrgContext = (PFXIO_FORWARD_PROGRESS_CONTEXT)
273 Irp->Tail.Overlay.DriverContext[FX_IRP_QUEUE_CSQ_CONTEXT_ENTRY];
274
275 Irp->Tail.Overlay.DriverContext[FX_IRP_QUEUE_CSQ_CONTEXT_ENTRY] = NULL;
276
277 m_FwdPrgContext->m_PendedReserveLock.Acquire(&irql);
278
279 RemoveEntryList(&Irp->Tail.Overlay.ListEntry);
280 InitializeListHead(&Irp->Tail.Overlay.ListEntry);
281
282 m_FwdPrgContext->m_PendedReserveLock.Release(irql);
283
284 Irp->IoStatus.Status = STATUS_CANCELLED;
285 Irp->IoStatus.Information = 0;
286 IoCompleteRequest(Irp, IO_NO_INCREMENT);
287
288 return;
289 }
290
291 VOID
FlushQueuedDpcs(VOID)292 FxIoQueue::FlushQueuedDpcs(
293 VOID
294 )
295 /*++
296
297 Routine Description:
298
299 This is the kernel mode routine to flush queued DPCs.
300
301 Arguments:
302
303 Return Value:
304
305 --*/
306 {
307 KeFlushQueuedDpcs();
308 }
309
310
311 VOID
InsertQueueDpc(VOID)312 FxIoQueue::InsertQueueDpc(
313 VOID
314 )
315 /*++
316
317 Routine Description:
318
319 This is the kernel mode routine to insert a dpc.
320
321 Arguments:
322
323 Return Value:
324
325 --*/
326 {
327 KeInsertQueueDpc(&m_Dpc, NULL, NULL);
328 }
329
330 _Must_inspect_result_
331 NTSTATUS
GetReservedRequest(__in MdIrp Irp,__deref_out_opt FxRequest ** ReservedRequest)332 FxIoQueue::GetReservedRequest(
333 __in MdIrp Irp,
334 __deref_out_opt FxRequest **ReservedRequest
335 )
336 /*++
337
338 Routine Description:
339 Use the policy configured on the queue to decide whether to allocate a
340 reserved request.
341
342 --*/
343 {
344 WDF_IO_FORWARD_PROGRESS_ACTION action;
345 FxRequest *pRequest;
346 KIRQL oldIrql;
347 NTSTATUS status;
348 PFX_DRIVER_GLOBALS pFxDriverGlobals;
349
350 pFxDriverGlobals = GetDriverGlobals();
351
352 *ReservedRequest = NULL;
353 action = WdfIoForwardProgressActionInvalid;
354
355 switch (m_FwdProgContext->m_Policy) {
356
357 case WdfIoForwardProgressReservedPolicyPagingIO:
358 if (IsPagingIo(Irp)) {
359 action = WdfIoForwardProgressActionUseReservedRequest;
360 }
361 else {
362 action = WdfIoForwardProgressActionFailRequest;
363 }
364
365 break;
366 case WdfIoForwardProgressReservedPolicyAlwaysUseReservedRequest:
367
368 action = WdfIoForwardProgressActionUseReservedRequest;
369 break;
370
371 case WdfIoForwardProgressReservedPolicyUseExamine:
372
373 //
374 // Call the driver to figure out what action to take.
375 //
376 if (m_FwdProgContext->m_IoExamineIrp.Method != NULL) {
377
378 action = m_FwdProgContext->m_IoExamineIrp.Invoke(GetHandle(), Irp);
379
380 //
381 // Make sure the use has returned a valid action
382 //
383 if((action < WdfIoForwardProgressActionFailRequest ||
384 action > WdfIoForwardProgressActionUseReservedRequest)) {
385
386 status = STATUS_UNSUCCESSFUL;
387 DoTraceLevelMessage(
388 pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
389 "EvtIoExamineIrp callback on WDFQUEUE %p returned an "
390 "invalid action %d, %!STATUS!",
391 GetHandle(), action, status);
392
393 FxVerifierDbgBreakPoint(pFxDriverGlobals);
394
395 return status;
396 }
397
398 }
399 break;
400
401 default:
402 ASSERTMSG("Invalid forward progress setting ", FALSE);
403 break;
404 }
405
406 if (action == WdfIoForwardProgressActionFailRequest) {
407 status = STATUS_UNSUCCESSFUL;
408 DoTraceLevelMessage(
409 pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
410 "Forward action on WDFQUEUE %p says that framework should fail "
411 "the Irp %p, %!STATUS!",
412 GetHandle(), Irp, status);
413
414 return status;
415 }
416
417 pRequest = NULL;
418
419 m_FwdProgContext->m_PendedReserveLock.Acquire(&oldIrql);
420
421 if (!IsListEmpty(&m_FwdProgContext->m_ReservedRequestList)) {
422 PLIST_ENTRY pEntry;
423
424 pEntry = RemoveHeadList(&m_FwdProgContext->m_ReservedRequestList);
425
426 pRequest = FxRequest::_FromOwnerListEntry(FxListEntryForwardProgress,
427 pEntry);
428 ASSERT(pRequest != NULL);
429 ASSERT(pRequest->GetRefCnt() == 1);
430
431 InsertTailList(&m_FwdProgContext->m_ReservedRequestInUseList,
432 pRequest->GetListEntry(FxListEntryForwardProgress));
433
434 pRequest->m_Irp.SetIrp(Irp);
435 pRequest->AssignMemoryBuffers(m_Device->GetIoType());
436
437 if (pFxDriverGlobals->FxVerifierOn) {
438 pRequest->SetVerifierFlags(FXREQUEST_FLAG_RESERVED_REQUEST_ASSOCIATED_WITH_IRP);
439 }
440
441 //
442 // if *ReservedRequest is non-null the caller needs to free the
443 // previous request it allocated.
444 //
445 *ReservedRequest = pRequest;
446
447 status = STATUS_SUCCESS;
448 }
449 else {
450
451 status = QueueForwardProgressIrpLocked(Irp);
452 ASSERT(*ReservedRequest == NULL);
453 }
454
455 m_FwdProgContext->m_PendedReserveLock.Release(oldIrql);
456
457 return status;
458 }
459
460 _Must_inspect_result_
461 NTSTATUS
AssignForwardProgressPolicy(__in PWDF_IO_QUEUE_FORWARD_PROGRESS_POLICY Policy)462 FxIoQueue::AssignForwardProgressPolicy(
463 __in PWDF_IO_QUEUE_FORWARD_PROGRESS_POLICY Policy
464 )
465 /*++
466
467 Routine Description:
468 Configure the queue for forward Progress.
469
470 --*/
471 {
472 NTSTATUS status;
473 FxRequest *pRequest;
474 ULONG index;
475 PFX_DRIVER_GLOBALS pFxDriverGlobals;
476
477 pFxDriverGlobals = GetDriverGlobals();
478
479 //
480 // If the queue is not a top level queue then return an error
481 //
482 status = STATUS_SUCCESS;
483
484 //
485 // From v1.11 any queue can be top level (see WdfDeviceWdmDispatchIrpToIoQueue API).
486 //
487 if (pFxDriverGlobals->IsVersionGreaterThanOrEqualTo(1, 11) == FALSE) {
488 if (m_PkgIo->IsTopLevelQueue(this) == FALSE) {
489 status = STATUS_INVALID_PARAMETER;
490 DoTraceLevelMessage(
491 pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
492 "Setting Forward progress policy on non-top level queue %!STATUS!",
493 status);
494
495 return status;
496 }
497 }
498
499 m_FwdProgContext = (PFXIO_FORWARD_PROGRESS_CONTEXT)
500 FxPoolAllocate(GetDriverGlobals(),
501 NonPagedPool,
502 sizeof(FXIO_FORWARD_PROGRESS_CONTEXT)
503 );
504 if (m_FwdProgContext == NULL) {
505 status = STATUS_INSUFFICIENT_RESOURCES;
506 DoTraceLevelMessage(
507 pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
508 "Could not allocate memory for forward progress context %!STATUS!",
509 status);
510
511 return status;
512 }
513
514 *m_FwdProgContext = {};
515
516 //
517 // Initialize the things which will not fail first
518 //
519 m_FwdProgContext->m_Policy = Policy->ForwardProgressReservedPolicy;
520
521 m_FwdProgContext->m_NumberOfReservedRequests =
522 Policy->TotalForwardProgressRequests;
523
524 m_FwdProgContext->m_IoReservedResourcesAllocate.Method =
525 Policy->EvtIoAllocateResourcesForReservedRequest;
526
527 m_FwdProgContext->m_IoResourcesAllocate.Method =
528 Policy->EvtIoAllocateRequestResources;
529
530 m_FwdProgContext->m_IoExamineIrp.Method =
531 Policy->ForwardProgressReservePolicySettings.Policy.\
532 ExaminePolicy.EvtIoWdmIrpForForwardProgress;
533
534 InitializeListHead(&m_FwdProgContext->m_ReservedRequestList);
535 InitializeListHead(&m_FwdProgContext->m_ReservedRequestInUseList);
536 InitializeListHead(&m_FwdProgContext->m_PendedIrpList);
537
538 m_FwdProgContext->m_PendedReserveLock.Initialize();
539
540 for (index = 0;
541 index < m_FwdProgContext->m_NumberOfReservedRequests;
542 index++)
543 {
544 status = AllocateReservedRequest(&pRequest);
545 if (!NT_SUCCESS(status)) {
546 break;
547 }
548
549 InsertTailList(&m_FwdProgContext->m_ReservedRequestList,
550 pRequest->GetListEntry(FxListEntryForwardProgress));
551 }
552
553 //
554 // Since we allow forward progress policy to be set even after AddDevice
555 // when the queue has already started dispatching, we need to make
556 // sure all checks which determine whether the queue is configured for
557 // forward progress succeed. Setting Forward progress on the queue as the
558 // last operation helps with that.
559 //
560 if (NT_SUCCESS(status)) {
561 m_SupportForwardProgress = TRUE;
562 }
563 else {
564 FreeAllReservedRequests(FALSE);
565
566 m_FwdProgContext->m_PendedReserveLock.Uninitialize();
567
568 FxPoolFree(m_FwdProgContext);
569 m_FwdProgContext = NULL;
570 }
571
572 return status;
573 }
574