1 /*++
2
3 Copyright (c) Microsoft Corporation
4
5 Module Name:
6
7 FxPkgIo.cpp
8
9 Abstract:
10
11 This module implements the I/O package for the driver frameworks.
12
13 Author:
14
15
16
17 Environment:
18
19 Both kernel and user mode
20
21 Revision History:
22
23
24
25 --*/
26
27 #include "ioprivshared.hpp"
28
29 // Tracing support
30 extern "C" {
31 #if defined(EVENT_TRACING)
32 #include "FxPkgIo.tmh"
33 #endif
34 }
35
36 //
37 // This package is initialized by the FxPkgIo::Install virtual method
38 // being invoked.
39 //
40 // A reference is held on it by the FxDevice which owns it. When the
41 // FxDevice is destroyed, its destructor FxDevice::~FxDevice will release
42 // its reference to this package, so that FxPkgIo::~FxPkgIo can run.
43 //
44 // There is no other package remove, or un-install call.
45 //
46
FxPkgIo(__in PFX_DRIVER_GLOBALS FxDriverGlobals,__in CfxDevice * Device)47 FxPkgIo::FxPkgIo(
48 __in PFX_DRIVER_GLOBALS FxDriverGlobals,
49 __in CfxDevice *Device
50 ) :
51 FxPackage(FxDriverGlobals, Device, FX_TYPE_PACKAGE_IO),
52 m_InCallerContextCallback(FxDriverGlobals)
53 {
54 LARGE_INTEGER tickCount;
55
56 m_Device = Device;
57
58 m_DefaultQueue = NULL;
59
60 RtlZeroMemory(m_DispatchTable, sizeof(m_DispatchTable));
61
62 m_Filter = FALSE;
63
64 m_PowerStateOn = FALSE;
65
66 m_QueuesAreShuttingDown = FALSE;
67
68 InitializeListHead(&m_IoQueueListHead);
69
70 InitializeListHead(&m_DynamicDispatchInfoListHead);
71
72 Mx::MxQueryTickCount(&tickCount);
73
74 m_RandomSeed = tickCount.LowPart;
75
76 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO,
77 "Constructed FxPkgIo 0x%p",this);
78 }
79
~FxPkgIo()80 FxPkgIo::~FxPkgIo()
81 {
82 PLIST_ENTRY next;
83
84 m_DefaultQueue = NULL;
85
86 m_Device = NULL;
87
88 while (!IsListEmpty(&m_DynamicDispatchInfoListHead)) {
89 next = RemoveHeadList(&m_DynamicDispatchInfoListHead);
90 FxIrpDynamicDispatchInfo* info;
91 info = CONTAINING_RECORD(next, FxIrpDynamicDispatchInfo, ListEntry);
92 InitializeListHead(next);
93 delete info;
94 }
95
96 ASSERT(IsListEmpty(&m_IoQueueListHead));
97
98 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGIO,
99 "Destroyed FxPkgIo 0x%p",this);
100 }
101
102 _Must_inspect_result_
103 NTSTATUS
Dispatch(__inout MdIrp Irp)104 FxPkgIo::Dispatch(
105 __inout MdIrp Irp
106 )
107 {
108 FxIrp fxIrp(Irp);
109 FX_TRACK_DRIVER(GetDriverGlobals());
110
111 DoTraceLevelMessage(
112 GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGIO,
113 "WDFDEVICE 0x%p !devobj 0x%p %!IRPMJ!, IRP_MN %x, IRP 0x%p",
114 m_Device->GetHandle(), m_Device->GetDeviceObject(),
115 fxIrp.GetMajorFunction(),
116 fxIrp.GetMinorFunction(), Irp);
117
118 return DispatchStep1(Irp, m_DynamicDispatchInfoListHead.Flink);
119 }
120
121 _Must_inspect_result_
122 NTSTATUS
FX_VF_METHOD(FxPkgIo,VerifyDispatchContext)123 FX_VF_METHOD(FxPkgIo, VerifyDispatchContext) (
124 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals,
125 _In_ WDFCONTEXT DispatchContext
126 )
127 {
128 NTSTATUS status = STATUS_SUCCESS;
129 BOOLEAN ctxValid;
130 PLIST_ENTRY next;
131
132 PAGED_CODE_LOCKED();
133
134 //
135 // Make sure context is valid.
136 //
137 ctxValid = (PLIST_ENTRY)DispatchContext ==
138 &m_DynamicDispatchInfoListHead ?
139 TRUE : FALSE;
140
141 for (next = m_DynamicDispatchInfoListHead.Flink;
142 next != &m_DynamicDispatchInfoListHead;
143 next = next->Flink) {
144 if ((PLIST_ENTRY)DispatchContext == next) {
145 ctxValid = TRUE;
146 break;
147 }
148 }
149
150 if (FALSE == ctxValid) {
151 status = STATUS_INVALID_PARAMETER;
152 DoTraceLevelMessage(
153 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
154 "DispatchContext 0x%p is invalid, %!STATUS!",
155 DispatchContext, status);
156 FxVerifierDbgBreakPoint(FxDriverGlobals);
157 }
158
159 return status;
160 }
161
162 _Must_inspect_result_
163 NTSTATUS
164 __fastcall
DispatchStep1(__inout MdIrp Irp,__in WDFCONTEXT DispatchContext)165 FxPkgIo::DispatchStep1(
166 __inout MdIrp Irp,
167 __in WDFCONTEXT DispatchContext
168 )
169 /*++
170
171 Routine Description:
172
173 Checks for any registered dynamic dispatch callbacks that handles this type of request, else
174 selects the default queue based on the IRP's major code.
175
176 Arguments:
177
178 Irp - WDM request.
179
180 DispatchContext - Is the next FxIrpDynamicDispatchInfo element.
181
182 Return Value:
183
184 Irp's status.
185
186 --*/
187
188 {
189 NTSTATUS status;
190 FxIrp fxIrp(Irp);
191
192 ASSERT(((UCHAR)(ULONG_PTR)DispatchContext & FX_IN_DISPATCH_CALLBACK) == 0);
193
194 ASSERT(fxIrp.GetMajorFunction() <= IRP_MJ_MAXIMUM_FUNCTION);
195
196 //
197 // Look for I/O dynamic dispatch callbacks.
198 //
199 if ((PLIST_ENTRY)DispatchContext != &m_DynamicDispatchInfoListHead) {
200 int index;
201 index = FxIrpDynamicDispatchInfo::Mj2Index(fxIrp.GetMajorFunction());
202
203 //
204 // Only read/writes/ctrls/internal_ctrls IRPs are allowed, i.e., request cannot
205 // IRP type in its callback.
206 //
207 if (index >= (int)FxIrpDynamicDispatchInfo::DynamicDispatchMax) {
208 status = STATUS_INVALID_PARAMETER;
209 DoTraceLevelMessage(
210 GetDriverGlobals(), TRACE_LEVEL_ERROR, TRACINGIO,
211 "Driver cannot change the IRP type in its dispatch "
212 "callback Irp 0x%p, %!IRPMJ!, IRP_MN %x, Device 0x%p, "
213 "%!STATUS!",
214 Irp, fxIrp.GetMajorFunction(), fxIrp.GetMinorFunction(),
215 m_Device->GetHandle(), status);
216 FxVerifierDbgBreakPoint(GetDriverGlobals());
217 goto CompleteIrp;
218 }
219
220 //
221 // Verifier checks.
222 //
223 status = VerifyDispatchContext(GetDriverGlobals(), DispatchContext);
224 if( !NT_SUCCESS(status)){
225 goto CompleteIrp;
226 }
227
228 do {
229 FxIrpDynamicDispatchInfo* info;
230
231 info = CONTAINING_RECORD(DispatchContext,
232 FxIrpDynamicDispatchInfo,
233 ListEntry);
234 //
235 // Advance to next node.
236 //
237 DispatchContext = (WDFCONTEXT)(((PLIST_ENTRY)DispatchContext)->Flink);
238 ASSERT(((UCHAR)(ULONG_PTR)DispatchContext & FX_IN_DISPATCH_CALLBACK) == 0);
239
240 ASSERT(fxIrp.GetMajorFunction() == IRP_MJ_READ ||
241 fxIrp.GetMajorFunction() == IRP_MJ_WRITE ||
242 fxIrp.GetMajorFunction() == IRP_MJ_DEVICE_CONTROL ||
243 fxIrp.GetMajorFunction() == IRP_MJ_INTERNAL_DEVICE_CONTROL);
244
245 //
246 // If registered, invoke dispatch callback for this major function.
247 //
248 ASSERT(index < (int)FxIrpDynamicDispatchInfo::DynamicDispatchMax);
249 if (NULL != info->Dispatch[index].EvtDeviceDynamicDispatch){
250 return info->Dispatch[index].EvtDeviceDynamicDispatch(
251 m_Device->GetHandle(),
252 fxIrp.GetMajorFunction(),
253 fxIrp.GetMinorFunction(),
254 fxIrp.GetParameterIoctlCode(),
255 info->Dispatch[index].DriverContext,
256 reinterpret_cast<PIRP> (fxIrp.GetIrp()),
257 (WDFCONTEXT)((ULONG_PTR)DispatchContext |
258 FX_IN_DISPATCH_CALLBACK)
259 );
260 }
261 } while ((PLIST_ENTRY)DispatchContext !=
262 &m_DynamicDispatchInfoListHead);
263 }
264
265 //
266 // Only now push these local variables on the stack, this is to keep the
267 // stack from growing unnecessarily in the dynamic dispatch path above.
268 //
269 FxIoQueue* queue;
270 FxIoInCallerContext* ioInCallerCtx;
271
272 //
273 // Get the queue from the dispatch-table
274 //
275 queue = m_DispatchTable[fxIrp.GetMajorFunction()];
276 if (queue == NULL) {
277 ioInCallerCtx = GetIoInCallerContextCallback(NULL);
278 if (ioInCallerCtx->m_Method == NULL) {
279 //
280 // No queue configured yet, fail request unless the driver is a filter.
281 //
282 if (m_Filter) {
283 goto Forward;
284 }
285
286 status = STATUS_INVALID_DEVICE_REQUEST;
287 DoTraceLevelMessage(
288 GetDriverGlobals(), TRACE_LEVEL_ERROR, TRACINGIO,
289 "No queue configured for WDFDEVICE 0x%p, failing IRP 0x%p,"
290 " %!STATUS!",
291 m_Device->GetHandle(), Irp, status);
292
293 goto CompleteIrp;
294 }
295 }
296 else {
297 ioInCallerCtx = GetIoInCallerContextCallback(queue->GetCxDeviceInfo());
298 }
299
300 //
301 // If the driver is filter and queue is a default-queue then before
302 // calling the queue, we should make sure the queue can dispatch
303 // requests to the driver. If the queue cannot dispatch request,
304 // we should forward it down to the lower driver ourself.
305 // This is to cover the scenario where the driver has registered only
306 // type specific handler and expect the framework to auto-forward other
307 // requests.
308 //
309 if (m_Filter &&
310 ioInCallerCtx->m_Method == NULL &&
311 queue == m_DefaultQueue &&
312 queue->IsIoEventHandlerRegistered((WDF_REQUEST_TYPE)fxIrp.GetMajorFunction()) == FALSE) {
313 //
314 // Default queue doesn't have callback events registered to
315 // handle this request. So forward it down.
316 //
317 goto Forward;
318 }
319
320 //
321 // Finally queue request.
322 //
323 return DispatchStep2(Irp, ioInCallerCtx, queue);
324
325 Forward:
326
327 fxIrp.SkipCurrentIrpStackLocation();
328 return fxIrp.CallDriver(m_Device->GetAttachedDevice());
329
330 CompleteIrp:
331
332 fxIrp.SetStatus(status);
333 fxIrp.SetInformation(0);
334 fxIrp.CompleteRequest(IO_NO_INCREMENT);
335
336 return status;
337 }
338
339 _Must_inspect_result_
340 NTSTATUS
341 __fastcall
DispatchStep2(__inout MdIrp Irp,__in_opt FxIoInCallerContext * IoInCallerCtx,__in_opt FxIoQueue * Queue)342 FxPkgIo::DispatchStep2(
343 __inout MdIrp Irp,
344 __in_opt FxIoInCallerContext* IoInCallerCtx,
345 __in_opt FxIoQueue* Queue
346 )
347 {
348 NTSTATUS status;
349 FxRequest* request;
350 BOOLEAN isForwardProgressQueue;
351 BOOLEAN inCriticalRegion;
352 PWDF_OBJECT_ATTRIBUTES reqAttribs;
353 FxIrp fxIrp(Irp);
354
355 request = NULL;
356 inCriticalRegion = FALSE;
357 isForwardProgressQueue = Queue != NULL && Queue->IsForwardProgressQueue();
358
359 ASSERT(fxIrp.GetMajorFunction() <= IRP_MJ_MAXIMUM_FUNCTION);
360 ASSERT((IoInCallerCtx != NULL && IoInCallerCtx->m_Method != NULL) ||
361 Queue != NULL);
362 //
363 // The request inserted into the queue can be retrieved and processed
364 // by an arbitrary thread. So we need to make sure that such a thread doesn't
365 // get suspended and deadlock the driver and potentially the system by
366 // entering critical region.The KeEnterCriticalRegion temporarily disables
367 // the delivery of normal kernel APCs used to suspend a thread.
368 // Kernel APCs queued to this thread will get executed when we leave the
369 // critical region.
370 //
371 if (Mx::MxGetCurrentIrql() <= APC_LEVEL) {
372 Mx::MxEnterCriticalRegion();
373 inCriticalRegion = TRUE;
374 }
375
376 if (Queue != NULL && Queue->GetCxDeviceInfo() != NULL) {
377 reqAttribs = &Queue->GetCxDeviceInfo()->RequestAttributes;
378 }
379 else {
380 reqAttribs = m_Device->GetRequestAttributes();
381 }
382
383 status = FxRequest::_CreateForPackage(m_Device, reqAttribs, Irp, &request);
384
385 //
386 // Check if it is forward progress queue and the EnhancedVerifierOption for
387 // testing forward progress are set.
388 //
389 if (isForwardProgressQueue &&
390 NT_SUCCESS(status) &&
391 IsFxVerifierTestForwardProgress(GetDriverGlobals())) {
392 //
393 // This function returns STATUS_INSUFFICIENT_RESOURCES
394 // if testing forward progress is enabled and free's the passed in request.
395 //
396 status = VerifierFreeRequestToTestForwardProgess(request);
397 }
398
399 if (!NT_SUCCESS(status)) {
400 if (m_Filter && Queue == NULL) {
401 goto CompleteIrp;
402 }
403
404 if (isForwardProgressQueue) {
405 status = Queue->GetReservedRequest(Irp, &request);
406 if (status == STATUS_PENDING) {
407 goto IrpIsGone;
408 }
409 else if (!NT_SUCCESS(status)) {
410 goto CompleteIrp;
411 }
412 }
413 else {
414 //
415 // Fail the request
416 //
417 DoTraceLevelMessage(
418 GetDriverGlobals(), TRACE_LEVEL_ERROR, TRACINGIO,
419 "Could not create WDFREQUEST, %!STATUS!", status);
420
421 goto CompleteIrp;
422 }
423 }
424 else {
425 if (isForwardProgressQueue) {
426 status = Queue->InvokeAllocateResourcesCallback(request);
427 if (!NT_SUCCESS(status)) {
428 //
429 // Failure of the callback means the driver wasn't able to
430 // allocate resources for the request. In that case free the
431 // request allocated earlier and use the reserved one.
432 //
433 request->FreeRequest();
434 request = NULL;
435
436 status = Queue->GetReservedRequest(Irp, &request);
437 if (status == STATUS_PENDING) {
438 goto IrpIsGone;
439 }
440 else if (!NT_SUCCESS(status)) {
441 goto CompleteIrp;
442 }
443 }
444 }
445 }
446
447 //
448 // Since we can't guarantee the callback to be called in the context of the
449 // caller for reserved requests, we will skip calling InCallerContextCallback
450 // for reserverd request.
451 //
452 if (IoInCallerCtx != NULL &&
453 IoInCallerCtx->m_Method != NULL &&
454 request->IsReserved() == FALSE) {
455
456 request->SetInternalContext(Queue);
457 status = DispathToInCallerContextCallback(IoInCallerCtx, request, Irp);
458
459 //
460 // The driver is responsible for calling WdfDeviceEnqueueRequest to
461 // insert it back into the I/O processing pipeline, or completing it.
462 //
463 goto IrpIsGone;
464 }
465
466 ASSERT(Queue != NULL);
467 status = Queue->QueueRequest(request);
468 goto IrpIsGone;
469
470 CompleteIrp:
471
472 fxIrp.SetStatus(status);
473 fxIrp.SetInformation(0);
474 fxIrp.CompleteRequest(IO_NO_INCREMENT);
475 //
476 // fallthrough
477 //
478 IrpIsGone:
479
480 if (inCriticalRegion) {
481 Mx::MxLeaveCriticalRegion();
482 }
483
484 return status;
485 }
486
487 _Must_inspect_result_
488 NTSTATUS
InitializeDefaultQueue(__in CfxDevice * Device,__inout FxIoQueue * Queue)489 FxPkgIo::InitializeDefaultQueue(
490 __in CfxDevice * Device,
491 __inout FxIoQueue * Queue
492 )
493
494 /*++
495
496 Routine Description:
497
498 Make the input queue as the default queue. There can be
499 only one queue as the default queue.
500
501 The default queue is the place all requests go to
502 automatically if a specific queue was not configured
503 for them.
504
505 Arguments:
506
507
508 Return Value:
509
510 NTSTATUS
511
512 --*/
513 {
514 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals();
515 ULONG index;
516
517 if (m_DefaultQueue != NULL) {
518 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
519 "Default Queue Already Configured for "
520 "FxPkgIo 0x%p, WDFDEVICE 0x%p %!STATUS!",this,
521 Device->GetHandle(), STATUS_UNSUCCESSFUL);
522 return STATUS_UNSUCCESSFUL;
523 }
524
525 for (index=0; index <= IRP_MJ_MAXIMUM_FUNCTION; index++) {
526 if (m_DispatchTable[index] == NULL) {
527 m_DispatchTable[index] = Queue;
528 }
529 }
530
531 m_DefaultQueue = Queue;
532
533 //
534 // Default queue can't be deleted. So mark the object to fail WdfObjectDelete on
535 // the default queue.
536 //
537 Queue->MarkNoDeleteDDI();
538 return STATUS_SUCCESS;
539 }
540
541 __inline
542 FxDriver*
GetDriver(VOID)543 FxPkgIo::GetDriver(
544 VOID
545 )
546 {
547 return m_Device->GetDriver();
548 }
549
550 _Must_inspect_result_
551 NTSTATUS
FX_VF_METHOD(FxPkgIo,VerifyEnqueueRequestUpdateFlags)552 FX_VF_METHOD(FxPkgIo, VerifyEnqueueRequestUpdateFlags) (
553 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals,
554 _In_ FxRequest* Request,
555 _Inout_ SHORT* OrigVerifierFlags
556 )
557 {
558 NTSTATUS status = STATUS_SUCCESS;
559
560 PAGED_CODE_LOCKED();
561
562 KIRQL irql;
563
564 Request->Lock(&irql);
565
566 *OrigVerifierFlags = Request->GetVerifierFlagsLocked();
567
568 status = Request->VerifyRequestIsInCallerContext(FxDriverGlobals);
569 if (NT_SUCCESS(status)) {
570 status = Request->VerifyRequestIsDriverOwned(FxDriverGlobals);
571 }
572
573 if (NT_SUCCESS(status)) {
574 Request->ClearVerifierFlagsLocked(
575 FXREQUEST_FLAG_DRIVER_INPROCESS_CONTEXT |
576 FXREQUEST_FLAG_DRIVER_OWNED);
577 }
578
579 Request->Unlock(irql);
580 return status;
581 }
582
583 VOID
FX_VF_METHOD(FxPkgIo,VerifyEnqueueRequestRestoreFlags)584 FX_VF_METHOD(FxPkgIo, VerifyEnqueueRequestRestoreFlags) (
585 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals,
586 _In_ FxRequest* Request,
587 _In_ SHORT OrigVerifierFlags
588 )
589 {
590 UNREFERENCED_PARAMETER(FxDriverGlobals);
591 KIRQL irql;
592
593 PAGED_CODE_LOCKED();
594
595 Request->Lock(&irql);
596 Request->ClearVerifierFlagsLocked(~OrigVerifierFlags);
597 Request->SetVerifierFlagsLocked(OrigVerifierFlags);
598 Request->Unlock(irql);
599 }
600
601
602 //
603 // This inserts a request into the I/O processing pipeline
604 //
605 _Must_inspect_result_
606 NTSTATUS
EnqueueRequest(__in CfxDevice * Device,__inout FxRequest * pRequest)607 FxPkgIo::EnqueueRequest(
608 __in CfxDevice* Device,
609 __inout FxRequest* pRequest
610 )
611 {
612 NTSTATUS status;
613 FxIoQueue* pQueue;
614 FxIrp* Irp = NULL;
615 FxRequestCompletionState oldState;
616 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals();
617 SHORT origVerifierFlags = 0;
618
619 //
620 // Request is owned by the driver, and has a reference count of == 1
621 // (or > 1 if EvtIoInCallerContext callback took an additional reference),
622 // with a FxPkgIoInProcessRequestComplete callback registered.
623 //
624 ASSERT(pRequest->GetRefCnt() >= 1);
625
626 Irp = pRequest->GetFxIrp();
627
628 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO,
629 "WDFREQUEST 0x%p", pRequest->GetObjectHandle());
630
631 status = VerifyEnqueueRequestUpdateFlags(FxDriverGlobals,
632 pRequest,
633 &origVerifierFlags);
634 if (!NT_SUCCESS(status)) {
635 return status;
636 }
637
638 //
639 // Get the associated queue
640 //
641 pQueue = (FxIoQueue*) pRequest->GetInternalContext();
642 if (NULL == pQueue) {
643 pQueue = m_DispatchTable[Irp->GetMajorFunction()];
644 if (pQueue == NULL) {
645 //
646 // No queue configured yet, fail request unless the driver is a filter.
647 //
648 if (m_Filter) {
649 goto Forward;
650 }
651
652 status = STATUS_INVALID_DEVICE_REQUEST;
653
654 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
655 "No queue configured for WDFDEVICE 0x%p, "
656 "failing WDFREQUEST 0x%p %!STATUS!",
657 Device->GetHandle(),
658 pRequest->GetObjectHandle(),
659 status);
660
661 FxVerifierDbgBreakPoint(FxDriverGlobals);
662
663 //
664 // Return it back to the driver to decide the outcome
665 //
666 goto Error;
667 }
668 }
669
670 //
671 // If the queue is a default-queue and driver is a filter then before
672 // calling the queue we should make sure the queue can dispatch
673 // requests to the driver. If the queue cannot dispatch request,
674 // we should forward it down to the lower driver ourself.
675 if (m_Filter &&
676 pQueue == m_DefaultQueue &&
677 pQueue->IsIoEventHandlerRegistered((WDF_REQUEST_TYPE)Irp->GetMajorFunction()) == FALSE) {
678 //
679 // Default queue doesn't have callback events registered to
680 // handle this request. So forward it down.
681 //
682 goto Forward;
683 }
684
685 pQueue->AddRef();
686
687 // Must add a reference before releasing the callback and its reference
688 pRequest->ADDREF(FXREQUEST_STATE_TAG);
689
690 // Release the callback
691 oldState = pRequest->SetCompletionState(FxRequestCompletionStateNone);
692 ASSERT(oldState != FxRequestCompletionStateNone);
693 UNREFERENCED_PARAMETER(oldState);
694
695 status = pQueue->QueueRequestFromForward(pRequest);
696
697 pQueue->Release();
698
699 //
700 // If not successfull, must place the request back
701 // to the state it was in on entry so that the driver
702 // can decide what to do next with it
703 //
704 if (!NT_SUCCESS(status)) {
705
706 //
707 // If the request comes back to us, it should still
708 // have a reference count of 1
709 //
710 oldState = pRequest->SetCompletionState(FxRequestCompletionStateIoPkg);
711
712 ASSERT(oldState == FxRequestCompletionStateNone);
713 UNREFERENCED_PARAMETER(oldState);
714
715 //
716 // Release the reference count on the request since
717 // the callback will hold the only one that gets
718 // decremented when the request is completed
719 //
720 pRequest->RELEASE(FXREQUEST_STATE_TAG);
721 goto Error;
722 }
723 else {
724 //
725 // On success, can not touch the request since it
726 // may have already been completed
727 //
728 }
729
730 return status;
731
732 Forward:
733
734 //
735 // Cannot send-and-forget a request with a formatted IO context.
736 //
737 if (pRequest->HasContext()) {
738 status = STATUS_INVALID_DEVICE_REQUEST;
739
740 DoTraceLevelMessage(
741 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
742 "Cannot send-and-forget WDFREQUEST 0x%p with formatted IO"
743 " context for filter WDFDEVICE 0x%p, %!STATUS!",
744 pRequest->GetObjectHandle(),
745 Device->GetHandle(),
746 status );
747
748 FxVerifierDbgBreakPoint(FxDriverGlobals);
749 goto Error;
750 }
751
752 //
753 // This will skip the current stack location and perform
754 // early dispose on the request.
755 //
756 pRequest->PreProcessSendAndForget();
757
758 (VOID)Irp->CallDriver(Device->GetAttachedDevice());
759
760 //
761 // This will delete the request and free the memory back
762 // to the device lookaside list.
763 //
764 pRequest->PostProcessSendAndForget();
765
766 //
767 // Return a success status in this code path even if the previous call
768 // to send the request to the lower driver failed. The status code returned
769 // by this function should reflect the status of enqueuing the request and
770 // not the status returned by the lower driver.
771 //
772 return STATUS_SUCCESS;
773
774 Error:
775
776 //
777 // If not successful, we must set the original verifier flags.
778 //
779 VerifyEnqueueRequestRestoreFlags(FxDriverGlobals, pRequest, origVerifierFlags);
780
781 return status;
782 }
783
784 _Must_inspect_result_
785 NTSTATUS
ConfigureDynamicDispatching(__in UCHAR MajorFunction,__in_opt FxCxDeviceInfo * CxDeviceInfo,__in PFN_WDFDEVICE_WDM_IRP_DISPATCH EvtDeviceWdmIrpDispatch,__in_opt WDFCONTEXT DriverContext)786 FxPkgIo::ConfigureDynamicDispatching(
787 __in UCHAR MajorFunction,
788 __in_opt FxCxDeviceInfo* CxDeviceInfo,
789 __in PFN_WDFDEVICE_WDM_IRP_DISPATCH EvtDeviceWdmIrpDispatch,
790 __in_opt WDFCONTEXT DriverContext
791 )
792 {
793 NTSTATUS status;
794 PFX_DRIVER_GLOBALS fxDriverGlobals;
795 PLIST_ENTRY next;
796 FxIrpDynamicDispatchInfo* dispatchInfo;
797 LONG mjIndex;
798 CCHAR driverIndex;
799 BOOLEAN addNew;
800
801 fxDriverGlobals = GetDriverGlobals();
802 addNew = TRUE;
803
804 mjIndex = FxIrpDynamicDispatchInfo::Mj2Index(MajorFunction);
805
806 //
807 // Indirect MajorFunction validation.
808 //
809 if (mjIndex >= FxIrpDynamicDispatchInfo::DynamicDispatchMax) {
810 status = STATUS_INVALID_PARAMETER;
811 DoTraceLevelMessage(fxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
812 "Invalid MajorFunction %!IRPMJ!, %!STATUS!",
813 MajorFunction, status);
814 goto Done;
815 }
816
817 //
818 // Get driver I/O device path index.
819 //
820 driverIndex = FxDevice::GetCxDriverIndex(CxDeviceInfo);
821
822 //
823 // Insert new info into correct slot in the I/O path.
824 // Index goes from higher to lower (..., 2, 1, 0) b/c cx's callback is called before
825 // client's one.
826 //
827 for (next = m_DynamicDispatchInfoListHead.Flink;
828 next != &m_DynamicDispatchInfoListHead;
829 next = next->Flink) {
830
831 CCHAR curIndex = 0;
832
833 dispatchInfo = CONTAINING_RECORD(next,
834 FxIrpDynamicDispatchInfo,
835 ListEntry);
836 //
837 // Get current I/O device path index.
838 //
839 curIndex = FxDevice::GetCxDriverIndex(dispatchInfo->CxDeviceInfo);
840 if (driverIndex == curIndex) {
841 //
842 // Found it.
843 //
844 if (dispatchInfo->Dispatch[mjIndex].EvtDeviceDynamicDispatch != NULL) {
845 status = STATUS_INVALID_PARAMETER;
846 DoTraceLevelMessage(
847 fxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDEVICE,
848 "Driver %p has already set a dispatch callback for "
849 "%!IRPMJ!, %!STATUS!",
850 CxDeviceInfo == NULL ?
851 GetDriver()->GetHandle() :
852 CxDeviceInfo->Driver->GetHandle(),
853 MajorFunction, status);
854 goto Done;
855 }
856
857 dispatchInfo->Dispatch[mjIndex].DriverContext = DriverContext;
858 dispatchInfo->Dispatch[mjIndex].EvtDeviceDynamicDispatch =
859 EvtDeviceWdmIrpDispatch;
860
861 ASSERT(dispatchInfo->CxDeviceInfo == CxDeviceInfo);
862
863 addNew = FALSE;
864 break;
865 }
866 else if (driverIndex > curIndex) {
867 //
868 // Not found (past valid range), add one before current.
869 //
870 break;
871 }
872 else if (driverIndex < curIndex) {
873 //
874 // Keep looking, too high.
875 //
876 continue;
877 }
878 }
879
880 if (addNew) {
881 dispatchInfo = new(fxDriverGlobals) FxIrpDynamicDispatchInfo();
882 if (dispatchInfo == NULL) {
883 status = STATUS_INSUFFICIENT_RESOURCES;
884 DoTraceLevelMessage(
885 fxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDEVICE,
886 "Couldn't create object DynamicDispatchInfo, %!STATUS!",
887 status);
888 goto Done;
889 }
890
891 dispatchInfo->CxDeviceInfo = CxDeviceInfo;
892 dispatchInfo->Dispatch[mjIndex].DriverContext = DriverContext;
893 dispatchInfo->Dispatch[mjIndex].EvtDeviceDynamicDispatch =
894 EvtDeviceWdmIrpDispatch;
895
896 //
897 // We must insert it before 'next' element.
898 //
899 InsertTailList(next, &dispatchInfo->ListEntry);
900 }
901
902 status = STATUS_SUCCESS;
903
904 Done:
905 return status;
906 }
907
908 _Must_inspect_result_
909 NTSTATUS
ConfigureForwarding(__inout FxIoQueue * TargetQueue,__in WDF_REQUEST_TYPE RequestType)910 FxPkgIo::ConfigureForwarding(
911 __inout FxIoQueue* TargetQueue,
912 __in WDF_REQUEST_TYPE RequestType
913 )
914 {
915 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals();
916 KIRQL irql;
917 NTSTATUS status;
918
919 ASSERT(RequestType <= IRP_MJ_MAXIMUM_FUNCTION);
920
921 if(TargetQueue->IsIoEventHandlerRegistered(RequestType) == FALSE){
922 status = STATUS_INVALID_DEVICE_REQUEST;
923 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
924 "Must have EvtIoDefault or %!WDF_REQUEST_TYPE! "
925 "specific dispatch event registered for "
926 "WDFQUEUE 0x%p, %!STATUS!", RequestType,
927 TargetQueue->GetObjectHandle(),
928 status);
929 FxVerifierDbgBreakPoint(FxDriverGlobals);
930 return status;
931 }
932
933 // Lock IoPackage data structure
934
935 Lock(&irql);
936
937 if (TargetQueue == m_DefaultQueue) {
938 status = STATUS_INVALID_DEVICE_REQUEST;
939 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
940 "Default WDFQUEUE 0x%p cannot be configured to "
941 "dispatch specific type of request, %!STATUS!",
942 TargetQueue->GetObjectHandle(),
943 status);
944 FxVerifierDbgBreakPoint(FxDriverGlobals);
945 Unlock(irql);
946 return status;
947 }
948
949 // Error if already has an entry
950 if (m_DispatchTable[RequestType] != NULL &&
951 m_DispatchTable[RequestType] != m_DefaultQueue) {
952 status = STATUS_WDF_BUSY;
953 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
954 "%!WDF_REQUEST_TYPE! is already configured for"
955 "WDFQUEUE 0x%p, %!STATUS!", RequestType,
956 TargetQueue->GetObjectHandle(),
957 status);
958 FxVerifierDbgBreakPoint(FxDriverGlobals);
959 Unlock(irql);
960 return status;
961 }
962
963 //
964 // We don't take an extra reference count since we already
965 // have one from our associated list (DriverQueues)
966 //
967 m_DispatchTable[RequestType] = TargetQueue;
968
969 //
970 // Queues configured to auto-dispatch requests cannot be deleted
971 //
972 TargetQueue->MarkNoDeleteDDI();
973
974 Unlock(irql);
975
976 return STATUS_SUCCESS;
977 }
978
979 _Must_inspect_result_
980 NTSTATUS
CreateQueue(__in PWDF_IO_QUEUE_CONFIG Config,__in PWDF_OBJECT_ATTRIBUTES QueueAttributes,__in_opt FxDriver * Caller,__deref_out FxIoQueue ** ppQueue)981 FxPkgIo::CreateQueue(
982 __in PWDF_IO_QUEUE_CONFIG Config,
983 __in PWDF_OBJECT_ATTRIBUTES QueueAttributes,
984 __in_opt FxDriver* Caller,
985 __deref_out FxIoQueue** ppQueue
986 )
987 {
988 PFX_DRIVER_GLOBALS pFxDriverGlobals;
989 FxObject* pParent;
990 FxIoQueue* pQueue;
991 NTSTATUS status;
992 FxDriver* pDriver;
993
994 pParent = NULL;
995 pQueue = NULL;
996 pDriver = NULL;
997 pFxDriverGlobals = GetDriverGlobals();
998
999 if (QueueAttributes != NULL && QueueAttributes->ParentObject != NULL) {
1000 CfxDeviceBase* pSearchDevice;
1001
1002 FxObjectHandleGetPtr(pFxDriverGlobals,
1003 QueueAttributes->ParentObject,
1004 FX_TYPE_OBJECT,
1005 (PVOID*)&pParent);
1006
1007 pSearchDevice = FxDeviceBase::_SearchForDevice(pParent, NULL);
1008
1009 if (pSearchDevice == NULL) {
1010 status = STATUS_INVALID_DEVICE_REQUEST;
1011
1012 DoTraceLevelMessage(
1013 pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
1014 "QueueAttributes->ParentObject 0x%p must have WDFDEVICE as an "
1015 "eventual ancestor, %!STATUS!",
1016 QueueAttributes->ParentObject, status);
1017
1018 return status;
1019 }
1020 else if (pSearchDevice != m_DeviceBase) {
1021 status = STATUS_INVALID_DEVICE_REQUEST;
1022
1023 DoTraceLevelMessage(
1024 pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
1025 "Attributes->ParentObject 0x%p ancestor is WDFDEVICE %p, but "
1026 "not the same WDFDEVICE 0x%p passed to WdfIoQueueCreate, "
1027 "%!STATUS!",
1028 QueueAttributes->ParentObject, pSearchDevice->GetHandle(),
1029 m_Device->GetHandle(), status);
1030
1031 return status;
1032 }
1033 }
1034 else {
1035 //
1036 // By default, use the package as the parent
1037 //
1038 pParent = this;
1039 }
1040
1041 //
1042 // v1.11 and up: get driver object if driver handle is specified.
1043 // Client driver can also specify a driver handle, the end result in this case is
1044 // a PkgIoContext that is NULL (see below), i.e., the same as if driver handle
1045 // was NULL.
1046 //
1047 if (Config->Size > sizeof(WDF_IO_QUEUE_CONFIG_V1_9) &&
1048 Config->Driver != NULL) {
1049
1050 FxObjectHandleGetPtr(GetDriverGlobals(),
1051 Config->Driver,
1052 FX_TYPE_DRIVER,
1053 (PVOID*)&pDriver);
1054 }
1055
1056 status = FxIoQueue::_Create(pFxDriverGlobals,
1057 QueueAttributes,
1058 Config,
1059 Caller,
1060 this,
1061 m_PowerStateOn,
1062 &pQueue
1063 );
1064
1065 if (!NT_SUCCESS(status)) {
1066 ASSERT(pQueue == NULL);
1067 return status;
1068 }
1069
1070 //
1071 // Class extension support: associate queue with a specific cx layer.
1072 //
1073 if (pDriver != NULL) {
1074 pQueue->SetCxDeviceInfo(m_Device->GetCxDeviceInfo(pDriver));
1075 }
1076
1077 status = pQueue->Commit(QueueAttributes, NULL, pParent);
1078 if (!NT_SUCCESS(status)) {
1079 pQueue->DeleteFromFailedCreate();
1080 return status;
1081 }
1082
1083 AddIoQueue(pQueue);
1084 *ppQueue = pQueue;
1085
1086 return status;
1087 }
1088
1089
1090 VOID
RemoveQueueReferences(__inout FxIoQueue * pQueue)1091 FxPkgIo::RemoveQueueReferences(
1092 __inout FxIoQueue* pQueue
1093 )
1094 /*++
1095
1096 Routine Description:
1097
1098 This is called from FxIoQueue::Dispose to remove
1099 the queue from the transaction list.
1100
1101 Since this acquires the FxPkgIo lock, it assumes that
1102 no calls are made into FxIoQueue while holding the
1103 FxPkgIo lock.
1104
1105 Arguments:
1106
1107 None
1108
1109 Return Value:
1110 None
1111 --*/
1112 {
1113 // Remove it from transacation list
1114 RemoveIoQueue(pQueue);
1115 return;
1116 }
1117
1118 _Must_inspect_result_
1119 NTSTATUS
SetFilter(__in BOOLEAN Value)1120 FxPkgIo::SetFilter(
1121 __in BOOLEAN Value
1122 )
1123 {
1124 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals();
1125
1126 if (m_DefaultQueue != NULL) {
1127 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
1128 "I/O Package already has a default queue. "
1129 "SetFilter must be called before creating "
1130 "a default queue %!STATUS!",
1131 STATUS_INVALID_DEVICE_REQUEST);
1132 FxVerifierDbgBreakPoint(FxDriverGlobals);
1133 return STATUS_INVALID_DEVICE_REQUEST;
1134 }
1135
1136 m_Filter = Value;
1137
1138 return STATUS_SUCCESS;
1139 }
1140
1141 _Must_inspect_result_
1142 NTSTATUS
StopProcessingForPower(__in FxIoStopProcessingForPowerAction Action)1143 FxPkgIo::StopProcessingForPower(
1144 __in FxIoStopProcessingForPowerAction Action
1145 )
1146
1147 /*++
1148
1149 Routine Description:
1150
1151 Stops all PowerManaged queues automatic I/O processing due to
1152 a power event that requires I/O to stop.
1153
1154 This is called on a PASSIVE_LEVEL thread that can block until
1155 I/O has been stopped, or completed/cancelled.
1156
1157 Arguments:
1158
1159 Action -
1160
1161 FxIoStopProcessingForPowerHold:
1162 the function returns when the driver has acknowledged that it has
1163 stopped all I/O processing, but may have outstanding "in-flight" requests
1164 that have not been completed.
1165
1166 FxIoStopProcessingForPowerPurgeManaged:
1167 the function returns when all requests from a power managed queue have
1168 been completed and/or cancelled., and there are no more in-flight requests.
1169
1170 FxIoStopProcessingForPowerPurgeNonManaged:
1171 the function returns when all requests from a non-power managed queue have
1172 been completed and/or cancelled., and there are no more in-flight requests.
1173 Only called during device-remove.
1174
1175 Return Value:
1176
1177 NTSTATUS
1178
1179 --*/
1180
1181 {
1182 KIRQL irql;
1183 FxIoQueue* queue;
1184 SINGLE_LIST_ENTRY queueList, *ple;
1185
1186 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_INFORMATION, TRACINGIO,
1187 "Perform %!FxIoStopProcessingForPowerAction! for all queues of "
1188 "WDFDEVICE 0x%p", Action, m_Device->GetHandle());
1189
1190 queueList.Next = NULL;
1191
1192 Lock(&irql);
1193 //
1194 // Device is moving into low power state. So any new queues
1195 // created after this point would start off as powered off.
1196 //
1197 m_PowerStateOn = FALSE;
1198
1199 //
1200 // If queues are shutting down, any new queue created after
1201 // this point would not accept any requests.
1202 //
1203 switch(Action) {
1204 case FxIoStopProcessingForPowerPurgeManaged:
1205 case FxIoStopProcessingForPowerPurgeNonManaged:
1206 m_QueuesAreShuttingDown = TRUE;
1207 break;
1208 }
1209
1210 GetIoQueueListLocked(&queueList, FxIoQueueIteratorListPowerOff);
1211
1212 Unlock(irql);
1213
1214 //
1215 // If the power action is hold then first change the state of all the queues
1216 // to PowerStartTransition. This will prevent the queues from dispatching
1217 // new requests before we start asking each queue to stop processing
1218 // inflight requests.
1219 //
1220 if(Action == FxIoStopProcessingForPowerHold) {
1221
1222 //
1223 // Walk the list without popping entries because we need to scan
1224 // the list again.
1225 //
1226 for(ple = queueList.Next; ple != NULL; ple = ple->Next) {
1227
1228 queue = FxIoQueue::_FromPowerSListEntry(ple);
1229
1230 queue->StartPowerTransitionOff();
1231 }
1232 }
1233
1234 //
1235 // Ask the queues to stop processing inflight requests.
1236 //
1237 for (ple = PopEntryList(&queueList); ple != NULL;
1238 ple = PopEntryList(&queueList)) {
1239
1240 queue = FxIoQueue::_FromPowerSListEntry(ple);
1241
1242 queue->StopProcessingForPower(Action);
1243
1244 ple->Next = NULL;
1245
1246 queue->RELEASE(IO_ITERATOR_POWER_TAG);
1247 }
1248
1249 return STATUS_SUCCESS;
1250 }
1251
1252 _Must_inspect_result_
1253 NTSTATUS
ResumeProcessingForPower()1254 FxPkgIo::ResumeProcessingForPower()
1255
1256 /*++
1257
1258 Routine Description:
1259
1260 Resumes all PowerManaged queues for automatic I/O processing due to
1261 a power event that allows I/O to resume.
1262
1263 Non-PowerManaged queues are left alone.
1264
1265 Arguments:
1266
1267 Return Value:
1268
1269 NTSTATUS
1270
1271 --*/
1272
1273 {
1274 KIRQL irql;
1275 FxIoQueue* queue;
1276 SINGLE_LIST_ENTRY queueList, *ple;
1277
1278 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_INFORMATION, TRACINGIO,
1279 "Power resume all queues of WDFDEVICE 0x%p",
1280 m_Device->GetHandle());
1281
1282 queueList.Next = NULL;
1283
1284 Lock(&irql);
1285
1286 GetIoQueueListLocked(&queueList, FxIoQueueIteratorListPowerOn);
1287 //
1288 // Change the state so that new queues created while we
1289 // are resuming the existing queues can be in a powered-on state.
1290 //
1291 m_PowerStateOn = TRUE;
1292
1293 //
1294 // Change the accepting state so that new queues created while we
1295 // are resuming the existing queues can be accept request.
1296 //
1297 m_QueuesAreShuttingDown = FALSE;
1298
1299 Unlock(irql);
1300
1301 //
1302 // We will power-up the queues in two steps. First we will resume
1303 // the power of all the queues and then we will start dispatching I/Os.
1304 // This is to avoid a queue being powered up at the begining of the list
1305 // trying to forward a request to another queue lower in the list that's
1306 // not powered up yet.
1307 //
1308 for(ple = queueList.Next; ple != NULL; ple = ple->Next) {
1309
1310 queue = FxIoQueue::_FromPowerSListEntry(ple);
1311
1312 queue->ResumeProcessingForPower();
1313 }
1314
1315 for (ple = PopEntryList(&queueList);
1316 ple != NULL;
1317 ple = PopEntryList(&queueList)) {
1318
1319 queue = FxIoQueue::_FromPowerSListEntry(ple);
1320
1321 queue->StartPowerTransitionOn();
1322
1323 ple->Next = NULL;
1324
1325 queue->RELEASE(IO_ITERATOR_POWER_TAG);
1326 }
1327
1328 return STATUS_SUCCESS;
1329 }
1330
1331 VOID
ResetStateForRestart(VOID)1332 FxPkgIo::ResetStateForRestart(
1333 VOID
1334 )
1335 /*++
1336
1337 Routine Description:
1338 This is called on a device which has been restarted from the removed
1339 state. It will reset purged queues so that they can accept new requests
1340 when ResumeProcessingForPower is called afterwards.
1341
1342 Arguments:
1343 None
1344
1345 Return Value:
1346 None
1347
1348 --*/
1349 {
1350 KIRQL irql;
1351 FxIoQueue* queue;
1352 SINGLE_LIST_ENTRY queueList, *ple;
1353
1354 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_INFORMATION, TRACINGIO,
1355 "Restart queues from purged state for WDFDEVICE 0x%p due to "
1356 "device restart", m_Device->GetHandle());
1357
1358 queueList.Next = NULL;
1359
1360 Lock(&irql);
1361
1362 GetIoQueueListLocked(&queueList, FxIoQueueIteratorListPowerOn);
1363
1364 Unlock(irql);
1365
1366 for (ple = PopEntryList(&queueList);
1367 ple != NULL;
1368 ple = PopEntryList(&queueList)) {
1369
1370 queue = FxIoQueue::_FromPowerSListEntry(ple);
1371
1372 queue->ResetStateForRestart();
1373
1374 ple->Next = NULL;
1375
1376 queue->RELEASE(IO_ITERATOR_POWER_TAG);
1377 }
1378
1379 Lock(&irql);
1380
1381 m_PowerStateOn = TRUE;
1382
1383 m_QueuesAreShuttingDown = FALSE;
1384
1385 Unlock(irql);
1386
1387 return;
1388
1389 }
1390
1391 _Must_inspect_result_
1392 NTSTATUS
FlushAllQueuesByFileObject(__in MdFileObject FileObject)1393 FxPkgIo::FlushAllQueuesByFileObject(
1394 __in MdFileObject FileObject
1395 )
1396
1397 /*++
1398
1399 Routine Description:
1400
1401 Enumerate all the queues and cancel the requests that have
1402 the same fileobject as the Cleanup IRP.
1403
1404 We are making an assumption that cleanup irps are sent only
1405 at passive-level.
1406
1407 Return Value:
1408
1409 NTSTATUS
1410
1411 --*/
1412 {
1413 FxIoQueue* queue = NULL;
1414 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
1415 FxIoQueueNode flushBookmark(FxIoQueueNodeTypeBookmark);
1416 KIRQL irql;
1417
1418 if(Mx::MxGetCurrentIrql() != PASSIVE_LEVEL) {
1419
1420 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO,
1421 "Currently framework allow flushing of queues "
1422 "by fileobject on cleanup only at PASSIVE_LEVEL");
1423
1424 FxVerifierDbgBreakPoint(pFxDriverGlobals);
1425 return STATUS_SUCCESS;
1426 }
1427
1428 //
1429 // Iterate through the queue list and flush each one.
1430 //
1431 Lock(&irql);
1432 queue = GetFirstIoQueueLocked(&flushBookmark, IO_ITERATOR_FLUSH_TAG);
1433 Unlock(irql);
1434
1435 while(queue != NULL) {
1436
1437 queue->FlushByFileObject(FileObject);
1438
1439 queue->RELEASE(IO_ITERATOR_FLUSH_TAG);
1440
1441 Lock(&irql);
1442 queue = GetNextIoQueueLocked(&flushBookmark, IO_ITERATOR_FLUSH_TAG);
1443 Unlock(irql);
1444 }
1445
1446 return STATUS_SUCCESS;
1447 }
1448
1449 static
1450 VOID
GetIoQueueList_ProcessQueueListEntry(PLIST_ENTRY QueueLE,PSINGLE_LIST_ENTRY SListHead,PVOID Tag)1451 GetIoQueueList_ProcessQueueListEntry(
1452 PLIST_ENTRY QueueLE,
1453 PSINGLE_LIST_ENTRY SListHead,
1454 PVOID Tag
1455 )
1456 {
1457 FxIoQueueNode* listNode;
1458 FxIoQueue* queue;
1459
1460 //
1461 // Skip any nodes that are not queues. They can be bookmarks for
1462 // in-progress flush operations.
1463 //
1464 listNode = FxIoQueueNode::_FromListEntry(QueueLE);
1465 if (listNode->IsNodeType(FxIoQueueNodeTypeQueue) == FALSE) {
1466 return;
1467 }
1468
1469 queue = FxIoQueue::_FromIoPkgListEntry(QueueLE);
1470 PushEntryList(SListHead, &queue->m_PowerSListEntry);
1471
1472 //
1473 // Add a reference since the request will be touched outside of the
1474 // lock being held. We will use the enumerant value as the tag.
1475 //
1476 queue->ADDREF(Tag);
1477 }
1478
1479 VOID
GetIoQueueListLocked(__in PSINGLE_LIST_ENTRY SListHead,__inout FxIoIteratorList ListType)1480 FxPkgIo::GetIoQueueListLocked(
1481 __in PSINGLE_LIST_ENTRY SListHead,
1482 __inout FxIoIteratorList ListType
1483 )
1484 /*++
1485
1486 Routine Description:
1487
1488 Called to make a temporary list of queues for iteration purpose.
1489 Function is called with the FxPkg lock held.
1490
1491 --*/
1492 {
1493 PLIST_ENTRY listHead, le;
1494
1495 listHead = &m_IoQueueListHead;
1496
1497 if (FxIoQueueIteratorListPowerOn == ListType ||
1498 (FxIoQueueIteratorListPowerOff == ListType && // backwards compatibility order.
1499 m_Device->IsCxInIoPath() == FALSE)) {
1500 //
1501 // Power up: first client driver's queues then cx's queues.
1502 // List is already sorted with client driver's queue first.
1503 // Since we are inserting into the head of the single list head, if we walked
1504 // over the list from first to last, we would reverse the entries. By walking
1505 // the list backwards, we build the single list head in the order of m_IoQueueListHead.
1506 //
1507 for (le = listHead->Blink; le != listHead; le = le->Blink) {
1508 GetIoQueueList_ProcessQueueListEntry(le,
1509 SListHead,
1510 IO_ITERATOR_POWER_TAG);
1511 }
1512 }
1513 else if (FxIoQueueIteratorListPowerOff == ListType) {
1514 //
1515 // Power down: first cx's queues then client driver's queues.
1516 // List is already sorted with client driver's queue first.
1517 // Since we are inserting into the head of the single list head, if we walked
1518 // over the list from last to first, we would reverse the entries. By walking
1519 // the list forwards, we build the single list head in the desired order
1520 //
1521 for (le = listHead->Flink; le != listHead; le = le->Flink) {
1522 GetIoQueueList_ProcessQueueListEntry(le,
1523 SListHead,
1524 IO_ITERATOR_POWER_TAG);
1525 }
1526 }
1527 else {
1528 ASSERT(FALSE);
1529 }
1530 }
1531
1532 VOID
AddIoQueue(__inout FxIoQueue * IoQueue)1533 FxPkgIo::AddIoQueue(
1534 __inout FxIoQueue* IoQueue
1535 )
1536 {
1537 PLIST_ENTRY listHead, le;
1538 FxIoQueue* queue;
1539 KIRQL irql;
1540 FxIoQueueNode* listNode;
1541 CCHAR queueIndex, curIndex;
1542
1543 listHead = &m_IoQueueListHead;
1544 queueIndex = FxDevice::GetCxDriverIndex(IoQueue->GetCxDeviceInfo());
1545 Lock(&irql);
1546
1547 ASSERT(IoQueue->m_IoPkgListNode.IsNodeType(FxIoQueueNodeTypeQueue));
1548
1549 //
1550 // Insert new queue in sorted list; search from last to first.
1551 //
1552 for (le = listHead->Blink; le != listHead; le = le->Blink) {
1553 //
1554 // Skip any nodes that are not queues. They can be bookmarks for
1555 // in-progress flush operations.
1556 //
1557 listNode = FxIoQueueNode::_FromListEntry(le);
1558 if (listNode->IsNodeType(FxIoQueueNodeTypeQueue) == FALSE) {
1559 continue;
1560 }
1561
1562 //
1563 // Get current queue's driver index.
1564 //
1565 queue = FxIoQueue::_FromIoPkgListEntry(le);
1566 curIndex = FxDevice::GetCxDriverIndex(queue->GetCxDeviceInfo());
1567 //
1568 // Queues are inserted in order at the end of its allowed range.
1569 //
1570 if (curIndex < queueIndex || curIndex == queueIndex) {
1571 break;
1572 }
1573 }
1574
1575 InsertHeadList(le, &IoQueue->m_IoPkgListNode.m_ListEntry);
1576
1577 if (m_PowerStateOn) {
1578 IoQueue->SetPowerState(FxIoQueuePowerOn);
1579 } else {
1580 IoQueue->SetPowerState(FxIoQueuePowerOff);
1581 if (m_QueuesAreShuttingDown) {
1582 // Clear accept requests
1583 IoQueue->SetStateForShutdown();
1584 }
1585 }
1586
1587 Unlock(irql);
1588
1589 return;
1590 }
1591
1592 VOID
RemoveIoQueue(__inout FxIoQueue * IoQueue)1593 FxPkgIo::RemoveIoQueue(
1594 __inout FxIoQueue* IoQueue
1595 )
1596 {
1597 KIRQL irql;
1598
1599 Lock(&irql);
1600
1601 RemoveEntryList(&IoQueue->m_IoPkgListNode.m_ListEntry);
1602 ASSERT(IoQueue->m_IoPkgListNode.IsNodeType(FxIoQueueNodeTypeQueue));
1603
1604 InitializeListHead(&IoQueue->m_IoPkgListNode.m_ListEntry);
1605
1606 Unlock(irql);
1607 }
1608
1609 FxIoQueue*
GetFirstIoQueueLocked(__in FxIoQueueNode * QueueBookmark,__in PVOID Tag)1610 FxPkgIo::GetFirstIoQueueLocked(
1611 __in FxIoQueueNode* QueueBookmark,
1612 __in PVOID Tag
1613 )
1614 /*++
1615
1616 Routine Description:
1617
1618 Inserts the provided bookmark (FxIoQueueNode) at the beginning
1619 of the IO Package's queue list, and calls GetNextIoQueueLocked
1620 to retrieve the first queue and to advance the bookmark.
1621
1622 Function is called with the FxPkg lock held.
1623
1624 Return Value:
1625
1626 NULL if there are no queues in list else
1627 FxIoQueue* reference to first queue in list.
1628
1629 --*/
1630 {
1631 ASSERT(QueueBookmark->IsNodeType(FxIoQueueNodeTypeBookmark));
1632 ASSERT(IsListEmpty(&QueueBookmark->m_ListEntry));
1633
1634 InsertHeadList(&m_IoQueueListHead, &QueueBookmark->m_ListEntry);
1635
1636 return GetNextIoQueueLocked(QueueBookmark, Tag);
1637 }
1638
1639 FxIoQueue*
GetNextIoQueueLocked(__in FxIoQueueNode * QueueBookmark,__in PVOID Tag)1640 FxPkgIo::GetNextIoQueueLocked(
1641 __in FxIoQueueNode* QueueBookmark,
1642 __in PVOID Tag
1643 )
1644 /*++
1645
1646 Routine Description:
1647
1648 Moves the provided bookmark ahead in the IO Package's queue list
1649 and returns the next available queue (if any).
1650
1651 Return Value:
1652
1653 NULL if there are no more queues in list else
1654 FxIoQueue* reference to the next queue in list.
1655
1656 --*/
1657 {
1658 PLIST_ENTRY ple = NULL;
1659 FxIoQueue* queue = NULL;
1660 FxIoQueueNode* listNode = NULL;
1661
1662 ASSERT(QueueBookmark->IsNodeType(FxIoQueueNodeTypeBookmark));
1663 ASSERT(IsListEmpty(&QueueBookmark->m_ListEntry) == FALSE);
1664
1665 //
1666 // Try to advance bookmark to next location.
1667 //
1668 ple = QueueBookmark->m_ListEntry.Flink;
1669 RemoveEntryList(&QueueBookmark->m_ListEntry);
1670 InitializeListHead(&QueueBookmark->m_ListEntry);
1671
1672 for (; ple != &m_IoQueueListHead; ple = ple->Flink) {
1673 //
1674 // Skip any nodes that are not queues. These nodes can be
1675 // bookmarks for in-progress flush operations.
1676 //
1677 listNode = FxIoQueueNode::_FromListEntry(ple);
1678 if (listNode->IsNodeType(FxIoQueueNodeTypeQueue)) {
1679
1680 //
1681 // This entry is a real queue.
1682 //
1683 queue = FxIoQueue::_FromIoPkgListEntry(ple);
1684 queue->ADDREF(Tag);
1685
1686 //
1687 // Insert bookmark after this entry.
1688 //
1689 InsertHeadList(ple, &QueueBookmark->m_ListEntry);
1690
1691 break;
1692 }
1693 }
1694
1695 return queue;
1696 }
1697
1698 NTSTATUS
DispathToInCallerContextCallback(__in FxIoInCallerContext * InCallerContextInfo,__in FxRequest * Request,__inout MdIrp Irp)1699 FxPkgIo::DispathToInCallerContextCallback(
1700 __in FxIoInCallerContext *InCallerContextInfo,
1701 __in FxRequest *Request,
1702 __inout MdIrp Irp
1703 )
1704 {
1705 PFX_DRIVER_GLOBALS pFxDriverGlobals;
1706 FxRequestCompletionState oldState;
1707 FxIrp fxIrp(Irp);
1708
1709 pFxDriverGlobals = GetDriverGlobals();
1710
1711 //
1712 // Mark the IRP pending since we are going
1713 // to return STATUS_PENDING regardless of whether
1714 // the driver completes the request right away
1715 // or not
1716 //
1717 fxIrp.MarkIrpPending();
1718
1719 if (pFxDriverGlobals->FxVerifierOn) {
1720 Request->SetVerifierFlags(FXREQUEST_FLAG_DRIVER_INPROCESS_CONTEXT |
1721 FXREQUEST_FLAG_DRIVER_OWNED);
1722 }
1723
1724 //
1725 // Set a completion callback to manage the reference
1726 // count on the request
1727 //
1728 oldState = Request->SetCompletionState(FxRequestCompletionStateIoPkg);
1729
1730 ASSERT(oldState == FxRequestCompletionStateNone);
1731 UNREFERENCED_PARAMETER(oldState);
1732
1733 //
1734 // Release the reference count on the request since
1735 // the callback will hold the only one that gets
1736 // decremented when the request is completed
1737 //
1738 Request->RELEASE(FXREQUEST_STATE_TAG);
1739
1740 Request->SetPresented();
1741
1742 //
1743 // Drivers that use this API are responsible for handling
1744 // all locking, threading, and IRQL level issues...
1745 //
1746 InCallerContextInfo->Invoke(m_Device->GetHandle(),
1747 Request->GetHandle());
1748 //
1749 // The driver is responsible for calling WdfDeviceEnqueueRequest to insert
1750 // it back into the I/O processing pipeline, or completing it.
1751 //
1752
1753 return STATUS_PENDING;
1754 }
1755
1756 _Must_inspect_result_
1757 NTSTATUS
VerifierFreeRequestToTestForwardProgess(__in FxRequest * Request)1758 FxPkgIo::VerifierFreeRequestToTestForwardProgess(
1759 __in FxRequest* Request
1760 )
1761 {
1762 BOOLEAN failAllocation;
1763 PFX_DRIVER_GLOBALS pFxDriverGlobals;
1764
1765 pFxDriverGlobals = GetDriverGlobals();
1766 failAllocation = FALSE;
1767 //
1768 // forwardProgressTestFailAll takes precedence over forwardProgressTestFailRandom
1769 //
1770 if (IsFxVerifierTestForwardProgressFailAll(pFxDriverGlobals)) {
1771 failAllocation = TRUE;
1772 }
1773 else if (IsFxVerifierTestForwardProgressFailRandom(pFxDriverGlobals)) {
1774 //
1775 // Modulo 17 makes the probability of failure ~6% just like verifier.exe
1776 //
1777 failAllocation = (FxRandom(&m_RandomSeed) % 17 == 0);
1778 }
1779
1780 if (failAllocation) {
1781 //
1782 // Don't use DeleteObject() here as the Request wasn't presented to the
1783 // driver and the cleanup /dispose callback can be invoked unless
1784 // we use FreeRequest() which clears the cleanup /dispose callbacks
1785 //
1786 Request->FreeRequest();
1787
1788 return STATUS_INSUFFICIENT_RESOURCES;
1789 }
1790 else {
1791 return STATUS_SUCCESS;
1792 }
1793 }
1794
1795
1796