1 /*++ 2 3 Copyright (c) Microsoft Corporation 4 5 Module Name: 6 7 FxIoQueue.cpp 8 9 Abstract: 10 11 This module implements the FxIoQueue object and C interfaces 12 13 Author: 14 15 16 17 18 Revision History: 19 20 21 22 23 --*/ 24 25 #include "ioprivshared.hpp" 26 #include "fxioqueue.hpp" 27 28 extern "C" { 29 #if defined(EVENT_TRACING) 30 #include "FxIoQueue.tmh" 31 #endif 32 } 33 34 // 35 // Public constructors 36 // 37 FxIoQueue::FxIoQueue( 38 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 39 __in FxPkgIo* PkgIo 40 ) : 41 FxNonPagedObject(FX_TYPE_QUEUE, sizeof(FxIoQueue), FxDriverGlobals), 42 m_CallbackSpinLock(FxDriverGlobals), 43 m_CallbackMutexLock(FxDriverGlobals), 44 m_IoPkgListNode(FxIoQueueNodeTypeQueue) 45 { 46 m_Configured = FALSE; 47 m_Disposing = FALSE; 48 m_PowerManaged = FALSE; 49 m_PowerState = FxIoQueuePowerOn; 50 m_PowerReferenced = FALSE; 51 m_AllowZeroLengthRequests = FALSE; 52 m_IsDevicePowerPolicyOwner = FALSE; 53 m_Type = WdfIoQueueDispatchSequential; 54 55 // A newly created queue can not accept requests until initialized 56 m_QueueState = (FX_IO_QUEUE_STATE)0; 57 58 // 59 // Set our Cancel callbacks 60 // 61 m_Queue.Initialize(this, _IrpCancelForQueue); 62 63 m_DriverCancelable.Initialize(this, _IrpCancelForDriver); 64 65 InitializeListHead(&m_Cancelled); 66 67 InitializeListHead(&m_CanceledOnQueueList); 68 69 InitializeListHead(&m_DriverOwned); 70 71 InitializeListHead(&m_PowerNotify); 72 73 InitializeListHead(&m_PowerDriverNotified); 74 75 m_PowerSListEntry.Next = NULL; 76 77 // 78 // We do not reference count the I/O package instance 79 // since it contains us. The fact we exist, the I/O 80 // package instance exists. 81 // 82 m_PkgIo = PkgIo; 83 m_CxDeviceInfo = NULL; 84 85 m_Device = PkgIo->GetDevice(); 86 87 m_IsDevicePowerPolicyOwner = (m_Device->IsPnp() && 88 m_Device->m_PkgPnp->IsPowerPolicyOwner()); 89 90 m_Dispatching = 0L; 91 92 m_TransitionFromEmpty = FALSE; 93 m_ForceTransitionFromEmptyWhenAddingNewRequest = FALSE; 94 95 m_DriverIoCount = 0L; 96 m_TwoPhaseCompletions = 0L; 97 98 m_SystemWorkItem = NULL; 99 100 m_IdleComplete.Method = NULL; 101 m_IdleCompleteContext = NULL; 102 103 m_PurgeComplete.Method = NULL; 104 m_PurgeCompleteContext = NULL; 105 106 m_ReadyNotify.Method = NULL; 107 m_ReadyNotifyContext = NULL; 108 109 m_CallbackLockPtr = NULL; 110 m_CallbackLockObjectPtr = NULL; 111 112 #if FX_IS_KERNEL_MODE 113 114 // Initialize the DPC used for deferrals 115 KeInitializeDpc( 116 &m_Dpc, 117 _DeferredDispatchDpcThunk, 118 this 119 ); 120 #endif 121 122 m_DpcQueued = FALSE; 123 124 m_WorkItemQueued = FALSE; 125 126 m_RequeueDeferredDispatcher = FALSE; 127 128 m_Deleted = FALSE; 129 m_SupportForwardProgress = FALSE; 130 m_PassiveLevel = FALSE; 131 132 m_ExecutionLevel = WdfExecutionLevelInheritFromParent; 133 m_SynchronizationScope = WdfSynchronizationScopeInheritFromParent; 134 135 m_FwdProgContext = NULL; 136 MarkPassiveDispose(ObjectDoNotLock); 137 m_MaxParallelQueuePresentedRequests = (ULONG)-1; 138 139 return; 140 } 141 142 FxIoQueue::~FxIoQueue() 143 { 144 ASSERT(m_SystemWorkItem == NULL); 145 146 if (m_PkgIo != NULL) { 147 m_PkgIo = NULL; 148 } 149 150 ASSERT(IsListEmpty(&m_Cancelled)); 151 ASSERT(IsListEmpty(&m_CanceledOnQueueList)); 152 ASSERT(IsListEmpty(&m_DriverOwned)); 153 ASSERT(IsListEmpty(&m_PowerNotify)); 154 ASSERT(IsListEmpty(&m_PowerDriverNotified)); 155 ASSERT(!m_PowerReferenced); 156 ASSERT(!m_DpcQueued); 157 ASSERT(!m_WorkItemQueued); 158 ASSERT(!m_RequeueDeferredDispatcher); 159 ASSERT(m_TwoPhaseCompletions == 0); 160 } 161 162 _Must_inspect_result_ 163 NTSTATUS 164 FxIoQueue::_Create( 165 __in PFX_DRIVER_GLOBALS DriverGlobals, 166 __in_opt PWDF_OBJECT_ATTRIBUTES Attributes, 167 __in PWDF_IO_QUEUE_CONFIG Config, 168 __in_opt FxDriver* Caller, 169 __in FxPkgIo* PkgIo, 170 __in BOOLEAN InitialPowerStateOn, 171 __deref_out FxIoQueue** Object 172 ) 173 { 174 NTSTATUS status; 175 FxIoQueue* pQueue; 176 177 *Object = NULL; 178 179 pQueue = new(DriverGlobals, Attributes) FxIoQueue(DriverGlobals, PkgIo); 180 181 if (pQueue == NULL) { 182 status = STATUS_INSUFFICIENT_RESOURCES; 183 DoTraceLevelMessage(DriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 184 "Memory allocation failed: %!STATUS!", status); 185 return status; 186 } 187 188 // 189 // Initialize it, creating the handle to pass the driver 190 // and configuring its callbacks and queue type 191 // 192 status = pQueue->Initialize(Config, 193 Attributes, 194 Caller, 195 InitialPowerStateOn); 196 if (!NT_SUCCESS(status)) { 197 DoTraceLevelMessage(DriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 198 "Could not configure queue: %!STATUS!", status); 199 goto Done; 200 } 201 Done: 202 if (NT_SUCCESS(status)) { 203 *Object = pQueue; 204 } 205 else { 206 // 207 // Release our newly allocated Queue object 208 // 209 pQueue->DeleteFromFailedCreate(); 210 } 211 212 return status; 213 } 214 215 _Must_inspect_result_ 216 NTSTATUS 217 FxIoQueue::Initialize( 218 __in PWDF_IO_QUEUE_CONFIG pConfig, 219 __in_opt PWDF_OBJECT_ATTRIBUTES QueueAttributes, 220 __in_opt FxDriver* Caller, 221 __in BOOLEAN InitialPowerStateOn 222 ) 223 224 /*++ 225 226 Routine Description: 227 228 Initialize the IoQueue after creating. 229 230 This creates the handle required for passing to the driver. 231 232 Arguments: 233 234 Returns: 235 236 NTSTATUS 237 238 --*/ 239 240 { 241 NTSTATUS Status; 242 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 243 244 Status = m_PowerIdle.Initialize(NotificationEvent, FALSE); 245 if (!NT_SUCCESS(Status)) { 246 return Status; 247 } 248 249 Status = m_FinishDisposing.Initialize(NotificationEvent, FALSE); 250 if (!NT_SUCCESS(Status)) { 251 return Status; 252 } 253 254 #if (FX_CORE_MODE==FX_CORE_USER_MODE) 255 Status = m_RequestWaitEventUm.Initialize(SynchronizationEvent, FALSE); 256 if (!NT_SUCCESS(Status)) { 257 return Status; 258 } 259 #endif 260 261 MarkDisposeOverride(ObjectDoNotLock); 262 263 // 264 // Set the execution level and callback synchronization based on 265 // configuration 266 // 267 Status = ConfigureConstraints(QueueAttributes, Caller); 268 if (!NT_SUCCESS(Status)) { 269 return Status; 270 } 271 272 // 273 // Validate dispatch type. 274 // 275 if (pConfig->DispatchType <= WdfIoQueueDispatchInvalid || 276 pConfig->DispatchType >= WdfIoQueueDispatchMax) { 277 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 278 "Invalid dispatch type " 279 "specified %d, Queue 0x%p %!STATUS!", 280 pConfig->DispatchType, 281 GetObjectHandle(), 282 STATUS_INVALID_PARAMETER); 283 return STATUS_INVALID_PARAMETER; 284 } 285 286 // 287 // If not a manual queue, must set at least IoStart, or one of 288 // read|write|devicecontrol 289 // 290 if ((pConfig->DispatchType != WdfIoQueueDispatchManual) && 291 (pConfig->EvtIoDefault == NULL)) { 292 293 if ((pConfig->EvtIoDefault == NULL) && 294 (pConfig->EvtIoRead == NULL) && 295 (pConfig->EvtIoWrite == NULL) && 296 (pConfig->EvtIoDeviceControl == NULL) && 297 (pConfig->EvtIoInternalDeviceControl == NULL)) { 298 299 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 300 "At least one of EvtIoDefault|EvtIoRead|EvtIoWrite|" 301 "EvtIoDeviceControl|EvtIoInternalDeviceControl " 302 "must be set %!STATUS!", STATUS_WDF_NO_CALLBACK); 303 return STATUS_WDF_NO_CALLBACK; 304 } 305 } 306 307 // 308 // A manual queue should not set any callback function 309 // pointers since they will not get invoked. 310 // 311 if (pConfig->DispatchType == WdfIoQueueDispatchManual) { 312 313 if ((pConfig->EvtIoDefault != NULL) || 314 (pConfig->EvtIoRead != NULL) || 315 (pConfig->EvtIoWrite != NULL) || 316 (pConfig->EvtIoDeviceControl != NULL) || 317 (pConfig->EvtIoInternalDeviceControl != NULL)) { 318 319 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 320 "Cannot set io callback events " 321 "on a manual WDFQUEUE 0x%p %!STATUS!", 322 GetObjectHandle(), 323 STATUS_INVALID_PARAMETER); 324 return STATUS_INVALID_PARAMETER; 325 } 326 } 327 328 // 329 // For version less than v1.9 m_MaxParallelQueuePresentedRequests is set to 330 // -1 by the FxIoQueue Constructor. 331 // By checking > below we mean v1.9 and above (public API already did the official 332 // validation). 333 // 334 if (pConfig->Size > sizeof(WDF_IO_QUEUE_CONFIG_V1_7)) { 335 if (pConfig->Settings.Parallel.NumberOfPresentedRequests != 0 && 336 (pConfig->DispatchType == WdfIoQueueDispatchSequential || 337 pConfig->DispatchType == WdfIoQueueDispatchManual)) { 338 Status = STATUS_INVALID_PARAMETER; 339 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 340 "Cannot have NumberOfPresentedRequests other " 341 "than 0 on a Sequential or manual WDFQUEUE 0x%p." 342 "Make Sure you set NumberOfPresentedRequests" 343 " to 0, %!STATUS!", 344 GetObjectHandle(), 345 Status 346 ); 347 return Status; 348 349 } 350 else{ 351 m_MaxParallelQueuePresentedRequests = 352 pConfig->Settings.Parallel.NumberOfPresentedRequests; 353 } 354 } 355 356 // 357 // Initialize our workitem if we have to support passive callbacks 358 // 359 if (m_PassiveLevel) { 360 361 Status = FxSystemWorkItem::_Create(FxDriverGlobals, 362 m_Device->GetDeviceObject(), 363 &m_SystemWorkItem 364 ); 365 366 if (!NT_SUCCESS(Status)) { 367 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 368 "Could not allocate workitem: %!STATUS!", Status); 369 return Status; 370 } 371 } 372 373 m_Type = pConfig->DispatchType; 374 375 switch(pConfig->PowerManaged) { 376 377 case WdfUseDefault: 378 if(m_Device->IsFilter()){ 379 m_PowerManaged = FALSE; 380 } else { 381 m_PowerManaged = TRUE; 382 } 383 break; 384 385 case WdfTrue: 386 m_PowerManaged = TRUE; 387 break; 388 389 case WdfFalse: 390 m_PowerManaged = FALSE; 391 break; 392 default: 393 ASSERTMSG("Invalid value in WDF_IO_QUEUE_CONFIG PowerManaged field\n", FALSE); 394 break; 395 } 396 397 // 398 // Queues for NonPnp devices can't be power managed. 399 // 400 if(m_Device->IsLegacy()) { 401 m_PowerManaged = FALSE; 402 } 403 404 // 405 // If power managed queue, ensure its initial power state 406 // is same as the device. 407 // 408 if (m_PowerManaged) { 409 if (InitialPowerStateOn) { 410 m_PowerState = FxIoQueuePowerOn; 411 } 412 else { 413 m_PowerState = FxIoQueuePowerOff; 414 } 415 } else { 416 m_PowerState = FxIoQueuePowerOn; 417 } 418 419 m_AllowZeroLengthRequests = pConfig->AllowZeroLengthRequests; 420 421 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 422 "EvtIoDefault 0x%p, EvtIoRead 0x%p, EvtIoWrite 0x%p, " 423 "EvtIoDeviceControl 0x%p for WDFQUEUE 0x%p", 424 pConfig->EvtIoDefault, 425 pConfig->EvtIoRead, 426 pConfig->EvtIoWrite, 427 pConfig->EvtIoDeviceControl, GetObjectHandle()); 428 429 m_IoDefault.Method = pConfig->EvtIoDefault; 430 m_IoStop.Method = pConfig->EvtIoStop; 431 m_IoResume.Method = pConfig->EvtIoResume; 432 m_IoRead.Method = pConfig->EvtIoRead; 433 m_IoWrite.Method = pConfig->EvtIoWrite; 434 m_IoDeviceControl.Method = pConfig->EvtIoDeviceControl; 435 m_IoInternalDeviceControl.Method = pConfig->EvtIoInternalDeviceControl; 436 m_IoCanceledOnQueue.Method = pConfig->EvtIoCanceledOnQueue; 437 438 439 // A newly created queue can accept and dispatch requests once initialized 440 SetState((FX_IO_QUEUE_SET_STATE)(FxIoQueueSetAcceptRequests|FxIoQueueSetDispatchRequests)); 441 442 m_Configured = TRUE; 443 444 return STATUS_SUCCESS; 445 } 446 447 BOOLEAN 448 FxIoQueue::Dispose( 449 ) 450 /*++ 451 452 Routine Description: 453 454 Should be called at PASSIVE_LEVEL because of the synchronous call 455 to drain requests, workitems, and dpcs associated with this queue. 456 457 Arguments: 458 459 Returns: 460 461 TRUE or FALSE 462 463 --*/ 464 { 465 KIRQL irql; 466 467 if (IsCommitted() == FALSE) { 468 // 469 // We called DeleteFromFailedCreate because we couldn't commit the 470 // object. 471 // 472 goto End; 473 } 474 475 // 476 // If object is commited means we are added to the FxPkgIo queue list. 477 // 478 // 479 // Purge the queue asynchrnously without providing any callback. That way, 480 // we allow the driver to have an outstanding purge request while the delete 481 // is in progress. 482 // 483 (VOID) QueuePurge(TRUE, TRUE, NULL, NULL); 484 485 Lock(&irql); 486 487 // 488 // Mark that this queue is disposing 489 // 490 491 ASSERT(m_Disposing == FALSE); 492 493 m_Disposing = TRUE; 494 495 // 496 // Just make sure the state hasn't changed after the purge. 497 // 498 ASSERT(IsState(WdfIoQueueAcceptRequests) == FALSE); 499 500 // 501 // Call the FxPkgIo to remove its references to this queue 502 // 503 // Note: We are holding the queue lock to prevent races, and 504 // FxPkgIo never calls FxIoQueue methods while holding 505 // its lock. 506 // 507 m_PkgIo->RemoveQueueReferences(this); 508 509 DispatchEvents(irql); 510 511 // 512 // Wait for the finished event to be signalled. This event will 513 // be signalled when the queue is in a disposed state and there 514 // are no more pending events. 515 // 516 GetDriverGlobals()->WaitForSignal(m_FinishDisposing.GetSelfPointer(), 517 "waiting for the queue to be deleted, WDFQUEUE", GetHandle(), 518 GetDriverGlobals()->FxVerifierDbgWaitForSignalTimeoutInSec, 519 WaitSignalBreakUnderVerifier); 520 521 522 ASSERT(m_Deleted == TRUE); 523 524 ASSERT(m_Queue.GetRequestCount() == 0); 525 ASSERT(m_DriverIoCount == 0); 526 527 if (IsForwardProgressQueue()) { 528 FreeAllReservedRequests(TRUE); 529 ASSERT(IsListEmpty(&m_FwdProgContext->m_ReservedRequestList)); 530 ASSERT(IsListEmpty(&m_FwdProgContext->m_PendedIrpList)); 531 } 532 533 if (m_FwdProgContext != NULL) { 534 m_FwdProgContext->m_PendedReserveLock.Uninitialize(); 535 FxPoolFree(m_FwdProgContext); 536 m_FwdProgContext = NULL; 537 } 538 539 // 540 // Rundown the workitem. 541 // 542 if (m_SystemWorkItem != NULL) { 543 m_SystemWorkItem->DeleteObject(); 544 m_SystemWorkItem = NULL; 545 } 546 547 // 548 // Rundown the DPCs 549 // 550 if (m_DpcQueued) { 551 FlushQueuedDpcs(); 552 } 553 554 // 555 // All callbacks into the device driver acquire and release a 556 // reference on the queue. This ensures that the queue will 557 // not actually complete deleting until return from any 558 // outstanding event callbacks into the device driver. 559 // 560 // See DispatchRequestToDriver() 561 // 562 End: 563 564 FxNonPagedObject::Dispose(); // __super call 565 566 return TRUE; 567 } 568 569 _Must_inspect_result_ 570 NTSTATUS 571 FxIoQueue::ConfigureConstraints( 572 __in_opt PWDF_OBJECT_ATTRIBUTES ObjectAttributes, 573 __in_opt FxDriver* Caller 574 ) 575 576 /*++ 577 578 Routine Description: 579 580 Configures the locking and threading model for the 581 Queue according to parameters specified by the device 582 driver when it initialized. 583 584 Arguments: 585 586 Returns: 587 588 NTSTATUS 589 590 --*/ 591 592 { 593 WDF_EXECUTION_LEVEL ParentLevel; 594 WDF_SYNCHRONIZATION_SCOPE ParentScope; 595 BOOLEAN AutomaticLockingRequired; 596 597 AutomaticLockingRequired = FALSE; 598 ASSERT(m_Device != NULL); 599 600 // 601 // Initialize both spin and mutex locks 602 // 603 m_CallbackSpinLock.Initialize(this); 604 m_CallbackMutexLock.Initialize(this); 605 606 // 607 // If WDF_OBJECT_ATTRIBUTES is specified, these override any 608 // default settings. 609 // 610 if (ObjectAttributes != NULL) { 611 m_ExecutionLevel = ObjectAttributes->ExecutionLevel; 612 m_SynchronizationScope = ObjectAttributes->SynchronizationScope; 613 } 614 615 // 616 // If no WDFQUEUE specific attributes are specified, we 617 // get them from WDFDEVICE, which allows WDFDEVICE to 618 // provide a default for all WDFQUEUE's created. 619 // 620 m_Device->GetConstraints(&ParentLevel, &ParentScope); 621 ASSERT(ParentLevel != WdfExecutionLevelInheritFromParent); 622 ASSERT(ParentScope != WdfSynchronizationScopeInheritFromParent); 623 624 if (m_ExecutionLevel == WdfExecutionLevelInheritFromParent) { 625 m_ExecutionLevel = ParentLevel; 626 } 627 628 if (m_SynchronizationScope == WdfSynchronizationScopeInheritFromParent) { 629 m_SynchronizationScope = ParentScope; 630 } 631 632 // 633 // For backward compatibility purposes always have a lock associated with the 634 // object even for WdfSynchronizationScopeNone. This is so that we return a non-null 635 // presentation lock for the WDFQUEUE object. 636 // 637 if (m_ExecutionLevel == WdfExecutionLevelPassive) { 638 // 639 // Mark FxObject as passive level to ensure that Dispose and Destroy 640 // callbacks are passive to the driver 641 // 642 MarkPassiveCallbacks(ObjectDoNotLock); 643 m_PassiveLevel = TRUE; 644 645 // 646 // Passive Callbacks constraint has been set, we use a mutex for the 647 // callback lock. 648 // 649 m_CallbackLockPtr = &m_CallbackMutexLock; 650 m_CallbackLockObjectPtr = this; 651 } 652 else { 653 // 654 // If no passive level constraint is specified, then spinlocks 655 // are used for callbacks since they are lightweight and work with 656 // DPC's and Timer's 657 // 658 m_CallbackLockPtr = &m_CallbackSpinLock; 659 m_CallbackLockObjectPtr = this; 660 } 661 662 // 663 // Configure synchronization scope 664 // 665 if (m_SynchronizationScope == WdfSynchronizationScopeDevice) { 666 NTSTATUS status; 667 668 // 669 // WDF extensions are not allowed to use this type of synchronization. 670 // 671 if (Caller != NULL && Caller != GetDriverGlobals()->Driver) { 672 status = STATUS_INVALID_PARAMETER; 673 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_ERROR, TRACINGIO, 674 "WDFQUEUE 0x%p Synchronization scope is set to " 675 "device; WDF extension drivers are not allowed " 676 "to use this type of synchronization, %!STATUS!", 677 GetObjectHandle(), status); 678 return status; 679 } 680 681 // 682 // If we inherit the Sync. scope from parent or device 683 // and if the parent/device has Exec. Level different from Queue 684 // then disallow that case. 685 // FUTURE PROOF NOTE: Adding a new Execution Level will need reevaluation 686 // of the check below. 687 // 688 if (ParentLevel != m_ExecutionLevel) { 689 status = STATUS_INVALID_PARAMETER; 690 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_ERROR, TRACINGIO, 691 "WDFQUEUE 0x%p Synchronization scope is set to device" 692 " but the Device ExecutionLevel: 0x%x" 693 " doesn't match Queue ExecutionLevel: 0x%x, %!STATUS!", 694 GetObjectHandle(), ParentLevel, 695 m_ExecutionLevel, status); 696 return status; 697 } 698 // 699 // Per device automatic callback synchronization, so we update our 700 // callback lock ptr to point to the devices lock 701 // 702 AutomaticLockingRequired = TRUE; 703 704 // 705 // Get the callback lock and object from the device 706 // 707 m_CallbackLockPtr = m_Device->GetCallbackLockPtr(&m_CallbackLockObjectPtr); 708 } 709 else if (m_SynchronizationScope == WdfSynchronizationScopeQueue) { 710 // 711 // Per object automatic serialization 712 // 713 AutomaticLockingRequired = TRUE; 714 715 // m_CallbackLockPtr has been set above in execution level constraint 716 } 717 718 719 if (AutomaticLockingRequired) { 720 // 721 // If automatic locking has been configured, set the lock 722 // on the FxCallback object delegates 723 // 724 m_IoDefault.SetCallbackLockPtr(m_CallbackLockPtr); 725 m_IoStop.SetCallbackLockPtr(m_CallbackLockPtr); 726 m_IoResume.SetCallbackLockPtr(m_CallbackLockPtr); 727 m_IoRead.SetCallbackLockPtr(m_CallbackLockPtr); 728 m_IoWrite.SetCallbackLockPtr(m_CallbackLockPtr); 729 m_IoDeviceControl.SetCallbackLockPtr(m_CallbackLockPtr); 730 m_IoInternalDeviceControl.SetCallbackLockPtr(m_CallbackLockPtr); 731 m_PurgeComplete.SetCallbackLockPtr(m_CallbackLockPtr); 732 m_ReadyNotify.SetCallbackLockPtr(m_CallbackLockPtr); 733 m_IoCanceledOnQueue.SetCallbackLockPtr(m_CallbackLockPtr); 734 735 m_IoCancelCallbackLockPtr = m_CallbackLockPtr; 736 } 737 else { 738 // 739 // No automatic locking specified 740 // 741 m_IoDefault.SetCallbackLockPtr(NULL); 742 m_IoStop.SetCallbackLockPtr(NULL); 743 m_IoResume.SetCallbackLockPtr(NULL); 744 m_IoRead.SetCallbackLockPtr(NULL); 745 m_IoWrite.SetCallbackLockPtr(NULL); 746 m_IoDeviceControl.SetCallbackLockPtr(NULL); 747 m_IoInternalDeviceControl.SetCallbackLockPtr(NULL); 748 m_PurgeComplete.SetCallbackLockPtr(NULL); 749 m_ReadyNotify.SetCallbackLockPtr(NULL); 750 m_IoCanceledOnQueue.SetCallbackLockPtr(NULL); 751 752 m_IoCancelCallbackLockPtr = NULL; 753 754 } 755 756 return STATUS_SUCCESS; 757 } 758 759 WDF_IO_QUEUE_STATE 760 FxIoQueue::GetState( 761 __out_opt PULONG pQueueCount, 762 __out_opt PULONG pDriverCount 763 ) 764 { 765 int stat; 766 ULONG QueueCount, DriverCount; 767 768 // Get request counts 769 GetRequestCount(&QueueCount, &DriverCount); 770 771 if (pQueueCount ) *pQueueCount = QueueCount; 772 773 if (pDriverCount ) *pDriverCount = DriverCount; 774 775 // 776 // First fill in the values that are kept up to date at runtime 777 // 778 stat = (int)m_QueueState & (int)(WdfIoQueueAcceptRequests | WdfIoQueueDispatchRequests); 779 780 // 781 // Set additional information bits from information retrieved 782 // from other sources. It's cheaper to get this info at the infrequent 783 // GetStatus time, rather than keep the bits up to date at each 784 // request and queue transition. 785 // 786 if (QueueCount == 0) { 787 stat = stat | (int)WdfIoQueueNoRequests; 788 } 789 790 if (DriverCount == 0) { 791 stat = stat | (int)WdfIoQueueDriverNoRequests; 792 } 793 794 if(m_PowerManaged) { 795 796 if (m_PowerState != FxIoQueuePowerOn) { 797 stat = stat | (int)WdfIoQueuePnpHeld; 798 } 799 } 800 801 return (WDF_IO_QUEUE_STATE)stat; 802 } 803 804 VOID 805 FxIoQueue::SetState( 806 __in FX_IO_QUEUE_SET_STATE NewStatus 807 ) 808 { 809 int AllowedBits; 810 811 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 812 813 // 814 // Only allow setting of valid bits 815 // 816 AllowedBits = (int)(FxIoQueueSetAcceptRequests | 817 FxIoQueueClearAcceptRequests | 818 FxIoQueueSetDispatchRequests | 819 FxIoQueueClearDispatchRequests | 820 FxIoQueueSetShutdown | 821 FxIoQueueClearShutdown 822 ); 823 824 if ((int)NewStatus & ~AllowedBits) { 825 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 826 "Invalid WDFQUEUE 0x%p state", 827 GetObjectHandle()); 828 FxVerifierDbgBreakPoint(FxDriverGlobals); 829 return; 830 } 831 832 // 833 // Clear the high bit used to prevent accidental mixing of 834 // WDF_IO_QUEUE_STATE and FX_IO_QUEUE_SET_STATE 835 // 836 NewStatus = (FX_IO_QUEUE_SET_STATE)((int)NewStatus & 0x7FFFFFFF); 837 838 if (NewStatus & (int)FxIoQueueClearShutdown) { 839 m_QueueState = (FX_IO_QUEUE_STATE)((int)m_QueueState & ~(int)FxIoQueueShutdown); 840 } 841 842 if (NewStatus & (int)FxIoQueueSetShutdown) { 843 m_QueueState = (FX_IO_QUEUE_STATE)((int)m_QueueState | (int)FxIoQueueShutdown); 844 } 845 846 if (NewStatus & (int)FxIoQueueSetAcceptRequests) { 847 if (IsState(FxIoQueueShutdown) == FALSE) { 848 m_QueueState = (FX_IO_QUEUE_STATE)((int)m_QueueState | (int)WdfIoQueueAcceptRequests); 849 } 850 else { 851 DoTraceLevelMessage(FxDriverGlobals, 852 TRACE_LEVEL_INFORMATION, TRACINGIO, 853 "WDFQUEUE 0x%p is shut down, preventing queue " 854 "from accepting requests", 855 GetObjectHandle()); 856 } 857 } 858 859 if (NewStatus & (int)FxIoQueueClearAcceptRequests) { 860 m_QueueState = (FX_IO_QUEUE_STATE)((int)m_QueueState & ~(int)WdfIoQueueAcceptRequests); 861 } 862 863 if (NewStatus & (int)FxIoQueueSetDispatchRequests) { 864 m_QueueState = (FX_IO_QUEUE_STATE)((int)m_QueueState | (int)WdfIoQueueDispatchRequests); 865 // 866 // If the queue is allowed to dispatch new requests, we must clear this flag. 867 // See also WdfIoQueueStopAndPurge for more info about the flag. 868 // 869 m_CancelDispatchedRequests = FALSE; 870 } 871 872 if (NewStatus & (int)FxIoQueueClearDispatchRequests) { 873 m_QueueState = (FX_IO_QUEUE_STATE)((int)m_QueueState & ~(int)WdfIoQueueDispatchRequests); 874 } 875 876 return; 877 } 878 879 _Must_inspect_result_ 880 NTSTATUS 881 FX_VF_METHOD(FxIoQueue, VerifyGetRequestUpdateFlags) ( 882 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals, 883 _In_ FxRequest* TagRequest 884 ) 885 { 886 KIRQL irql; 887 NTSTATUS status; 888 889 PAGED_CODE_LOCKED(); 890 891 if (TagRequest != NULL) { 892 // 893 // WdfIoQueueRetrieveFoundRequest is only valid on manual queues. 894 // v1.11 and above: driver is not required to find the request 895 // using WdfIoQueueFindRequest. 896 // v1.9 and below: driver is required to find the request 897 // using WdfIoQueueFindRequest. 898 // 899 if (FxDriverGlobals->IsVersionGreaterThanOrEqualTo(1,11)) { 900 if (m_Type != WdfIoQueueDispatchManual) { 901 status = STATUS_INVALID_DEVICE_REQUEST; 902 DoTraceLevelMessage( 903 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 904 "WdfIoQueueRetrieveFoundRequest is allowed " 905 "only on a manual queue 0x%p, %!STATUS!", 906 GetHandle(), status); 907 FxVerifierDbgBreakPoint(FxDriverGlobals); 908 return status; 909 } 910 } 911 else { 912 // 913 // Legacy validation. 914 // 915 TagRequest->Lock(&irql); 916 status = TagRequest->VerifyRequestIsTagRequest(FxDriverGlobals); 917 TagRequest->Unlock(irql); 918 if (!NT_SUCCESS(status)) { 919 return status; 920 } 921 } 922 } 923 924 Lock(&irql); 925 if ((m_Type == WdfIoQueueDispatchSequential) && (m_DriverIoCount == 0)) { 926 927 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 928 "Driver called WdfIoQueueRetrieveNextRequest on a sequential WDFQUEUE 0x%p with no " 929 "outstanding requests. This can cause a race with automatically dispatched " 930 "requests. Call WdfIoQueueRetrieveNextRequest before completing the current request(s)", 931 GetObjectHandle()); 932 933 FxVerifierDbgBreakPoint(FxDriverGlobals); 934 935 // Allow them to continue, though this is a race condition in their driver 936 } 937 Unlock(irql); 938 939 return STATUS_SUCCESS; 940 } 941 942 VOID 943 FX_VF_METHOD(FxIoQueue, VerifyGetRequestRestoreFlags)( 944 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals, 945 _In_ FxRequest* pRequest 946 ) 947 { 948 UNREFERENCED_PARAMETER(FxDriverGlobals); 949 KIRQL irql; 950 951 PAGED_CODE_LOCKED(); 952 pRequest->Lock(&irql); 953 954 pRequest->ClearVerifierFlagsLocked(FXREQUEST_FLAG_TAG_REQUEST); 955 pRequest->SetVerifierFlagsLocked(FXREQUEST_FLAG_DRIVER_OWNED); 956 957 pRequest->Unlock(irql); 958 } 959 960 _Must_inspect_result_ 961 NTSTATUS 962 FxIoQueue::GetRequest( 963 __in_opt MdFileObject FileObject, 964 __in_opt FxRequest* TagRequest, 965 __deref_out FxRequest** pOutRequest 966 ) 967 /*++ 968 969 Routine Description: 970 971 This method is called by 972 973 WdfIoQueueRetrieveNextRequest 974 WdfIoQueueRetrieveRequestByFileObject 975 WdfIoQueueRetrieveFoundRequest 976 977 to retrieve a request from the queue. 978 979 Arguments: 980 981 Returns: 982 983 NTSTATUS 984 985 --*/ 986 { 987 NTSTATUS status; 988 FxRequest* pRequest = NULL; 989 FxRequestCompletionState oldState; 990 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 991 KIRQL irql; 992 993 status = VerifyGetRequestUpdateFlags(pFxDriverGlobals, TagRequest); 994 if(!NT_SUCCESS(status)){ 995 return status; 996 } 997 998 // 999 // Don't allow on parallel queues 1000 // 1001 if ((m_Type != WdfIoQueueDispatchManual) && 1002 (m_Type != WdfIoQueueDispatchSequential)) { 1003 status = STATUS_INVALID_DEVICE_STATE; 1004 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 1005 "Cannot be called on a parallel WDFQUEUE 0x%p, %!STATUS!", 1006 GetObjectHandle(), status); 1007 return status; 1008 } 1009 1010 Lock(&irql); 1011 1012 // 1013 // Only if the queue state allows requests to be retrieved. 1014 // It's okay to retrieve requests while the queue is in a transitioning state. 1015 // 1016 if (m_PowerState == FxIoQueuePowerOff) { 1017 status = STATUS_WDF_PAUSED; 1018 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 1019 "WDFQUEUE 0x%p is powered off, %!STATUS!", 1020 GetObjectHandle(), status); 1021 Unlock(irql); 1022 return status; 1023 } 1024 1025 // 1026 // See if the queue is (still) processing requests 1027 // 1028 if (!IsState(WdfIoQueueDispatchRequests)) { 1029 status = STATUS_WDF_PAUSED; 1030 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 1031 "WDFQUEUE 0x%p is stopped, %!STATUS!", 1032 GetObjectHandle(), status); 1033 Unlock(irql); 1034 return status; 1035 } 1036 1037 #pragma warning(disable:4127) 1038 while (TRUE) { 1039 1040 #pragma warning(default:4127) 1041 // 1042 // Get the next FxRequest from the cancel safe queue 1043 // 1044 status = FxRequest::GetNextRequest(&m_Queue, FileObject, TagRequest, &pRequest); 1045 if (!NT_SUCCESS(status)) { 1046 // 1047 // This code address the following race condition: 1048 // 1) Queue has only one request (count 1). 1049 // 2) Request in queue is cancelled. 1050 // 3) Request's cancellation logic starts to run on thread 1. 1051 // 4) But before cancellation logic gets the queue's lock 1052 // thread 2 calls WdfIoQueueRetrieveNextRequest. 1053 // 5) WdfIoQueueRetrieveNextRequest returns no more requests. 1054 // Driver waits for the ReadyNotify callback. (count 1) 1055 // 6) Thread 3 adds a new request in queue. (count 1->2) 1056 // 7) Thread 1 finally runs. (count 2->1). 1057 // 8) At this point driver stops responding b/c it never receives ReadyNotify. 1058 // 1059 // This code below forces the queue logic to send a ReadyNotify 1060 // callback the next time a new request is added (in step 6 above). 1061 // 1062 if (STATUS_NO_MORE_ENTRIES == status && 1063 NULL == FileObject && // WdfIoQueueRetrieveNextRequest 1064 NULL == TagRequest && // WdfIoQueueRetrieveNextRequest 1065 m_Queue.GetRequestCount() > 0L) { 1066 1067 m_ForceTransitionFromEmptyWhenAddingNewRequest = TRUE; 1068 } 1069 1070 Unlock(irql); 1071 return status; 1072 } 1073 1074 // 1075 // If we don't allow zero length read/write's to the driver, 1076 // complete it now with success and attempt to get another 1077 // request from the queue. 1078 // 1079 if (!m_AllowZeroLengthRequests) { 1080 1081 1082 1083 1084 1085 (VOID)pRequest->GetCurrentIrpStackLocation(); 1086 1087 FxIrp* pIrp = pRequest->GetFxIrp(); 1088 UCHAR majorFunction = pIrp->GetMajorFunction(); 1089 1090 if ((majorFunction == IRP_MJ_READ) && 1091 (pIrp->GetParameterReadLength() == 0)) { 1092 1093 Unlock(irql); 1094 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 1095 "Zero length WDFREQUEST 0x%p completed automatically by WDFQUEUE 0x%p", 1096 pRequest->GetHandle(),GetObjectHandle()); 1097 pRequest->CompleteWithInformation(STATUS_SUCCESS, 0); 1098 pRequest->RELEASE(FXREQUEST_COMPLETE_TAG); 1099 1100 Lock(&irql); 1101 1102 // Get another request from the queue 1103 continue; 1104 } 1105 else if ((majorFunction == IRP_MJ_WRITE) && 1106 (pIrp->GetParameterWriteLength() == 0)) { 1107 1108 Unlock(irql); 1109 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 1110 "Zero length WDFREQUEST 0x%p completed automatically by WDFQUEUE 0x%p", 1111 pRequest->GetHandle(), GetObjectHandle()); 1112 1113 pRequest->CompleteWithInformation(STATUS_SUCCESS, 0); 1114 pRequest->RELEASE(FXREQUEST_COMPLETE_TAG); 1115 1116 Lock(&irql); 1117 1118 // Get another request from the queue 1119 continue; 1120 } 1121 } 1122 1123 break; 1124 } 1125 1126 // Increase the driver owned request count 1127 InsertInDriverOwnedList(pRequest); 1128 1129 Unlock(irql); 1130 1131 // 1132 // We don't need to check for PurgeComplete since 1133 // we are giving the request to the driver 1134 // 1135 1136 // pRequest is not cancellable now 1137 1138 // 1139 // We are now going to return the request 1140 // to the driver, and it must complete it. 1141 // 1142 1143 // 1144 // Set a completion event, this takes a reference 1145 // 1146 oldState = pRequest->SetCompletionState(FxRequestCompletionStateQueue); 1147 ASSERT(oldState == FxRequestCompletionStateNone); 1148 UNREFERENCED_PARAMETER(oldState); 1149 1150 // 1151 // Track that we have given the request to the driver 1152 // 1153 VerifyGetRequestRestoreFlags(pFxDriverGlobals, pRequest); 1154 1155 pRequest->SetPresented(); 1156 1157 // 1158 // Release our original reference. The FxRequest::Complete 1159 // will release the final one since we have registered a completion 1160 // callback handler 1161 // 1162 // We now have one reference count on the FxRequest object until 1163 // its completion routine runs since the completion event made 1164 // an extra reference, and will dereference it when it fires, or 1165 // its canceled. 1166 // 1167 1168 pRequest->RELEASE(FXREQUEST_STATE_TAG); 1169 1170 // Return it to the driver 1171 *pOutRequest = pRequest; 1172 1173 return STATUS_SUCCESS; 1174 } 1175 1176 _Must_inspect_result_ 1177 NTSTATUS 1178 FX_VF_METHOD(FxIoQueue, VerifyPeekRequest) ( 1179 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals, 1180 _In_ FxRequest* TagRequest 1181 ) 1182 { 1183 NTSTATUS status; 1184 KIRQL irql; 1185 1186 PAGED_CODE_LOCKED(); 1187 1188 TagRequest->Lock(&irql); 1189 status = TagRequest->VerifyRequestIsTagRequest(FxDriverGlobals); 1190 TagRequest->Unlock(irql); 1191 1192 return status; 1193 } 1194 1195 _Must_inspect_result_ 1196 NTSTATUS 1197 FxIoQueue::PeekRequest( 1198 __in_opt FxRequest* TagRequest, 1199 __in_opt MdFileObject FileObject, 1200 __out_opt PWDF_REQUEST_PARAMETERS Parameters, 1201 __deref_out FxRequest** pOutRequest 1202 ) 1203 /*++ 1204 1205 Routine Description: 1206 1207 This method is called by WdfIoQueueFindRequest to 1208 look for a specific request from the queue. If tagrequest 1209 is not specified then this method will return the very 1210 first request from the queue. 1211 1212 If the fileobject is specified then fileobject is also 1213 used as one of the constrain for returing the request. 1214 1215 Important point to remember is that only request information 1216 is returned to the caller. The request is still present in 1217 the queue. 1218 1219 If the request is returned, there is an additional reference 1220 taken on the queue to prevent it from deletion while the 1221 caller is using the request handle. The caller has to 1222 explicitly drop the reference once he is done using the 1223 request handle. 1224 1225 Arguments: 1226 1227 Returns: 1228 1229 NTSTATUS 1230 1231 --*/ 1232 1233 { 1234 NTSTATUS status; 1235 FxRequest* pRequest = NULL; 1236 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 1237 KIRQL irql; 1238 1239 // 1240 // FindRequest is allowed only on a manual queue. 1241 // 1242 if (m_Type != WdfIoQueueDispatchManual) { 1243 status = STATUS_INVALID_DEVICE_REQUEST; 1244 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 1245 "FindRequest is allowed only on a manaul queue 0x%p, %!STATUS!", 1246 GetHandle(), status); 1247 FxVerifierDbgBreakPoint(pFxDriverGlobals); 1248 return status; 1249 } 1250 1251 if (TagRequest != NULL) { 1252 status = VerifyPeekRequest(pFxDriverGlobals, TagRequest); 1253 if (!NT_SUCCESS(status)) { 1254 return status; 1255 } 1256 } 1257 1258 // 1259 // Get the next FxRequest from the cancel safe queue 1260 // 1261 // If success, it will return a referenced FxRequest in 1262 // which the caller must release the reference. 1263 // 1264 Lock(&irql); 1265 1266 status = FxRequest::PeekRequest( 1267 &m_Queue, 1268 TagRequest, 1269 FileObject, 1270 Parameters, 1271 &pRequest 1272 ); 1273 1274 // 1275 // This code address the following potential race condition: 1276 // 1) Queue has only one request (count 1). 1277 // 2) Request in queue is cancelled. 1278 // 3) Request's cancellation logic starts to run on thread 1. 1279 // 4) But before cancellation logic gets the queue's lock 1280 // thread 2 calls WdfIoQueueFindRequest to find any request. 1281 // 5) WdfIoQueueFindRequest returns no more requests. 1282 // Driver waits for the ReadyNotify callback. (count 1) 1283 // 6) Thread 3 adds a new request in queue. (count 1->2) 1284 // 7) Thread 1 finally runs. (count 2->1). 1285 // 8) At this point driver stops responding b/c it never receives ReadyNotify. 1286 // 1287 // This code below forces the queue logic to send a ReadyNotify 1288 // callback the next time a new request is added (in step 6 above). 1289 // 1290 if (STATUS_NO_MORE_ENTRIES == status && 1291 NULL == FileObject && // WdfIoQueueFindRequest(any request) 1292 NULL == TagRequest && // WdfIoQueueFindRequest(any request) 1293 m_Queue.GetRequestCount() > 0L) { 1294 1295 m_ForceTransitionFromEmptyWhenAddingNewRequest = TRUE; 1296 } 1297 1298 Unlock(irql); 1299 1300 if (!NT_SUCCESS(status)) { 1301 return status; 1302 } 1303 1304 // 1305 // Mark it as a tag request to detect abuse since its not 1306 // driver owned. 1307 // 1308 if (pFxDriverGlobals->FxVerifierOn) { 1309 pRequest->SetVerifierFlags(FXREQUEST_FLAG_TAG_REQUEST); 1310 } 1311 1312 // Return it to the driver 1313 *pOutRequest = pRequest; 1314 1315 return status; 1316 } 1317 1318 SHORT 1319 FX_VF_METHOD(FxIoQueue, VerifyForwardRequestUpdateFlags) ( 1320 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals, 1321 _In_ FxRequest* Request 1322 ) 1323 { 1324 UNREFERENCED_PARAMETER(FxDriverGlobals); 1325 SHORT OldFlags = 0; 1326 KIRQL irql; 1327 1328 PAGED_CODE_LOCKED(); 1329 1330 Request->Lock(&irql); 1331 1332 // Save old flags to put them back if forward fails 1333 OldFlags = Request->GetVerifierFlagsLocked(); 1334 1335 // 1336 // Set that the request was forwarded. This effects 1337 // cancel behavior. 1338 // 1339 Request->SetVerifierFlagsLocked(FXREQUEST_FLAG_FORWARDED); 1340 1341 ASSERT((Request->GetVerifierFlagsLocked() & FXREQUEST_FLAG_DRIVER_OWNED) != 0); 1342 1343 // Set that the request is no longer driver owned 1344 Request->ClearVerifierFlagsLocked( 1345 FXREQUEST_FLAG_DRIVER_OWNED | FXREQUEST_FLAG_DRIVER_DISPATCH); 1346 1347 Request->Unlock(irql); 1348 1349 return OldFlags; 1350 } 1351 1352 _Must_inspect_result_ 1353 NTSTATUS 1354 FxIoQueue::ForwardRequestWorker( 1355 __in FxRequest* Request, 1356 __in FxIoQueue* DestQueue 1357 ) 1358 { 1359 NTSTATUS status; 1360 FxRequestCompletionState oldState; 1361 PLIST_ENTRY ple; 1362 SHORT OldFlags; 1363 KIRQL irql; 1364 1365 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 1366 1367 OldFlags = 0; 1368 1369 // 1370 // The request has only one reference, held by the completion 1371 // callback function. We need to take another one before cancelling 1372 // this function, otherwise we will lose the request object 1373 // 1374 Request->ADDREF(FXREQUEST_STATE_TAG); 1375 1376 // 1377 // Cancel its current completion event for this queue 1378 // 1379 oldState = Request->SetCompletionState(FxRequestCompletionStateNone); 1380 ASSERT(oldState == FxRequestCompletionStateQueue); 1381 1382 OldFlags = VerifyForwardRequestUpdateFlags(FxDriverGlobals, Request); 1383 1384 // 1385 // Remove it from this queues driver owned list. 1386 // 1387 // This must be done before forward since new queue will 1388 // use the list entry in the FxRequest 1389 // 1390 // We can't use RemoveFromDriverOwnedList since we want the 1391 // m_DriverIoCount to be left alone in case the forward fails. 1392 // If we don't, another thread can run when we drop the lock, notice 1393 // that there are no more requests, and raise the purged and empty 1394 // events. But if the forward fails, the request will wind up back 1395 // on the queue! So m_DriverIoCount is used as a gate to prevent 1396 // these events from firing until we are really sure this queue 1397 // is done with the request. 1398 // 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 Lock(&irql); 1411 ple = Request->GetListEntry(FxListEntryDriverOwned); 1412 RemoveEntryList(ple); 1413 InitializeListHead(ple); 1414 Unlock(irql); 1415 1416 // 1417 // Attempt to pass the request onto the target queue 1418 // 1419 status = DestQueue->QueueRequestFromForward(Request); 1420 if (!NT_SUCCESS(status)) { 1421 1422 // 1423 // Target queue did not accept the request, so we 1424 // restore the original completion callback function 1425 // and flags 1426 // 1427 oldState = Request->SetCompletionState(oldState); 1428 ASSERT(oldState == FxRequestCompletionStateNone); 1429 UNREFERENCED_PARAMETER(oldState); 1430 1431 if (FxDriverGlobals->FxVerifierOn) { 1432 Request->SetVerifierFlags(OldFlags); 1433 } 1434 1435 // Release the extra reference we took 1436 Request->RELEASE(FXREQUEST_STATE_TAG); 1437 1438 Lock(&irql); 1439 // Place it back on the driver owned list 1440 InsertTailList(&m_DriverOwned, ple); 1441 Unlock(irql); 1442 } 1443 else { 1444 1445 Lock(&irql); 1446 1447 // Request is no longer part of the I/O count for this queue 1448 m_DriverIoCount--; 1449 1450 ASSERT(m_DriverIoCount >= 0); 1451 1452 // 1453 // Don't run the event dispatcher if we are called from a 1454 // dispath routine in order to prevent stack recursion. 1455 // Since some other thread (possibly this thread higher on 1456 // the stack) is running the dispatcher, no events will get lost. 1457 // 1458 // 1459 // This returns with the IoQueue lock released 1460 // 1461 DispatchInternalEvents(irql); 1462 1463 // 1464 // We don't dereference the request object since the new IoQueue 1465 // will release it when it is done. 1466 // 1467 } 1468 1469 return status; 1470 } 1471 1472 _Must_inspect_result_ 1473 NTSTATUS 1474 FX_VF_METHOD(FxIoQueue, VerifyForwardRequestToParent) ( 1475 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals, 1476 _In_ FxIoQueue* DestQueue, 1477 _In_ FxRequest* Request 1478 ) 1479 { 1480 KIRQL irql; 1481 NTSTATUS status; 1482 1483 PAGED_CODE_LOCKED(); 1484 1485 if (m_Device->m_ParentDevice == NULL) { 1486 status = STATUS_INVALID_DEVICE_REQUEST; 1487 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 1488 "No parent device for WDFQUEUE 0x%p Device, %!STATUS!", 1489 DestQueue->m_Device->GetHandle(), status); 1490 FxVerifierDbgBreakPoint(FxDriverGlobals); 1491 goto Done; 1492 } 1493 1494 Request->Lock(&irql); 1495 1496 status = Request->VerifyRequestIsDriverOwned(FxDriverGlobals); 1497 1498 if (NT_SUCCESS(status)) { 1499 status = Request->VerifyRequestIsNotCancelable(FxDriverGlobals); 1500 } 1501 1502 Request->Unlock(irql); 1503 1504 if (!NT_SUCCESS(status)) { 1505 goto Done; 1506 } 1507 1508 if (DestQueue == this) { 1509 status = STATUS_INVALID_DEVICE_REQUEST; 1510 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 1511 "Cannot forward a request to the same WDFQUEUE 0x%p" 1512 " %!STATUS!", GetObjectHandle(), status); 1513 FxVerifierDbgBreakPoint(FxDriverGlobals); 1514 goto Done; 1515 } 1516 1517 if (m_Device->m_ParentDevice != DestQueue->m_Device) { 1518 status = STATUS_INVALID_DEVICE_REQUEST; 1519 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 1520 "Cannot forward a request to " 1521 "a different WDFDEVICE 0x%p which is not the " 1522 "parent, %!STATUS!", 1523 DestQueue->m_Device->GetHandle(), 1524 status); 1525 FxVerifierDbgBreakPoint(FxDriverGlobals); 1526 goto Done; 1527 } 1528 1529 if (Request->IsReserved()) { 1530 status = STATUS_INVALID_DEVICE_REQUEST; 1531 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 1532 "Cannot forward reserved WDFREQUEST 0x%p to a " 1533 "parent WDFDEVICE 0x%p, %!STATUS!", 1534 Request->GetHandle(), 1535 DestQueue->m_Device->GetHandle(), 1536 status); 1537 FxVerifierDbgBreakPoint(FxDriverGlobals); 1538 goto Done; 1539 } 1540 1541 // 1542 // Make sure the child device is a PDO 1543 // 1544 ASSERT(m_Device->IsPdo()); 1545 1546 // 1547 // Check if the WdfPdoInitSetForwardRequestToParent was called to increase 1548 // the StackSize of the child Device to include the stack size of the 1549 // parent Device 1550 // 1551 if (m_Device->IsPnp() 1552 && 1553 m_Device->GetPdoPkg()->m_AllowForwardRequestToParent == FALSE) { 1554 status = STATUS_INVALID_DEVICE_REQUEST; 1555 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 1556 "WdfPdoInitSetForwardRequestToParent not called on " 1557 "WDFDEVICE 0x%p, %!STATUS!", m_Device->GetHandle(), 1558 status); 1559 FxVerifierDbgBreakPoint(FxDriverGlobals); 1560 goto Done; 1561 } 1562 1563 Done: 1564 return status; 1565 } 1566 1567 _Must_inspect_result_ 1568 NTSTATUS 1569 FxIoQueue::ForwardRequestToParent( 1570 __in FxIoQueue* DestQueue, 1571 __in FxRequest* Request, 1572 __in PWDF_REQUEST_FORWARD_OPTIONS ForwardOptions 1573 ) 1574 1575 /*++ 1576 1577 Routine Description: 1578 1579 ForwardRequest is called from the drivers EvtIoDefault routine 1580 and the following conditions apply: 1581 1582 Request is not on a CSQ and not cancellable 1583 1584 Request is FXREQUEST_FLAG_DRIVER_OWNED 1585 1586 m_DriverIoCount has been incremented to reflect the request 1587 1588 Request has an I/O completion callback function pointing to 1589 FxIoQueueRequestComplete with the context for this Queue 1590 1591 The Request has one reference count from the I/O completion callback. 1592 1593 If a driver calls this API, it will not complete the request 1594 as a result of this queues EvtIoDefault, and does not own 1595 the request until it has been re-presented by the Destination 1596 Queue. 1597 1598 Arguments: 1599 1600 Returns: 1601 1602 NTSTATUS 1603 1604 --*/ 1605 { 1606 NTSTATUS status; 1607 PFX_DRIVER_GLOBALS pFxDriverGlobals; 1608 BOOLEAN forwardRequestToParent; 1609 FxIrp* pIrp; 1610 1611 UNREFERENCED_PARAMETER(ForwardOptions); 1612 1613 pFxDriverGlobals = GetDriverGlobals(); 1614 1615 forwardRequestToParent = Request->m_ForwardRequestToParent; 1616 1617 status = VerifyForwardRequestToParent(pFxDriverGlobals, 1618 DestQueue, 1619 Request); 1620 if(!NT_SUCCESS(status)){ 1621 return status; 1622 } 1623 1624 pIrp = Request->GetFxIrp(); 1625 1626 pIrp->CopyCurrentIrpStackLocationToNext(); 1627 pIrp->SetNextIrpStackLocation(); 1628 1629 // 1630 // Save a pointer to the device object for this request so that it can 1631 // be used later in completion. 1632 // 1633 pIrp->SetCurrentDeviceObject(m_Device->m_ParentDevice->GetDeviceObject()); 1634 1635 Request->SetDeviceBase((CfxDeviceBase *)m_Device->m_ParentDevice); 1636 Request->m_ForwardRequestToParent = TRUE; 1637 1638 status = ForwardRequestWorker(Request, DestQueue); 1639 1640 // 1641 // Undo the actions of changing the FxDevice and 1642 // changing the deviceObject and stack location in the IRP 1643 // 1644 if (!NT_SUCCESS(status)) { 1645 Request->SetDeviceBase((CfxDeviceBase *)m_Device); 1646 pIrp = Request->GetFxIrp(); 1647 pIrp->SkipCurrentIrpStackLocation(); 1648 ASSERT(pIrp->GetDeviceObject() == m_Device->GetDeviceObject()); 1649 1650 // 1651 // Set the value of m_ForwardRequestToParent to the previous 1652 // value so that if the Request has been forwarded to Parent 1653 // successfully but fails to be forwarded to the grandparent 1654 // from the parent then we free it back using ExFreePool 1655 // instead of the Lookaside buffer . 1656 // 1657 Request->m_ForwardRequestToParent = forwardRequestToParent; 1658 } 1659 1660 return status; 1661 } 1662 1663 _Must_inspect_result_ 1664 NTSTATUS 1665 FX_VF_METHOD(FxIoQueue, VerifyForwardRequest) ( 1666 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals, 1667 _In_ FxIoQueue* pDestQueue, 1668 _In_ FxRequest* pRequest 1669 ) 1670 { 1671 NTSTATUS status; 1672 KIRQL irql; 1673 1674 PAGED_CODE_LOCKED(); 1675 1676 pRequest->Lock(&irql); 1677 1678 status = pRequest->VerifyRequestIsDriverOwned(FxDriverGlobals); 1679 if (NT_SUCCESS(status)) { 1680 status = pRequest->VerifyRequestIsNotCancelable(FxDriverGlobals); 1681 } 1682 1683 pRequest->Unlock(irql); 1684 1685 if (!NT_SUCCESS(status)) { 1686 return status; 1687 } 1688 1689 if (pDestQueue == this) { 1690 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 1691 "Cannot forward a request to the same WDFQUEUE 0x%p" 1692 " %!STATUS!", 1693 GetObjectHandle(), 1694 STATUS_INVALID_DEVICE_REQUEST); 1695 FxVerifierDbgBreakPoint(FxDriverGlobals); 1696 return STATUS_INVALID_DEVICE_REQUEST; 1697 } 1698 1699 if ((m_Device != pDestQueue->m_Device)) { 1700 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 1701 "Cannot forward a request to a different WDFDEVICE 0x%p", 1702 pDestQueue->m_Device->GetHandle()); 1703 FxVerifierDbgBreakPoint(FxDriverGlobals); 1704 return STATUS_INVALID_DEVICE_REQUEST; 1705 } 1706 1707 return status; 1708 } 1709 1710 _Must_inspect_result_ 1711 NTSTATUS 1712 FxIoQueue::ForwardRequest( 1713 __in FxIoQueue* pDestQueue, 1714 __in FxRequest* pRequest 1715 ) 1716 /*++ 1717 1718 Routine Description: 1719 1720 ForwardRequest is called from the drivers EvtIoDefault routine 1721 and the following conditions apply: 1722 1723 Request is not on a CSQ and not cancellable 1724 1725 Request is FXREQUEST_FLAG_DRIVER_OWNED 1726 1727 m_DriverIoCount has been incremented to reflect the request 1728 1729 Request has an I/O completion callback function pointing to 1730 FxIoQueueRequestComplete with the context for this Queue 1731 1732 The Request has one reference count from the I/O completion callback. 1733 1734 If a driver calls this API, it will not complete the request 1735 as a result of this queues EvtIoDefault, and does not own 1736 the request until it has been re-presented by the Destination 1737 Queue. 1738 1739 Arguments: 1740 1741 Returns: 1742 1743 NTSTATUS 1744 1745 --*/ 1746 { 1747 NTSTATUS status; 1748 1749 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 1750 1751 status = VerifyForwardRequest(FxDriverGlobals, pDestQueue, pRequest); 1752 if (!NT_SUCCESS(status)) { 1753 return status; 1754 } 1755 1756 status = ForwardRequestWorker(pRequest, pDestQueue); 1757 return status; 1758 } 1759 1760 _Must_inspect_result_ 1761 NTSTATUS 1762 FX_VF_METHOD(FxIoQueue, VerifyQueueDriverCreatedRequest) ( 1763 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals, 1764 _In_ FxRequest* Request, 1765 _Inout_ SHORT* OldFlags 1766 ) 1767 { 1768 NTSTATUS status; 1769 KIRQL irql; 1770 1771 PAGED_CODE_LOCKED(); 1772 1773 Request->Lock(&irql); 1774 1775 *OldFlags = Request->GetVerifierFlagsLocked(); 1776 ASSERT((FXREQUEST_FLAG_DRIVER_DISPATCH & (*OldFlags)) == 0); 1777 1778 status = Request->VerifyRequestIsNotCancelable(FxDriverGlobals); 1779 if (NT_SUCCESS(status)) { 1780 // Clear the driver owned flag. 1781 ASSERT((FXREQUEST_FLAG_DRIVER_OWNED & (*OldFlags)) != 0); 1782 Request->ClearVerifierFlagsLocked(FXREQUEST_FLAG_DRIVER_OWNED); 1783 } 1784 1785 Request->Unlock(irql); 1786 return status; 1787 } 1788 1789 _Must_inspect_result_ 1790 NTSTATUS 1791 FxIoQueue::QueueDriverCreatedRequest( 1792 __in FxRequest* Request, 1793 __in BOOLEAN ParentQueue 1794 ) 1795 /*++ 1796 1797 Routine Description: 1798 1799 Insert a driver-created-request into this queue. 1800 The following conditions apply: 1801 1802 Request is not on a CSQ and not cancellable. 1803 1804 Request doesn't have the FXREQUEST_FLAG_DRIVER_OWNED set yet. 1805 1806 Request doesn't have an I/O completion callback function pointing to 1807 FxIoQueueRequestComplete since the original queue is NULL. 1808 1809 The Request has one reference count from WdfRequestCreate[FromIrp]. 1810 1811 On a successful return, the request is owned by the queue. Driver can complete 1812 this request only after it has been re-presented to the driver. 1813 1814 Arguments: 1815 1816 Request - Driver created request (already validated by public API) to 1817 insert in queue. 1818 1819 ParentQueue - TRUE if the queue is owned by the parent device. 1820 1821 Returns: 1822 1823 NTSTATUS 1824 1825 --*/ 1826 { 1827 NTSTATUS status; 1828 CfxDeviceBase * origDeviceBase; 1829 SHORT oldFlags = 0; 1830 FxIrp* fxIrp; 1831 1832 PFX_DRIVER_GLOBALS fxDriverGlobals = GetDriverGlobals(); 1833 fxIrp = Request->GetFxIrp(); 1834 1835 status = VerifyQueueDriverCreatedRequest(fxDriverGlobals, Request, &oldFlags); 1836 if(!NT_SUCCESS(status)) { 1837 return status; 1838 } 1839 1840 ASSERT(Request->SetCompletionState(FxRequestCompletionStateNone) == 1841 FxRequestCompletionStateNone); 1842 1843 // 1844 // If this is the parent queue, we need to adjust the IRP's stack. 1845 // 1846 if (ParentQueue) { 1847 1848 // 1849 // IRP should not have a completion routine set yet. 1850 // 1851 1852 ASSERT(fxIrp->GetNextCompletionRoutine() == NULL); 1853 1854 fxIrp->CopyCurrentIrpStackLocationToNext(); 1855 fxIrp->SetNextIrpStackLocation(); 1856 1857 // 1858 // Save a pointer to the device object for this request so that it can 1859 // be used later in completion. 1860 // 1861 fxIrp->SetCurrentDeviceObject(m_Device->GetDeviceObject()); 1862 } 1863 1864 origDeviceBase = Request->GetDeviceBase(); 1865 Request->SetDeviceBase((CfxDeviceBase *)m_Device); 1866 1867 // 1868 // Attempt to insert the request into the queue 1869 // 1870 status = QueueRequestFromForward(Request); 1871 if (!NT_SUCCESS(status)) { 1872 // 1873 // Request was not accepted, restore the original DeviceBase and flags. 1874 // 1875 ASSERT(Request->SetCompletionState(FxRequestCompletionStateNone) == 1876 FxRequestCompletionStateNone); 1877 1878 // 1879 // Restore original device/info. 1880 // 1881 Request->SetDeviceBase(origDeviceBase); 1882 1883 if (fxDriverGlobals->FxVerifierOn) { 1884 Request->SetVerifierFlags(oldFlags); 1885 } 1886 1887 // 1888 // If this is the parent queue, we need to adjust the IRP's stack. 1889 // 1890 if (ParentQueue) { 1891 fxIrp->SkipCurrentIrpStackLocation(); 1892 // 1893 // There is no completion routine. See above assert. 1894 // 1895 Request->m_Irp.ClearNextStack(); 1896 } 1897 } 1898 1899 return status; 1900 } 1901 1902 _Must_inspect_result_ 1903 NTSTATUS 1904 FX_VF_METHOD(FxIoQueue, VerifyRequeue) ( 1905 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals, 1906 _In_ FxRequest* pRequest 1907 ) 1908 { 1909 NTSTATUS status = STATUS_SUCCESS; 1910 KIRQL irql; 1911 1912 PAGED_CODE_LOCKED(); 1913 1914 pRequest->Lock(&irql); 1915 1916 status = pRequest->VerifyRequestIsDriverOwned(FxDriverGlobals); 1917 if (NT_SUCCESS(status)) { 1918 status = pRequest->VerifyRequestIsNotCancelable(FxDriverGlobals); 1919 } 1920 1921 if (NT_SUCCESS(status)) { 1922 pRequest->ClearVerifierFlagsLocked(FXREQUEST_FLAG_DRIVER_OWNED | 1923 FXREQUEST_FLAG_DRIVER_DISPATCH); 1924 } 1925 pRequest->Unlock(irql); 1926 1927 return status; 1928 } 1929 1930 1931 _Must_inspect_result_ 1932 NTSTATUS 1933 FxIoQueue::Requeue( 1934 __in FxRequest* pRequest 1935 ) 1936 { 1937 NTSTATUS status; 1938 FxRequestCompletionState oldState; 1939 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 1940 KIRQL irql; 1941 1942 status = VerifyRequeue(FxDriverGlobals, pRequest); 1943 if (!NT_SUCCESS(status)) { 1944 return status; 1945 } 1946 1947 // 1948 // Requeue is allowed only on Manual queue. 1949 // 1950 if(pRequest->GetCurrentQueue()->m_Type != WdfIoQueueDispatchManual) { 1951 1952 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 1953 "Requeue is allowed only for " 1954 "a manual queue, WDFREQUEST 0x%p " 1955 "%!STATUS!", 1956 pRequest, 1957 STATUS_INVALID_DEVICE_REQUEST); 1958 FxVerifierDbgBreakPoint(FxDriverGlobals); 1959 return STATUS_INVALID_DEVICE_REQUEST; 1960 } 1961 1962 // 1963 // The request has only one reference, held by the completion 1964 // callback function. We need to take another one before cancelling 1965 // this function, otherwise we will lose the request object 1966 // 1967 pRequest->ADDREF(FXREQUEST_STATE_TAG); 1968 1969 // Cancel the request complete callback (deletes a reference) 1970 oldState = pRequest->SetCompletionState(FxRequestCompletionStateNone); 1971 ASSERT(oldState == FxRequestCompletionStateQueue); 1972 UNREFERENCED_PARAMETER(oldState); 1973 1974 Lock(&irql); 1975 1976 // 1977 // We are going to place the request back on the queue 1978 // 1979 1980 1981 // Driver did not accept the I/O 1982 RemoveFromDriverOwnedList(pRequest); 1983 1984 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 1985 "WDFREQUEST 0x%p", pRequest->GetHandle()); 1986 1987 // 1988 // Check if we need to delete this request. 1989 // 1990 if (m_CancelDispatchedRequests) { 1991 // 1992 // Do not requeue this request. 1993 // 1994 status = STATUS_CANCELLED; 1995 } 1996 else { 1997 // 1998 // Place the request back at the head of the main queue 1999 // so as not to re-order requests 2000 // 2001 status = pRequest->InsertHeadIrpQueue(&m_Queue, NULL); 2002 } 2003 2004 if (!NT_SUCCESS(status)) { 2005 2006 // Request did not get placed in queue 2007 ASSERT(status == STATUS_CANCELLED); 2008 // 2009 // Let the caller think the request is requeued successfully 2010 // because this is no different from the request cancelling 2011 // while it's in the queue. By returning STATUS_CANCELLED 2012 // the caller can't take any recovery action anyways 2013 // because the request is gone. 2014 // 2015 status = STATUS_SUCCESS; 2016 2017 // 2018 // We must add a reference since the CancelForQueue path 2019 // assumes we were on the FxIrpQueue with the extra reference 2020 // 2021 pRequest->ADDREF(FXREQUEST_QUEUE_TAG); 2022 2023 // 2024 // Mark the request as cancelled, place it on the cancel list, 2025 // and schedule the cancel event to the driver 2026 // 2027 CancelForQueue(pRequest, irql); 2028 2029 Lock(&irql); 2030 } 2031 else { 2032 // Check if went from no requests to have requests 2033 CheckTransitionFromEmpty(); 2034 } 2035 2036 // 2037 // Visit the DispatchEvent so that we can deliver EvtIoReadyNotify 2038 // 2039 DispatchEvents(irql); 2040 2041 return status; 2042 } 2043 2044 _Must_inspect_result_ 2045 NTSTATUS 2046 FX_VF_METHOD(FxIoQueue, VerifyRequestCancelable) ( 2047 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals, 2048 _In_ FxRequest* pRequest, 2049 _In_ BOOLEAN Cancelable 2050 ) 2051 { 2052 NTSTATUS status; 2053 KIRQL irql; 2054 2055 PAGED_CODE_LOCKED(); 2056 2057 pRequest->Lock(&irql); 2058 2059 // Make sure the driver owns the request 2060 status = pRequest->VerifyRequestIsDriverOwned(FxDriverGlobals); 2061 if (!NT_SUCCESS(status)) { 2062 goto Done; 2063 } 2064 2065 if (Cancelable) { 2066 // 2067 // Make sure the request is not cancelable for it to be made 2068 // cancelable. 2069 // 2070 status = pRequest->VerifyRequestIsNotCancelable(FxDriverGlobals); 2071 if (!NT_SUCCESS(status)) { 2072 goto Done; 2073 } 2074 } 2075 else { 2076 // 2077 // Make sure the request is cancelable for it to be made 2078 // uncancelable. 2079 // 2080 status = pRequest->VerifyRequestIsCancelable(FxDriverGlobals); 2081 if (!NT_SUCCESS(status)) { 2082 goto Done; 2083 } 2084 } 2085 2086 Done: 2087 pRequest->Unlock(irql); 2088 return status; 2089 } 2090 2091 _Must_inspect_result_ 2092 NTSTATUS 2093 FxIoQueue::RequestCancelable( 2094 __in FxRequest* pRequest, 2095 __in BOOLEAN Cancelable, 2096 __in_opt PFN_WDF_REQUEST_CANCEL EvtRequestCancel, 2097 __in BOOLEAN FailIfIrpIsCancelled 2098 ) 2099 /*++ 2100 2101 Routine Description: 2102 2103 This is called to mark or unmark the request cancelable. 2104 2105 Arguments: 2106 2107 FxRequest* - Request that is completing 2108 2109 Cancelable - if TRUE, mark the request cancellable 2110 if FALSE, mark the request not cancelable 2111 if it's previously marked canceelable. 2112 2113 EvtRequestCancel - points to driver provided cancel routine 2114 if the cancelable flag is TRUE. 2115 2116 FailIfIrpIsCancelled - if FALSE and the IRP is already cancelled, 2117 call the provided cancel routine and 2118 return success. 2119 if TRUE and the IRP is already cancelled, 2120 return STATUS_CANCELLED. 2121 2122 Returns: 2123 2124 NTSTATUS 2125 2126 --*/ 2127 { 2128 NTSTATUS status; 2129 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 2130 KIRQL irql; 2131 2132 status = VerifyRequestCancelable(FxDriverGlobals, pRequest, Cancelable); 2133 if(!NT_SUCCESS(status)) { 2134 return status; 2135 } 2136 2137 if (Cancelable) { 2138 2139 if (FxDriverGlobals->FxVerifierOn) { 2140 pRequest->SetVerifierFlags(FXREQUEST_FLAG_DRIVER_CANCELABLE); 2141 } 2142 // 2143 // Set the Request for cancel status by inserting in the driver owned 2144 // CSQ. Note: This could fire the cancel callback right away 2145 // if the IRP was already cancelled. 2146 // 2147 2148 ASSERT(EvtRequestCancel); 2149 2150 Lock(&irql); 2151 2152 pRequest->m_CancelRoutine.m_Cancel = EvtRequestCancel; 2153 2154 // 2155 // Check if we need to delete this request. 2156 // 2157 if (m_CancelDispatchedRequests) { 2158 // 2159 // Purge is in progress, cancel this request. 2160 // 2161 status = STATUS_CANCELLED; 2162 } 2163 else { 2164 status = pRequest->InsertTailIrpQueue(&m_DriverCancelable, NULL); 2165 } 2166 2167 if (NT_SUCCESS(status)) { 2168 Unlock(irql); 2169 } 2170 else if (FailIfIrpIsCancelled == FALSE) { 2171 2172 ASSERT(status == STATUS_CANCELLED); 2173 2174 // This is not an error to the driver 2175 status = STATUS_SUCCESS; 2176 2177 pRequest->m_Canceled = TRUE; 2178 2179 Unlock(irql); 2180 2181 // 2182 // We must add a reference since the CancelForDriver path 2183 // assumes we were on the FxIrpQueue with the extra reference 2184 // 2185 pRequest->ADDREF(FXREQUEST_QUEUE_TAG); 2186 2187 // 2188 // Mark the request as cancelled, place it on the cancel list, 2189 // and schedule the cancel event to the driver 2190 // 2191 CancelForDriver(pRequest); 2192 } 2193 else { 2194 2195 ASSERT(status == STATUS_CANCELLED); 2196 2197 pRequest->m_CancelRoutine.m_Cancel = NULL; 2198 2199 // 2200 // Let the caller complete the request with STATUS_CANCELLED. 2201 // 2202 Unlock(irql); 2203 2204 if (FxDriverGlobals->FxVerifierOn) { 2205 pRequest->ClearVerifierFlags(FXREQUEST_FLAG_DRIVER_CANCELABLE); 2206 } 2207 } 2208 2209 return status; 2210 } 2211 else { 2212 // 2213 // This can return STATUS_CANCELLED if the request 2214 // has been canceled already 2215 // 2216 Lock(&irql); 2217 status = pRequest->RemoveFromIrpQueue(&m_DriverCancelable); 2218 2219 if (NT_SUCCESS(status)) { 2220 pRequest->m_CancelRoutine.m_Cancel = NULL; 2221 } 2222 else { 2223 // 2224 // In the failure case, the cancel routine has won the race and will 2225 // be invoked on another thread. 2226 // 2227 DO_NOTHING(); 2228 } 2229 Unlock(irql); 2230 2231 if (FxDriverGlobals->FxVerifierOn) { 2232 2233 // We got the request back, can clear the cancelable flag 2234 if (NT_SUCCESS(status)) { 2235 pRequest->ClearVerifierFlags(FXREQUEST_FLAG_DRIVER_CANCELABLE); 2236 } 2237 } 2238 2239 return status; 2240 } 2241 } 2242 2243 _Must_inspect_result_ 2244 NTSTATUS 2245 FxIoQueue::QueueRequest( 2246 __in FxRequest* pRequest 2247 ) 2248 2249 /*++ 2250 2251 Routine Description: 2252 2253 Enqueue a request to the end of the queue. 2254 2255 Note: This routine owns the final disposition of 2256 the Request object, and must handle it and 2257 dereference even if the driver does not. 2258 2259 Arguments: 2260 2261 pRequest - Pointer to Request object 2262 2263 Returns: 2264 2265 NTSTATUS 2266 --*/ 2267 2268 { 2269 NTSTATUS Status; 2270 KIRQL irql; 2271 MdIrp pIrp; 2272 FxIrp* pFxIrp; 2273 2274 // Get IoQueue Object Lock 2275 Lock(&irql); 2276 2277 ASSERT(pRequest->GetRefCnt() == 1); 2278 2279 // 2280 // If the request is reserved, take an additional reference. This reference 2281 // will be released when the request is completed. This additional reference 2282 // enables us to detect 2 to 1 transition in the completion path so that 2283 // we can reclaim the reserved request for reuse. 2284 // 2285 if (pRequest->IsReserved()) { 2286 pRequest->ADDREF(FXREQUEST_FWDPRG_TAG); 2287 } 2288 2289 // 2290 // If queue is not taking new requests, fail now 2291 // 2292 if (!IsState(WdfIoQueueAcceptRequests)) { 2293 2294 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGIO, 2295 "WDFQUEUE 0x%p is not accepting requests, " 2296 "state is %!WDF_IO_QUEUE_STATE!, %s" 2297 "completing WDFREQUEST 0x%p %!STATUS!", 2298 GetObjectHandle(), m_QueueState, 2299 IsState(FxIoQueueShutdown) ? 2300 "power stopping (Drain) in progress," : "", 2301 pRequest->GetHandle(), 2302 STATUS_INVALID_DEVICE_STATE); 2303 2304 // Must release IoQueue object Lock 2305 Unlock(irql); 2306 2307 Status = STATUS_INVALID_DEVICE_STATE; 2308 2309 // Complete it with error 2310 pRequest->CompleteWithInformation(Status, 0); 2311 2312 // Dereference request object 2313 pRequest->RELEASE(FXREQUEST_COMPLETE_TAG); 2314 2315 return Status; 2316 } 2317 2318 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGIO, 2319 "Queuing WDFREQUEST 0x%p on WDFQUEUE 0x%p", 2320 pRequest->GetHandle(),GetObjectHandle()); 2321 2322 (VOID)pRequest->GetIrp(&pIrp); 2323 2324 pFxIrp = pRequest->GetFxIrp(); 2325 2326 pFxIrp->MarkIrpPending(); 2327 2328 // 2329 // If the request is reserved, we may be called to dispatch 2330 // a pending reserved IRP from within the context of the completion routine. 2331 // So to avoid recursion, we will insert the request in the queue and try 2332 // to dispatch in the return path. If the request is not reserved then we 2333 // will dispatch it directly because this path is meant for dispatching new 2334 // incoming I/O. There is no concern for running into recursion in that 2335 // scenario. 2336 // 2337 if (pRequest->IsReserved() && m_Dispatching != 0) { 2338 InsertNewRequestLocked(&pRequest, irql); 2339 Unlock(irql); 2340 } 2341 else { 2342 DispatchEvents(irql, pRequest); 2343 } 2344 2345 // We always return status pending through the frameworks 2346 return STATUS_PENDING; 2347 } 2348 2349 _Must_inspect_result_ 2350 NTSTATUS 2351 FxIoQueue::QueueRequestFromForward( 2352 __in FxRequest* pRequest 2353 ) 2354 2355 /*++ 2356 2357 Routine Description: 2358 2359 Enqueue a request to the end of the queue. 2360 2361 This is an internal version that does not fail 2362 the request if it can not be enqueued. 2363 2364 Arguments: 2365 2366 pRequest - Pointer to Request object 2367 2368 Returns: 2369 2370 STATUS_SUCCESS on success 2371 --*/ 2372 2373 { 2374 NTSTATUS status; 2375 KIRQL irql; 2376 BOOLEAN fromIo; 2377 2378 // Get IoQueue Object Lock 2379 Lock(&irql); 2380 2381 // 2382 // If queue is not taking new requests, fail now 2383 // 2384 if (!IsState(WdfIoQueueAcceptRequests)) { 2385 2386 status = STATUS_WDF_BUSY; 2387 2388 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_WARNING, TRACINGIO, 2389 "WDFQUEUE 0x%p is not accepting requests " 2390 "state is %!WDF_IO_QUEUE_STATE!, %s" 2391 "WDFREQUEST 0x%p %!STATUS!", 2392 GetObjectHandle(), m_QueueState, 2393 IsState(FxIoQueueShutdown) ? 2394 "power stopping (Drain) in progress," : "", 2395 pRequest->GetHandle(), status); 2396 2397 Unlock(irql); 2398 2399 return status; 2400 } 2401 #if FX_VERBOSE_TRACE 2402 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGIO, 2403 "Queuing WDFREQUEST 0x%p on WDFQUEUE 0x%p", 2404 pRequest->GetHandle(), GetObjectHandle()); 2405 #endif 2406 // 2407 // The Request has one reference count, and no completion 2408 // callback function. It has been completely removed from 2409 // its previous queue. 2410 // 2411 2412 // 2413 // Cache this info b/c the request can be delete and freed by the time we use it. 2414 // 2415 fromIo = pRequest->IsAllocatedFromIo(); 2416 2417 // 2418 // Insert it in the Cancel Safe Queue 2419 // 2420 // This will mark the IRP pending 2421 // 2422 status = pRequest->InsertTailIrpQueue(&m_Queue, NULL); 2423 2424 if (!NT_SUCCESS(status)) { 2425 2426 pRequest->SetCurrentQueue(this); 2427 2428 ASSERT(status == STATUS_CANCELLED); 2429 2430 // 2431 // We must add a reference since the CancelForQueue path 2432 // assumes we were on the FxIrpQueue with the extra reference 2433 // 2434 pRequest->ADDREF(FXREQUEST_QUEUE_TAG); 2435 2436 // 2437 // Mark the request as cancelled, place it on the cancel list, 2438 // and schedule the cancel event to the driver 2439 // 2440 CancelForQueue(pRequest, irql); 2441 2442 Lock(&irql); 2443 } 2444 else { 2445 pRequest->SetCurrentQueue(this); 2446 2447 // Check if went from no requests to have requests 2448 CheckTransitionFromEmpty(); 2449 } 2450 2451 // 2452 // If the request is driver-created, we may be called to dispatch 2453 // a request from within the context of the completion routine. 2454 // So to avoid recursion, we will try to dispatch in the return path. 2455 // If the request is not driver-created then we will dispatch it directly because 2456 // this path is meant for dispatching new incoming I/O. There is no concern for 2457 // running into recursion in that scenario. 2458 // 2459 if (fromIo == FALSE && m_Dispatching != 0) { 2460 Unlock(irql); 2461 } 2462 else { 2463 // 2464 // Attempt to dispatch any new requests. 2465 // 2466 // This releases, and re-acquires the IoQueue lock 2467 // 2468 DispatchEvents(irql); 2469 } 2470 2471 return STATUS_SUCCESS; 2472 } 2473 2474 VOID 2475 FxIoQueue::DeferredDispatchRequestsFromDpc( 2476 ) 2477 2478 /*++ 2479 2480 Routine Description: 2481 2482 Dispatch requests from the queue to the driver 2483 from within the m_Dpc 2484 2485 Arguments: 2486 2487 Returns: 2488 2489 --*/ 2490 2491 { 2492 KIRQL irql; 2493 2494 Lock(&irql); 2495 2496 ASSERT(m_DpcQueued != FALSE); 2497 2498 m_RequeueDeferredDispatcher = FALSE; 2499 2500 DispatchEvents(irql); 2501 2502 // 2503 // DispatchEvents drops the lock before returning. So reacquire the lock. 2504 // 2505 Lock(&irql); 2506 2507 if (m_Deleted == FALSE && m_RequeueDeferredDispatcher) { 2508 InsertQueueDpc(); 2509 } else { 2510 m_RequeueDeferredDispatcher = FALSE; 2511 m_DpcQueued = FALSE; 2512 } 2513 2514 Unlock(irql); 2515 2516 return; 2517 } 2518 2519 VOID 2520 FxIoQueue::DeferredDispatchRequestsFromWorkerThread( 2521 ) 2522 2523 /*++ 2524 2525 Routine Description: 2526 2527 Dispatch requests from the queue to the driver 2528 from within the m_WorkItem. 2529 2530 Arguments: 2531 2532 Returns: 2533 2534 --*/ 2535 2536 { 2537 KIRQL irql; 2538 2539 Lock(&irql); 2540 2541 ASSERT(m_WorkItemQueued != FALSE); 2542 2543 m_RequeueDeferredDispatcher = FALSE; 2544 2545 DispatchEvents(irql); 2546 2547 // 2548 // DispatchEvents drops the lock before returning. So reacquire 2549 // the lock. 2550 // 2551 Lock(&irql); 2552 2553 if (m_Deleted == FALSE && 2554 m_RequeueDeferredDispatcher && 2555 m_SystemWorkItem->Enqueue(_DeferredDispatchThreadThunk, this)) { 2556 // 2557 // Workitem is queued. 2558 // 2559 DO_NOTHING(); 2560 } else { 2561 m_RequeueDeferredDispatcher = FALSE; 2562 m_WorkItemQueued = FALSE; 2563 } 2564 2565 Unlock(irql); 2566 2567 return; 2568 } 2569 2570 NTSTATUS 2571 FxIoQueue::InsertNewRequestLocked( 2572 __deref_in FxRequest** Request, 2573 __in KIRQL PreviousIrql 2574 ) 2575 /*++ 2576 2577 Routine Description: 2578 2579 Purpose of this function is to insert the request that's dispatched 2580 by the IoPkg into FxIrpQueue. This function has been added to improve 2581 the performance of queueing logic. Prior to version 1.7, when a 2582 request is dispatched to a queue, it was first inserted into queue, 2583 various checks for the readiness of queue made, and then the request 2584 is removed from the queue to be presented to the driver. 2585 2586 To improve the I/O performance, dispatching logic has been changed 2587 such that the request will not be inserted into the queue if the queue 2588 is ready to dispatch the request. If the queue is not ready or if there 2589 are other events to be dispatched before dispatching the new incoming request, 2590 we will queue the request first using this function before releasing the lock 2591 so that we don't change the ordering of requests in the queue. 2592 2593 --*/ 2594 { 2595 NTSTATUS status; 2596 2597 status = (*Request)->InsertTailIrpQueue(&m_Queue, NULL); 2598 2599 if (!NT_SUCCESS(status)) { 2600 // 2601 // Request was never presented to the driver 2602 // so there is no need to call CancelForQueue 2603 // in this case. 2604 // 2605 ASSERT(status == STATUS_CANCELLED); 2606 2607 Unlock(PreviousIrql); 2608 2609 (*Request)->CompleteWithInformation(status, 0); 2610 2611 (*Request)->RELEASE(FXREQUEST_COMPLETE_TAG); 2612 2613 Lock(&PreviousIrql); 2614 } 2615 else { 2616 (*Request)->SetCurrentQueue(this); 2617 2618 // Check if went from no requests to have requests 2619 CheckTransitionFromEmpty(); 2620 } 2621 2622 // 2623 // Request is either inserted into the queue or completed. Clear 2624 // the field to prevent touching the request. 2625 // 2626 *Request = NULL; 2627 2628 return status; 2629 } 2630 2631 _Must_inspect_result_ 2632 BOOLEAN 2633 FxIoQueue::CanThreadDispatchEventsLocked( 2634 __in KIRQL PreviousIrql 2635 ) 2636 /*++ 2637 2638 Routine Description: 2639 2640 Dispatch events and requests from the queue to the driver. 2641 2642 The IoQueue object lock must be held on entry. This routine 2643 should not drop and reacquire the lock to ensure the request 2644 is not queued out of order. 2645 2646 Returns: 2647 2648 TRUE - if the thread meets all the sychronization and 2649 execution contraints to dispatch the events. 2650 FALSE - if the dispatching of events to be defered to 2651 another thread - either DPC or workitem. 2652 --*/ 2653 { 2654 // 2655 // If the current irql is not at passive-level and the queue is configured 2656 // to receive events only at passive-level then we should queue a 2657 // workitem to defer the processing. 2658 // 2659 if ((PreviousIrql > PASSIVE_LEVEL) && m_PassiveLevel) { 2660 ASSERT(PreviousIrql <= DISPATCH_LEVEL); 2661 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_WARNING, TRACINGIO, 2662 "Current thread 0x%p is not at the passive-level" 2663 " %!irql!, posting to worker thread for WDFQUEUE" 2664 " 0x%p", 2665 Mx::MxGetCurrentThread(), 2666 PreviousIrql, 2667 GetObjectHandle()); 2668 // 2669 // We only need to post this once 2670 // 2671 if (m_WorkItemQueued == FALSE) { 2672 2673 m_WorkItemQueued = TRUE; 2674 2675 if (!m_SystemWorkItem->Enqueue(_DeferredDispatchThreadThunk, this)) { 2676 ASSERT(FALSE); 2677 m_WorkItemQueued = FALSE; 2678 } 2679 } 2680 2681 return FALSE; 2682 } 2683 2684 // 2685 // If the current thread is holding the presentation lock, we 2686 // must defer to a DPC or work item. 2687 // This is the result of the device driver calling 2688 // WdfRequestForwardToIoQueue, or WdfIoQueueStart/Stop from 2689 // within I/O dispatch handler. This can also occur if a driver 2690 // attempts to forward a request among a circular series of Queues 2691 // that are configured to have locking constraints. 2692 // 2693 if (m_CallbackLockPtr && m_CallbackLockPtr->IsOwner()) { 2694 2695 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_WARNING, TRACINGIO, 2696 "Presentation lock for WDFQUEUE 0x%p is " 2697 "already held, deferring to dpc or workitem", 2698 GetObjectHandle()); 2699 2700 if (m_PassiveLevel) { 2701 2702 if(m_WorkItemQueued == FALSE) { 2703 2704 m_WorkItemQueued = TRUE; 2705 2706 if (!m_SystemWorkItem->Enqueue(_DeferredDispatchThreadThunk, this)) { 2707 ASSERT(FALSE); 2708 m_WorkItemQueued = FALSE; 2709 } 2710 } 2711 } 2712 else { 2713 // 2714 // We only need to post this once 2715 // 2716 if (m_DpcQueued == FALSE) { 2717 2718 m_DpcQueued = TRUE; 2719 2720 InsertQueueDpc(); 2721 } 2722 } 2723 2724 return FALSE; 2725 } 2726 2727 return TRUE; 2728 } 2729 2730 _Releases_lock_(this->m_SpinLock.m_Lock) 2731 __drv_requiresIRQL(DISPATCH_LEVEL) 2732 BOOLEAN 2733 FxIoQueue::DispatchEvents( 2734 __in __drv_restoresIRQL KIRQL PreviousIrql, 2735 __in_opt FxRequest* NewRequest 2736 ) 2737 /*++ 2738 2739 Routine Description: 2740 2741 Dispatch events and requests from the queue to the driver. 2742 2743 The IoQueue object lock must be held on entry, but this routine can release 2744 and re-acquire the lock multiple times while processing. 2745 2746 It returns to the caller with the lock released, but queue state may have 2747 changed. 2748 2749 The main processing loop checks for various Queue state change events 2750 delivering them to the driver, and then finally any WDFREQUEST objects 2751 that are pending in the Queue. 2752 2753 The design also handles the recursive case with the m_Dispatching 2754 field so that a driver that completes requests from within the 2755 callback does not cause a stack or lock recursion. 2756 2757 All event callbacks to the device driver are provided though the 2758 FxCallback object which manages lock acquire and release as required 2759 by the locking model. 2760 2761 In addition these may be passive or dispatch level locks. 2762 If configured for passive level callbacks, 2763 must defer to a work item if current thread is DISPATCH_LEVEL 2764 when not owning the current FxIoQueue lock 2765 2766 Arguments: 2767 2768 NewRequest - This is a new incoming request from the driver above. 2769 It will be either presented to the driver or saved into 2770 a queue if the conditions are not right to dispatch. 2771 2772 Returns: 2773 2774 FALSE if the queue is in a deleted state else TRUE. 2775 Caller should check for return value only if it's waiting 2776 on the some events to be invoked by this call. 2777 2778 --*/ 2779 { 2780 FxRequest* pRequest; 2781 ULONG totalIoCount; 2782 NTSTATUS status; 2783 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 2784 2785 if (m_Deleted) { 2786 ASSERT(NewRequest == NULL); 2787 Unlock(PreviousIrql); 2788 return FALSE; 2789 } 2790 2791 // 2792 // The design of the I/O Queue allows all "events" to notify the driver of 2793 // to be deferred until the opportune time to deliver them. 2794 // Depending on the drivers configured locking and threading 2795 // mode, this may have to be deferred to a worker thread or a DPC 2796 // to be in a compatible IRQL level, or to prevent a lock recursion 2797 // when a parent objects lock is in effect. 2798 // 2799 2800 if (CanThreadDispatchEventsLocked(PreviousIrql) == FALSE) { 2801 // 2802 // Previous workitem or Dpc might be running the DispatchEvents right now. 2803 // But it may be at a point where it might miss out to process the event 2804 // that we have been asked to dispatch. This is possible because the 2805 // DispatchEvent is reentrant as it acquires and drops lock along 2806 // the way. So we make a note of this, so that when the current Dpc or 2807 // workItem runs to completion, it will requeue itself to handle our message. 2808 // 2809 m_RequeueDeferredDispatcher = TRUE; 2810 2811 // 2812 // Queue the request in to FxIrpQueue and return. 2813 // 2814 InsertNewRequest(&NewRequest, PreviousIrql); 2815 Unlock(PreviousIrql); 2816 return TRUE; 2817 } 2818 2819 // 2820 // This must be incremented before attempting to deliver any 2821 // events to the driver. This prevents recursion on the presentation lock, 2822 // and limits the stack depth in a Start/Complete/Start/... recursion 2823 // 2824 m_Dispatching++; 2825 2826 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 2827 "Thread %p is processing WDFQUEUE 0x%p", 2828 Mx::MxGetCurrentThread(), GetObjectHandle()); 2829 2830 // 2831 // At this point all constaints such as IRQL level, locks held, 2832 // and stack recursion protection has been satisfied, and we can 2833 // make callbacks into the device driver. 2834 // 2835 // Process events and requests until we either have an empty queue, 2836 // the driver stops taking requests, or some queue state does not 2837 // allow the driver to take new requests 2838 // 2839 #pragma warning(disable:4127) 2840 while (TRUE) { 2841 #pragma warning(default:4127) 2842 // 2843 // totoalIoCount is sum of requests pending in the queue and requests 2844 // currently owned by the driver. 2845 // 2846 totalIoCount = m_Queue.GetRequestCount() + m_DriverIoCount; 2847 2848 // 2849 // Increment the count if there is a new request to be dispatched. 2850 // 2851 totalIoCount += ((NewRequest != NULL) ? 1 : 0); 2852 2853 if (!IsListEmpty(&this->m_Cancelled)) { 2854 status = InsertNewRequest(&NewRequest, PreviousIrql); 2855 if (!NT_SUCCESS(status)) { 2856 continue; // totalIoCount may be zero now. 2857 } 2858 2859 // 2860 // This can drop and re-acquire the queue lock 2861 // ProcessCancelledRequests returns FALSE if the queue is 2862 // notifying driver about power state changes. 2863 // 2864 if(ProcessCancelledRequests(&PreviousIrql)) { 2865 continue; 2866 } 2867 } 2868 2869 if (!IsListEmpty(&this->m_CanceledOnQueueList)) { 2870 status = InsertNewRequest(&NewRequest, PreviousIrql); 2871 if (!NT_SUCCESS(status)) { 2872 continue; // totalIoCount may be zero now. 2873 } 2874 2875 // 2876 // This can drop and re-acquire the queue lock 2877 // ProcessCancelledRequests returns FALSE if the queue is 2878 // notifying driver about power state changes. 2879 // 2880 if (ProcessCancelledRequestsOnQueue(&PreviousIrql)) { 2881 continue; 2882 } 2883 } 2884 2885 if (m_IdleComplete.Method != NULL && 2886 m_Dispatching == 1L && 2887 m_DriverIoCount == 0L) { 2888 2889 InsertNewRequest(&NewRequest, PreviousIrql); 2890 2891 // no more driver owned requests, we can clear the following flag: 2892 m_CancelDispatchedRequests = FALSE; 2893 2894 // This can drop and re-acquire the queue lock 2895 ProcessIdleComplete(&PreviousIrql); 2896 continue; 2897 } 2898 2899 if (m_PurgeComplete.Method != NULL && 2900 totalIoCount == 0L && 2901 m_Dispatching == 1L) { 2902 2903 InsertNewRequest(&NewRequest, PreviousIrql); 2904 2905 // no more driver owned requests, we can clear the following flag: 2906 m_CancelDispatchedRequests = FALSE; 2907 2908 // This can drop and re-acquire the queue lock 2909 ProcessPurgeComplete(&PreviousIrql); 2910 continue; 2911 } 2912 2913 if (m_IsDevicePowerPolicyOwner && 2914 m_PowerManaged && 2915 m_PowerReferenced && 2916 totalIoCount == 0L && 2917 m_Dispatching == 1L) { 2918 2919 // 2920 // Queue has no requests, and is going idle. Notify 2921 // PNP/Power. 2922 // 2923 m_Device->m_PkgPnp->PowerDereference(); 2924 m_PowerReferenced = FALSE; 2925 continue; 2926 } 2927 2928 // 2929 // Look for power state transitions 2930 // 2931 if (m_PowerState != FxIoQueuePowerOn && 2932 m_PowerState != FxIoQueuePowerOff) { 2933 2934 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 2935 "WDFQUEUE 0x%p Power Transition State " 2936 "%!FxIoQueuePowerState!", GetObjectHandle(), 2937 m_PowerState); 2938 2939 status = InsertNewRequest(&NewRequest, PreviousIrql); 2940 if (!NT_SUCCESS(status)) { 2941 continue; // totalIoCount may be zero now. 2942 } 2943 2944 // Process intermediate power state 2945 // This can drop and re-acquire the queue lock 2946 if (ProcessPowerEvents(&PreviousIrql)) { 2947 continue; 2948 } 2949 else { 2950 2951 // 2952 // Return, awaiting some response from the driver 2953 // 2954 goto Done; 2955 } 2956 } 2957 else { 2958 // Queue is either in PowerOn or PowerOff state 2959 DO_NOTHING(); 2960 } 2961 2962 // 2963 // Check for queue disposing should be made after processing all 2964 // the events. 2965 // 2966 if (m_Disposing && 2967 totalIoCount == 0L && 2968 m_Dispatching == 1L) { 2969 2970 m_Deleted = TRUE; 2971 2972 // 2973 // After this point, no other thread will be able to dispatch 2974 // events from this queue. Also threads that are about to call 2975 // this function as soon as we drop the lock below should have 2976 // a reference on the queue to prevent queue object from being 2977 // freed when we signal the dispose thread to run through. 2978 // 2979 Unlock(PreviousIrql); 2980 2981 m_FinishDisposing.Set(); 2982 return TRUE; 2983 } 2984 2985 2986 // 2987 // Return if power is off, can't deliver any request oriented events 2988 // to the driver. 2989 // 2990 if (m_PowerState == FxIoQueuePowerOff) { 2991 status = InsertNewRequest(&NewRequest, PreviousIrql); 2992 if (!NT_SUCCESS(status)) { 2993 continue; // totalIoCount may be zero now. 2994 } 2995 2996 goto Done; 2997 } 2998 2999 // 3000 // See if the queue is (still) processing requests 3001 // 3002 if (!IsState(WdfIoQueueDispatchRequests)) { 3003 3004 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_WARNING, TRACINGIO, 3005 "WDFQUEUE 0x%p not in dispatching state, " 3006 "current state is %!WDF_IO_QUEUE_STATE!", 3007 GetObjectHandle(), m_QueueState); 3008 3009 status = InsertNewRequest(&NewRequest, PreviousIrql); 3010 if (!NT_SUCCESS(status)) { 3011 continue; // totalIoCount may be zero now. 3012 } 3013 3014 goto Done; 3015 } 3016 3017 // 3018 // A manual dispatch queue can have a request ready notification 3019 // 3020 if (m_Type == WdfIoQueueDispatchManual) { 3021 3022 status = InsertNewRequest(&NewRequest, PreviousIrql); 3023 if (!NT_SUCCESS(status)) { 3024 continue; // totalIoCount may be zero now. 3025 } 3026 3027 if (m_ReadyNotify.Method != NULL && m_TransitionFromEmpty) { 3028 3029 // This can drop and re-acquire the lock to callback to the driver 3030 ProcessReadyNotify(&PreviousIrql); 3031 continue; 3032 } 3033 3034 goto Done; 3035 } 3036 3037 if (m_Type == WdfIoQueueDispatchSequential && m_DriverIoCount > 0) { 3038 status = InsertNewRequest(&NewRequest, PreviousIrql); 3039 if (!NT_SUCCESS(status)) { 3040 continue; // totalIoCount may be zero now. 3041 } 3042 3043 goto Done; 3044 } 3045 3046 // 3047 // For counted Queue's dont dispatch request to driver if the 3048 // m_DriverIoCount exceeds the one set by the driver writer. 3049 // 3050 if (m_Type == WdfIoQueueDispatchParallel && 3051 (ULONG)m_DriverIoCount >= m_MaxParallelQueuePresentedRequests) { 3052 status = InsertNewRequest(&NewRequest, PreviousIrql); 3053 if (!NT_SUCCESS(status)) { 3054 continue; // totalIoCount may be zero now. 3055 } 3056 3057 goto Done; 3058 } 3059 3060 // 3061 // If there is a request in the queue, then retrieve that. 3062 // 3063 pRequest = NULL; 3064 if (m_Queue.GetRequestCount() > 0L) { 3065 pRequest = FxRequest::GetNextRequest(&m_Queue); 3066 } 3067 3068 // 3069 // The request from the queue should be dispatched first 3070 // to preserve the ordering. 3071 // 3072 if (pRequest != NULL) { 3073 InsertNewRequest(&NewRequest, PreviousIrql); 3074 } 3075 else { 3076 // 3077 // If there is no request in the queue then dispatch 3078 // the incoming one. 3079 // 3080 pRequest = NewRequest; 3081 if (pRequest != NULL) { 3082 pRequest->SetCurrentQueue(this); 3083 SetTransitionFromEmpty(); 3084 NewRequest = NULL; 3085 } 3086 else { 3087 goto Done; 3088 } 3089 } 3090 3091 // 3092 // pRequest is not cancellable now 3093 // 3094 InsertInDriverOwnedList(pRequest); 3095 3096 Unlock(PreviousIrql); 3097 3098 DispatchRequestToDriver(pRequest); 3099 3100 Lock(&PreviousIrql); 3101 } 3102 3103 Done: 3104 m_Dispatching--; 3105 Unlock(PreviousIrql); 3106 return TRUE; 3107 } 3108 3109 VOID 3110 FxIoQueue::DispatchRequestToDriver( 3111 __in FxRequest* pRequest 3112 ) 3113 3114 /*++ 3115 3116 Routine Description: 3117 3118 Dispatch the next request to the driver. 3119 3120 The IoQueue object lock is *not* held. 3121 3122 It returns to the caller with the lock *not* held. 3123 3124 This is called by DispatchRequests(), and should not be 3125 called directly in order to maintain queue processing model. 3126 3127 Arguments: 3128 3129 Returns: 3130 3131 --*/ 3132 3133 { 3134 PFX_DRIVER_GLOBALS FxDriverGlobals; 3135 NTSTATUS Status; 3136 FxRequestCompletionState oldState; 3137 WDFREQUEST hRequest; 3138 FxIrp* pIrp; 3139 3140 FxDriverGlobals = GetDriverGlobals(); 3141 3142 3143 3144 3145 3146 (VOID)pRequest->GetCurrentIrpStackLocation(); 3147 3148 pIrp = pRequest->GetFxIrp(); 3149 3150 // The Irp does not have a cancel function right now 3151 #if (FX_CORE_MODE == FX_CORE_KERNEL_MODE) 3152 ASSERT(pIrp->GetCurrentIrpStackLocation() != NULL); 3153 #endif 3154 3155 // 3156 // Set our completion callback on the request now before 3157 // calling the driver since the driver can complete the 3158 // request in the callback handler, and to avoid races with 3159 // the drivers completion thread. 3160 // 3161 // This takes a reference on the request object. 3162 // 3163 oldState = pRequest->SetCompletionState(FxRequestCompletionStateQueue); 3164 ASSERT(oldState == FxRequestCompletionStateNone); 3165 UNREFERENCED_PARAMETER(oldState); 3166 3167 if (FxDriverGlobals->FxVerifierOn) { 3168 // 3169 // If the verifier is on, we do not release the extra 3170 // reference so we can mark the request as no longer 3171 // being dispatched to the driver on return from the 3172 // event callback to the driver 3173 // 3174 3175 ASSERT((pRequest->GetVerifierFlags() & FXREQUEST_FLAG_DRIVER_OWNED) == 0); 3176 3177 // Mark the request as being "owned" by the driver 3178 pRequest->SetVerifierFlags(FXREQUEST_FLAG_DRIVER_OWNED | 3179 FXREQUEST_FLAG_DRIVER_DISPATCH); 3180 } 3181 else { 3182 3183 // 3184 // Release our original reference. The FxRequest::Complete 3185 // will release the final one since we have registered a completion 3186 // callback handler 3187 // 3188 // We now have one reference count on the FxRequest object until 3189 // its completion routine runs since the completion event made 3190 // an extra reference, and will dereference it when it fires, or 3191 // its canceled. 3192 // 3193 3194 pRequest->RELEASE(FXREQUEST_STATE_TAG); 3195 } 3196 3197 // 3198 // Attempt to dispatch it to the driver 3199 // 3200 3201 // 3202 // Note: A driver that changes its callback pointers at runtime 3203 // could run into a race here since we released the queue 3204 // lock. Currently, changing parameters on a processing 3205 // queue is undefined. 3206 // 3207 // The C DDI's force the callbacks to be registered at 3208 // queue creation time and avoid this race. 3209 // 3210 3211 hRequest = pRequest->GetHandle(); 3212 3213 UCHAR majorFunction = pIrp->GetMajorFunction(); 3214 3215 if ((majorFunction == IRP_MJ_READ) && m_IoRead.Method) { 3216 ULONG readLength = pIrp->GetParameterReadLength(); 3217 3218 // 3219 // Complete zero length reads with STATUS_SUCCESS unless the 3220 // driver specified it wants them delivered. 3221 // 3222 if ((readLength == 0) && 3223 !m_AllowZeroLengthRequests) { 3224 3225 DoTraceLevelMessage( 3226 FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 3227 "Zero length WDFREQUEST 0x%p completed automatically " 3228 "by WDFQUEUE 0x%p", hRequest, GetObjectHandle()); 3229 3230 pRequest->Complete(STATUS_SUCCESS); 3231 if (FxDriverGlobals->FxVerifierOn) { 3232 // 3233 // Release the reference taken in the call to SetCompletionState 3234 // at the top of the function. 3235 // 3236 pRequest->RELEASE(FXREQUEST_STATE_TAG); 3237 } 3238 return; 3239 } 3240 3241 pRequest->SetPresented(); 3242 3243 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 3244 "Calling driver EvtIoRead for WDFREQUEST 0x%p", 3245 hRequest); 3246 3247 m_IoRead.Invoke( 3248 GetHandle(), 3249 hRequest, 3250 readLength 3251 ); 3252 } 3253 else if ((majorFunction == IRP_MJ_WRITE) && m_IoWrite.Method) { 3254 ULONG writeLength = pIrp->GetParameterWriteLength(); 3255 3256 // 3257 // Complete zero length writes with STATUS_SUCCESS unless the 3258 // driver specified it wants them delivered. 3259 // 3260 if ((writeLength == 0) && 3261 !m_AllowZeroLengthRequests) { 3262 3263 DoTraceLevelMessage( 3264 FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 3265 "Zero length WDFREQUEST 0x%p completed automatically " 3266 "by WDFQUEUE 0x%p", hRequest, GetObjectHandle()); 3267 3268 pRequest->Complete(STATUS_SUCCESS); 3269 3270 if (FxDriverGlobals->FxVerifierOn) { 3271 // 3272 // Release the reference taken in the call to SetCompletionState 3273 // at the top of the function. 3274 // 3275 pRequest->RELEASE(FXREQUEST_STATE_TAG); 3276 } 3277 return; 3278 } 3279 3280 pRequest->SetPresented(); 3281 3282 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 3283 "Calling driver EvtIoWrite for WDFREQUEST 0x%p", 3284 pRequest->GetObjectHandle()); 3285 3286 m_IoWrite.Invoke( 3287 GetHandle(), 3288 hRequest, 3289 writeLength 3290 ); 3291 } 3292 else if ((majorFunction == IRP_MJ_DEVICE_CONTROL) && m_IoDeviceControl.Method) { 3293 3294 pRequest->SetPresented(); 3295 3296 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 3297 "Calling driver EvtIoDeviceControl for " 3298 "WDFREQUEST 0x%p", hRequest); 3299 3300 m_IoDeviceControl.Invoke( 3301 GetHandle(), 3302 hRequest, 3303 pIrp->GetParameterIoctlOutputBufferLength(), 3304 pIrp->GetParameterIoctlInputBufferLength(), 3305 pIrp->GetParameterIoctlCode() 3306 ); 3307 } 3308 3309 else if ( (majorFunction == IRP_MJ_INTERNAL_DEVICE_CONTROL) && m_IoInternalDeviceControl.Method) { 3310 3311 pRequest->SetPresented(); 3312 3313 DoTraceLevelMessage( 3314 FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 3315 "Calling driver EvtIoInternalDeviceControl for WDFREQUEST 0x%p", 3316 hRequest); 3317 3318 m_IoInternalDeviceControl.Invoke( 3319 GetHandle(), 3320 hRequest, 3321 pIrp->GetParameterIoctlOutputBufferLength(), 3322 pIrp->GetParameterIoctlInputBufferLength(), 3323 pIrp->GetParameterIoctlCode() 3324 ); 3325 } 3326 else { 3327 3328 // 3329 // If we have an IoStart registered, call it 3330 // 3331 if (m_IoDefault.Method) { 3332 3333 DoTraceLevelMessage( 3334 FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 3335 "Calling driver EvtIoDefault for WDFREQUEST 0x%p", hRequest); 3336 3337 3338 // 3339 // If we don't allow zero length requests, we must dig in whether 3340 // its a read or a write 3341 // 3342 if (!m_AllowZeroLengthRequests) { 3343 3344 if (majorFunction == IRP_MJ_READ) { 3345 3346 if (pIrp->GetParameterReadLength() == 0) { 3347 3348 DoTraceLevelMessage( 3349 FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 3350 "Zero length WDFREQUEST 0x%p completed automatically " 3351 "by WDFQUEUE 0x%p", hRequest, GetObjectHandle()); 3352 3353 pRequest->Complete(STATUS_SUCCESS); 3354 if (FxDriverGlobals->FxVerifierOn) { 3355 // 3356 // Release the reference taken in the call to SetCompletionState 3357 // at the top of the function. 3358 // 3359 pRequest->RELEASE(FXREQUEST_STATE_TAG); 3360 } 3361 return; 3362 } 3363 } 3364 else if (majorFunction == IRP_MJ_WRITE) { 3365 3366 if (pIrp->GetParameterWriteLength() == 0) { 3367 3368 pRequest->Complete(STATUS_SUCCESS); 3369 3370 DoTraceLevelMessage( 3371 FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 3372 "Zero length WDFREQUEST 0x%p completed automatically " 3373 "by WDFQUEUE 0x%p", hRequest, GetObjectHandle()); 3374 3375 if (FxDriverGlobals->FxVerifierOn) { 3376 // 3377 // Release the reference taken in the call to SetCompletionState 3378 // at the top of the function. 3379 // 3380 pRequest->RELEASE(FXREQUEST_STATE_TAG); 3381 } 3382 return; 3383 } 3384 } 3385 } 3386 3387 pRequest->SetPresented(); 3388 3389 m_IoDefault.Invoke(GetHandle(), hRequest); 3390 } 3391 else { 3392 Status = STATUS_INVALID_DEVICE_REQUEST; 3393 3394 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 3395 "Driver has no event callback " 3396 "for %!WDF_REQUEST_TYPE!, completing WDFREQUEST 0x%p with " 3397 "%!STATUS!", 3398 majorFunction, 3399 pRequest, 3400 Status); 3401 3402 pRequest->Complete(Status); 3403 3404 if (FxDriverGlobals->FxVerifierOn) { 3405 // 3406 // Release our extra verifier reference now 3407 // 3408 // Release the reference taken in the call to SetCompletionState 3409 // at the top of the function. 3410 // 3411 pRequest->RELEASE(FXREQUEST_STATE_TAG); 3412 } 3413 3414 return; 3415 } 3416 } 3417 3418 // ****************************** 3419 // Request may now be a freed object unless verifier is on. Only touch 3420 // request if verifier is on. 3421 // ****************************** 3422 3423 if (FxDriverGlobals->FxVerifierOn) { 3424 3425 // 3426 // If the request has been forwarded, don't clear this 3427 // since the new queue may already be dispatching in a new thread or DPC 3428 // 3429 if ((pRequest->GetVerifierFlags() & FXREQUEST_FLAG_FORWARDED) == 0x0) { 3430 pRequest->ClearVerifierFlags(FXREQUEST_FLAG_DRIVER_DISPATCH); 3431 } 3432 3433 // 3434 // Release our extra verifier reference now 3435 // 3436 // Release the reference taken in the call to SetCompletionState 3437 3438 // at the top of the function. 3439 // 3440 pRequest->RELEASE(FXREQUEST_STATE_TAG); 3441 } 3442 3443 // Driver accepted a request 3444 return; 3445 } 3446 3447 3448 // 3449 // Register a callback when the Queue has a request. 3450 // 3451 // Only valid for a manual Queue. 3452 // 3453 _Must_inspect_result_ 3454 NTSTATUS 3455 FxIoQueue::ReadyNotify( 3456 __in PFN_WDF_IO_QUEUE_STATE QueueReady, 3457 __in_opt WDFCONTEXT Context 3458 ) 3459 { 3460 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 3461 KIRQL irql; 3462 NTSTATUS status; 3463 3464 // Only valid for a manually dispatched Queue 3465 if (m_Type != WdfIoQueueDispatchManual) { 3466 status = STATUS_INVALID_DEVICE_REQUEST; 3467 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 3468 "WDFQUEUE 0x%p is " 3469 "not a Manual queue, ReadyNotify is only valid " 3470 "on a manual Queue, %!STATUS!", 3471 GetObjectHandle(), status); 3472 FxVerifierDbgBreakPoint(FxDriverGlobals); 3473 return status; 3474 } 3475 3476 Lock(&irql); 3477 3478 // If the queue is deleted, requests will not be serviced anymore 3479 if (m_Deleted) { 3480 Unlock(irql); 3481 return STATUS_DELETE_PENDING; 3482 } 3483 3484 if (QueueReady != NULL) { 3485 3486 // 3487 // Only one ReadyNotify registration per Queue is allowed 3488 // 3489 if (m_ReadyNotify.Method != NULL) { 3490 status = STATUS_INVALID_DEVICE_REQUEST; 3491 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 3492 "WDFQUEUE 0x%p " 3493 "already has a ReadyNotify callback 0x%p" 3494 "registered, %!STATUS!",GetObjectHandle(), 3495 &m_ReadyNotify, status); 3496 FxVerifierDbgBreakPoint(FxDriverGlobals); 3497 Unlock(irql); 3498 return status; 3499 } 3500 3501 m_ReadyNotify.Method = QueueReady; 3502 m_ReadyNotifyContext = Context; 3503 } 3504 else { 3505 3506 // 3507 // A request to cancel ready notifications 3508 // 3509 3510 // If already cancelled, the driver is confused, notify it 3511 if (m_ReadyNotify.Method == NULL) { 3512 status = STATUS_INVALID_DEVICE_REQUEST; 3513 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 3514 "WDFQUEUE 0x%p " 3515 "does not have a ReadyNotify to cancel, %!STATUS!", 3516 GetObjectHandle(), status); 3517 FxVerifierDbgBreakPoint(FxDriverGlobals); 3518 Unlock(irql); 3519 return status; 3520 } 3521 3522 // 3523 // The queue should be stopped from dispatching requests to 3524 // avoid missing state transistions between clear and set. 3525 // 3526 if(IsState(WdfIoQueueDispatchRequests)) { 3527 status = STATUS_INVALID_DEVICE_REQUEST; 3528 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 3529 "WDFQUEUE 0x%p " 3530 "should be stopped before clearing ReadyNotify callback " 3531 "0x%p registered, %!STATUS!",GetObjectHandle(), 3532 &m_ReadyNotify, status); 3533 FxVerifierDbgBreakPoint(FxDriverGlobals); 3534 Unlock(irql); 3535 return status; 3536 3537 } 3538 3539 m_ReadyNotify.Method = NULL; 3540 m_ReadyNotifyContext = NULL; 3541 } 3542 3543 // 3544 // Check for ready notification since there may already be an event 3545 // 3546 DispatchEvents(irql); 3547 3548 return STATUS_SUCCESS; 3549 } 3550 3551 VOID 3552 FxIoQueue::QueueStart( 3553 ) 3554 { 3555 KIRQL irql; 3556 3557 Lock(&irql); 3558 3559 SetState((FX_IO_QUEUE_SET_STATE)(FxIoQueueSetAcceptRequests | FxIoQueueSetDispatchRequests) ); 3560 3561 // 3562 // We should set the flag to notify the driver on queue start in case 3563 // the driver stops the queue while the ReadyNotify callback is executing. 3564 // If that happens, the request will be left in the manual queue with 3565 // m_TransitionFromEmpty cleared. 3566 // 3567 if (m_Queue.GetRequestCount() > 0L) { 3568 m_TransitionFromEmpty = TRUE; 3569 m_ForceTransitionFromEmptyWhenAddingNewRequest = FALSE; 3570 } 3571 3572 // 3573 // We may have transitioned to a status that resumes 3574 // processing, so call dispatch function. 3575 // 3576 3577 DispatchEvents(irql); 3578 3579 return; 3580 } 3581 3582 _Must_inspect_result_ 3583 NTSTATUS 3584 FxIoQueue::QueueIdle( 3585 __in BOOLEAN CancelRequests, 3586 __in_opt PFN_WDF_IO_QUEUE_STATE IdleComplete, 3587 __in_opt WDFCONTEXT Context 3588 ) 3589 3590 /*++ 3591 3592 Routine Description: 3593 3594 Idle (stop) the Queue. 3595 3596 If CancelRequests == TRUE, 3597 1) any requests in the Queue that have not been presented to the device driver are 3598 completed with STATUS_CANCELLED. 3599 2) any requests that the driver is operating on that are cancelable will have an 3600 I/O Cancel done on them. 3601 3) any forward progress queued IRPs are completed with STATUS_CANCELLED. 3602 3603 Arguments: 3604 3605 Returns: 3606 3607 --*/ 3608 3609 { 3610 PFX_DRIVER_GLOBALS fxDriverGlobals = GetDriverGlobals(); 3611 KIRQL irql; 3612 NTSTATUS status; 3613 LIST_ENTRY fwrIrpList = {0}; 3614 FxRequest* request; 3615 3616 3617 Lock(&irql); 3618 3619 // If the queue is deleted, requests will not be serviced anymore 3620 if (m_Deleted) { 3621 status = STATUS_DELETE_PENDING; 3622 DoTraceLevelMessage(fxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 3623 "WDFQUEUE 0x%p is already deleted, %!STATUS!", 3624 GetObjectHandle(), status); 3625 Unlock(irql); 3626 3627 return status; 3628 } 3629 3630 // 3631 // If a IdleComplete callback is supplied, we must register it up 3632 // front since a transition empty could occur in another thread. 3633 // 3634 if (IdleComplete != NULL) { 3635 3636 // 3637 // Only one Idle or Purge Complete callback can be outstanding 3638 // at a time per Queue 3639 // 3640 if (m_IdleComplete.Method != NULL) { 3641 status = STATUS_INVALID_DEVICE_REQUEST; 3642 DoTraceLevelMessage(fxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 3643 "WDFQUEUE 0x%p already has a " 3644 "IdleComplete callback registered 0x%p, " 3645 "%!STATUS!", GetObjectHandle(), 3646 m_IdleComplete.Method, 3647 status); 3648 Unlock(irql); 3649 3650 return status; 3651 } 3652 3653 m_IdleComplete.Method = IdleComplete; 3654 m_IdleCompleteContext = Context; 3655 } 3656 3657 // Set Accept request and Clear dispatch requests 3658 SetState((FX_IO_QUEUE_SET_STATE)(FxIoQueueSetAcceptRequests | FxIoQueueClearDispatchRequests)); 3659 3660 // 3661 // Get ready to cancel current queued requests. Note that we don't want to 3662 // prevent new requests from being queue, i.e., it is legal for an upper 3663 // driver can resend another request in its completion routine. 3664 // 3665 if (CancelRequests) { 3666 // 3667 // Driver wants to abort/complete all queued request and cancel or 3668 // wait for all requests the driver is currently handling. Thus we must 3669 // prevent the driver from requeuing stale requests. 3670 // The 'cancel driver requests' field gives us this ability. 3671 // It is set here, and cleared when: 3672 // (a) Driver doesn't own any more requests, or 3673 // (b) the driver calls WdfIoQueueStart again (dispatch gate is opened). 3674 // When set, the framework automatically deletes any request that the 3675 // driver requeues. 3676 // 3677 m_CancelDispatchedRequests = TRUE; 3678 3679 request = NULL; // Initial tag used by PeekRequest. 3680 #pragma warning(disable:4127) 3681 while (TRUE) { 3682 #pragma warning(default:4127) 3683 status = FxRequest::PeekRequest(&m_Queue, // in:queue 3684 request, // in:tag. 3685 NULL, // in:file_obj 3686 NULL, // out:parameters 3687 &request); // out:request. 3688 if (status != STATUS_SUCCESS) { 3689 ASSERT(status != STATUS_NOT_FOUND); 3690 break; 3691 } 3692 3693 // 3694 // Tag this request and release the extra ref that Peek() takes. 3695 // 3696 request->m_Canceled = TRUE; 3697 3698 #pragma prefast(suppress:__WARNING_PASSING_FUNCTION_UNEXPECTED_NULL, "This is the tag value used in the ADDREF of Peek()") 3699 request->RELEASE(NULL); 3700 } 3701 3702 // 3703 // Move forward progress IRPs to a temp list; we use this logic to 3704 // allow new IRPs to be pended to the original list. 3705 // 3706 if (IsForwardProgressQueue()) { 3707 InitializeListHead(&fwrIrpList); 3708 GetForwardProgressIrps(&fwrIrpList, NULL); 3709 } 3710 } 3711 3712 // Unlock queue lock 3713 Unlock(irql); 3714 3715 if (CancelRequests) { 3716 #pragma warning(disable:4127) 3717 while (TRUE) { 3718 #pragma warning(default:4127) 3719 // 3720 // Get the next FxRequest from the cancel safe queue 3721 // 3722 Lock(&irql); 3723 request = FxRequest::GetNextRequest(&m_Queue); 3724 if (request == NULL) { 3725 DoTraceLevelMessage( 3726 fxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 3727 "All WDFQUEUE 0x%p requests cancelled", 3728 GetObjectHandle()); 3729 Unlock(irql); 3730 break; 3731 } 3732 3733 // Irp is not cancellable now 3734 3735 // 3736 // Make sure to purged requests only if: 3737 // (a) the request was present when we started this operation. 3738 // (b) any following request that is marked as cancelled. 3739 // 3740 if (request->IsCancelled() == FALSE) { 3741 status = request->InsertHeadIrpQueue(&m_Queue, NULL); 3742 if (NT_SUCCESS(status)) { 3743 Unlock(irql); 3744 break; 3745 } 3746 3747 ASSERT(status == STATUS_CANCELLED); 3748 } 3749 DoTraceLevelMessage( 3750 fxDriverGlobals, TRACE_LEVEL_INFORMATION, TRACINGIO, 3751 "Cancelling WDFREQUEST 0x%p, WDFQUEUE 0x%p", 3752 request->GetHandle(),GetObjectHandle()); 3753 3754 // 3755 // We must add a reference since the CancelForQueue path 3756 // assumes we were on the FxIrpQueue with the extra reference 3757 // 3758 request->ADDREF(FXREQUEST_QUEUE_TAG); 3759 3760 // 3761 // Mark the request as cancelled, place it on the cancel list, 3762 // and schedule the cancel event to the driver 3763 // 3764 CancelForQueue(request, irql); 3765 } 3766 3767 // 3768 // Walk the driver cancelable list cancelling the requests. 3769 // 3770 #pragma warning(disable:4127) 3771 while (TRUE) { 3772 #pragma warning(default:4127) 3773 // 3774 // Get the next request of driver cancelable requests 3775 // 3776 Lock(&irql); 3777 request = FxRequest::GetNextRequest(&m_DriverCancelable); 3778 if (request == NULL) { 3779 DoTraceLevelMessage( 3780 fxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 3781 "All driver cancellable requests cancelled " 3782 " in WDFQUEUE 0x%p", 3783 GetObjectHandle()); 3784 Unlock(irql); 3785 break; 3786 } 3787 3788 request->m_Canceled = TRUE; 3789 3790 Unlock(irql); 3791 3792 // 3793 // If the driver follows the pattern of removing cancel status 3794 // from the request before completion, then there is no race 3795 // with this routine since we will not be able to retrieve any 3796 // requests the driver has made non-cancellable in preparation 3797 // for completion. 3798 // 3799 request->ADDREF(FXREQUEST_QUEUE_TAG); 3800 3801 CancelForDriver(request); 3802 3803 // The request could have been completed and released by the driver 3804 } 3805 3806 // 3807 // Cleanup forward progress IRP list. 3808 // 3809 if (IsForwardProgressQueue()) { 3810 CancelIrps(&fwrIrpList); 3811 } 3812 } 3813 3814 // 3815 // Since we set that no new requests may be dispatched, 3816 // if both m_Queue.GetRequestCount(), m_DriverIoCount == 0, and 3817 // m_Dispatch == 0, right now the queue is completely idle. 3818 // 3819 3820 // 3821 // We check if our m_PurgeComplete callback is still set 3822 // since it may have been called by another thread when 3823 // we dropped the lock above 3824 // 3825 Lock(&irql); 3826 DispatchEvents(irql); 3827 3828 // 3829 // If the driver registered an IdleComplete callback, and it was 3830 // not idle in the above check, it will be called when the final 3831 // callback handler from the device driver returns. 3832 // 3833 return STATUS_SUCCESS; 3834 } 3835 3836 _Must_inspect_result_ 3837 NTSTATUS 3838 FxIoQueue::QueueIdleSynchronously( 3839 __in BOOLEAN CancelRequests 3840 ) 3841 /*++ 3842 3843 Routine Description: 3844 3845 Idle the Queue and wait for the driver-owned requests to complete. 3846 3847 Arguments: 3848 3849 CancelRequests - If TRUE, functions tries to cancel outstanding requests. 3850 3851 Returns: 3852 3853 --*/ 3854 { 3855 NTSTATUS status; 3856 #if (FX_CORE_MODE==FX_CORE_USER_MODE) 3857 MxEvent* event = this->m_RequestWaitEventUm.GetSelfPointer(); 3858 #else 3859 MxEvent eventOnStack; 3860 // 3861 // Note that initialize always succeeds in KM so return is not checked. 3862 // 3863 eventOnStack.Initialize(NotificationEvent, FALSE); 3864 MxEvent* event = eventOnStack.GetSelfPointer(); 3865 #endif 3866 3867 status = QueueIdle(CancelRequests, _IdleComplete, event->GetSelfPointer()); 3868 3869 if(NT_SUCCESS(status)) { 3870 3871 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGIO, 3872 "Waiting for %d requests to complete " 3873 "on WDFQUEUE 0x%p", 3874 m_DriverIoCount, 3875 GetObjectHandle()); 3876 3877 Mx::MxEnterCriticalRegion(); 3878 3879 GetDriverGlobals()->WaitForSignal(event->GetSelfPointer(), 3880 "waiting for queue to stop, WDFQUEUE", GetHandle(), 3881 GetDriverGlobals()->FxVerifierDbgWaitForSignalTimeoutInSec, 3882 WaitSignalBreakUnderVerifier); 3883 3884 3885 Mx::MxLeaveCriticalRegion(); 3886 } 3887 3888 return status; 3889 3890 } 3891 3892 _Must_inspect_result_ 3893 NTSTATUS 3894 FxIoQueue::QueuePurge( 3895 __in BOOLEAN CancelQueueRequests, 3896 __in BOOLEAN CancelDriverRequests, 3897 __in_opt PFN_WDF_IO_QUEUE_STATE PurgeComplete, 3898 __in_opt WDFCONTEXT Context 3899 ) 3900 /*++ 3901 3902 Routine Description: 3903 3904 Purge the Queue. 3905 3906 If CancelQueueRequests == TRUE, any requests in the 3907 Queue that have not been presented to the device driver are 3908 completed with STATUS_CANCELLED. 3909 3910 If CancelDriverRequests == TRUE, any requests that the 3911 driver is operating on that are cancelable will have an 3912 I/O Cancel done on them. 3913 3914 Arguments: 3915 3916 Returns: 3917 3918 --*/ 3919 { 3920 FxRequest* pRequest; 3921 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 3922 KIRQL irql; 3923 NTSTATUS status; 3924 3925 Lock(&irql); 3926 3927 // 3928 // If the Queue is deleted, there can't be any requests 3929 // to purge, and the queue is no longer executing its 3930 // event dispatch loop, so we would stop responding if we 3931 // registered now. 3932 // 3933 // We could try and silently succeed this, but if we do, we 3934 // must invoke the PurgeComplete callback, and without our 3935 // queue state machine excuting, we can not ensure any 3936 // callback constraints are handled such as locking, queueing 3937 // to passive level, etc. So we just fail to indicate to the 3938 // driver we *will not* be invoking its PurgeComplete function. 3939 // 3940 if (m_Deleted) { 3941 status = STATUS_DELETE_PENDING; 3942 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 3943 "WDFQUEUE 0x%p is already deleted %!STATUS!", 3944 GetObjectHandle(), status); 3945 Unlock(irql); 3946 3947 return status; 3948 } 3949 3950 // 3951 // If a PurgeComplete callback is supplied, we must register it up 3952 // front since a transition empty could occur in another thread. 3953 // 3954 if (PurgeComplete != NULL) { 3955 3956 // 3957 // Only one PurgeComplete callback can be outstanding 3958 // at a time per Queue 3959 // 3960 if (m_PurgeComplete.Method != NULL) { 3961 status = STATUS_INVALID_DEVICE_REQUEST; 3962 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 3963 "WDFQUEUE 0x%p already has a " 3964 "PurgeComplete callback registered 0x%p " 3965 "%!STATUS!", GetObjectHandle(), 3966 m_PurgeComplete.Method, status); 3967 Unlock(irql); 3968 3969 return status; 3970 } 3971 3972 m_PurgeComplete.Method = PurgeComplete; 3973 m_PurgeCompleteContext = Context; 3974 } 3975 3976 // Clear accept requests 3977 SetState(FxIoQueueClearAcceptRequests); 3978 3979 if (CancelQueueRequests && CancelDriverRequests && 3980 FxDriverGlobals->IsVersionGreaterThanOrEqualTo(1,11)) { 3981 // 3982 // Driver wants to abort/complete all queued request and cancel or 3983 // wait for all requests the driver is currently handling. Thus we must 3984 // prevent the driver from requeuing stale requests. 3985 // This flag is set here, and cleared when: 3986 // (a) Driver doesn't own any more requests, or 3987 // (b) the driver calls WdfIoQueueStart again (dispatch gate is opened). 3988 // When set, the framework automatically deletes any request that the 3989 // driver requeues. 3990 // For compatibility we do this only for drivers v1.11 and above. 3991 // 3992 m_CancelDispatchedRequests = TRUE; 3993 } 3994 3995 // Unlock queue lock 3996 Unlock(irql); 3997 3998 if (CancelQueueRequests) { 3999 #pragma warning(disable:4127) 4000 while (TRUE) { 4001 #pragma warning(default:4127) 4002 // 4003 // Get the next FxRequest from the cancel safe queue 4004 // 4005 Lock(&irql); 4006 pRequest = FxRequest::GetNextRequest(&m_Queue); 4007 if (pRequest == NULL) { 4008 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 4009 "All WDFQUEUE 0x%p requests cancelled", 4010 GetObjectHandle()); 4011 Unlock(irql); 4012 break; 4013 } 4014 4015 // Irp is not cancellable now 4016 4017 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_INFORMATION, TRACINGIO, 4018 "Cancelling WDFREQUEST 0x%p, WDFQUEUE 0x%p", 4019 pRequest->GetHandle(),GetObjectHandle()); 4020 4021 // 4022 // We must add a reference since the CancelForQueue path 4023 // assumes we were on the FxIrpQueue with the extra reference 4024 pRequest->ADDREF(FXREQUEST_QUEUE_TAG); 4025 4026 // 4027 // Mark the request as cancelled, place it on the cancel list, 4028 // and schedule the cancel event to the driver 4029 // 4030 CancelForQueue(pRequest, irql); 4031 4032 } 4033 } 4034 4035 if (CancelDriverRequests) { 4036 4037 // 4038 // Walk the driver cancelable list cancelling 4039 // the requests. 4040 // 4041 #pragma warning(disable:4127) 4042 while (TRUE) { 4043 #pragma warning(default:4127) 4044 // 4045 // Get the next request of driver cancelable requests 4046 // 4047 Lock(&irql); 4048 pRequest = FxRequest::GetNextRequest(&m_DriverCancelable); 4049 if (pRequest == NULL) { 4050 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 4051 "All driver cancellable requests cancelled " 4052 " in WDFQUEUE 0x%p", 4053 GetObjectHandle()); 4054 Unlock(irql); 4055 break; 4056 } 4057 4058 pRequest->m_Canceled = TRUE; 4059 4060 Unlock(irql); 4061 4062 // 4063 // If the driver follows the pattern of removing cancel status 4064 // from the request before completion, then there is no race 4065 // with this routine since we will not be able to retrieve any 4066 // requests the driver has made non-cancellable in preparation 4067 // for completion. 4068 // 4069 pRequest->ADDREF(FXREQUEST_QUEUE_TAG); 4070 4071 CancelForDriver(pRequest); 4072 4073 // The request could have been completed and released by the driver 4074 } 4075 } 4076 4077 if (IsForwardProgressQueue()) { 4078 PurgeForwardProgressIrps(NULL); 4079 } 4080 4081 // 4082 // Since we set that no new requests may be enqueued, 4083 // if both m_Queue.GetRequestCount() and m_DriverIoCount == 0 right 4084 // now the queue is completely purged. 4085 // 4086 4087 // 4088 // We check if our m_PurgeComplete callback is still set 4089 // since it may have been called by another thread when 4090 // we dropped the lock above 4091 // 4092 Lock(&irql); 4093 4094 DispatchEvents(irql); 4095 4096 // 4097 // If the driver registered a PurgeComplete callback, and it was 4098 // not empty in the above check, it will be called when a 4099 // request complete from the device driver completes the 4100 // final request. 4101 // 4102 return STATUS_SUCCESS; 4103 } 4104 4105 _Must_inspect_result_ 4106 NTSTATUS 4107 FxIoQueue::QueuePurgeSynchronously( 4108 ) 4109 /*++ 4110 4111 Routine Description: 4112 4113 Purge the queue and wait for it to complete. 4114 When this call returns, there are no requests in the queue or device 4115 driver and the queue state is set to reject new requests. 4116 4117 --*/ 4118 { 4119 NTSTATUS status; 4120 4121 #if (FX_CORE_MODE==FX_CORE_USER_MODE) 4122 MxEvent* event = this->m_RequestWaitEventUm.GetSelfPointer(); 4123 #else 4124 MxEvent eventOnStack; 4125 // 4126 // Note that initialize always succeeds in KM so return is not checked. 4127 // 4128 eventOnStack.Initialize(NotificationEvent, FALSE); 4129 MxEvent* event = eventOnStack.GetSelfPointer(); 4130 #endif 4131 4132 status = QueuePurge(TRUE, TRUE, _PurgeComplete, event->GetSelfPointer()); 4133 4134 if(NT_SUCCESS(status)) { 4135 4136 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGIO, 4137 "Waiting for %d requests to complete " 4138 "on WDFQUEUE 0x%p", 4139 (m_DriverIoCount + m_Queue.GetRequestCount()), 4140 GetObjectHandle()); 4141 4142 Mx::MxEnterCriticalRegion(); 4143 4144 GetDriverGlobals()->WaitForSignal(event->GetSelfPointer(), 4145 "waiting for queue to purge, WDFQUEUE", GetHandle(), 4146 GetDriverGlobals()->FxVerifierDbgWaitForSignalTimeoutInSec, 4147 WaitSignalBreakUnderVerifier); 4148 4149 Mx::MxLeaveCriticalRegion(); 4150 } 4151 4152 return status; 4153 4154 } 4155 4156 _Must_inspect_result_ 4157 NTSTATUS 4158 FxIoQueue::QueueDrain( 4159 __in_opt PFN_WDF_IO_QUEUE_STATE PurgeComplete, 4160 __in_opt WDFCONTEXT Context 4161 ) 4162 { 4163 // 4164 // We drain the queue by calling QueuePurge with CancelQueueRequests 4165 // and CancelDriverRequests == FALSE. The Queue will reject new 4166 // requests, but allow the device driver to continue processing 4167 // requests currently on the Queue. The DrainComplete callback is 4168 // invoked when there are no requests in Queue or device driver. 4169 // 4170 4171 return QueuePurge(FALSE, FALSE, PurgeComplete, Context); 4172 4173 } 4174 4175 _Must_inspect_result_ 4176 NTSTATUS 4177 FxIoQueue::QueueDrainSynchronously( 4178 ) 4179 /*++ 4180 4181 Routine Description: 4182 4183 Drain the queue and wait for it to complete. 4184 When this call returns, there are no requests in the queue or device 4185 driver and the queue state is set to reject new requests. 4186 4187 --*/ 4188 { 4189 NTSTATUS status; 4190 #if (FX_CORE_MODE==FX_CORE_USER_MODE) 4191 MxEvent* event = this->m_RequestWaitEventUm.GetSelfPointer(); 4192 #else 4193 MxEvent eventOnStack; 4194 // 4195 // Note that initialize always succeeds in KM so return is not checked. 4196 // 4197 eventOnStack.Initialize(NotificationEvent, FALSE); 4198 MxEvent* event = eventOnStack.GetSelfPointer(); 4199 #endif 4200 4201 status = QueueDrain(_PurgeComplete, event->GetSelfPointer()); 4202 4203 if(NT_SUCCESS(status)) { 4204 4205 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGIO, 4206 "Waiting for %d requests to complete " 4207 "on WDFQUEUE 0x%p", 4208 (m_DriverIoCount + m_Queue.GetRequestCount()), 4209 GetObjectHandle()); 4210 4211 Mx::MxEnterCriticalRegion(); 4212 4213 GetDriverGlobals()->WaitForSignal(event->GetSelfPointer(), 4214 "waiting for queue to drain, WDFQUEUE", GetHandle(), 4215 GetDriverGlobals()->FxVerifierDbgWaitForSignalTimeoutInSec, 4216 WaitSignalBreakUnderVerifier); 4217 4218 Mx::MxLeaveCriticalRegion(); 4219 } 4220 4221 return status; 4222 4223 } 4224 4225 4226 VOID 4227 FxIoQueue::GetRequestCount( 4228 __out_opt PULONG pQueuedRequests, 4229 __out_opt PULONG pDriverPendingRequests 4230 ) 4231 /*++ 4232 4233 Routine Description: 4234 4235 Return the count of requests currently on the queue 4236 and owned by the driver. 4237 4238 Arguments: 4239 4240 Returns: 4241 4242 --*/ 4243 { 4244 if (pQueuedRequests != NULL) { 4245 *pQueuedRequests = m_Queue.GetRequestCount(); 4246 } 4247 4248 if (pDriverPendingRequests != NULL) { 4249 *pDriverPendingRequests = m_DriverIoCount; 4250 } 4251 4252 return; 4253 } 4254 4255 VOID 4256 FxIoQueue::FlushByFileObject( 4257 __in MdFileObject FileObject 4258 ) 4259 /*++ 4260 4261 Routine Description: 4262 4263 Scan the queue and cancel all the requests that have 4264 the same fileobject as the input argument. 4265 4266 This function is called when the IoPkg receives a 4267 IRP_MJ_CLEANUP requests. 4268 4269 Additional reference is already taken on the object by the caller 4270 to prevent the queue from being deleted. 4271 4272 Return Value: 4273 4274 --*/ 4275 { 4276 FxRequest* pRequest = NULL; 4277 NTSTATUS status; 4278 KIRQL irql; 4279 4280 if (IsForwardProgressQueue()) { 4281 PurgeForwardProgressIrps(FileObject); 4282 } 4283 4284 Lock(&irql); 4285 4286 #pragma warning(disable:4127) 4287 while (TRUE) { 4288 #pragma warning(default:4127) 4289 4290 // 4291 // Get the next FxRequest from the cancel safe queue 4292 // 4293 status = FxRequest::GetNextRequest(&m_Queue, FileObject, NULL, &pRequest); 4294 if(status == STATUS_NO_MORE_ENTRIES) { 4295 break; 4296 } 4297 if(!NT_SUCCESS(status)) { 4298 ASSERTMSG("GetNextRequest failed\n", FALSE); 4299 break; 4300 } 4301 4302 // 4303 // We must add a reference since the CancelForQueue path 4304 // assumes we were on the FxIrpQueue with the extra reference 4305 // 4306 pRequest->ADDREF(FXREQUEST_QUEUE_TAG); 4307 4308 // 4309 // Mark the request as cancelled, place it on the cancel list, 4310 // and schedule the cancel event to the driver 4311 // 4312 CancelForQueue(pRequest, irql); 4313 4314 // 4315 // Reacquire the lock because CancelForQueue visits the dispatch-loop 4316 // and releases the lock. 4317 // 4318 Lock(&irql); 4319 } 4320 4321 DispatchEvents(irql); 4322 4323 return; 4324 4325 } 4326 4327 _Releases_lock_(this->m_SpinLock.m_Lock) 4328 VOID 4329 FxIoQueue::CancelForQueue( 4330 __in FxRequest* pRequest, 4331 __in __drv_restoresIRQL KIRQL PreviousIrql 4332 ) 4333 /*++ 4334 4335 Routine Description: 4336 4337 This routine performs the actions when notified of a cancel 4338 on a request that has not been presented to the driver 4339 4340 Return Value: 4341 4342 NTSTATUS 4343 4344 --*/ 4345 { 4346 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 4347 FxRequestCompletionState oldState; 4348 4349 // This is not an error, but want to make sure cancel testing works 4350 if (FxDriverGlobals->FxVerifierOn) { 4351 4352 // Clear cancellable status, otherwise verifier in FxRequest::Complete will complain 4353 pRequest->ClearVerifierFlags(FXREQUEST_FLAG_DRIVER_CANCELABLE); 4354 4355 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_WARNING, TRACINGIO, 4356 "WDFREQUEST 0x%p " 4357 "was cancelled while on WDFQUEUE 0x%p", 4358 pRequest->GetHandle(),GetObjectHandle()); 4359 } 4360 4361 pRequest->m_Canceled = TRUE; 4362 4363 pRequest->MarkRemovedFromIrpQueue(); 4364 4365 // 4366 // Drop the extra reference taken when it was added to the queue 4367 // because the request is now leaving the queue. 4368 // 4369 pRequest->RELEASE(FXREQUEST_QUEUE_TAG); 4370 4371 // 4372 // If the driver has registered m_CanceledOnQueue callback, and if 4373 // the request was ever presented to the driver then we need to 4374 // notify the driver 4375 // 4376 if(m_IoCanceledOnQueue.Method && pRequest->m_Presented) { 4377 4378 // 4379 // Set the state to indicate the request has come from a queue. 4380 // 4381 oldState = pRequest->SetCompletionState(FxRequestCompletionStateQueue); 4382 ASSERT(oldState == FxRequestCompletionStateNone); 4383 UNREFERENCED_PARAMETER(oldState); 4384 4385 // Insert it on the driver owned list 4386 InsertInDriverOwnedList(pRequest); 4387 4388 if (FxDriverGlobals->FxVerifierOn) { 4389 ASSERT((pRequest->GetVerifierFlags() & FXREQUEST_FLAG_DRIVER_OWNED) == 0); 4390 pRequest->SetVerifierFlags(FXREQUEST_FLAG_DRIVER_OWNED); 4391 } 4392 4393 // 4394 // Also insert the request in to m_CanceledOnQueueList so 4395 // that we can notify the driver when we visit the DispatchEvents 4396 // 4397 InsertTailList(&m_CanceledOnQueueList, pRequest->GetListEntry(FxListEntryQueueOwned)); 4398 4399 // 4400 // Release the reference taken in the call to SetCompletionState. 4401 // 4402 pRequest->RELEASE(FXREQUEST_STATE_TAG); 4403 } else { 4404 4405 Unlock(PreviousIrql); 4406 4407 // Its gone from our list, so complete it cancelled 4408 pRequest->CompleteWithInformation(STATUS_CANCELLED, 0); 4409 4410 // Dereference the request objects final reference 4411 pRequest->RELEASE(FXREQUEST_COMPLETE_TAG); 4412 4413 Lock(&PreviousIrql); 4414 } 4415 4416 // This may have caused the queue to be emptied 4417 DispatchInternalEvents(PreviousIrql); 4418 4419 return; 4420 } 4421 4422 VOID 4423 FxIoQueue::_IrpCancelForQueue( 4424 __in FxIrpQueue* IrpQueue, 4425 __in MdIrp Irp, 4426 __in PMdIoCsqIrpContext CsqContext, 4427 __in KIRQL Irql 4428 ) 4429 /*++ 4430 4431 Routine Description: 4432 This is our Cancel Safe Queue Callback from FxIrpQueue notifying us of an 4433 I/O cancellation on the main (pre driver) Queue. 4434 4435 Note this callback is called with the queue lock held. 4436 4437 Arguments: 4438 IrpQueue - Queue the request was on 4439 4440 Irp - the irp being cancelled 4441 4442 CsqContext - the context associated with the irp 4443 4444 Return Value: 4445 None 4446 4447 --*/ 4448 { 4449 FxIoQueue* ioQueue; 4450 FxRequest* pRequest; 4451 4452 ioQueue = CONTAINING_RECORD(IrpQueue, FxIoQueue, m_Queue); 4453 pRequest = FxRequest::RetrieveFromCsqContext(CsqContext); 4454 4455 // 4456 // Must reference the queue since this could be the final 4457 // request on a deleting queue 4458 // 4459 ioQueue->ADDREF(Irp); 4460 4461 // 4462 // We cannot drop the lock here because we may have to insert the 4463 // request in the driver owned list if the driver has registered 4464 // for canceled-on-queue callback. If we drop the lock and if the request 4465 // happens to be last request, the delete will run thru and put 4466 // the state of the queue to deleted state and prevent further dispatching 4467 // of requests. 4468 // 4469 ioQueue->CancelForQueue(pRequest, Irql); 4470 4471 ioQueue->RELEASE(Irp); 4472 } 4473 4474 VOID 4475 FX_VF_METHOD(FxIoQueue, VerifyValidateCompletedRequest)( 4476 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals, 4477 _In_ FxRequest* Request 4478 ) 4479 { 4480 UNREFERENCED_PARAMETER(FxDriverGlobals); 4481 4482 PAGED_CODE_LOCKED(); 4483 4484 PLIST_ENTRY pEntry; 4485 KIRQL irql; 4486 4487 Request->Lock(&irql); 4488 4489 (VOID) Request->VerifyRequestIsDriverOwned(FxDriverGlobals); 4490 Request->ClearVerifierFlagsLocked(FXREQUEST_FLAG_DRIVER_OWNED); 4491 4492 Request->Unlock(irql); 4493 4494 // Driver no longer owns it once completed 4495 4496 // Request can't be on a cancel list 4497 pEntry = Request->GetListEntry(FxListEntryQueueOwned); 4498 if( !IsListEmpty(pEntry) ) { 4499 DoTraceLevelMessage( 4500 GetDriverGlobals(), TRACE_LEVEL_ERROR, TRACINGIO, 4501 "WDFREQUEST 0x%p is on a cancellation list for WDFQUEUE 0x%p", 4502 Request->GetHandle(), GetObjectHandle()); 4503 FxVerifierDbgBreakPoint(GetDriverGlobals()); 4504 } 4505 } 4506 4507 VOID 4508 FX_VF_METHOD(FxIoQueue, VerifyCancelForDriver) ( 4509 _In_ PFX_DRIVER_GLOBALS FxDriverGlobals, 4510 _In_ FxRequest* Request 4511 ) 4512 { 4513 PLIST_ENTRY pEntry; 4514 4515 PAGED_CODE_LOCKED(); 4516 4517 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_WARNING, TRACINGIO, 4518 "WDFREQUEST 0x%p " 4519 "was cancelled in driver for WDFQUEUE 0x%p", 4520 Request->GetHandle(), GetObjectHandle()); 4521 4522 // Verifier code assures this is available to the cancel processing 4523 pEntry = Request->GetListEntry(FxListEntryQueueOwned); 4524 if (!IsListEmpty(pEntry)) { 4525 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 4526 "WDFREQUEST 0x%p is " 4527 "already on list, FxRequest::m_ListEntry is busy!, " 4528 "WDFQUEUE 0x%p", 4529 Request->GetHandle(), GetObjectHandle()); 4530 FxVerifierDbgBreakPoint(FxDriverGlobals); 4531 } 4532 } 4533 4534 VOID 4535 FxIoQueue::CancelForDriver( 4536 __in FxRequest* pRequest 4537 ) 4538 /*++ 4539 4540 Routine Description: 4541 4542 This is called when a driver-owned cancelable request is canceled. 4543 This routine will add the request to m_Canceled list so that the 4544 dispatcher and call the driver cancel-routine to notify the driver. 4545 4546 Queue lock is not held. 4547 4548 Arguments: 4549 4550 pRequest - is a driver owned cancelable request. 4551 4552 Return Value: 4553 4554 --*/ 4555 { 4556 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 4557 KIRQL irql; 4558 4559 // This is not an error, but want to make sure cancel testing works 4560 VerifyCancelForDriver(FxDriverGlobals, pRequest); 4561 4562 // 4563 // We are called with no locks held, but 4564 // can be in arbitrary thread context from 4565 // a cancel occuring from another driver within 4566 // a driver stack. 4567 // 4568 4569 // 4570 // The Csq has removed this request from the driver pending 4571 // queue, and it no longer has a cancel function if the 4572 // driver does not accept the cancel right now. 4573 // 4574 // When callside eventually goes to remove it from the queue 4575 // by CsqContext, the Csq's will return NULL. 4576 // 4577 // Irp and FxRequest is still valid until the driver calls 4578 // WdfRequestComplete either as a result of this cancel 4579 // callback, or at its leasure if it chooses to ignore it. 4580 // 4581 // The insert of FxRequest onto the FxIrpQueue took out a 4582 // reference, and when an IRP gets cancelled, we are responsible 4583 // for this final dereference after calling into the driver. 4584 // 4585 4586 // 4587 // Cancellations are dispatched as events to the device driver 4588 // using the standard DispatchEvents processing loop. In order 4589 // to support this, we must defer it by linking this request 4590 // into a list of cancelled requests. 4591 // 4592 // The requests will be removed from this list and cancel notified 4593 // to the device driver by the processing loop. 4594 // 4595 4596 pRequest->MarkRemovedFromIrpQueue(); 4597 4598 // 4599 // Queue it on the cancelled list 4600 // 4601 Lock(&irql); 4602 4603 InsertTailList(&m_Cancelled, pRequest->GetListEntry(FxListEntryQueueOwned)); 4604 4605 // 4606 // Visit the event dispatcher 4607 // 4608 DispatchInternalEvents(irql); 4609 4610 return; 4611 } 4612 4613 VOID 4614 FxIoQueue::_IrpCancelForDriver( 4615 __in FxIrpQueue* IrpQueue, 4616 __in MdIrp Irp, 4617 __in PMdIoCsqIrpContext CsqContext, 4618 __in KIRQL Irql 4619 ) 4620 /*++ 4621 4622 Routine Description: 4623 This is our Cancel Safe Queue Callback from FxIrpQueue notifying us of an 4624 I/O cancellation on a driver owned request (driver queue) 4625 4626 Note this callback is called with the queue lock held. 4627 4628 Arguments: 4629 IrpQueue - Queue the request was on 4630 4631 Irp - the irp being cancelled 4632 4633 CsqContext - the context associated with the irp 4634 4635 4636 Return Value: 4637 None 4638 4639 --*/ 4640 { 4641 FxIoQueue* ioQueue; 4642 FxRequest* pRequest; 4643 4644 ioQueue = CONTAINING_RECORD(IrpQueue, FxIoQueue, m_DriverCancelable); 4645 pRequest = FxRequest::RetrieveFromCsqContext(CsqContext); 4646 4647 pRequest->m_Canceled = TRUE; 4648 4649 // 4650 // Must reference the queue since this could be the final 4651 // request on a deleting queue. 4652 // 4653 ioQueue->ADDREF(Irp); 4654 4655 // 4656 // We can drop the lock because this request is a driver owned request and 4657 // it is tracked by m_DriverIoCount. As a result delete will be blocked 4658 // until the request is completed. 4659 // 4660 ioQueue->Unlock(Irql); 4661 4662 ioQueue->CancelForDriver(pRequest); 4663 4664 ioQueue->RELEASE(Irp); 4665 } 4666 4667 __drv_requiresIRQL(DISPATCH_LEVEL) 4668 VOID 4669 FxIoQueue::ProcessIdleComplete( 4670 __out PKIRQL PreviousIrql 4671 ) 4672 /*++ 4673 4674 Routine Description: 4675 4676 Handle IdleComplete. 4677 Calls back the driver if conditions are met. 4678 Called with Queue lock held, but can drop and re-acquire 4679 it when delivering the event callback to the device driver. 4680 4681 Arguments: 4682 IrpQueue - Queue the request was on 4683 4684 Irp - the irp being cancelled 4685 4686 CsqContext - the context associated with the irp 4687 4688 Return Value: 4689 None 4690 4691 --*/ 4692 { 4693 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 4694 WDFCONTEXT ctx; 4695 FxIoQueueIoState callback; 4696 4697 4698 callback = m_IdleComplete; 4699 ctx = m_IdleCompleteContext; 4700 4701 m_IdleComplete.Method = NULL; 4702 m_IdleCompleteContext = NULL; 4703 4704 Unlock(*PreviousIrql); 4705 4706 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 4707 "WDFQUEUE 0x%p is idle, calling driver callback", 4708 GetHandle()); 4709 4710 // Notify driver by callback 4711 if (callback.Method != NULL) { 4712 callback.Invoke(GetHandle(), ctx); 4713 } 4714 4715 Lock(PreviousIrql); 4716 4717 return; 4718 } 4719 4720 __drv_requiresIRQL(DISPATCH_LEVEL) 4721 VOID 4722 FxIoQueue::ProcessPurgeComplete( 4723 __out PKIRQL PreviousIrql 4724 ) 4725 /*++ 4726 4727 Routine Description: 4728 4729 4730 Handle PurgeComplete. 4731 4732 Calls back the driver if conditions are met. 4733 4734 Called with Queue lock held, but can drop and re-acquire 4735 it when delivering the event callback to the device driver. 4736 4737 4738 Arguments: 4739 4740 Return Value: 4741 None 4742 4743 --*/ 4744 { 4745 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 4746 WDFCONTEXT ctx; 4747 FxIoQueueIoState callback; 4748 4749 callback = m_PurgeComplete; 4750 ctx = m_PurgeCompleteContext; 4751 4752 m_PurgeComplete.Method = NULL; 4753 m_PurgeCompleteContext = NULL; 4754 4755 Unlock(*PreviousIrql); 4756 4757 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 4758 "WDFQUEUE 0x%p is purged, calling driver callback", 4759 GetObjectHandle()); 4760 4761 // Notify driver by callback 4762 if (callback.Method != NULL) { 4763 callback.Invoke(GetHandle(), ctx); 4764 } 4765 4766 Lock(PreviousIrql); 4767 4768 return; 4769 } 4770 4771 __drv_requiresIRQL(DISPATCH_LEVEL) 4772 VOID 4773 FxIoQueue::ProcessReadyNotify( 4774 __out PKIRQL PreviousIrql 4775 ) 4776 /*++ 4777 4778 Routine Description: 4779 4780 Callback the driver for a Queue ready notify 4781 Called with the Queue lock held, and may release 4782 and re-acquire it in calling back the driver 4783 4784 Arguments: 4785 4786 Return Value: 4787 None 4788 4789 --*/ 4790 { 4791 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 4792 WDFCONTEXT ctx; 4793 FxIoQueueIoState callback; 4794 4795 // 4796 // A callback to the driver "consumes" the notification. 4797 // Since we drop the lock when we call the driver, there is 4798 // a chance for the queue to be stopped or powered-off before 4799 // the driver tries to retrieve the request. So make sure 4800 // to set this flag when you power-on or start the queue to 4801 // avoid abandoning the requests in the queue. 4802 // 4803 m_TransitionFromEmpty = FALSE; 4804 4805 // 4806 // Save a local copy since another thread could 4807 // cancel the ready notification out from under us 4808 // when we drop the lock 4809 // 4810 callback = m_ReadyNotify; 4811 4812 ctx = m_ReadyNotifyContext; 4813 4814 Unlock(*PreviousIrql); 4815 4816 if (callback.Method != NULL) { 4817 callback.Invoke(GetHandle(), ctx); 4818 } 4819 else { 4820 if (FxDriverGlobals->FxVerifierOn) { 4821 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 4822 "ReadyNotify notify method is NULL " 4823 "on WDFQUEUE 0x%p", GetObjectHandle()); 4824 FxVerifierDbgBreakPoint(FxDriverGlobals); 4825 } 4826 } 4827 4828 Lock(PreviousIrql); 4829 4830 return; 4831 } 4832 4833 __drv_requiresIRQL(DISPATCH_LEVEL) 4834 BOOLEAN 4835 FxIoQueue::ProcessCancelledRequests( 4836 __out PKIRQL PreviousIrql 4837 ) 4838 /*++ 4839 4840 Routine Description: 4841 4842 Process any cancelled requests 4843 Called with the Queue lock held 4844 Can drop and re-acquire the queue lock 4845 Returns with the Queue lock held 4846 4847 Arguments: 4848 4849 Return Value: 4850 None 4851 4852 --*/ 4853 { 4854 PLIST_ENTRY pEntry; 4855 FxRequest* pRequest; 4856 4857 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 4858 4859 if (IsPowerStateNotifyingDriver()) { 4860 // 4861 // We will not process cancelled request while the driver is being 4862 // notified to stop processing request to avoid double completion 4863 // of the request. 4864 // 4865 return FALSE; 4866 } 4867 4868 while (!IsListEmpty(&m_Cancelled)) { 4869 pEntry = m_Cancelled.Flink; 4870 4871 RemoveEntryList(pEntry); 4872 4873 // FxRequest ensures its not on any list on checked builds 4874 InitializeListHead(pEntry); 4875 4876 pRequest = FxRequest::_FromOwnerListEntry(FxListEntryQueueOwned, pEntry); 4877 4878 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_INFORMATION, TRACINGIO, 4879 "Calling CancelRoutine routine " 4880 "for WDFREQUEST 0x%p on WDFQUEUE 0x%p", 4881 pRequest->GetHandle(), GetObjectHandle()); 4882 4883 if (FxDriverGlobals->FxVerifierOn) { 4884 4885 // Set cancelled status, otherwise verifier in FxRequest::Complete will complain 4886 pRequest->SetVerifierFlags(FXREQUEST_FLAG_CANCELLED); 4887 } 4888 4889 Unlock(*PreviousIrql); 4890 4891 if (FxDriverGlobals->FxVerifierOn) { 4892 ASSERT((pRequest->GetVerifierFlags() & FXREQUEST_FLAG_DRIVER_OWNED) != 0); 4893 } 4894 4895 // 4896 // Notify the driver of cancel desire 4897 // 4898 pRequest->m_CancelRoutine.InvokeCancel( 4899 m_IoCancelCallbackLockPtr, 4900 pRequest->GetHandle() 4901 ); 4902 4903 // 4904 // Release the reference that FxRequest took out on itself 4905 // when it was placed onto the FxIrpQueue. It is now leaving 4906 // the FxIrpQueue due to cancel, and we own this final 4907 // release. 4908 // 4909 pRequest->RELEASE(FXREQUEST_QUEUE_TAG); 4910 4911 Lock(PreviousIrql); 4912 } 4913 4914 return TRUE; 4915 } 4916 4917 __drv_requiresIRQL(DISPATCH_LEVEL) 4918 BOOLEAN 4919 FxIoQueue::ProcessCancelledRequestsOnQueue( 4920 __out PKIRQL PreviousIrql 4921 ) 4922 /*++ 4923 4924 Routine Description: 4925 4926 Process any cancelled requests 4927 Called with the Queue lock held 4928 Can drop and re-acquire the queue lock 4929 Returns with the Queue lock held 4930 4931 Arguments: 4932 4933 Return Value: 4934 None 4935 4936 --*/ 4937 { 4938 PLIST_ENTRY pEntry; 4939 FxRequest* pRequest; 4940 4941 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 4942 4943 if (IsPowerStateNotifyingDriver()) { 4944 // 4945 // We will not process cancelled request while the driver is being 4946 // notified to stop/resume processing request to avoid double 4947 // completion of the request. 4948 // 4949 return FALSE; 4950 } 4951 4952 while (!IsListEmpty(&m_CanceledOnQueueList)) { 4953 pEntry = m_CanceledOnQueueList.Flink; 4954 4955 RemoveEntryList(pEntry); 4956 4957 // FxRequest ensures its not on any list on checked builds 4958 InitializeListHead(pEntry); 4959 4960 pRequest = FxRequest::_FromOwnerListEntry(FxListEntryQueueOwned, pEntry); 4961 4962 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_INFORMATION, TRACINGIO, 4963 "Calling CanceledOnQueue routine " 4964 "for WDFREQUEST 0x%p on WDFQUEUE 0x%p", 4965 pRequest->GetHandle(), GetObjectHandle()); 4966 4967 if (FxDriverGlobals->FxVerifierOn) { 4968 4969 // Set cancelled status, otherwise verifier in FxRequest::Complete will complain 4970 pRequest->SetVerifierFlags(FXREQUEST_FLAG_CANCELLED); 4971 } 4972 4973 Unlock(*PreviousIrql); 4974 4975 if (FxDriverGlobals->FxVerifierOn) { 4976 ASSERT((pRequest->GetVerifierFlags() & FXREQUEST_FLAG_DRIVER_OWNED) != 0); 4977 } 4978 4979 // 4980 // Notify the driver 4981 // 4982 m_IoCanceledOnQueue.Invoke(GetHandle(), pRequest->GetHandle()); 4983 4984 Lock(PreviousIrql); 4985 } 4986 4987 return TRUE; 4988 } 4989 4990 __drv_requiresIRQL(DISPATCH_LEVEL) 4991 BOOLEAN 4992 FxIoQueue::ProcessPowerEvents( 4993 __out PKIRQL PreviousIrql 4994 ) 4995 /*++ 4996 4997 Routine Description: 4998 4999 Processes the power state machine for I/O queues. 5000 5001 Called with Queue lock held, but can drop and re-acquire 5002 it when it has to deliver event callbacks to the device driver. 5003 5004 This can modify queue state as it transits the state machine, and 5005 is called from the main event processing loop DispatchEvents(). 5006 5007 Handling "in-flight" I/O requests in the device driver due 5008 to a power state transition (stopping) is tricky, and 5009 involves a complex state machine. 5010 5011 The device driver must ensure that all in-flight I/O is stopped 5012 before it can release the thread calling into the driver to 5013 perform the power state transition, otherwise a system crash 5014 can result from accessing hardware after resources have been removed. 5015 5016 In WDF, this burden is placed on FxIoQueue, so that the device driver 5017 does not have to implement the more complex aspects of this code, 5018 but can rely on notifications from the queue serialized under the 5019 IRQL level and locking it has configured. 5020 5021 5022 Implementation of FxIoQueue Power state machine: 5023 ------------------------------------------ 5024 5025 Since we must drop our lock to callback into the device 5026 driver for power notifications, the processing must occur 5027 as a state machine with three lists. 5028 5029 On entry to the power stop state, any "in-flight" I/O requests 5030 are recorded on the m_DriverOwned list using 5031 FxRequest::FxListEntryDriverOwned for linkage. 5032 5033 All of the requests on m_DriverOwned are moved to m_PowerNotify 5034 while holding the lock, with m_DriverOwned cleared. The state is changed 5035 to indicate that the driver is now being notified of requests. 5036 5037 While in the driver notification state, requests are taken from the 5038 m_PowerNotify list, and moved on to the m_PowerDriverNotified list while 5039 under the lock, and the request is notified to the device driver 5040 by the callback while dropping the lock and re-acquiring the lock. 5041 5042 As the driver acknowledges the power notification, it calls 5043 WdfRequestStopAcknowledge (FxIoQueue::StopAcknowledge) which moves the 5044 request from the m_PowerDriverNotified list back to the m_DriverOwned 5045 list. 5046 5047 The device driver could also complete requests, in which case they 5048 just dis-appear from the lists by the completion code doing 5049 a RemoveEntryList on FxRequest::FxListEntryDriverOwned. 5050 5051 This occurs until the m_PowerNotify list is empty, in which case 5052 the state is changed to driver notified. 5053 5054 While in the driver notified state, the queue event processing continues 5055 until the m_PowerDriverNotified list is empty, and when it is, the 5056 stopped state is set, and the event m_PowerIdle is set. This releases 5057 the thread waiting on the power state transition in which all of the 5058 device drivers "in-flight" I/O has been stopped and accounted for. 5059 5060 State Transitions: 5061 -------------- 5062 5063 During Stop: 5064 5065 FxIoQueuePowerOn 5066 --> FxIoQueuePowerStartingTransition 5067 --> FxIoQueuePowerStopping 5068 --> FxIoQueuePowerStoppingNotifyingDriver 5069 --> FxIoQueuePowerStoppingDriverNotified 5070 --> FxIoQueuePowerOff 5071 5072 During Purge: 5073 5074 FxIoQueuePowerPurge 5075 --> FxIoQueuePowerPurgeNotifyingDriver 5076 --> FxIoQueuePowerPurgeDriverNotified 5077 --> FxIoQueuePowerOff 5078 5079 5080 During Resume: 5081 5082 FxIoQueuePowerOff 5083 --> FxIoQueuePowerRestarting 5084 --> FxIoQueuePowerRestartingNotifyingDriver 5085 --> FxIoQueuePowerRestartingDriverNotified 5086 --> FxIoQueuePowerOn 5087 5088 Arguments: 5089 5090 Return Value: 5091 5092 TRUE - Continue processing the event loop 5093 FALSE - Stop processing event loop 5094 5095 --*/ 5096 { 5097 PLIST_ENTRY Entry; 5098 FxRequest* pRequest; 5099 5100 PFX_DRIVER_GLOBALS FxDriverGlobals = GetDriverGlobals(); 5101 5102 switch(m_PowerState) { 5103 5104 case FxIoQueuePowerStartingTransition: 5105 if (m_Dispatching == 1) { 5106 5107 // 5108 // If we are the last routine actively dispatching callbacks to 5109 // the device driver under a Power stop, then we must set the 5110 // event specifying no more callbacks are active. 5111 // 5112 m_PowerIdle.Set(); 5113 } 5114 5115 return FALSE; 5116 5117 case FxIoQueuePowerStopping: 5118 // 5119 // Set state to FxIoQueuePowerPurgeNotifyingDriver 5120 // 5121 m_PowerState = FxIoQueuePowerStoppingNotifyingDriver; 5122 5123 // This should be empty on entry to this state 5124 ASSERT(IsListEmpty(&this->m_PowerNotify)); 5125 5126 if (!IsListEmpty(&m_DriverOwned)) { 5127 5128 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5129 "Power Stop: WDFQUEUE 0x%p is powering off " 5130 "with in-flight requests", 5131 GetObjectHandle()); 5132 5133 5134 // Ensure the logic of m_DriverOwned is correct 5135 ASSERT(m_DriverOwned.Flink->Blink == &m_DriverOwned); 5136 ASSERT(m_DriverOwned.Blink->Flink == &m_DriverOwned); 5137 5138 // 5139 // Move all requests on m_DriverOwned to m_PowerNotify 5140 // 5141 m_PowerNotify.Flink = m_DriverOwned.Flink; 5142 m_PowerNotify.Blink = m_DriverOwned.Blink; 5143 m_PowerNotify.Flink->Blink = &m_PowerNotify; 5144 m_PowerNotify.Blink->Flink = &m_PowerNotify; 5145 5146 // This is now empty 5147 InitializeListHead(&m_DriverOwned); 5148 } 5149 else { 5150 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5151 "Power Stop: WDFQUEUE 0x%p is powering off without " 5152 "in-flight requests",GetObjectHandle()); 5153 } 5154 5155 // 5156 // Return to main processing loop which will callback to 5157 // process notifications 5158 // 5159 return TRUE; 5160 5161 case FxIoQueuePowerPurge: 5162 // 5163 // Set state to FxIoQueuePowerPurgeNotifyingDriver 5164 // 5165 m_PowerState = FxIoQueuePowerPurgeNotifyingDriver; 5166 5167 // This should be empty on entry to this state 5168 ASSERT(IsListEmpty(&this->m_PowerNotify)); 5169 5170 if (!IsListEmpty(&m_DriverOwned)) { 5171 5172 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5173 "Power Stop: WDFQUEUE 0x%p is purging with " 5174 "in-flight requests",GetObjectHandle()); 5175 5176 // Ensure the logic of m_DriverOwned is correct 5177 ASSERT(m_DriverOwned.Flink->Blink == &m_DriverOwned); 5178 ASSERT(m_DriverOwned.Blink->Flink == &m_DriverOwned); 5179 5180 // 5181 // Move all requests on m_DriverOwned to m_PowerNotify 5182 // 5183 m_PowerNotify.Flink = m_DriverOwned.Flink; 5184 m_PowerNotify.Blink = m_DriverOwned.Blink; 5185 m_PowerNotify.Flink->Blink = &m_PowerNotify; 5186 m_PowerNotify.Blink->Flink = &m_PowerNotify; 5187 5188 // This is now empty 5189 InitializeListHead(&m_DriverOwned); 5190 } 5191 else { 5192 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5193 "Power purge: WDFQUEUE 0x%p is purging without " 5194 "in-flight requests", GetObjectHandle()); 5195 } 5196 5197 // 5198 // Return to main processing loop which will callback to 5199 // process notifications 5200 // 5201 return TRUE; 5202 5203 case FxIoQueuePowerStoppingNotifyingDriver: { 5204 5205 FxIoQueueIoStop stopCallback; 5206 5207 // 5208 // If the list of requests to notify the driver about is 5209 // empty, change to the notified state. 5210 // 5211 if (IsListEmpty(&m_PowerNotify)) { 5212 m_PowerState = FxIoQueuePowerStoppingDriverNotified; 5213 5214 // 5215 // Return to main processing loop which will callback to 5216 // process the wait/signaling for the driver to acknowledge 5217 // all stops. 5218 // 5219 return TRUE; 5220 } 5221 5222 // 5223 // Notify each entry in m_PowerNotify into the driver 5224 // 5225 5226 // Remove from the notify list, place it on the driver notified list 5227 Entry = RemoveHeadList(&m_PowerNotify); 5228 5229 InsertTailList(&m_PowerDriverNotified, Entry); 5230 5231 // Retrieve the FxRequest 5232 pRequest = FxRequest::_FromOwnerListEntry(FxListEntryDriverOwned, Entry); 5233 5234 stopCallback = m_IoStop; 5235 5236 // 5237 // Notify driver by callback. 5238 // 5239 // If no callback is registered, the power thread will in effect 5240 // wait until the driver completes or cancels all requests. 5241 // 5242 if (stopCallback.Method != NULL && pRequest->m_Canceled == FALSE) { 5243 ULONG ActionFlags = WdfRequestStopActionSuspend; 5244 5245 if(pRequest->IsInIrpQueue(&m_DriverCancelable)) { 5246 ActionFlags |= WdfRequestStopRequestCancelable; 5247 } 5248 5249 DoTraceLevelMessage( 5250 FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5251 "Power Stop Notifying Driver, WDFQUEUE 0x%p, WDFREQUEST 0x%p", 5252 GetObjectHandle(), pRequest->GetObjectHandle()); 5253 5254 // Driver could be calling RequestComplete as we attempt to stop 5255 pRequest->ADDREF(FXREQUEST_HOLD_TAG); 5256 5257 Unlock(*PreviousIrql); 5258 5259 if (FxDriverGlobals->FxVerifierOn) { 5260 pRequest->SetVerifierFlags(FXREQUEST_FLAG_DRIVER_IN_EVTIOSTOP_CONTEXT); 5261 } 5262 5263 stopCallback.Invoke(GetHandle(), pRequest->GetHandle(), ActionFlags); 5264 5265 pRequest->RELEASE(FXREQUEST_HOLD_TAG); 5266 5267 Lock(PreviousIrql); 5268 } 5269 5270 // 5271 // As they are acknowledged, they will move back to m_DriverOwned. 5272 // 5273 // If the driver completes them, they go away. 5274 // 5275 5276 // Return to main processing loop and continue processing notifications 5277 return TRUE; 5278 } 5279 5280 case FxIoQueuePowerPurgeNotifyingDriver: { 5281 5282 FxIoQueueIoStop stopCallback; 5283 5284 // 5285 // If the list of requests to notify the driver about is 5286 // empty, change to the notified state. 5287 // 5288 if (IsListEmpty(&m_PowerNotify)) { 5289 m_PowerState = FxIoQueuePowerPurgeDriverNotified; 5290 5291 // 5292 // Return to main processing loop which will callback to 5293 // process the wait/signaling for the driver to acknowledge 5294 // all stops. 5295 // 5296 return TRUE; 5297 } 5298 5299 // 5300 // Notify each entry in m_PowerNotify into the driver 5301 // 5302 5303 // Remove from the notify list, place it on the driver notified list 5304 Entry = RemoveHeadList(&m_PowerNotify); 5305 5306 InsertTailList(&m_PowerDriverNotified, Entry); 5307 5308 // Retrieve the FxRequest 5309 pRequest = FxRequest::_FromOwnerListEntry(FxListEntryDriverOwned, Entry); 5310 5311 stopCallback = m_IoStop; 5312 5313 // 5314 // Make sure power stop state is cleared before invoking the stop callback. 5315 // 5316 pRequest->ClearPowerStopState(); 5317 5318 // 5319 // Notify driver by callback. 5320 // 5321 // If no callback is registered, the power thread will in effect 5322 // wait until the driver completes or cancels all requests. 5323 // 5324 if (stopCallback.Method != NULL && pRequest->m_Canceled == FALSE) { 5325 ULONG ActionFlags = WdfRequestStopActionPurge; 5326 5327 if(pRequest->IsInIrpQueue(&m_DriverCancelable)) { 5328 ActionFlags |= WdfRequestStopRequestCancelable; 5329 } 5330 5331 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5332 "Power Purge Notifying Driver " 5333 "WDFQUEUE 0x%p, WDFREQUEST 0x%p", 5334 GetObjectHandle(),pRequest->GetHandle()); 5335 5336 // Driver could be calling RequestComplete as we attempt to stop 5337 pRequest->ADDREF(FXREQUEST_HOLD_TAG); 5338 5339 Unlock(*PreviousIrql); 5340 5341 if (FxDriverGlobals->FxVerifierOn) { 5342 pRequest->SetVerifierFlags(FXREQUEST_FLAG_DRIVER_IN_EVTIOSTOP_CONTEXT); 5343 } 5344 5345 stopCallback.Invoke(GetHandle(), pRequest->GetHandle(), ActionFlags); 5346 5347 pRequest->RELEASE(FXREQUEST_HOLD_TAG); 5348 5349 Lock(PreviousIrql); 5350 } 5351 5352 // 5353 // As they are acknowledged, they will move back to m_DriverOwned. 5354 // 5355 // If the driver completes them, they go away. 5356 // 5357 5358 // Return to main processing loop and continue processing notifications 5359 return TRUE; 5360 } 5361 5362 case FxIoQueuePowerStoppingDriverNotified: 5363 case FxIoQueuePowerPurgeDriverNotified: { 5364 5365 PLIST_ENTRY thisEntry, nextEntry, listHead; 5366 LIST_ENTRY acknowledgedList; 5367 BOOLEAN continueProcessing = FALSE; 5368 5369 InitializeListHead(&acknowledgedList); 5370 5371 // 5372 // First move all the acknowledged requests to local list and then 5373 // process the local list. We have do that in two steps because 5374 // ProcessAcknowledgedRequests drops and reacquires the lock. 5375 // 5376 listHead = &m_PowerDriverNotified; 5377 5378 for (thisEntry = listHead->Flink; 5379 thisEntry != listHead; 5380 thisEntry = nextEntry) { 5381 5382 // Retrieve the FxRequest 5383 pRequest = FxRequest::_FromOwnerListEntry(FxListEntryDriverOwned, thisEntry); 5384 5385 nextEntry = thisEntry->Flink; 5386 5387 if (pRequest->IsPowerStopAcknowledged()) { 5388 RemoveEntryList(thisEntry); 5389 InsertTailList(&acknowledgedList, thisEntry); 5390 } 5391 } 5392 5393 // 5394 // Process all the acknowledged request from the local list. 5395 // 5396 while (!IsListEmpty(&acknowledgedList)) 5397 { 5398 thisEntry = RemoveHeadList(&acknowledgedList); 5399 pRequest = FxRequest::_FromOwnerListEntry(FxListEntryDriverOwned, thisEntry); 5400 ProcessAcknowledgedRequests(pRequest, PreviousIrql); 5401 5402 // 5403 // When this function drops the lock, other threads may attempt 5404 // to dispatch but fail since we are currently dispatching. 5405 // We need to be sure to process any pending events that other 5406 // threads initiated but could not be dispatch. The acknowledged 5407 // list will eventually be cleared out, allowing exit paths from 5408 // this function to return control to the driver. 5409 // 5410 continueProcessing = TRUE; 5411 } 5412 5413 // 5414 // Check to see if there are any unacknowledged requests. 5415 // 5416 if (!IsListEmpty(&m_PowerDriverNotified)) { 5417 5418 // 5419 // If there are still entries on the list, we potentially return 5420 // FALSE to tell the main event dispatching loop to return to 5421 // the device driver, since we are awaiting response from 5422 // the driver while in this state. 5423 // 5424 5425 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5426 "Power Stop: Waiting for Driver to complete or " 5427 "acknowledge in-flight requests on WDFQUEUE 0x%p", 5428 GetObjectHandle()); 5429 5430 return continueProcessing; 5431 } 5432 5433 // 5434 // Check to see if there are any requests in the middle of two-phase-completion. 5435 // If so, bail out and wait. 5436 // 5437 if (m_TwoPhaseCompletions != 0) { 5438 5439 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5440 "Power Stop: Waiting for Driver to complete or " 5441 "acknowledge in-flight requests on WDFQUEUE 0x%p", 5442 GetObjectHandle()); 5443 5444 return continueProcessing; 5445 } 5446 5447 // 5448 // All the requests are acknowledged. We will signal the pnp thread waiting 5449 // in StopProcessingForPower to continue if we are the last one to visit 5450 // the dispatch event loop. 5451 // 5452 // 5453 if ( m_Dispatching == 1) { 5454 5455 // 5456 // If we are the last routine actively dispatching callbacks to 5457 // the device driver under a Power stop, then we must set the 5458 // event specifying no more callbacks are active. 5459 // 5460 5461 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5462 "Power Stop: WDFQUEUE 0x%p is now powered off with no " 5463 "in-flight requests",GetObjectHandle()); 5464 5465 m_PowerState = FxIoQueuePowerOff; 5466 5467 m_PowerIdle.Set(); 5468 5469 return TRUE; 5470 } 5471 5472 // 5473 // The driver has acknowledged all requests, and the 5474 // notification list is empty. But, there are still outstanding 5475 // dispatch calls into the driver (m_Dispatching != 1), so we potentially 5476 // return false here to hopefully unwind to the final dispatch routine, 5477 // which will set the power off state. 5478 // 5479 5480 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5481 "Power Stop: Driver has acknowledged all in-flight " 5482 "requests, but WDFQUEUE 0x%p has outstanding callbacks", 5483 GetObjectHandle()); 5484 5485 return continueProcessing; 5486 } 5487 case FxIoQueuePowerRestarting: 5488 5489 // 5490 // Power is being resumed to the device. We notify the 5491 // device driver by an event callback for all driver 5492 // owned requests that it has idled due to a previous 5493 // power stop. 5494 // 5495 m_PowerState = FxIoQueuePowerRestartingNotifyingDriver; 5496 5497 // This should be empty on entry to this state 5498 ASSERT(IsListEmpty(&this->m_PowerNotify)); 5499 5500 if (!IsListEmpty(&m_DriverOwned)) { 5501 5502 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5503 "Power Resume: Driver has power paused requests " 5504 "on WDFQUEUE 0x%p",GetObjectHandle()); 5505 5506 // Ensure the logic of m_DriverOwned is correct 5507 ASSERT(m_DriverOwned.Flink->Blink == &m_DriverOwned); 5508 ASSERT(m_DriverOwned.Blink->Flink == &m_DriverOwned); 5509 5510 // 5511 // Move all requests on m_DriverOwned to m_PowerNotify 5512 // 5513 m_PowerNotify.Flink = m_DriverOwned.Flink; 5514 m_PowerNotify.Blink = m_DriverOwned.Blink; 5515 m_PowerNotify.Flink->Blink = &m_PowerNotify; 5516 m_PowerNotify.Blink->Flink = &m_PowerNotify; 5517 5518 // This is now empty 5519 InitializeListHead(&m_DriverOwned); 5520 } 5521 else { 5522 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5523 "Power Resume: Driver has no power paused requests " 5524 "on WDFQUEUE 0x%p", GetObjectHandle()); 5525 } 5526 5527 // 5528 // Return to main processing loop which will callback to 5529 // process notifications 5530 // 5531 return TRUE; 5532 5533 5534 case FxIoQueuePowerRestartingNotifyingDriver: { 5535 5536 FxIoQueueIoResume resumeCallback; 5537 5538 // 5539 // If the list of requests to notify the driver about is 5540 // empty, change to the notified state. 5541 // 5542 if (IsListEmpty(&m_PowerNotify)) { 5543 5544 m_PowerState = FxIoQueuePowerRestartingDriverNotified; 5545 5546 // 5547 // Return to main processing loop which will callback to 5548 // process the next state 5549 // 5550 return TRUE; 5551 } 5552 5553 // 5554 // Notify each entry in m_PowerNotify into the driver, placing them 5555 // back on the m_DriverOwned list. 5556 // 5557 5558 // Remove from the notify list, place it on the driver owned list 5559 Entry = RemoveHeadList(&m_PowerNotify); 5560 5561 InsertTailList(&m_DriverOwned, Entry); 5562 5563 // Retrieve the FxRequest 5564 pRequest = FxRequest::_FromOwnerListEntry(FxListEntryDriverOwned, Entry); 5565 5566 resumeCallback = m_IoResume; 5567 5568 // Notify driver by callback 5569 if (resumeCallback.Method != NULL && pRequest->m_Canceled == FALSE) { 5570 5571 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5572 "Power Resume, Notifying Driver, WDFQUEUE 0x%p, " 5573 "WDFREQUEST 0x%p", 5574 GetObjectHandle(), 5575 pRequest->GetObjectHandle()); 5576 5577 // Driver could be calling RequestComplete as we attempt to resume 5578 pRequest->ADDREF(FXREQUEST_HOLD_TAG); 5579 5580 Unlock(*PreviousIrql); 5581 5582 resumeCallback.Invoke(GetHandle(), pRequest->GetHandle()); 5583 5584 pRequest->RELEASE(FXREQUEST_HOLD_TAG); 5585 5586 Lock(PreviousIrql); 5587 } 5588 else { 5589 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5590 "Power Resume: Driver has no callback for " 5591 "EvtIoResume registered on WDFQUEUE 0x%p",GetObjectHandle()); 5592 } 5593 5594 // Return to main processing loop and continue processing notifications 5595 return TRUE; 5596 } 5597 5598 case FxIoQueuePowerRestartingDriverNotified: 5599 5600 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5601 "Power Resume: WDFQUEUE 0x%p is now powered on and " 5602 "I/O has resumed",GetObjectHandle()); 5603 5604 // Power state has resumed 5605 m_PowerState = FxIoQueuePowerOn; 5606 5607 // 5608 // We will resume dispatching I/O after all the queues 5609 // are moved into PowerOn state. 5610 // 5611 return FALSE; 5612 5613 5614 default: 5615 // Called on invalid state 5616 ASSERT(FALSE); 5617 return FALSE; 5618 } 5619 5620 /* NOTREACHED*/ 5621 } 5622 5623 __drv_requiresIRQL(DISPATCH_LEVEL) 5624 VOID 5625 FxIoQueue::ProcessAcknowledgedRequests( 5626 __in FxRequest* Request, 5627 __out PKIRQL PreviousIrql 5628 ) 5629 /*++ 5630 5631 Routine Description: 5632 5633 Process requests that are acknowledged by the driver. 5634 5635 Called with the queue lock held. This function can drop 5636 and reacquire the lock if needed. 5637 5638 Return Value: 5639 5640 --*/ 5641 { 5642 PLIST_ENTRY Entry; 5643 BOOLEAN requeue; 5644 PFX_DRIVER_GLOBALS pFxDriverGlobals; 5645 5646 pFxDriverGlobals = GetDriverGlobals(); 5647 5648 ASSERT(Request->IsPowerStopAcknowledged()); 5649 5650 requeue = Request->IsPowerStopAcknowledgedWithRequeue(); 5651 5652 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 5653 "Acknowledging WDFREQUEST %p on WDFQUEUE %p %s requeue option", 5654 Request->GetObjectHandle(), GetObjectHandle(), 5655 (requeue ? "with" : "without")); 5656 5657 Request->ClearPowerStopState(); 5658 5659 // 5660 // Remove the request from the m_PowerDriverNotified list and 5661 // place it back on the m_DriverOwned list. 5662 // 5663 // N.B. Our caller guarantees that we have already been removed, thus we 5664 // must not explicitly remove here. 5665 // 5666 Entry = Request->GetListEntry(FxListEntryDriverOwned); 5667 5668 InitializeListHead(Entry); 5669 5670 InsertTailList(&this->m_DriverOwned, Entry); 5671 5672 if (pFxDriverGlobals->FxVerifierOn) { 5673 // 5674 // As soon as we drop the lock below the request may get completed. 5675 // So take an addition reference so that we can safely clear the 5676 // flag. 5677 // 5678 Request->ADDREF(FXREQUEST_HOLD_TAG); 5679 } 5680 5681 Unlock(*PreviousIrql); 5682 5683 if (pFxDriverGlobals->FxVerifierOn) { 5684 Request->ClearVerifierFlags(FXREQUEST_FLAG_DRIVER_IN_EVTIOSTOP_CONTEXT); 5685 Request->RELEASE(FXREQUEST_HOLD_TAG); 5686 } 5687 5688 if (requeue) { 5689 FxRequestCompletionState oldState; 5690 NTSTATUS status; 5691 5692 if (pFxDriverGlobals->FxVerifierOn) { 5693 Request->ClearVerifierFlags(FXREQUEST_FLAG_DRIVER_OWNED | 5694 FXREQUEST_FLAG_DRIVER_DISPATCH); 5695 } 5696 5697 // 5698 // If the device driver requests it back on the queue, we will place it 5699 // on the front and it will be re-delivered in the normal EvtIoDefault, ... path. 5700 // 5701 // EvtIoResume *will not* be called on power resume for this request. 5702 // 5703 5704 // 5705 // The request has only one reference, held by the completion 5706 // callback function. We need to take another one before cancelling 5707 // this function, otherwise we will lose the request object 5708 // 5709 Request->ADDREF(FXREQUEST_STATE_TAG); 5710 5711 // Cancel the request complete callback (deletes a reference) 5712 oldState = Request->SetCompletionState(FxRequestCompletionStateNone); 5713 ASSERT(oldState == FxRequestCompletionStateQueue); 5714 UNREFERENCED_PARAMETER(oldState); 5715 5716 Lock(PreviousIrql); 5717 5718 // 5719 // We are going to place the request back on the queue 5720 // 5721 5722 // Driver is returning I/O 5723 RemoveFromDriverOwnedList(Request); 5724 5725 // 5726 // Check if we need to delete this request. 5727 // 5728 if (m_CancelDispatchedRequests) { 5729 // 5730 // Do not requeue this request. 5731 // 5732 status = STATUS_CANCELLED; 5733 } 5734 else { 5735 // 5736 // Place the request back at the head of the main queue 5737 // so as not to re-order requests 5738 // 5739 status = Request->InsertHeadIrpQueue(&m_Queue, NULL); 5740 } 5741 5742 if (!NT_SUCCESS(status)) { 5743 5744 // Request not placed in queue, cancel it 5745 ASSERT(status == STATUS_CANCELLED); 5746 5747 status = STATUS_SUCCESS; 5748 5749 // 5750 // We must add a reference since the CancelForQueue path 5751 // assumes we were on the FxIrpQueue with the extra reference 5752 // 5753 Request->ADDREF(FXREQUEST_QUEUE_TAG); 5754 5755 // 5756 // Mark the request as cancelled, place it on the cancel list, 5757 // and schedule the cancel event to the driver 5758 // 5759 CancelForQueue(Request, *PreviousIrql); 5760 5761 // 5762 // Reacquire the lock because CancelForQueue visits the dispatch-loop 5763 // and releases the lock. 5764 // 5765 Lock(PreviousIrql); 5766 } 5767 else { 5768 // Check if went from no requests to have requests 5769 CheckTransitionFromEmpty(); 5770 } 5771 5772 } else { 5773 Lock(PreviousIrql); 5774 } 5775 5776 return; 5777 } 5778 5779 VOID 5780 FxIoQueue::StartPowerTransitionOff( 5781 ) 5782 /*++ 5783 5784 Routine Description: 5785 5786 Purpose of this routine is to put the queue in state that would 5787 prevent any new requests from being dispatched to the driver. 5788 5789 Arguments: 5790 5791 Return Value: 5792 5793 VOID 5794 5795 --*/ 5796 { 5797 KIRQL irql; 5798 BOOLEAN result; 5799 5800 if(m_PowerManaged == FALSE) { 5801 return; 5802 } 5803 5804 Lock(&irql); 5805 5806 if (m_Deleted == FALSE) { 5807 ASSERT(m_PowerState == FxIoQueuePowerOn); 5808 } 5809 5810 m_PowerState = FxIoQueuePowerStartingTransition; 5811 5812 // We must wait on the current thread until the queue is actually idle 5813 m_PowerIdle.Clear(); 5814 5815 // 5816 // Run the event dispatching loop before waiting on the event 5817 // in case this thread actually performs the transition 5818 // 5819 result = DispatchEvents(irql); 5820 if(result) { 5821 // 5822 // This is called from a kernel mode PNP thread, so we do not need 5823 // a KeEnterCriticalRegion() 5824 // 5825 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGIO, 5826 "Waiting for all threads to stop dispatching requests" 5827 " so that WDFQUEUE 0x%p can be powered off", 5828 GetObjectHandle()); 5829 5830 GetDriverGlobals()->WaitForSignal(m_PowerIdle.GetSelfPointer(), 5831 "waiting for all threads to stop dispatching requests so " 5832 "that queue can be powered off, WDFQUEUE", GetHandle(), 5833 GetDriverGlobals()->FxVerifierDbgWaitForSignalTimeoutInSec, 5834 WaitSignalBreakUnderVerifier); 5835 } 5836 5837 return; 5838 } 5839 5840 VOID 5841 FxIoQueue::StopProcessingForPower( 5842 __in FxIoStopProcessingForPowerAction Action 5843 ) 5844 /*++ 5845 5846 Routine Description: 5847 5848 Stops automatic I/O processing due to a power event that requires I/O to stop. 5849 5850 This is called on a PASSIVE_LEVEL thread that can block until 5851 I/O has been stopped, or completed/cancelled. 5852 5853 Additional reference is already taken on the object by the caller 5854 to prevent the queue from being deleted. 5855 5856 5857 Arguments: 5858 5859 Action - 5860 5861 FxIoStopProcessingForPowerHold: 5862 the function returns when the driver has acknowledged that it has 5863 stopped all I/O processing, but may have outstanding "in-flight" requests 5864 that have not been completed. 5865 5866 FxIoStopProcessingForPowerPurgeManaged: 5867 the function returns when all requests from a power managed queue have 5868 been completed and/or cancelled., and there are no more in-flight requests. 5869 5870 FxIoStopProcessingForPowerPurgeNonManaged: 5871 the function returns when all requests from a non-power managed queue have 5872 been completed and/or cancelled., and there are no more in-flight requests. 5873 5874 Return Value: 5875 5876 NTSTATUS 5877 5878 --*/ 5879 { 5880 KIRQL irql; 5881 BOOLEAN result; 5882 5883 switch (Action) { 5884 case FxIoStopProcessingForPowerPurgeNonManaged: 5885 5886 // 5887 // If power managed, leave it alone 5888 // 5889 if(m_PowerManaged == TRUE) { 5890 // Should be powered off by now. 5891 ASSERT(m_PowerState == FxIoQueuePowerOff); 5892 return; 5893 } 5894 5895 // 5896 // Queue is being shut down. This flag prevents the following: 5897 // (1) a race condition where a dispatch queue handler changes the 5898 // state of the queue to accept_requests while we are in the 5899 // middle of a power stopping (purge) operation 5900 5901 // (2) another thread calling Stop or Start on a queue that is in the 5902 // middle of a power stopping (purge) operation. 5903 // 5904 Lock(&irql); 5905 SetStateForShutdown(); 5906 Unlock(irql); 5907 5908 QueuePurge(TRUE, TRUE, NULL, NULL); 5909 5910 Lock(&irql); 5911 // 5912 // Queue must be in PowerOn state. 5913 // 5914 ASSERT(m_PowerState == FxIoQueuePowerOn); 5915 5916 m_PowerState = FxIoQueuePowerPurge; 5917 5918 break; 5919 5920 case FxIoStopProcessingForPowerPurgeManaged: 5921 5922 // 5923 // If not power managed, leave it alone 5924 // 5925 if(m_PowerManaged == FALSE) { 5926 ASSERT(m_PowerState == FxIoQueuePowerOn); 5927 return; 5928 } 5929 5930 // 5931 // Queue is being shut down. This flag prevents the following: 5932 // (1) a race condition where a dispatch queue handler changes the 5933 // state of the queue to accept_requests while we are in the 5934 // middle of a power stopping (purge) operation 5935 5936 // (2) another thread calling Stop or Start on a queue that is in the 5937 // middle of a power stopping (purge) operation. 5938 // 5939 Lock(&irql); 5940 SetStateForShutdown(); 5941 Unlock(irql); 5942 5943 QueuePurge(TRUE, TRUE, NULL, NULL); 5944 5945 Lock(&irql); 5946 5947 m_PowerState = FxIoQueuePowerPurge; 5948 5949 break; 5950 5951 case FxIoStopProcessingForPowerHold: 5952 // 5953 // If not power managed, leave it alone 5954 // 5955 if(m_PowerManaged == FALSE) { 5956 ASSERT(m_PowerState == FxIoQueuePowerOn); 5957 return; 5958 } 5959 5960 Lock(&irql); 5961 5962 m_PowerState = FxIoQueuePowerStopping; 5963 5964 break; 5965 5966 default: 5967 ASSERT(FALSE); 5968 return; 5969 } 5970 5971 // We must wait on the current thread until the queue is actually idle 5972 m_PowerIdle.Clear(); 5973 5974 // 5975 // Run the event dispatching loop before waiting on the event 5976 // in case this thread actually performs the transition 5977 // 5978 result = DispatchEvents(irql); 5979 if(result) { 5980 // 5981 // This is called from a kernel mode PNP thread, so we do not need 5982 // a KeEnterCriticalRegion() 5983 // 5984 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGIO, 5985 "Waiting for all inflight requests to be acknowledged " 5986 " on WDFQUEUE 0x%p", 5987 GetObjectHandle()); 5988 5989 GetDriverGlobals()->WaitForSignal(m_PowerIdle.GetSelfPointer(), 5990 "waiting for all inflight requests " 5991 "to be acknowledged on WDFQUEUE", 5992 GetHandle(), 5993 GetDriverGlobals()->FxVerifierDbgWaitForSignalTimeoutInSec, 5994 WaitSignalBreakUnderVerifier); 5995 } 5996 5997 return; 5998 } 5999 6000 VOID 6001 FxIoQueue::StartPowerTransitionOn( 6002 VOID 6003 ) 6004 /*++ 6005 Routine Description: Start dispatching I/Os 6006 6007 Arguments: VOID 6008 6009 Return Value: VOID 6010 --*/ 6011 { 6012 KIRQL irql; 6013 6014 if(m_PowerManaged == FALSE) { 6015 ASSERT(m_PowerState == FxIoQueuePowerOn); 6016 return; 6017 } 6018 6019 Lock(&irql); 6020 6021 if (m_Deleted == FALSE) { 6022 ASSERT(m_PowerState == FxIoQueuePowerOn); 6023 } 6024 6025 // 6026 // If there are requests in the queue when we power up, we 6027 // should set m_TransitionFromEmpty to trigger event-ready notify 6028 // callback on the manual queue to kick start processing of requests. 6029 // If we don't set, there is a posibility for abandoning the requests in the 6030 // the queue if the queue is powered off between the time we call 6031 // ProcessReadNotify and the call to retrieve requests made by the driver 6032 // because the retrieve call will fail and the request will be left in the 6033 // queue with m_TransitionFromEmpty state cleared. 6034 // 6035 if (m_Queue.GetRequestCount() > 0L) { 6036 m_TransitionFromEmpty = TRUE; 6037 m_ForceTransitionFromEmptyWhenAddingNewRequest = FALSE; 6038 } 6039 6040 DispatchEvents(irql); 6041 6042 return; 6043 } 6044 6045 VOID 6046 FxIoQueue::ResumeProcessingForPower( 6047 VOID 6048 ) 6049 /*++ 6050 6051 Routine Description: 6052 6053 Resumes a PowerManaged queue for automatic I/O processing due to 6054 a power event that allows I/O to resume. 6055 6056 Does nothing if its a non-power managed queue. 6057 6058 Additional reference is already taken on the object by the caller 6059 to prevent the queue from being deleted. 6060 6061 Arguments: 6062 6063 Return Value: 6064 6065 NTSTATUS 6066 6067 --*/ 6068 { 6069 KIRQL irql; 6070 6071 // 6072 // If not power managed, leave it alone 6073 // 6074 if (!m_PowerManaged) { 6075 ASSERT(m_PowerState == FxIoQueuePowerOn); 6076 return; 6077 } 6078 6079 Lock(&irql); 6080 6081 if (m_PowerState == FxIoQueuePowerOn) { 6082 Unlock(irql); 6083 return; 6084 } 6085 6086 ASSERT(m_PowerState == FxIoQueuePowerOff); 6087 6088 m_PowerState = FxIoQueuePowerRestarting; 6089 6090 // 6091 // We have transitioned to a status that resumes 6092 // processing, so call dispatch function. 6093 // 6094 6095 DispatchEvents(irql); 6096 6097 return; 6098 } 6099 6100 VOID 6101 FxIoQueue::SetStateForShutdown( 6102 VOID 6103 ) 6104 /*++ 6105 6106 Routine Description: 6107 The queue is shutting down. Disable WdfQueueStart/Stop from re-enabling 6108 the AcceptRequest bit. 6109 6110 Arguments: 6111 None 6112 6113 Return Value: 6114 None 6115 6116 --*/ 6117 { 6118 // 6119 // No need to take a lock since caller is responsible for providing the 6120 // required synchronization. 6121 // 6122 6123 // 6124 // Do not allow request to be queued. 6125 // 6126 SetState((FX_IO_QUEUE_SET_STATE)(FxIoQueueSetShutdown | FxIoQueueClearAcceptRequests)); 6127 } 6128 6129 VOID 6130 FxIoQueue::ResetStateForRestart( 6131 VOID 6132 ) 6133 /*++ 6134 6135 Routine Description: 6136 This is called on a device (PDO) which has been restarted from the removed 6137 state. It will reset purged queues so that it can accept new requests 6138 when ResumeProcessingForPower is called afterwards. 6139 6140 Additional reference is already taken on the object by the caller 6141 to prevent the queue from being deleted. 6142 6143 6144 Arguments: 6145 None 6146 6147 Return Value: 6148 None 6149 6150 --*/ 6151 { 6152 KIRQL irql; 6153 6154 Lock(&irql); 6155 6156 // 6157 // For non power managed queues, let us reset the m_PowerState to On 6158 // 6159 if (!m_PowerManaged) { 6160 m_PowerState = FxIoQueuePowerOn; 6161 } 6162 6163 // 6164 // Allow requests to be queued. 6165 // 6166 SetState((FX_IO_QUEUE_SET_STATE)(FxIoQueueClearShutdown | FxIoQueueSetAcceptRequests)); 6167 6168 // 6169 // No need to visit the DispatchEvents because the PowerState 6170 // is still off. 6171 // 6172 Unlock(irql); 6173 6174 return; 6175 } 6176 6177 BOOLEAN 6178 FxIoQueue::IsIoEventHandlerRegistered( 6179 __in WDF_REQUEST_TYPE RequestType 6180 ) 6181 /*++ 6182 6183 Routine Description: 6184 Given a request type, this function checks to see if the appropriate 6185 event handler is registered to receive dispatched requests. 6186 6187 Return Value: 6188 6189 TRUE - yes the queue is configured to dispatch requests of given RequestType 6190 FALSE - no, the queue cannot dispatch requests of given RequestType 6191 6192 --*/ 6193 { 6194 if(m_Type == WdfIoQueueDispatchManual) { 6195 // 6196 // Manual queues wouldn't have any IoEvent callbacks registered. 6197 // 6198 return TRUE; 6199 } 6200 6201 // 6202 // Default handler is a catch all handler. 6203 // 6204 if(m_IoDefault.Method != NULL) { 6205 return TRUE; 6206 } 6207 6208 // 6209 // Default handle is not registered. So check to see if request specific 6210 // handler is registered. 6211 // 6212 switch(RequestType) { 6213 case WdfRequestTypeRead: 6214 if(m_IoRead.Method == NULL) { 6215 return FALSE; 6216 } 6217 break; 6218 case WdfRequestTypeWrite: 6219 if(m_IoWrite.Method == NULL) { 6220 return FALSE; 6221 } 6222 break; 6223 case WdfRequestTypeDeviceControl: 6224 if(m_IoDeviceControl.Method == NULL) { 6225 return FALSE; 6226 } 6227 break; 6228 case WdfRequestTypeDeviceControlInternal: 6229 if(m_IoInternalDeviceControl.Method == NULL) { 6230 return FALSE; 6231 } 6232 break; 6233 case WdfRequestTypeCreate: // Fall through. Must have default handler. 6234 default: 6235 return FALSE; 6236 } 6237 6238 return TRUE; 6239 } 6240 6241 VOID 6242 FxIoQueue::_DeferredDispatchThreadThunk( 6243 __in PVOID Parameter 6244 ) 6245 /*++ 6246 6247 Routine Description: 6248 Thunk used when requests must be posted to a workitem. 6249 6250 --*/ 6251 { 6252 FxIoQueue* pQueue = (FxIoQueue*)Parameter; 6253 6254 PFX_DRIVER_GLOBALS FxDriverGlobals = pQueue->GetDriverGlobals(); 6255 6256 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 6257 "Dispatching requests from worker thread"); 6258 6259 pQueue->DeferredDispatchRequestsFromWorkerThread(); 6260 6261 return; 6262 } 6263 6264 6265 VOID 6266 FxIoQueue::_DeferredDispatchDpcThunk( 6267 __in PKDPC Dpc, 6268 __in PVOID DeferredContext, 6269 __in PVOID SystemArgument1, 6270 __in PVOID SystemArgument2 6271 ) 6272 /*++ 6273 6274 Routine Description: 6275 Thunk used when requests must be posted to a DPC. 6276 6277 --*/ 6278 { 6279 FxIoQueue* pQueue = (FxIoQueue*)DeferredContext; 6280 6281 PFX_DRIVER_GLOBALS FxDriverGlobals = pQueue->GetDriverGlobals(); 6282 6283 UNREFERENCED_PARAMETER(Dpc); 6284 UNREFERENCED_PARAMETER(SystemArgument1); 6285 UNREFERENCED_PARAMETER(SystemArgument2); 6286 6287 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGIO, 6288 "Dispatching requests from DPC"); 6289 6290 pQueue->DeferredDispatchRequestsFromDpc(); 6291 6292 return; 6293 } 6294 6295 VOID 6296 FxIoQueue::_PurgeComplete( 6297 __in WDFQUEUE Queue, 6298 __in WDFCONTEXT Context 6299 ) 6300 /*++ 6301 6302 Routine Description: 6303 Callback function when a queue purge completes 6304 6305 --*/ 6306 { 6307 MxEvent* event = (MxEvent*)Context; 6308 6309 UNREFERENCED_PARAMETER(Queue); 6310 6311 event->SetWithIncrement(EVENT_INCREMENT); 6312 6313 return; 6314 } 6315 6316 VOID 6317 FxIoQueue::_IdleComplete( 6318 __in WDFQUEUE Queue, 6319 __in WDFCONTEXT Context 6320 ) 6321 /*++ 6322 6323 Routine Description: 6324 Callback function when a stop completes 6325 6326 --*/ 6327 { 6328 MxEvent* event = (MxEvent*)Context; 6329 6330 UNREFERENCED_PARAMETER(Queue); 6331 6332 event->SetWithIncrement(EVENT_INCREMENT); 6333 6334 return; 6335 } 6336 6337 DECLSPEC_NORETURN 6338 VOID 6339 FxIoQueue::FatalError( 6340 __in NTSTATUS Status 6341 ) 6342 { 6343 WDF_QUEUE_FATAL_ERROR_DATA data; 6344 6345 RtlZeroMemory(&data, sizeof(data)); 6346 6347 data.Queue = GetHandle(); 6348 data.Request = NULL; 6349 data.Status = Status; 6350 6351 FxVerifierBugCheck(GetDriverGlobals(), 6352 WDF_QUEUE_FATAL_ERROR, 6353 (ULONG_PTR) &data); 6354 } 6355 6356 6357 _Must_inspect_result_ 6358 NTSTATUS 6359 FxIoQueue::AllocateReservedRequest( 6360 __deref_out FxRequest** Request 6361 ) 6362 /*++ 6363 6364 Routine Description: 6365 Called by Fxpkgio::Dispatch to allocate a reserved request if one is 6366 avaialble. 6367 6368 --*/ 6369 6370 { 6371 PFX_DRIVER_GLOBALS pFxDriverGlobals; 6372 NTSTATUS status; 6373 FxRequest* pRequest; 6374 PWDF_OBJECT_ATTRIBUTES reqAttribs; 6375 6376 pFxDriverGlobals = GetDriverGlobals(); 6377 *Request = NULL; 6378 reqAttribs = NULL; 6379 6380 // 6381 // Get the right context for this request object. 6382 // 6383 if (GetCxDeviceInfo() != NULL) { 6384 reqAttribs = &GetCxDeviceInfo()->RequestAttributes; 6385 } 6386 else { 6387 reqAttribs = m_Device->GetRequestAttributes(); 6388 } 6389 6390 // 6391 // FxRequest::_Create doesn't create a Request from the Device lookaside 6392 // hence we can't use that as the Request Context doesn't get associated 6393 // if the Request is not created from the lookaside. 6394 // 6395 status = FxRequest::_CreateForPackage(m_Device, reqAttribs, NULL, &pRequest); 6396 if (!NT_SUCCESS(status)) { 6397 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 6398 "Failure to allocate request %!STATUS!", status); 6399 return status; 6400 } 6401 6402 pRequest->SetReserved(); 6403 pRequest->SetCurrentQueue(this); 6404 6405 // 6406 // This is used to return the request to the correct queue if it was 6407 // forwarded 6408 // 6409 pRequest->SetForwardProgressQueue(this); 6410 pRequest->SetCompleted(FALSE); 6411 6412 if (m_FwdProgContext->m_IoReservedResourcesAllocate.Method != NULL) { 6413 6414 pRequest->SetPresented(); 6415 6416 status = m_FwdProgContext->m_IoReservedResourcesAllocate.Invoke( 6417 GetHandle(), 6418 pRequest->GetHandle()); 6419 if (!NT_SUCCESS(status)) { 6420 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGIO, 6421 "Failure from m_IoReservedResourcesAllocate callback %!STATUS!", 6422 status); 6423 pRequest->FreeRequest(); 6424 } 6425 } 6426 6427 if (NT_SUCCESS(status)) { 6428 *Request = pRequest; 6429 } 6430 6431 return status; 6432 } 6433 6434 6435 VOID 6436 FxIoQueue::CancelIrps( 6437 __in PLIST_ENTRY IrpListHead 6438 ) 6439 /*++ 6440 6441 Routine Description: 6442 6443 This function is called to purge (cancel) the specified list of IRPs. 6444 The IRP's Tail.Overlay.ListEntry field must be used to link these structs together. 6445 6446 --*/ 6447 { 6448 MdIrp irp; 6449 FxIrp fxIrp; 6450 PLIST_ENTRY entry; 6451 6452 while(!IsListEmpty(IrpListHead)) { 6453 6454 entry = RemoveHeadList(IrpListHead); 6455 6456 irp = fxIrp.GetIrpFromListEntry(entry); 6457 6458 fxIrp.SetIrp(irp); 6459 6460 fxIrp.SetInformation(0); 6461 fxIrp.SetStatus(STATUS_CANCELLED); 6462 fxIrp.CompleteRequest(IO_NO_INCREMENT); 6463 } 6464 } 6465 6466 VOID 6467 FxIoQueue::PurgeForwardProgressIrps( 6468 __in_opt MdFileObject FileObject 6469 ) 6470 /*++ 6471 6472 Routine Description: 6473 6474 This function is called when the queue is purged. 6475 6476 --*/ 6477 { 6478 LIST_ENTRY cleanupList; 6479 6480 InitializeListHead(&cleanupList); 6481 GetForwardProgressIrps(&cleanupList, FileObject); 6482 CancelIrps(&cleanupList); 6483 } 6484 6485 VOID 6486 FxIoQueue::VerifierVerifyFwdProgListsLocked( 6487 VOID 6488 ) 6489 /*++ 6490 6491 Routine Description: 6492 Called from dispose to Free all the reserved requests. 6493 6494 --*/ 6495 { 6496 ULONG countOfInUseRequests; 6497 ULONG countOfFreeRequests; 6498 PLIST_ENTRY thisEntry, nextEntry, listHead; 6499 6500 countOfInUseRequests = 0; 6501 6502 listHead = &m_FwdProgContext->m_ReservedRequestInUseList; 6503 6504 for(thisEntry = listHead->Flink; 6505 thisEntry != listHead; 6506 thisEntry = nextEntry) 6507 { 6508 nextEntry = thisEntry->Flink; 6509 countOfInUseRequests++; 6510 } 6511 6512 countOfFreeRequests = 0; 6513 6514 listHead = &m_FwdProgContext->m_ReservedRequestList; 6515 6516 for(thisEntry = listHead->Flink; 6517 thisEntry != listHead; 6518 thisEntry = nextEntry) 6519 { 6520 nextEntry = thisEntry->Flink; 6521 countOfFreeRequests++; 6522 } 6523 6524 ASSERT(countOfFreeRequests + countOfInUseRequests == 6525 m_FwdProgContext->m_NumberOfReservedRequests); 6526 return; 6527 } 6528