1 /*++ 2 3 Copyright (c) Microsoft Corporation 4 5 Module Name: 6 7 FxDmaTransaction.cpp 8 9 Abstract: 10 11 WDF DMA Transaction Object 12 13 Environment: 14 15 Kernel mode only. 16 17 Notes: 18 19 20 Revision History: 21 22 --*/ 23 24 #include "fxdmapch.hpp" 25 26 extern "C" { 27 // #include "FxDmaTransaction.tmh" 28 } 29 30 FxDmaTransactionBase::FxDmaTransactionBase( 31 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 32 __in USHORT ObjectSize, 33 __in USHORT ExtraSize, 34 __in FxDmaEnabler *DmaEnabler 35 ) : 36 FxNonPagedObject( 37 FX_TYPE_DMA_TRANSACTION, 38 ExtraSize == 0 ? ObjectSize : COMPUTE_OBJECT_SIZE(ObjectSize, ExtraSize), 39 FxDriverGlobals) 40 { 41 m_DmaEnabler = DmaEnabler; 42 m_EncodedRequest = NULL; 43 m_MaxFragmentLength = 0; 44 m_DmaDirection = WdfDmaDirectionReadFromDevice; 45 m_DmaAcquiredContext = NULL; 46 m_CurrentFragmentMdl = NULL; 47 m_CurrentFragmentOffset = 0; 48 m_StartOffset = NULL; 49 m_StartMdl = NULL; 50 m_Remaining = 0; 51 m_CurrentFragmentLength = 0; 52 m_TransactionLength = 0; 53 m_Transferred = 0; 54 m_Flags = 0; 55 56 m_DmaAcquiredFunction.Method.ProgramDma = NULL; 57 58 m_State = FxDmaTransactionStateCreated; 59 60 if (ExtraSize == 0) { 61 m_TransferContext = NULL; 62 } else { 63 m_TransferContext = WDF_PTR_ADD_OFFSET_TYPE( 64 this, 65 COMPUTE_RAW_OBJECT_SIZE(ObjectSize), 66 PVOID 67 ); 68 } 69 70 MarkDisposeOverride(ObjectDoNotLock); 71 } 72 73 BOOLEAN 74 FxDmaTransactionBase::Dispose( 75 VOID 76 ) 77 { 78 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 79 80 // 81 // Must not be in transfer state. 82 // 83 if (m_State == FxDmaTransactionStateTransfer) { 84 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 85 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! " 86 "is invalid", GetHandle(), m_State); 87 88 if (pFxDriverGlobals->IsVerificationEnabled(1, 9, OkForDownLevel)) { 89 FxVerifierBugCheck(pFxDriverGlobals, // globals 90 WDF_DMA_FATAL_ERROR, // type 91 (ULONG_PTR) GetObjectHandle(), // parm 2 92 (ULONG_PTR) m_State); // parm 3 93 } 94 } 95 96 m_State = FxDmaTransactionStateDeleted; 97 98 // 99 // Release resources for this Dma Transaction. 100 // 101 ReleaseResources(TRUE); 102 103 if (m_EncodedRequest != NULL) { 104 ClearRequest(); 105 } 106 107 return TRUE; 108 } 109 110 _Must_inspect_result_ 111 NTSTATUS 112 FxDmaTransactionBase::Initialize( 113 __in PFN_WDF_PROGRAM_DMA ProgramDmaFunction, 114 __in WDF_DMA_DIRECTION DmaDirection, 115 __in PMDL Mdl, 116 __in size_t Offset, 117 __in ULONG Length 118 ) 119 { 120 NTSTATUS status; 121 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 122 123 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 124 "Enter WDFDMATRANSACTION %p", GetHandle()); 125 // 126 // Must be in Reserve, Created or Released state. 127 // 128 if (m_State != FxDmaTransactionStateCreated && 129 m_State != FxDmaTransactionStateReserved && 130 m_State != FxDmaTransactionStateReleased) { 131 132 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 133 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! " 134 "is invalid", GetHandle(), m_State); 135 136 FxVerifierBugCheck(pFxDriverGlobals, // globals 137 WDF_DMA_FATAL_ERROR, // specific type 138 (ULONG_PTR) GetObjectHandle(), // parm 2 139 (ULONG_PTR) m_State); // parm 3 140 } 141 142 if (DmaDirection == WdfDmaDirectionReadFromDevice) { 143 m_AdapterInfo = m_DmaEnabler->GetReadDmaDescription(); 144 } else { 145 m_AdapterInfo = m_DmaEnabler->GetWriteDmaDescription(); 146 } 147 148 // 149 // Initialize the DmaTransaction object 150 // 151 152 m_MaxFragmentLength = m_AdapterInfo->MaximumFragmentLength; 153 m_DmaDirection = DmaDirection; 154 m_StartMdl = Mdl; 155 m_StartOffset = Offset; 156 m_CurrentFragmentMdl = Mdl; 157 m_CurrentFragmentOffset = Offset; 158 m_Remaining = Length; 159 m_TransactionLength = Length; 160 m_DmaAcquiredFunction.Method.ProgramDma = ProgramDmaFunction; 161 162 // 163 // If needed, initialize the transfer context. 164 // 165 166 if (m_DmaEnabler->UsesDmaV3()) { 167 m_DmaEnabler->InitializeTransferContext(GetTransferContext(), 168 m_DmaDirection); 169 } 170 171 status = InitializeResources(); 172 if (NT_SUCCESS(status)) { 173 m_State = FxDmaTransactionStateInitialized; 174 } else { 175 ReleaseForReuse(FALSE); 176 } 177 178 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 179 "Exit WDFDMATRANSACTION %p, %!STATUS!", 180 GetHandle(), status); 181 182 return status; 183 } 184 185 _Must_inspect_result_ 186 NTSTATUS 187 FxDmaTransactionBase::Execute( 188 __in PVOID Context 189 ) 190 { 191 NTSTATUS status; 192 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 193 194 // 195 // Must be in Initialized state. 196 // 197 if (m_State != FxDmaTransactionStateInitialized) { 198 199 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 200 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! " 201 "is invalid", GetHandle(), m_State); 202 203 FxVerifierBugCheck(pFxDriverGlobals, // globals 204 WDF_DMA_FATAL_ERROR, // specific type 205 (ULONG_PTR) GetObjectHandle(), // parm 2 206 (ULONG_PTR) m_State); // parm 3 207 } 208 209 // 210 // If this was initialized with a request, then reference the 211 // request now. 212 // 213 if (m_EncodedRequest != NULL) { 214 ReferenceRequest(); 215 } 216 217 // 218 // Set state to Transfer. 219 // This is necessary because the Execute path complete 220 // all the way to DmaCompleted before returning to this point. 221 // 222 m_State = FxDmaTransactionStateTransfer; 223 224 // 225 // Save the caller's context 226 // 227 m_DmaAcquiredContext = Context; 228 229 ASSERT(m_Transferred == 0); 230 ASSERT(m_CurrentFragmentLength == 0); 231 232 status = StartTransfer(); 233 if (!NT_SUCCESS(status)) { 234 m_State = FxDmaTransactionStateTransferFailed; 235 m_DmaAcquiredContext = NULL; 236 237 if (m_EncodedRequest != NULL) { 238 ReleaseButRetainRequest(); 239 } 240 } 241 242 // 243 // StartTransfer results in a call to the EvtProgramDma routine 244 // where driver could complete and delete the object. So 245 // don't touch the object beyond this point. 246 // 247 248 return status; 249 } 250 251 BOOLEAN 252 FxDmaTransactionBase::DmaCompleted( 253 __in size_t TransferredLength, 254 __out NTSTATUS * ReturnStatus, 255 __in FxDmaCompletionType CompletionType 256 ) 257 { 258 BOOLEAN hasTransitioned; 259 NTSTATUS status; 260 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 261 WDFDMATRANSACTION dmaTransaction; 262 263 // 264 // In the case of partial completion, we will start a new transfer 265 // from with in this function by calling StageTransfer. After that 266 // call, we lose ownership of the object. Since we need the handle 267 // for tracing purposes, we will save the value in a local variable and 268 // use that. 269 // 270 dmaTransaction = GetHandle(); 271 272 if (pFxDriverGlobals->FxVerifierOn) { 273 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 274 "Enter WDFDMATRANSACTION %p, length %d", 275 dmaTransaction, (ULONG)TransferredLength); 276 } 277 278 // 279 // Must be in Transfer state. 280 // 281 if (m_State != FxDmaTransactionStateTransfer) { 282 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 283 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! " 284 "is invalid", dmaTransaction, m_State); 285 286 FxVerifierBugCheck(pFxDriverGlobals, // globals 287 WDF_DMA_FATAL_ERROR, // specific type 288 (ULONG_PTR) dmaTransaction, // parm 2 289 (ULONG_PTR) m_State); // parm 3 290 } 291 292 if (TransferredLength > m_CurrentFragmentLength) { 293 status = STATUS_INVALID_PARAMETER; 294 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 295 "WDFDMATRANSACTION %p Transfered Length %I64d can't be more " 296 "than the length asked to transfer %I64d " 297 "%!STATUS!", dmaTransaction, TransferredLength, 298 m_CurrentFragmentLength, status); 299 FxVerifierDbgBreakPoint(pFxDriverGlobals); 300 goto End; 301 } 302 303 304 if (CompletionType == FxDmaCompletionTypePartial || 305 CompletionType == FxDmaCompletionTypeAbort) { 306 // 307 // Tally this DMA tranferred byte count into the accumulator. 308 // 309 m_Transferred += TransferredLength; 310 311 // 312 // Adjust the remaining length to account for the partial transfer. 313 // 314 m_Remaining += (m_CurrentFragmentLength - TransferredLength); 315 316 // 317 // Update CurrentDmaLength to reflect actual transfer because 318 // we need to FlushAdapterBuffers based on this value in 319 // TransferCompleted for packet based transfer. 320 // 321 m_CurrentFragmentLength = TransferredLength; 322 323 } else { 324 // 325 // Tally this DMA tranferred byte count into the accumulator. 326 // 327 m_Transferred += m_CurrentFragmentLength; 328 } 329 330 ASSERT(m_Transferred <= m_TransactionLength); 331 332 // 333 // Inform the derived object that transfer is completed so it 334 // can release resources specific to last transfer. 335 // 336 status = TransferCompleted(); 337 if (!NT_SUCCESS(status)) { 338 goto End; 339 } 340 341 // 342 // If remaining DmaTransaction length is zero or if the driver wants 343 // this to be the last transfer then free the map registers and 344 // change the state to completed. 345 // 346 if (m_Remaining == 0 || CompletionType == FxDmaCompletionTypeAbort) { 347 status = STATUS_SUCCESS; 348 goto End; 349 } 350 351 // 352 // Stage the next packet for this DmaTransaction... 353 // 354 status = StageTransfer(); 355 356 if (NT_SUCCESS(status)) { 357 // 358 // StageTransfer results in a call to the EvtProgramDma routine 359 // where driver could complete and delete the object. So 360 // don't touch the object beyond this point. 361 // 362 status = STATUS_MORE_PROCESSING_REQUIRED; 363 } 364 else { 365 // 366 // The error will be returned to the caller of 367 // WdfDmaTransactionDmaComplete*() 368 // 369 } 370 371 End: 372 373 if (status != STATUS_MORE_PROCESSING_REQUIRED) { 374 // 375 // Failed or succeeded. Either way free 376 // map registers and release the device. 377 // 378 if (NT_SUCCESS(status)) { 379 m_State = FxDmaTransactionStateTransferCompleted; 380 } else { 381 m_State = FxDmaTransactionStateTransferFailed; 382 } 383 384 if (pFxDriverGlobals->FxVerifierOn) { 385 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 386 "WDFDMATRANSACTION %p completed with status %!STATUS! - " 387 "releasing DMA resources", 388 GetHandle(), 389 status); 390 } 391 392 ReleaseResources(FALSE); 393 394 if (m_EncodedRequest != NULL) { 395 ReleaseButRetainRequest(); 396 } 397 398 m_CurrentFragmentLength = 0; 399 400 hasTransitioned = TRUE; 401 } else { 402 hasTransitioned = FALSE; 403 } 404 405 *ReturnStatus = status; 406 407 if (pFxDriverGlobals->FxVerifierOn) { 408 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 409 "Exit WDFDMATRANSACTION %p " 410 "Transitioned(%!BOOLEAN!)", 411 dmaTransaction, hasTransitioned); 412 } 413 414 return hasTransitioned; 415 } 416 417 VOID 418 FxDmaTransactionBase::ReleaseForReuse( 419 __in BOOLEAN ForceRelease 420 ) 421 { 422 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 423 424 if (ForceRelease == FALSE) 425 { 426 if (m_State == FxDmaTransactionStateReleased) { 427 428 // 429 // Double release is probably due to cancel during early in transaction 430 // initialization. DC2 on very slow machines shows this behavior. 431 // The double release case is rare and benign. 432 // 433 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_WARNING, TRACINGDMA, 434 "WDFDMATRANSACTION %p is already released, " 435 "%!STATUS!", GetHandle(), STATUS_SUCCESS); 436 437 return; // already released. 438 } 439 440 // 441 // Must not be in transfer state. 442 // 443 if (m_State == FxDmaTransactionStateTransfer) { 444 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 445 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! " 446 "is invalid (release transaction)", GetHandle(), m_State); 447 448 if (pFxDriverGlobals->IsVerificationEnabled(1, 11, OkForDownLevel)) { 449 FxVerifierBugCheck(pFxDriverGlobals, // globals 450 WDF_DMA_FATAL_ERROR, // type 451 (ULONG_PTR) GetObjectHandle(), // parm 2 452 (ULONG_PTR) m_State); // parm 3 453 } 454 } 455 } 456 457 m_State = FxDmaTransactionStateReleased; 458 459 ReleaseResources(ForceRelease); 460 461 // 462 // Except DMA enabler field and adapter info everything else should be 463 // cleared. Adapter info is cleared by ReleaseResources above. 464 // 465 m_DmaAcquiredContext = NULL; 466 467 if (m_EncodedRequest != NULL) { 468 ClearRequest(); 469 } 470 471 m_StartMdl = NULL; 472 m_CurrentFragmentMdl = NULL; 473 m_StartOffset = 0; 474 m_CurrentFragmentOffset = 0; 475 m_CurrentFragmentLength = 0; 476 m_Transferred = 0; 477 m_Remaining = 0; 478 m_MaxFragmentLength = 0; 479 m_TransactionLength = 0; 480 m_Flags = 0; 481 482 m_DmaAcquiredFunction.Method.ProgramDma = NULL; 483 484 } 485 486 VOID 487 FxDmaTransactionBase::SetImmediateExecution( 488 __in BOOLEAN Value 489 ) 490 { 491 if (m_State != FxDmaTransactionStateCreated && 492 m_State != FxDmaTransactionStateInitialized && 493 m_State != FxDmaTransactionStateReleased) { 494 DoTraceLevelMessage( 495 GetDriverGlobals(), TRACE_LEVEL_ERROR, TRACINGDMA, 496 "Must set immediate execution flag for WDFDMATRANSACTION " 497 "%p before calling AllocateResources or Execute (current " 498 "state is %!FxDmaTransactionState!)", 499 GetHandle(), 500 m_State 501 ); 502 FxVerifierDbgBreakPoint(GetDriverGlobals()); 503 } 504 505 if (Value) { 506 m_Flags |= DMA_SYNCHRONOUS_CALLBACK; 507 } 508 else { 509 m_Flags &= ~DMA_SYNCHRONOUS_CALLBACK; 510 } 511 } 512 513 BOOLEAN 514 FxDmaTransactionBase::CancelResourceAllocation( 515 VOID 516 ) 517 { 518 if ((m_State == FxDmaTransactionStateCreated) || 519 (m_State == FxDmaTransactionStateReleased) || 520 (m_State == FxDmaTransactionStateDeleted)) { 521 522 DoTraceLevelMessage( 523 GetDriverGlobals(), 524 TRACE_LEVEL_ERROR, 525 TRACINGDMA, 526 "WDFDMATRANSACTION %p cannot be cancelled in state " 527 "%!FxDmaTransactionState!", 528 GetHandle(), 529 m_State 530 ); 531 532 FxVerifierBugCheck(GetDriverGlobals(), 533 WDF_DMA_FATAL_ERROR, 534 (ULONG_PTR) GetObjectHandle(), 535 (ULONG_PTR) m_State); 536 // unreachable code 537 } 538 539 PDMA_OPERATIONS dmaOperations = 540 m_AdapterInfo->AdapterObject->DmaOperations; 541 542 BOOLEAN result; 543 544 result = dmaOperations->CancelAdapterChannel( 545 m_AdapterInfo->AdapterObject, 546 m_DmaEnabler->m_FDO, 547 GetTransferContext() 548 ); 549 550 if (result) { 551 m_State = FxDmaTransactionStateTransferFailed; 552 553 if (m_EncodedRequest != NULL) { 554 ReleaseButRetainRequest(); 555 } 556 } 557 558 return result; 559 } 560 561 VOID 562 FxDmaTransactionBase::_ComputeNextTransferAddress( 563 __in PMDL CurrentMdl, 564 __in size_t CurrentOffset, 565 __in ULONG Transferred, 566 __deref_out PMDL *NextMdl, 567 __out size_t *NextOffset 568 ) 569 /*++ 570 571 Routine Description: 572 573 This function computes the next mdl and offset given the current MDL, 574 offset and bytes transfered. 575 576 Arguments: 577 578 CurrentMdl - Mdl where the transfer currently took place. 579 580 CurrentVa - Current virtual address in the buffer 581 582 Transfered - Bytes transfered or to be transfered 583 584 NextMdl - Mdl where the next transfer will take place 585 586 NextVA - Offset within NextMdl where the transfer will start 587 588 --*/ 589 { 590 size_t transfered, mdlSize; 591 PMDL mdl; 592 593 mdlSize = MmGetMdlByteCount(CurrentMdl) - CurrentOffset; 594 595 if (Transferred < mdlSize) { 596 // 597 // We are still in the first MDL 598 // 599 *NextMdl = CurrentMdl; 600 *NextOffset = CurrentOffset + Transferred; 601 return; 602 } 603 604 // 605 // We have transfered the content of the first MDL. 606 // Move to the next one. 607 // 608 transfered = Transferred - mdlSize; 609 mdl = CurrentMdl->Next; 610 ASSERT(mdl != NULL); 611 612 while (transfered >= MmGetMdlByteCount(mdl)) { 613 // 614 // We have transfered the content of this MDL. 615 // Move to the next one. 616 // 617 transfered -= MmGetMdlByteCount(mdl); 618 mdl = mdl->Next; 619 ASSERT(mdl != NULL); 620 } 621 622 // 623 // This is the mdl where the last transfer occured. 624 // 625 *NextMdl = mdl; 626 *NextOffset = transfered; 627 628 return; 629 } 630 631 _Must_inspect_result_ 632 NTSTATUS 633 FxDmaTransactionBase::_CalculateRequiredMapRegisters( 634 __in PMDL Mdl, 635 __in size_t CurrentOffset, 636 __in ULONG Length, 637 __in ULONG AvailableMapRegisters, 638 __out_opt PULONG PossibleTransferLength, 639 __out PULONG MapRegistersRequired 640 ) 641 /*++ 642 643 Routine Description: 644 645 Used on Windows 2000 to compute number of map registered required 646 for this transfer. This is derived from HalCalculateScatterGatherListSize. 647 648 Arguments: 649 650 Mdl - Pointer to the MDL that describes the pages of memory that are being 651 read or written. 652 653 CurrentVa - Current virtual address in the buffer described by the MDL 654 that the transfer is being done to or from. 655 656 Length - Supplies the length of the transfer. 657 658 AvailableMapRegisters - Map registers available to do the transfer 659 660 PossibleTransferLength - Length that can transfered for the 661 662 MapRegistersRequired - Map registers required to the entire transfer 663 664 Return Value: 665 666 NTSTATUS 667 668 Notes: 669 670 --*/ 671 { 672 PMDL tempMdl; 673 ULONG requiredMapRegisters; 674 ULONG transferLength; 675 ULONG mdlLength; 676 ULONG pageOffset; 677 ULONG possTransferLength; 678 679 // 680 // Calculate the number of required map registers. 681 // 682 tempMdl = Mdl; 683 transferLength = (ULONG) MmGetMdlByteCount(tempMdl) - (ULONG) CurrentOffset; 684 mdlLength = transferLength; 685 686 pageOffset = BYTE_OFFSET(GetStartVaFromOffset(tempMdl, CurrentOffset)); 687 requiredMapRegisters = 0; 688 possTransferLength = 0; 689 690 // 691 // The virtual address should fit in the first MDL. 692 // 693 694 ASSERT(CurrentOffset <= tempMdl->ByteCount); 695 696 // 697 // Loop through chained MDLs, accumulating the required 698 // number of map registers. 699 // 700 701 while (transferLength < Length && tempMdl->Next != NULL) { 702 703 // 704 // With pageOffset and length, calculate number of pages spanned by 705 // the buffer. 706 // 707 requiredMapRegisters += (pageOffset + mdlLength + PAGE_SIZE - 1) >> 708 PAGE_SHIFT; 709 710 if (requiredMapRegisters <= AvailableMapRegisters) { 711 possTransferLength = transferLength; 712 } 713 714 tempMdl = tempMdl->Next; 715 pageOffset = tempMdl->ByteOffset; 716 mdlLength = tempMdl->ByteCount; 717 transferLength += mdlLength; 718 } 719 720 if ((transferLength + PAGE_SIZE) < (Length + pageOffset )) { 721 ASSERT(transferLength >= Length); 722 return STATUS_BUFFER_TOO_SMALL; 723 } 724 725 // 726 // Calculate the last number of map registers based on the requested 727 // length not the length of the last MDL. 728 // 729 730 ASSERT( transferLength <= mdlLength + Length ); 731 732 requiredMapRegisters += (pageOffset + Length + mdlLength - transferLength + 733 PAGE_SIZE - 1) >> PAGE_SHIFT; 734 735 if (requiredMapRegisters <= AvailableMapRegisters) { 736 possTransferLength += (Length + mdlLength - transferLength); 737 } 738 739 if (PossibleTransferLength != NULL) { 740 *PossibleTransferLength = possTransferLength; 741 } 742 743 ASSERT(*PossibleTransferLength); 744 745 *MapRegistersRequired = requiredMapRegisters; 746 747 return STATUS_SUCCESS; 748 } 749 750 // ---------------------------------------------------------------------------- 751 // ------------------- Scatter/Gather DMA Section ----------------------------- 752 // ---------------------------------------------------------------------------- 753 754 FxDmaScatterGatherTransaction::FxDmaScatterGatherTransaction( 755 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 756 __in USHORT ExtraSize, 757 __in FxDmaEnabler *DmaEnabler 758 ) : 759 FxDmaTransactionBase(FxDriverGlobals, 760 sizeof(FxDmaScatterGatherTransaction), 761 ExtraSize, 762 DmaEnabler) 763 { 764 m_LookasideBuffer = NULL; 765 m_SGList = NULL; 766 } 767 768 _Must_inspect_result_ 769 NTSTATUS 770 FxDmaScatterGatherTransaction::_Create( 771 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 772 __in PWDF_OBJECT_ATTRIBUTES Attributes, 773 __in FxDmaEnabler* DmaEnabler, 774 __out WDFDMATRANSACTION* Transaction 775 ) 776 { 777 FxDmaScatterGatherTransaction* pTransaction; 778 WDFOBJECT hTransaction; 779 NTSTATUS status; 780 781 pTransaction = new (FxDriverGlobals, Attributes, DmaEnabler->GetTransferContextSize()) 782 FxDmaScatterGatherTransaction(FxDriverGlobals, 783 DmaEnabler->GetTransferContextSize(), 784 DmaEnabler); 785 786 if (pTransaction == NULL) { 787 status = STATUS_INSUFFICIENT_RESOURCES; 788 DoTraceLevelMessage( 789 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 790 "Could not allocate memory for WDFTRANSACTION, %!STATUS!", status); 791 return status; 792 } 793 794 // 795 // Commit and apply the attributes 796 // 797 status = pTransaction->Commit(Attributes, &hTransaction, DmaEnabler); 798 799 if (NT_SUCCESS(status) && DmaEnabler->m_IsSGListAllocated) { 800 801 // 802 // Allocate buffer for SGList from lookaside list. 803 // 804 pTransaction->m_LookasideBuffer = (SCATTER_GATHER_LIST *) 805 FxAllocateFromNPagedLookasideList( 806 &DmaEnabler->m_SGList.ScatterGatherProfile.Lookaside 807 ); 808 809 if (pTransaction->m_LookasideBuffer == NULL) { 810 status = STATUS_INSUFFICIENT_RESOURCES; 811 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 812 "Unable to allocate memory for SG List, " 813 "WDFDMATRANSACTION %p, %!STATUS! ", 814 pTransaction->GetHandle(), status); 815 } 816 else { 817 // 818 // Take a reference on the enabler to ensure that it remains valid 819 // if the transaction's disposal is deferred. 820 // 821 DmaEnabler->ADDREF(pTransaction); 822 } 823 } 824 825 if (NT_SUCCESS(status)) { 826 *Transaction = (WDFDMATRANSACTION)hTransaction; 827 } 828 else { 829 // 830 // This will properly clean up the target's state and free it 831 // 832 pTransaction->DeleteFromFailedCreate(); 833 } 834 835 return status; 836 } 837 838 BOOLEAN 839 FxDmaScatterGatherTransaction::Dispose( 840 VOID 841 ) 842 { 843 BOOLEAN ret; 844 845 ret = FxDmaTransactionBase::Dispose(); // __super call 846 847 // 848 // Free Lookaside Buffer which held SGList 849 // 850 if (m_LookasideBuffer != NULL) { 851 852 FxFreeToNPagedLookasideList( 853 &m_DmaEnabler->m_SGList.ScatterGatherProfile.Lookaside, 854 m_LookasideBuffer 855 ); 856 m_LookasideBuffer = NULL; 857 m_DmaEnabler->RELEASE(this); 858 } 859 860 return ret; 861 } 862 863 _Must_inspect_result_ 864 NTSTATUS 865 FxDmaScatterGatherTransaction::InitializeResources( 866 VOID 867 ) 868 { 869 NTSTATUS status; 870 PMDL nextMdl; 871 size_t nextOffset; 872 ULONG mapRegistersRequired; 873 size_t remLength, transferLength, transferred, possibleLength=0; 874 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 875 876 status = STATUS_SUCCESS; 877 878 // 879 // If the caller has specified a limit on the number of scatter-gather 880 // elements each transfer can support then make sure it's within the 881 // limit by breaking up the whole transfer into m_MaxFragmentLength and 882 // computing the number of map-registers required for each fragment. 883 // This check may not be valid if the driver starts to do partial 884 // transfers. So driver that do partial transfer with sg-element limit 885 // should be capable of handling STATUS_WDF_TOO_FRAGMENTED failures during 886 // dma execution. 887 // 888 remLength = m_TransactionLength; 889 transferred = 0; 890 nextMdl = m_StartMdl; 891 nextOffset = m_StartOffset; 892 transferLength = 0; 893 894 while (remLength != 0) { 895 896 _ComputeNextTransferAddress(nextMdl, 897 nextOffset, 898 (ULONG) transferLength, 899 &nextMdl, 900 &nextOffset); 901 902 transferLength = FxSizeTMin(remLength, m_MaxFragmentLength); 903 904 status = _CalculateRequiredMapRegisters(nextMdl, 905 nextOffset, 906 (ULONG) transferLength, 907 m_AdapterInfo->NumberOfMapRegisters, 908 (PULONG) &possibleLength, 909 &mapRegistersRequired 910 ); 911 912 if (!NT_SUCCESS(status)) { 913 DoTraceLevelMessage( 914 pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 915 "CalculateScatterGatherList failed for " 916 "WDFDMATRANSACTION %p, %!STATUS!", GetHandle(), status); 917 FxVerifierDbgBreakPoint(pFxDriverGlobals); 918 return status; 919 } 920 921 if (mapRegistersRequired > m_DmaEnabler->m_MaxSGElements) { 922 status = STATUS_WDF_TOO_FRAGMENTED; 923 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 924 "WDFDMATRANSACTION %p for MDL %p is more fragmented (%d) " 925 "than the limit (%I64d) specified by the driver, %!STATUS! ", 926 GetHandle(), nextMdl, mapRegistersRequired, 927 m_DmaEnabler->m_MaxSGElements, status); 928 return status; 929 } 930 931 transferred += transferLength; 932 remLength -= transferLength; 933 } 934 935 return status; 936 } 937 938 VOID 939 FxDmaScatterGatherTransaction::ReleaseResources( 940 __in BOOLEAN /* ForceRelease */ 941 ) 942 { 943 if (m_SGList != NULL) { 944 PutScatterGatherList(m_SGList); 945 m_SGList = NULL; 946 } 947 m_AdapterInfo = NULL; 948 } 949 950 _Must_inspect_result_ 951 NTSTATUS 952 FxDmaScatterGatherTransaction::StartTransfer( 953 VOID 954 ) 955 { 956 ASSERT(m_CurrentFragmentMdl == m_StartMdl); 957 ASSERT(m_CurrentFragmentOffset == m_StartOffset); 958 ASSERT(m_CurrentFragmentLength == 0); 959 ASSERT(m_Transferred == 0); 960 961 return StageTransfer(); 962 } 963 964 _Must_inspect_result_ 965 NTSTATUS 966 FxDmaScatterGatherTransaction::StageTransfer( 967 VOID 968 ) 969 { 970 NTSTATUS status; 971 ULONG mapRegistersRequired; 972 WDFDMATRANSACTION dmaTransaction; 973 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 974 975 // 976 // Use an invalid value to make the function fail if the var is not 977 // updated correctly below. 978 // 979 mapRegistersRequired = 0xFFFFFFFF; 980 981 // 982 // Client driver could complete and delete the object in 983 // EvtProgramDmaFunction. So, save the handle because we need it 984 // for tracing after we invoke the callback. 985 // 986 dmaTransaction = GetHandle(); 987 988 if (pFxDriverGlobals->FxVerifierOn) { 989 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 990 "Enter WDFDMATRANSACTION %p ", GetHandle()); 991 } 992 993 // 994 // Given the first MDL and the bytes transfered, find the next MDL 995 // and byteoffset within that MDL. 996 // 997 _ComputeNextTransferAddress(m_CurrentFragmentMdl, 998 m_CurrentFragmentOffset, 999 (ULONG) m_CurrentFragmentLength, 1000 &m_CurrentFragmentMdl, 1001 &m_CurrentFragmentOffset); 1002 1003 // 1004 // Get the next possible transfer size. 1005 // 1006 m_CurrentFragmentLength = FxSizeTMin(m_Remaining, m_MaxFragmentLength); 1007 1008 // 1009 // Fix m_CurrentFragmentLength to meet the map registers limit. This is done 1010 // in case the MDL is a chained MDL for an highly fragmented buffer. 1011 // 1012 status = _CalculateRequiredMapRegisters(m_CurrentFragmentMdl, 1013 m_CurrentFragmentOffset, 1014 (ULONG) m_CurrentFragmentLength, 1015 m_AdapterInfo->NumberOfMapRegisters, 1016 (PULONG)&m_CurrentFragmentLength, 1017 &mapRegistersRequired); 1018 // 1019 // We have already validated the entire transfer during initialize 1020 // to see each transfer meets the sglimit. So this call shouldn't fail. 1021 // But, if the driver does partial transfer and changes the fragment 1022 // boundaries then it's possible for the sg-elements to vary. So, check 1023 // one more time to see if we are within the bounds before building 1024 // the sglist and calling into the driver. 1025 // 1026 ASSERT(NT_SUCCESS(status)); 1027 1028 if (mapRegistersRequired > m_DmaEnabler->m_MaxSGElements) { 1029 status = STATUS_WDF_TOO_FRAGMENTED; 1030 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 1031 "WDFDMATRANSACTION %p for MDL %p is more fragmented (%d) " 1032 "than the limit (%I64d) specified by the driver, %!STATUS! ", 1033 dmaTransaction, m_CurrentFragmentMdl, mapRegistersRequired, 1034 m_DmaEnabler->m_MaxSGElements, status); 1035 return status; 1036 } 1037 1038 1039 m_Remaining -= m_CurrentFragmentLength; 1040 1041 if (m_DmaEnabler->m_IsSGListAllocated) { 1042 1043 ASSERT(m_LookasideBuffer != NULL); 1044 status = BuildScatterGatherList(m_CurrentFragmentMdl, 1045 m_CurrentFragmentOffset, 1046 (ULONG) m_CurrentFragmentLength, 1047 #pragma prefast(suppress: __WARNING_CLASS_MISMATCH_NONE, "This warning requires a wrapper class for the DRIVER_LIST_CONTROL type.") 1048 _AdapterListControl, 1049 this, 1050 m_LookasideBuffer, 1051 (ULONG) m_AdapterInfo->PreallocatedSGListSize); 1052 1053 } else { 1054 1055 status = GetScatterGatherList(m_CurrentFragmentMdl, 1056 m_CurrentFragmentOffset, 1057 (ULONG) m_CurrentFragmentLength, 1058 #pragma prefast(suppress: __WARNING_CLASS_MISMATCH_NONE, "This warning requires a wrapper class for the DRIVER_LIST_CONTROL type.") 1059 _AdapterListControl, 1060 this); 1061 } 1062 1063 if (!NT_SUCCESS(status)) { 1064 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 1065 "Build or GetScatterGatherList failed for " 1066 "WDFDMATRANSACTION %p, %!STATUS!", 1067 dmaTransaction, status); 1068 // 1069 // Readjust remaining bytes transfered. 1070 // 1071 m_Remaining += m_CurrentFragmentLength; 1072 return status; 1073 } 1074 1075 // 1076 // Before GetScatterGatherList returns, _AdapterListControl can get called 1077 // on another thread and the driver could delete the transaction object. 1078 // So don't touch the object after this point. 1079 // 1080 1081 if (pFxDriverGlobals->FxVerifierOn) { 1082 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1083 "Exit WDFDMATRANSACTION %p, " 1084 "%!STATUS!", dmaTransaction, status); 1085 } 1086 1087 return status; 1088 } 1089 1090 1091 VOID 1092 FxDmaScatterGatherTransaction::_AdapterListControl( 1093 __in PDEVICE_OBJECT DeviceObject, 1094 __in PIRP Irp, // UNUSED 1095 __in PSCATTER_GATHER_LIST SgList, 1096 __in PVOID Context 1097 ) 1098 { 1099 PFX_DRIVER_GLOBALS pFxDriverGlobals; 1100 WDFDMATRANSACTION dmaTransaction; 1101 FxDmaScatterGatherTransaction * pDmaTransaction; 1102 1103 UNREFERENCED_PARAMETER(Irp); 1104 UNREFERENCED_PARAMETER(DeviceObject); 1105 1106 pDmaTransaction = (FxDmaScatterGatherTransaction*) Context; 1107 pFxDriverGlobals = pDmaTransaction->GetDriverGlobals(); 1108 dmaTransaction = pDmaTransaction->GetHandle(); 1109 1110 if (pFxDriverGlobals->FxVerifierOn) { 1111 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1112 "Enter WDFDMATRANSACTION %p", 1113 dmaTransaction); 1114 } 1115 1116 ASSERT(pDmaTransaction != NULL); 1117 ASSERT(pDmaTransaction->m_DmaAcquiredFunction.Method.ProgramDma != NULL); 1118 1119 ASSERT(SgList->NumberOfElements <= pDmaTransaction->m_DmaEnabler->GetMaxSGElements()); 1120 1121 pDmaTransaction->m_SGList = SgList; 1122 1123 // 1124 // We ignore the return value. The pattern we want the driver to follow is 1125 // that if it fails to program DMA transfer, it should call DmaCompletedFinal 1126 // to abort the transfer. 1127 // 1128 (VOID) pDmaTransaction->m_DmaAcquiredFunction.InvokeProgramDma( 1129 dmaTransaction, 1130 pDmaTransaction->m_DmaEnabler->m_DeviceBase->GetHandle(), 1131 pDmaTransaction->m_DmaAcquiredContext, 1132 pDmaTransaction->m_DmaDirection, 1133 SgList); 1134 1135 if (pFxDriverGlobals->FxVerifierOn) { 1136 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1137 "Exit WDFDMATRANSACTION %p", 1138 dmaTransaction); 1139 } 1140 } 1141 1142 _Must_inspect_result_ 1143 NTSTATUS 1144 FxDmaScatterGatherTransaction::TransferCompleted( 1145 VOID 1146 ) 1147 { 1148 // 1149 // All we have to do is release the scatter-gather list. 1150 // 1151 if (m_SGList != NULL) { 1152 1153 PutScatterGatherList(m_SGList); 1154 m_SGList = NULL; 1155 } 1156 1157 return STATUS_SUCCESS; 1158 } 1159 1160 1161 // ---------------------------------------------------------------------------- 1162 // ------------------- PACKET DMA SECTION ------------------------------------- 1163 // ---------------------------------------------------------------------------- 1164 1165 FxDmaPacketTransaction::FxDmaPacketTransaction( 1166 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 1167 __in USHORT ObjectSize, 1168 __in USHORT ExtraSize, 1169 __in FxDmaEnabler *DmaEnabler 1170 ) : 1171 FxDmaTransactionBase(FxDriverGlobals, ObjectSize, ExtraSize, DmaEnabler) 1172 { 1173 m_MapRegistersNeeded = 0; 1174 m_MapRegisterBase = NULL; 1175 m_MapRegisterBaseSet = FALSE; 1176 m_DeviceAddressOffset = 0; 1177 m_MapRegistersReserved = 0; 1178 1179 m_IsCancelled = FALSE; 1180 1181 m_TransferState.CurrentStagingThread = NULL; 1182 m_TransferState.RerunStaging = FALSE; 1183 m_TransferState.RerunCompletion = FALSE; 1184 m_TransferState.CompletionStatus = UNDEFINED_DMA_COMPLETION_STATUS; 1185 } 1186 1187 _Must_inspect_result_ 1188 NTSTATUS 1189 FxDmaPacketTransaction::_Create( 1190 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 1191 __in PWDF_OBJECT_ATTRIBUTES Attributes, 1192 __in FxDmaEnabler* DmaEnabler, 1193 __out WDFDMATRANSACTION* Transaction 1194 ) 1195 { 1196 FxDmaPacketTransaction* pTransaction; 1197 WDFOBJECT hTransaction; 1198 NTSTATUS status; 1199 1200 pTransaction = new (FxDriverGlobals, Attributes, DmaEnabler->GetTransferContextSize()) 1201 FxDmaPacketTransaction(FxDriverGlobals, 1202 sizeof(FxDmaPacketTransaction), 1203 DmaEnabler->GetTransferContextSize(), 1204 DmaEnabler); 1205 1206 if (pTransaction == NULL) { 1207 status = STATUS_INSUFFICIENT_RESOURCES; 1208 DoTraceLevelMessage( 1209 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 1210 "Could not allocate memory for WDFTRANSACTION, %!STATUS!", status); 1211 return status; 1212 } 1213 1214 // 1215 // Commit and apply the attributes 1216 // 1217 status = pTransaction->Commit(Attributes, &hTransaction, DmaEnabler); 1218 if (NT_SUCCESS(status)) { 1219 *Transaction = (WDFDMATRANSACTION)hTransaction; 1220 } 1221 else { 1222 // 1223 // This will properly clean up the target's state and free it 1224 // 1225 pTransaction->DeleteFromFailedCreate(); 1226 } 1227 1228 return status; 1229 } 1230 1231 _Must_inspect_result_ 1232 NTSTATUS 1233 FxDmaPacketTransaction::InitializeResources( 1234 VOID 1235 ) 1236 { 1237 KIRQL oldIrql; 1238 m_DeviceAddressOffset = 0; 1239 LockTransferState(&oldIrql); 1240 m_IsCancelled = FALSE; 1241 UnlockTransferState(oldIrql); 1242 return STATUS_SUCCESS; 1243 } 1244 1245 VOID 1246 FxDmaPacketTransaction::ReleaseResources( 1247 __in BOOLEAN ForceRelease 1248 ) 1249 { 1250 // 1251 // If the map register base hasn't been assigned, then just 1252 // skip this. 1253 // 1254 1255 if (IsMapRegisterBaseSet() == FALSE) { 1256 return; 1257 } 1258 1259 // 1260 // Map registers are reserved. Unless the caller is forcing 1261 // us to free them, just return. Otherwise updated the 1262 // number of map registers that FreeMapRegistersAndAdapter 1263 // is going to look at. 1264 // 1265 if ((m_MapRegistersReserved > 0) && (ForceRelease == FALSE)) 1266 { 1267 return; 1268 } 1269 1270 // 1271 // Free the map registers and release the device. 1272 // 1273 FreeMapRegistersAndAdapter(); 1274 1275 ClearMapRegisterBase(); 1276 1277 ReleaseDevice(); 1278 1279 m_AdapterInfo = NULL; 1280 m_MapRegistersReserved = 0; 1281 } 1282 1283 _Must_inspect_result_ 1284 NTSTATUS 1285 FxDmaPacketTransaction::ReserveAdapter( 1286 __in ULONG NumberOfMapRegisters, 1287 __in WDF_DMA_DIRECTION DmaDirection, 1288 __in PFN_WDF_RESERVE_DMA Callback, 1289 __in_opt PVOID Context 1290 ) 1291 { 1292 NTSTATUS status; 1293 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 1294 WDFDMATRANSACTION dmaTransaction = GetHandle(); 1295 1296 if (pFxDriverGlobals->FxVerifierOn) { 1297 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1298 "Enter WDFDMATRANSACTION %p", dmaTransaction); 1299 } 1300 1301 // 1302 // If caller doesn't supply a map register count then we get the count 1303 // out of the transaction. So the transaction must be initialized. 1304 // 1305 // Otherwise the transaction can't be executing. 1306 // 1307 if (NumberOfMapRegisters == 0) { 1308 if (m_State != FxDmaTransactionStateInitialized) { 1309 status = STATUS_INVALID_PARAMETER; 1310 1311 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 1312 "RequiredMapRegisters cannot be 0 because " 1313 "WDFDMATRANSACTION %p is not initialized (" 1314 "state is %!FxDmaTransactionState!) - %!STATUS!", 1315 GetHandle(), 1316 m_State, 1317 status); 1318 FxVerifierBugCheck(pFxDriverGlobals, // globals 1319 WDF_DMA_FATAL_ERROR, // specific type 1320 (ULONG_PTR) GetObjectHandle(), // parm 2 1321 (ULONG_PTR) m_State); // parm 3 1322 } 1323 } 1324 else if (m_State != FxDmaTransactionStateCreated && 1325 m_State != FxDmaTransactionStateInitialized && 1326 m_State != FxDmaTransactionStateReleased) { 1327 1328 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 1329 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! " 1330 "is invalid", dmaTransaction, m_State); 1331 1332 FxVerifierBugCheck(pFxDriverGlobals, // globals 1333 WDF_DMA_FATAL_ERROR, // specific type 1334 (ULONG_PTR) GetObjectHandle(), // parm 2 1335 (ULONG_PTR) m_State); // parm 3 1336 } 1337 1338 // 1339 // Must not already have reserved map registers 1340 // 1341 if (m_MapRegistersReserved != 0) 1342 { 1343 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 1344 "WDFDMATRANSACTION %p already has allocated map registers.", 1345 dmaTransaction); 1346 1347 FxVerifierBugCheck(pFxDriverGlobals, // globals 1348 WDF_DMA_FATAL_ERROR, // specific type 1349 (ULONG_PTR) GetObjectHandle(), // parm 2 1350 (ULONG_PTR) m_State); // parm 3 1351 } 1352 1353 // 1354 // Get the adapter 1355 // 1356 if (DmaDirection == WdfDmaDirectionReadFromDevice) { 1357 m_AdapterInfo = m_DmaEnabler->GetReadDmaDescription(); 1358 } else { 1359 m_AdapterInfo = m_DmaEnabler->GetWriteDmaDescription(); 1360 } 1361 1362 // 1363 // Save the number of map registers being reserved. 1364 // 1365 if (NumberOfMapRegisters != 0) { 1366 1367 // 1368 // Use the number the caller passed us 1369 // 1370 m_MapRegistersReserved = NumberOfMapRegisters; 1371 } 1372 else if (m_DmaEnabler->IsBusMaster() == FALSE) { 1373 1374 // 1375 // For system DMA use all the map registers we have 1376 // 1377 m_MapRegistersReserved = m_AdapterInfo->NumberOfMapRegisters; 1378 1379 } else { 1380 1381 // 1382 // Compute the number of map registers required based on 1383 // the MDL and length passed in 1384 // 1385 status = _CalculateRequiredMapRegisters( 1386 m_StartMdl, 1387 m_StartOffset, 1388 (ULONG) m_TransactionLength, 1389 m_AdapterInfo->NumberOfMapRegisters, 1390 NULL, 1391 &m_MapRegistersReserved 1392 ); 1393 1394 if (!NT_SUCCESS(status)) { 1395 ReleaseForReuse(TRUE); 1396 goto End; 1397 } 1398 } 1399 1400 // 1401 // Initialize the DmaTransaction object with enough data to 1402 // trick StartTransfer into allocating the adapter channel for us. 1403 // 1404 m_DmaDirection = DmaDirection; 1405 m_StartMdl = NULL; 1406 m_StartOffset = 0; 1407 m_CurrentFragmentMdl = NULL; 1408 m_CurrentFragmentOffset = 0; 1409 m_Remaining = 0; 1410 m_TransactionLength = 0; 1411 1412 // 1413 // Save the callback and context 1414 // 1415 m_DmaAcquiredFunction.Method.ReserveDma = Callback; 1416 m_DmaAcquiredContext = Context; 1417 1418 // 1419 // If needed, initialize the transfer context. 1420 // 1421 if (m_DmaEnabler->UsesDmaV3()) { 1422 m_DmaEnabler->InitializeTransferContext(GetTransferContext(), 1423 m_DmaDirection); 1424 } 1425 1426 status = InitializeResources(); 1427 if (NT_SUCCESS(status)) { 1428 // 1429 // Set the state to reserved so _AdapterControl knows which 1430 // callback to invoke. 1431 // 1432 m_State = FxDmaTransactionStateReserved; 1433 } else { 1434 ReleaseForReuse(TRUE); 1435 goto End; 1436 } 1437 1438 // 1439 // Start the adapter channel allocation through StartTransfer 1440 // 1441 status = StartTransfer(); 1442 1443 End: 1444 if (!NT_SUCCESS(status)) { 1445 m_State = FxDmaTransactionStateTransferFailed; 1446 m_DmaAcquiredFunction.Method.ReserveDma = NULL; 1447 m_DmaAcquiredContext = NULL; 1448 m_MapRegistersReserved = 0; 1449 1450 if (m_EncodedRequest != NULL) { 1451 ReleaseButRetainRequest(); 1452 } 1453 } 1454 1455 if (pFxDriverGlobals->FxVerifierOn) { 1456 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1457 "Exit WDFDMATRANSACTION %p, %!STATUS!", 1458 dmaTransaction, status); 1459 } 1460 1461 return status; 1462 } 1463 1464 VOID 1465 FxDmaPacketTransaction::ReleaseAdapter( 1466 VOID 1467 ) 1468 { 1469 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 1470 WDFDMATRANSACTION dmaTransaction = GetHandle(); 1471 1472 if (pFxDriverGlobals->FxVerifierOn) { 1473 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1474 "Enter WDFDMATRANSACTION %p", dmaTransaction); 1475 } 1476 1477 // 1478 // Must not be in invalid, created, transfer or deleted state 1479 // 1480 if (m_State == FxDmaTransactionStateInvalid || 1481 m_State == FxDmaTransactionStateCreated || 1482 m_State == FxDmaTransactionStateTransfer || 1483 m_State == FxDmaTransactionStateDeleted) { 1484 1485 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 1486 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! " 1487 "is invalid", dmaTransaction, m_State); 1488 1489 FxVerifierBugCheck(pFxDriverGlobals, // globals 1490 WDF_DMA_FATAL_ERROR, // specific type 1491 (ULONG_PTR) GetObjectHandle(), // parm 2 1492 (ULONG_PTR) m_State); // parm 3 1493 } 1494 1495 // 1496 // The caller wants to free the reserved map registers, so force their 1497 // release. 1498 // 1499 ReleaseForReuse(TRUE); 1500 1501 if (pFxDriverGlobals->FxVerifierOn) { 1502 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1503 "Exit WDFDMATRANSACTION %p", 1504 dmaTransaction); 1505 } 1506 1507 return; 1508 } 1509 1510 _Must_inspect_result_ 1511 NTSTATUS 1512 FxDmaPacketTransaction::StartTransfer( 1513 VOID 1514 ) 1515 { 1516 NTSTATUS status; 1517 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 1518 WDFDMATRANSACTION dmaTransaction; 1519 1520 // 1521 // Client driver could complete and delete the object in 1522 // EvtProgramDmaFunction. So, save the handle because we need it 1523 // for tracing after we invoke the callback. 1524 // 1525 dmaTransaction = GetHandle(); 1526 1527 if (pFxDriverGlobals->FxVerifierOn) { 1528 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1529 "Enter WDFDMATRANSACTION %p", 1530 dmaTransaction); 1531 1532 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1533 "Starting WDFDMATRANSACTION %p - MDL %#p, " 1534 "offset %I64x, length %I64x", 1535 dmaTransaction, 1536 m_StartMdl, 1537 m_StartOffset, 1538 m_TransactionLength); 1539 } 1540 1541 // 1542 // Reference the device when using DMA v2. For DMA v3 we can support 1543 // concurrent attempts to allocate the channel. 1544 // 1545 status = AcquireDevice(); 1546 if (!NT_SUCCESS(status)) { 1547 1548 NT_ASSERTMSG("AcquireDevice should never fail when DMAv3 is in use", 1549 m_DmaEnabler->UsesDmaV3()); 1550 1551 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 1552 "Only one transaction can be queued " 1553 "at one time on a packet based WDFDMAENABLER %p " 1554 "%!STATUS!", m_DmaEnabler->GetHandle(), 1555 status); 1556 FxVerifierDbgBreakPoint(pFxDriverGlobals); 1557 return status; 1558 } 1559 1560 // 1561 // Calculate the initial DMA transfer length. 1562 // 1563 m_CurrentFragmentLength = FxSizeTMin(m_Remaining, m_MaxFragmentLength); 1564 1565 m_CurrentFragmentOffset = m_StartOffset; 1566 1567 if (m_State == FxDmaTransactionStateReserved) { 1568 // 1569 // Caller is simply reserving the DMA adapter for later use. Ask for 1570 // as many map registers as the driver requested. 1571 // 1572 m_MapRegistersNeeded = m_MapRegistersReserved; 1573 1574 ASSERT(m_MapRegistersNeeded <= m_AdapterInfo->NumberOfMapRegisters); 1575 1576 status = AllocateAdapterChannel(FALSE); 1577 1578 } 1579 else { 1580 1581 if (m_DmaEnabler->IsBusMaster() == FALSE) { 1582 1583 // 1584 // Use as many map registers as we were granted. 1585 // 1586 m_MapRegistersNeeded = m_AdapterInfo->NumberOfMapRegisters; 1587 } else { 1588 1589 // 1590 // If the transfer is the size of the transaction then use the offset 1591 // to determine the number of map registers needed. If it's smaller 1592 // then use the worst-case offset to make sure we ask for enough MR's 1593 // to account for a bigger offset in one of the later transfers. 1594 // 1595 // Example: 1596 // Transaction is 8 KB and is page aligned 1597 // if max transfer is >= 8KB then this will be one transfer and only 1598 // requires two map registers. Even if the driver completes a partial 1599 // transfer and we have to do the rest in a second transfer it will 1600 // fit within two map registers becuase the overall transaction does 1601 // (and a partial transfer can't take more map registers than the 1602 // whole transaction would). 1603 // 1604 // If max transfer is 2KB then this nominally requires 4 2KB transfers. 1605 // In this case however, a partial completion of one of those transfers 1606 // would leave us attempting a second 2KB transfer starting on an 1607 // unaligned address. For example, we might transfer 2KB, then 1KB 1608 // then 2KB. Even though the first transfer was page aligned, the 1609 // 3rd transfer isn't and could cross a page boundary, requiring two 1610 // map registers rather than one. 1611 // 1612 // To account for this second case, ignore the actual MDL offset and 1613 // instead compute the maximum number of map registers than an N byte 1614 // transfer could take (with worst-case alignment). 1615 // 1616 // 1617 m_MapRegistersNeeded = 1618 (ULONG) ADDRESS_AND_SIZE_TO_SPAN_PAGES( 1619 ((m_CurrentFragmentLength == m_Remaining) ? 1620 GetStartVaFromOffset(m_CurrentFragmentMdl, 1621 m_CurrentFragmentOffset) : 1622 (PVOID)(ULONG_PTR) (PAGE_SIZE -1)), 1623 m_CurrentFragmentLength 1624 ); 1625 1626 1627 ASSERT(m_MapRegistersNeeded <= m_AdapterInfo->NumberOfMapRegisters); 1628 } 1629 1630 // 1631 // NOTE: the number of map registers needed for this transfer may 1632 // exceed the number that we've reserved. StageTransfer will 1633 // take care of fragmenting the transaction accordingly. 1634 // 1635 status = AllocateAdapterChannel(m_MapRegistersReserved > 0); 1636 } 1637 1638 if (!NT_SUCCESS(status)) { 1639 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 1640 "AllocateAdapterChannel failed for " 1641 "WDFDMATRANSACTION %p, %!STATUS!", 1642 dmaTransaction, status); 1643 ReleaseDevice(); 1644 } 1645 1646 // 1647 // Before AllocateAdapterChannel returns, _AdapterControl can get called 1648 // on another thread and the driver could delete the transaction object. 1649 // So don't touch the object after this point. 1650 // 1651 if (pFxDriverGlobals->FxVerifierOn) { 1652 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1653 "Exit WDFDMATRANSACTION %p, " 1654 "%!STATUS!", dmaTransaction, status); 1655 } 1656 1657 return status; 1658 } 1659 1660 IO_ALLOCATION_ACTION 1661 FxDmaPacketTransaction::_AdapterControl( 1662 __in PDEVICE_OBJECT DeviceObject, 1663 __in PIRP Irp, 1664 __in PVOID MapRegisterBase, 1665 __in PVOID Context 1666 ) 1667 { 1668 FxDmaPacketTransaction * pDmaTransaction; 1669 PFX_DRIVER_GLOBALS pFxDriverGlobals; 1670 IO_ALLOCATION_ACTION action; 1671 NTSTATUS status; 1672 1673 UNREFERENCED_PARAMETER(Irp); 1674 UNREFERENCED_PARAMETER(DeviceObject); 1675 1676 pDmaTransaction = (FxDmaPacketTransaction*) Context; 1677 ASSERT(pDmaTransaction); 1678 1679 pFxDriverGlobals = pDmaTransaction->GetDriverGlobals(); 1680 1681 // 1682 // Cache the return value while we can still touch the transaction 1683 // 1684 action = pDmaTransaction->GetAdapterControlReturnValue(); 1685 1686 // 1687 // Save the MapRegister base, unless it was previously set 1688 // during a reserve. 1689 // 1690 if (pDmaTransaction->IsMapRegisterBaseSet() == FALSE) { 1691 pDmaTransaction->SetMapRegisterBase(MapRegisterBase); 1692 } 1693 else { 1694 NT_ASSERTMSG("Caller was expected to use existing map register base", 1695 MapRegisterBase == pDmaTransaction->m_MapRegisterBase); 1696 } 1697 1698 if (pFxDriverGlobals->FxVerifierOn) { 1699 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1700 "Map registers for WDFDMATRANSACTION %p allocated " 1701 "(base %p)", 1702 pDmaTransaction->GetHandle(), 1703 MapRegisterBase); 1704 } 1705 1706 // 1707 // NOTE: KMDF used to call KeFlushIoBuffers() here to "ensure the 1708 // data buffers were flushed." However KeFlushIoBuffers did 1709 // nothing on x86 & amd64 (which are cache coherent WRT DMA) 1710 // and calling FlushAdapterBuffers() does any necessary 1711 // flushing anyway. Plus on non-cache-coherent architectures 1712 // (such as ARM) the flush operation has to be cache-line aligned 1713 // to avoid cache line tearing. So the flush is not necessary 1714 // and has been removed. 1715 1716 // 1717 // Check the state of the transaction. If it's reserve then call the 1718 // reserve callback and return. Otherwise stage the first fragment. 1719 // 1720 if (pDmaTransaction->m_State == FxDmaTransactionStateReserved) 1721 { 1722 FxDmaTransactionProgramOrReserveDma callback; 1723 1724 // 1725 // Save off and clear the callback before calling it. 1726 // 1727 callback = pDmaTransaction->m_DmaAcquiredFunction; 1728 pDmaTransaction->m_DmaAcquiredFunction.Clear(); 1729 1730 ASSERTMSG("Mismatch between map register counts", 1731 (pDmaTransaction->m_MapRegistersReserved == 1732 pDmaTransaction->m_MapRegistersNeeded)); 1733 1734 // 1735 // Invoke the callback. Note that from here the driver may initialize 1736 // and execute the transaction. 1737 // 1738 callback.InvokeReserveDma( 1739 pDmaTransaction->GetHandle(), 1740 pDmaTransaction->m_DmaAcquiredContext 1741 ); 1742 } 1743 else { 1744 1745 // 1746 // Stage next fragment 1747 // 1748 status = pDmaTransaction->StageTransfer(); 1749 1750 if (!NT_SUCCESS(status)) { 1751 1752 DMA_COMPLETION_STATUS dmaStatus = 1753 (status == STATUS_CANCELLED ? DmaCancelled : DmaError); 1754 FxDmaSystemTransaction* systemTransaction = (FxDmaSystemTransaction*) pDmaTransaction; 1755 1756 // 1757 // Map transfer failed. There will be no DMA completion callback 1758 // and no call to EvtProgramDma. And we have no way to hand this 1759 // status back directly to the driver. Fake a DMA completion with 1760 // the appropriate status. 1761 // 1762 // This should only happen for system DMA (and there most likely 1763 // only during cancelation, though we leave the possibility that 1764 // the DMA extension may fail the transfer) 1765 // 1766 ASSERTMSG("Unexpected failure of StageTransfer for packet based " 1767 "DMA", 1768 (pDmaTransaction->GetDmaEnabler()->IsBusMaster() == false)); 1769 1770 if (pFxDriverGlobals->FxVerifierOn) { 1771 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1772 "Invoking DmaCompleted callback %p (context %p) " 1773 "for WDFDMATRANSACTION %p (status %x) " 1774 "due to staging failure (%!STATUS!)", 1775 systemTransaction->m_TransferCompleteFunction.Method, 1776 systemTransaction->m_TransferCompleteContext, 1777 pDmaTransaction->GetHandle(), 1778 dmaStatus, 1779 status); 1780 } 1781 1782 pDmaTransaction->CallEvtDmaCompleted( 1783 status == STATUS_CANCELLED ? DmaCancelled : DmaError 1784 ); 1785 } 1786 } 1787 1788 // 1789 // Indicate that MapRegs are to be kept 1790 // 1791 return action; 1792 } 1793 1794 _Must_inspect_result_ 1795 NTSTATUS 1796 FxDmaPacketTransaction::StageTransfer( 1797 VOID 1798 ) 1799 { 1800 PSCATTER_GATHER_LIST sgList; 1801 1802 PFX_DRIVER_GLOBALS pFxDriverGlobals; 1803 UCHAR_MEMORY_ALIGNED sgListBuffer[sizeof(SCATTER_GATHER_LIST) 1804 + sizeof(SCATTER_GATHER_ELEMENT)]; 1805 WDFDMATRANSACTION dmaTransaction; 1806 1807 KIRQL oldIrql; 1808 BOOLEAN stagingNeeded; 1809 1810 NTSTATUS status = STATUS_SUCCESS; 1811 1812 // 1813 // Client driver could complete and delete the object in 1814 // EvtProgramDmaFunction. So, save the handle because we need it 1815 // for tracing after we invoke the callback. 1816 // 1817 pFxDriverGlobals = GetDriverGlobals(); 1818 dmaTransaction = GetHandle(); 1819 1820 if (pFxDriverGlobals->FxVerifierOn) { 1821 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1822 "Enter WDFDMATRANSACTION %p ", dmaTransaction); 1823 } 1824 1825 // 1826 // For packet base DMA, current and startMDL will always be 1827 // same. For V2 DMA we don't support MDL chains. For V3 DMA 1828 // we use the HAL's support for MDL chains and don't walk through 1829 // the MDL chain on our own. 1830 // 1831 ASSERT(m_CurrentFragmentMdl == m_StartMdl); 1832 1833 LockTransferState(&oldIrql); 1834 1835 if (m_TransferState.CurrentStagingThread != NULL) { 1836 1837 // 1838 // Staging in progress. Indicate that another staging will 1839 // be needed. 1840 // 1841 m_TransferState.RerunStaging = TRUE; 1842 1843 stagingNeeded = FALSE; 1844 1845 if (pFxDriverGlobals->FxVerifierOn) { 1846 DoTraceLevelMessage( 1847 pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1848 "Staging next fragment of WDFDMATRANSACTION %p " 1849 "deferred", 1850 dmaTransaction 1851 ); 1852 } 1853 } 1854 else { 1855 // 1856 // Staging isn't in progress anyplace else. Indicate that it's 1857 // running now so that any parallel attempt is blocked. 1858 // 1859 m_TransferState.CurrentStagingThread = KeGetCurrentThread(); 1860 1861 ASSERTMSG("The thread which was staging didn't clear " 1862 "RerunStaging", 1863 (m_TransferState.RerunStaging == FALSE)); 1864 1865 stagingNeeded = TRUE; 1866 } 1867 1868 UnlockTransferState(oldIrql); 1869 1870 // 1871 // Take a reference on the transaction so that we can safely 1872 // manipulate the transfer state even after it's destroyed. 1873 // 1874 AddRef(&pFxDriverGlobals); 1875 1876 // 1877 // Loop for as long as staging is required 1878 // 1879 while (stagingNeeded) { 1880 1881 // 1882 // Calculate length for this packet. 1883 // 1884 m_CurrentFragmentLength = FxSizeTMin(m_Remaining, m_MaxFragmentLength); 1885 1886 // 1887 // Calculate address for this packet. 1888 // 1889 m_CurrentFragmentOffset = m_StartOffset + m_Transferred; 1890 1891 // 1892 // Adjust the fragment length for the number of reserved map registers. 1893 // 1894 if ((m_MapRegistersReserved > 0) && 1895 (m_MapRegistersNeeded > m_MapRegistersReserved)) 1896 { 1897 size_t currentOffset = m_CurrentFragmentOffset; 1898 size_t currentPageOffset; 1899 PMDL mdl; 1900 1901 for (mdl = m_CurrentFragmentMdl; mdl != NULL; mdl = mdl->Next) 1902 { 1903 // 1904 // For packet/system transfers of chained MDLs, m_CurrentFragmentMdl 1905 // is never adjusted, and m_CurrentFragmentOFfset is the offset 1906 // into the entire chain. 1907 // 1908 // Locate the MDL which contains the current fragment. 1909 // 1910 ULONG mdlBytes = MmGetMdlByteCount(mdl); 1911 if (mdlBytes >= currentOffset) 1912 { 1913 // 1914 // This MDL is larger than the remaining offset, so it 1915 // contains the start address. 1916 // 1917 break; 1918 } 1919 1920 currentOffset -= mdlBytes; 1921 } 1922 1923 ASSERT(mdl != NULL); 1924 1925 // 1926 // Compute page offset from current MDL's initial page offset 1927 // and the offset into that MDL 1928 // 1929 1930 currentPageOffset = BYTE_OFFSET(MmGetMdlByteOffset(mdl) + 1931 currentOffset); 1932 1933 // 1934 // Compute the maximum number of bytes we can transfer with 1935 // the number of map registers we have reserved, taking into 1936 // account the offset of the first page. 1937 // 1938 size_t l = ((PAGE_SIZE * (m_MapRegistersReserved - 1)) + 1939 (PAGE_SIZE - currentPageOffset)); 1940 1941 m_CurrentFragmentLength = FxSizeTMin(m_CurrentFragmentLength, l); 1942 } 1943 1944 m_Remaining -= m_CurrentFragmentLength; 1945 1946 if (pFxDriverGlobals->FxVerifierOn) { 1947 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1948 "Initiating %s transfer for WDFDMATRANSACTION %p. " 1949 "Mdl %p, Offset %I64x, Length %I64x", 1950 m_Transferred == 0 ? "first" : "next", 1951 GetHandle(), 1952 m_CurrentFragmentMdl, 1953 m_Transferred, 1954 m_CurrentFragmentLength); 1955 } 1956 1957 // 1958 // Check for a pending cancellation. This can happen if the cancel 1959 // occurred between DMA completion and FlushAdapterBuffers - 1960 // FlushAdapterBuffers will clear the canceled bit in the transfer 1961 // context (TC), which would allow MapTransfer to succeed. 1962 // 1963 // An unprotected check of IsCancelled here is safe. A concurrent 1964 // cancel at this point would mark the TC cancelled such that 1965 // MapTransfer will fail. 1966 // 1967 if (m_IsCancelled == TRUE) { 1968 status = STATUS_CANCELLED; 1969 goto End; 1970 } 1971 1972 // 1973 // Profile specific work before mapping the transfer. if this 1974 // fails consider 'this' invalid. 1975 // 1976 if (PreMapTransfer() == FALSE) { 1977 status = STATUS_SUCCESS; 1978 goto End; 1979 } 1980 1981 // 1982 // Map this packet for transfer. 1983 // 1984 1985 // 1986 // For packet based DMA we use a single entry on-stack SGL. This 1987 // allows us to map multiple packet-based requests concurrently and 1988 // we know packet base DMA only requires a single SGL 1989 // 1990 // NOTE: It turns out the HAL doesn't handle chained MDLs in packet 1991 // mode correctly. It makes each MDL continguous, but returns 1992 // a list of SG elements - one for each MDL. That's scatter 1993 // gather DMA, not packet DMA. 1994 // 1995 // So it's actually very important in Win8 that we only use a 1996 // single entry SGL when calling MapTransferEx. This ensures 1997 // we only map the first MDL in the chain and thus get a 1998 // contiguous buffer 1999 // 2000 // For system DMA we use the SystemSGList stored in the DMA enabler 2001 // We can use a shared one in that case because we won't be mapping 2002 // multiple system DMA requests concurrently (HAL doesn't allow it) 2003 // 2004 FxDmaEnabler* enabler = GetDmaEnabler(); 2005 size_t sgListSize; 2006 2007 if (enabler->IsBusMaster()) { 2008 sgList = (PSCATTER_GATHER_LIST)sgListBuffer; 2009 sgListSize = sizeof(sgListBuffer); 2010 } else { 2011 sgList = enabler->m_SGList.SystemProfile.List; 2012 sgListSize = enabler->m_SGListSize; 2013 } 2014 2015 ULONG mappedBytes; 2016 2017 status = MapTransfer(sgList, 2018 (ULONG) sgListSize, 2019 GetTransferCompletionRoutine(), 2020 this, 2021 &mappedBytes); 2022 2023 NT_ASSERTMSG("Unexpected failure of MapTransfer", 2024 ((NT_SUCCESS(status) == TRUE) || 2025 (status == STATUS_CANCELLED))); 2026 2027 if (NT_SUCCESS(status)) { 2028 2029 NT_ASSERTMSG("unexpected number of mapped bytes", 2030 ((mappedBytes > 0) && 2031 (mappedBytes <= m_CurrentFragmentLength))); 2032 2033 // 2034 // Adjust the remaining byte count if the HAL mapped less data than we 2035 // requested. 2036 // 2037 if (mappedBytes < m_CurrentFragmentLength) { 2038 m_Remaining += m_CurrentFragmentLength - mappedBytes; 2039 m_CurrentFragmentLength = mappedBytes; 2040 } 2041 2042 // 2043 // Do client PFN_WDF_PROGRAM_DMA callback. 2044 // 2045 if (m_DmaAcquiredFunction.Method.ProgramDma != NULL) { 2046 2047 if (pFxDriverGlobals->FxVerifierOn) { 2048 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 2049 "Invoking ProgramDma callback %p (context %p) for " 2050 "WDFDMATRANSACTION %p.", 2051 m_DmaAcquiredFunction.Method.ProgramDma, 2052 m_DmaAcquiredContext, 2053 GetHandle() 2054 ); 2055 } 2056 2057 // 2058 // Call program DMA 2059 // 2060 (VOID) m_DmaAcquiredFunction.InvokeProgramDma( 2061 GetHandle(), 2062 m_DmaEnabler->m_DeviceBase->GetHandle(), 2063 m_DmaAcquiredContext, 2064 m_DmaDirection, 2065 sgList 2066 ); 2067 } 2068 } 2069 2070 End: 2071 // 2072 // Process any pending completion or nested staging. 2073 // 2074 { 2075 LockTransferState(&oldIrql); 2076 2077 // 2078 // While staging we could either have deferred a call to the 2079 // completion routine or deferred another call to stage the 2080 // next fragment. We should not ever have to do both - this 2081 // would imply that the driver didn't wait for its DMA completion 2082 // routine to run when calling TransferComplete*. 2083 // 2084 ASSERTMSG("driver called TransferComplete with pending DMA " 2085 "completion callback", 2086 !((m_TransferState.RerunCompletion == TRUE) && 2087 (m_TransferState.RerunStaging == TRUE))); 2088 2089 // 2090 // Check for pending completion. save the status away and clear it 2091 // before dropping the lock. 2092 // 2093 if (m_TransferState.RerunCompletion == TRUE) { 2094 DMA_COMPLETION_STATUS completionStatus; 2095 FxDmaSystemTransaction* systemTransaction = (FxDmaSystemTransaction*) this; 2096 2097 // 2098 // Save the completion status for when we drop the lock. 2099 // 2100 completionStatus = m_TransferState.CompletionStatus; 2101 2102 ASSERTMSG("completion needed, but status was not set or was " 2103 "already cleared", 2104 completionStatus != UNDEFINED_DMA_COMPLETION_STATUS); 2105 2106 ASSERTMSG("completion needed, but mapping failed so there shouldn't " 2107 "be any parallel work going on", 2108 NT_SUCCESS(status)); 2109 2110 // 2111 // Clear the completion needed state. 2112 // 2113 m_TransferState.RerunCompletion = FALSE; 2114 m_TransferState.CompletionStatus = UNDEFINED_DMA_COMPLETION_STATUS; 2115 2116 // 2117 // Drop the lock, call the completion routine, then take the 2118 // lock again. 2119 // 2120 UnlockTransferState(oldIrql); 2121 if (pFxDriverGlobals->FxVerifierOn) { 2122 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 2123 "Invoking DmaCompleted callback %p (context %p) " 2124 "for WDFDMATRANSACTION %p (status %x) " 2125 "after deferral", 2126 systemTransaction->m_TransferCompleteFunction.Method, 2127 systemTransaction->m_TransferCompleteContext, 2128 GetHandle(), 2129 completionStatus); 2130 } 2131 CallEvtDmaCompleted(completionStatus); 2132 LockTransferState(&oldIrql); 2133 2134 // 2135 // Staging is blocked, which means we aren't starting up the 2136 // next transfer. Therefore we cannot have a queued completion. 2137 // 2138 ASSERTMSG("RerunCompletion should not be set on an unstaged " 2139 "transaction", 2140 m_TransferState.RerunCompletion == FALSE); 2141 } 2142 2143 // 2144 // Capture whether another staging is needed. If none is needed 2145 // then we can clear staging in progress. 2146 // 2147 if (m_TransferState.RerunStaging == TRUE) { 2148 stagingNeeded = TRUE; 2149 m_TransferState.RerunStaging = FALSE; 2150 } 2151 else { 2152 m_TransferState.CurrentStagingThread = NULL; 2153 stagingNeeded = FALSE; 2154 } 2155 2156 UnlockTransferState(oldIrql); 2157 } 2158 2159 #if DBG 2160 if (!NT_SUCCESS(status)) { 2161 ASSERTMSG("MapTransfer returned an error - there should not be any " 2162 "deferred work.", 2163 (stagingNeeded == FALSE)); 2164 } 2165 #endif 2166 2167 } // while(stagingNeeded) 2168 2169 Release(&pFxDriverGlobals); 2170 2171 if (pFxDriverGlobals->FxVerifierOn) { 2172 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 2173 "Exit WDFDMATRANSACTION %p, " 2174 "%!STATUS!", dmaTransaction, 2175 status); 2176 } 2177 2178 return status; 2179 } 2180 2181 _Must_inspect_result_ 2182 NTSTATUS 2183 FxDmaPacketTransaction::TransferCompleted( 2184 VOID 2185 ) 2186 { 2187 NTSTATUS status; 2188 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 2189 2190 // 2191 // Flush the buffers 2192 // 2193 status = FlushAdapterBuffers(); 2194 2195 if (!NT_SUCCESS(status)) { 2196 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 2197 "FlushAdapterBuffers on WDFDMATRANSACTION %p " 2198 "failed, %!STATUS!", 2199 GetHandle(), status); 2200 FxVerifierDbgBreakPoint(pFxDriverGlobals); 2201 } 2202 2203 return status; 2204 } 2205 2206 // ---------------------------------------------------------------------------- 2207 // ------------------- SYSTEM DMA SECTION ------------------------------------- 2208 // ---------------------------------------------------------------------------- 2209 2210 FxDmaSystemTransaction::FxDmaSystemTransaction( 2211 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 2212 __in USHORT ExtraSize, 2213 __in FxDmaEnabler *DmaEnabler 2214 ) : 2215 FxDmaPacketTransaction(FxDriverGlobals, sizeof(FxDmaSystemTransaction), ExtraSize, DmaEnabler) 2216 { 2217 return; 2218 } 2219 2220 _Must_inspect_result_ 2221 NTSTATUS 2222 FxDmaSystemTransaction::_Create( 2223 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 2224 __in PWDF_OBJECT_ATTRIBUTES Attributes, 2225 __in FxDmaEnabler* DmaEnabler, 2226 __out WDFDMATRANSACTION* Transaction 2227 ) 2228 { 2229 FxDmaPacketTransaction* pTransaction; 2230 WDFOBJECT hTransaction; 2231 NTSTATUS status; 2232 2233 pTransaction = new (FxDriverGlobals, Attributes, DmaEnabler->GetTransferContextSize()) 2234 FxDmaSystemTransaction(FxDriverGlobals, DmaEnabler->GetTransferContextSize(), DmaEnabler); 2235 2236 if (pTransaction == NULL) { 2237 status = STATUS_INSUFFICIENT_RESOURCES; 2238 DoTraceLevelMessage( 2239 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 2240 "Could not allocate memory for WDFTRANSACTION, %!STATUS!", status); 2241 return status; 2242 } 2243 2244 // 2245 // Commit and apply the attributes 2246 // 2247 status = pTransaction->Commit(Attributes, &hTransaction, DmaEnabler); 2248 if (NT_SUCCESS(status)) { 2249 *Transaction = (WDFDMATRANSACTION)hTransaction; 2250 } 2251 else { 2252 // 2253 // This will properly clean up the target's state and free it 2254 // 2255 pTransaction->DeleteFromFailedCreate(); 2256 } 2257 2258 return status; 2259 } 2260 2261 BOOLEAN 2262 FxDmaSystemTransaction::PreMapTransfer( 2263 VOID 2264 ) 2265 { 2266 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 2267 BOOLEAN result = TRUE; 2268 2269 if (m_ConfigureChannelFunction.Method != NULL) { 2270 // 2271 // Invoke the callback. If it returns false then the driver has 2272 // completed the transaction in the callback and we must abort 2273 // processing. 2274 // 2275 2276 if (pFxDriverGlobals->FxVerifierOn) { 2277 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 2278 "Invoking ConfigureChannel callback %p (context " 2279 "%p) for WDFDMATRANSACTION %p.", 2280 m_ConfigureChannelFunction.Method, 2281 m_ConfigureChannelContext, 2282 GetHandle()); 2283 } 2284 2285 result = m_ConfigureChannelFunction.Invoke( 2286 GetHandle(), 2287 m_DmaEnabler->m_DeviceBase->GetHandle(), 2288 m_ConfigureChannelContext, 2289 m_CurrentFragmentMdl, 2290 m_CurrentFragmentOffset, 2291 m_CurrentFragmentLength 2292 ); 2293 } 2294 2295 return result; 2296 } 2297 2298 2299 PDMA_COMPLETION_ROUTINE 2300 FxDmaSystemTransaction::GetTransferCompletionRoutine( 2301 VOID 2302 ) 2303 { 2304 if (m_TransferCompleteFunction.Method == NULL) { 2305 return NULL; 2306 } 2307 else { 2308 return _SystemDmaCompletion; 2309 } 2310 } 2311 2312 VOID 2313 FxDmaSystemTransaction::CallEvtDmaCompleted( 2314 __in DMA_COMPLETION_STATUS Status 2315 ) 2316 { 2317 // 2318 // Call the TransferComplete callback to indicate that the 2319 // transfer was aborted. 2320 // 2321 m_TransferCompleteFunction.Invoke( 2322 GetHandle(), 2323 m_DmaEnabler->m_Device->GetHandle(), 2324 m_TransferCompleteContext, 2325 m_DmaDirection, 2326 Status 2327 ); 2328 } 2329 2330 VOID 2331 FxDmaTransactionBase::GetTransferInfo( 2332 __out_opt ULONG *MapRegisterCount, 2333 __out_opt ULONG *ScatterGatherElementCount 2334 ) 2335 { 2336 if (m_State != FxDmaTransactionStateInitialized) { 2337 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 2338 2339 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 2340 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! " 2341 "is invalid", GetHandle(), m_State); 2342 2343 FxVerifierBugCheck(pFxDriverGlobals, // globals 2344 WDF_DMA_FATAL_ERROR, // specific type 2345 (ULONG_PTR) GetObjectHandle(), // parm 2 2346 (ULONG_PTR) m_State); // parm 3 2347 } 2348 2349 DMA_TRANSFER_INFO info = {0}; 2350 2351 2352 2353 2354 2355 2356 if (m_DmaEnabler->UsesDmaV3()) { 2357 2358 // 2359 // Ask the HAL for information about the MDL and how many resources 2360 // it will require to transfer. 2361 // 2362 m_AdapterInfo->AdapterObject->DmaOperations->GetDmaTransferInfo( 2363 m_AdapterInfo->AdapterObject, 2364 m_StartMdl, 2365 m_StartOffset, 2366 (ULONG) this->m_TransactionLength, 2367 this->m_DmaDirection == WDF_DMA_DIRECTION::WdfDmaDirectionWriteToDevice, 2368 &info 2369 ); 2370 } else { 2371 size_t offset = m_StartOffset; 2372 size_t length = m_TransactionLength; 2373 2374 // 2375 // Walk through the MDL chain and make a worst-case computation of 2376 // the number of scatter gather entries and map registers the 2377 // transaction would require. 2378 // 2379 for(PMDL mdl = m_StartMdl; 2380 mdl != NULL && length != 0; 2381 mdl = mdl->Next) { 2382 2383 size_t byteCount = MmGetMdlByteCount(mdl); 2384 if (byteCount <= offset) { 2385 offset -= byteCount; 2386 } else { 2387 ULONG_PTR startVa = (ULONG_PTR) MmGetMdlVirtualAddress(mdl); 2388 2389 startVa += offset; 2390 byteCount -= offset; 2391 2392 info.V1.MapRegisterCount += 2393 (ULONG) ADDRESS_AND_SIZE_TO_SPAN_PAGES( 2394 startVa, 2395 min(byteCount, length) 2396 ); 2397 2398 length -= min(byteCount, length); 2399 } 2400 } 2401 2402 info.V1.ScatterGatherElementCount = info.V1.MapRegisterCount; 2403 } 2404 2405 if (ARGUMENT_PRESENT(MapRegisterCount)) { 2406 *MapRegisterCount = info.V1.MapRegisterCount; 2407 } 2408 2409 if (ARGUMENT_PRESENT(ScatterGatherElementCount)) { 2410 *ScatterGatherElementCount = info.V1.ScatterGatherElementCount; 2411 } 2412 2413 return; 2414 } 2415 2416 VOID 2417 FxDmaSystemTransaction::_SystemDmaCompletion( 2418 __in PDMA_ADAPTER /* DmaAdapter */, 2419 __in PDEVICE_OBJECT /* DeviceObject */, 2420 __in PVOID CompletionContext, 2421 __in DMA_COMPLETION_STATUS Status 2422 ) 2423 { 2424 FxDmaSystemTransaction* transaction = (FxDmaSystemTransaction*) CompletionContext; 2425 PFX_DRIVER_GLOBALS pFxDriverGlobals = transaction->GetDriverGlobals(); 2426 KIRQL oldIrql; 2427 BOOLEAN completionDeferred; 2428 2429 // 2430 // Lock the transfer state so that a staging or cancelling thread 2431 // cannot change it. 2432 // 2433 transaction->LockTransferState(&oldIrql); 2434 2435 ASSERTMSG("Completion state was already set", 2436 (transaction->m_TransferState.CompletionStatus == 2437 UNDEFINED_DMA_COMPLETION_STATUS)); 2438 ASSERTMSG("Deferred completion is already pending", 2439 (transaction->m_TransferState.RerunCompletion == FALSE)); 2440 2441 // 2442 // If a staging is in progress then defer the completion. 2443 // 2444 if (transaction->m_TransferState.CurrentStagingThread != NULL) { 2445 transaction->m_TransferState.CompletionStatus = Status; 2446 transaction->m_TransferState.RerunCompletion = TRUE; 2447 completionDeferred = TRUE; 2448 } 2449 else { 2450 completionDeferred = FALSE; 2451 } 2452 2453 transaction->UnlockTransferState(oldIrql); 2454 2455 // 2456 // Process the old state. 2457 // 2458 if (completionDeferred == TRUE) { 2459 // 2460 // The staging thread has not moved past EvtProgramDma. The staging thread 2461 // will detect the state change and call the completion routine. 2462 // 2463 // Nothing to do in this case. 2464 // 2465 if (pFxDriverGlobals->FxVerifierOn) { 2466 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 2467 "Deferring DmaCompleted callback for WDFDMATRANSACTION %p" 2468 "(status %x)", 2469 transaction->GetHandle(), 2470 Status); 2471 } 2472 } 2473 else { 2474 // 2475 // Completion occurred while the transfer was running or 2476 // being cancelled. Call the completion routine. 2477 // 2478 // Note: a cancel when in programming state leaves the 2479 // state as programming. that we're not in programming 2480 // means we don't need to worry about racing with 2481 // EvtProgramDma. 2482 // 2483 2484 if (pFxDriverGlobals->FxVerifierOn) { 2485 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 2486 "Invoking DmaCompleted callback %p (context %p) " 2487 "for WDFDMATRANSACTION %p (status %x)", 2488 transaction->m_TransferCompleteFunction.Method, 2489 transaction->m_TransferCompleteContext, 2490 transaction->GetHandle(), 2491 Status); 2492 } 2493 transaction->CallEvtDmaCompleted(Status); 2494 } 2495 } 2496 2497 VOID 2498 FxDmaSystemTransaction::StopTransfer( 2499 VOID 2500 ) 2501 { 2502 // 2503 // Mark the transfer cancelled so we have a record of it even if 2504 // a racing call to FlushAdapterBuffers clears the TC. 2505 // 2506 m_IsCancelled = TRUE; 2507 2508 // 2509 // Cancel the system DMA transfer. This arranges for one of two things 2510 // to happen: 2511 // * the next call to MapTransfer will fail 2512 // * the DMA completion routine will run 2513 // 2514 if (CancelMappedTransfer() == FALSE) { 2515 2516 // 2517 // The cancel failed. Someone has already stopped this transfer. 2518 // That's illegal. 2519 // 2520 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 2521 2522 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA, 2523 "WDFDMATRANSACTION %p has already been stopped", 2524 GetHandle()); 2525 2526 if (pFxDriverGlobals->IsVerificationEnabled(1, 11, OkForDownLevel)) { 2527 FxVerifierBugCheck(pFxDriverGlobals, // globals 2528 WDF_DMA_FATAL_ERROR, // type 2529 (ULONG_PTR) GetObjectHandle(), // parm 2 2530 (ULONG_PTR) m_State); // parm 3 2531 } 2532 } 2533 } 2534