1 // 2 // Copyright (C) Microsoft. All rights reserved. 3 // 4 #ifndef _FXDMATRANSACTION_HPP_ 5 #define _FXDMATRANSACTION_HPP_ 6 7 extern "C" { 8 #include "FxDmaTransaction.hpp.tmh" 9 } 10 11 #include "FxDmaTransactionCallbacks.hpp" 12 13 // 14 // This type is used to allocate scatter-gather list of 1 element on the stack. 15 // 16 typedef __declspec(align(MEMORY_ALLOCATION_ALIGNMENT))UCHAR UCHAR_MEMORY_ALIGNED; 17 18 // begin_wpp enum 19 20 // 21 // FxDmaTransactionStateCreated when the object is created. 22 // FxDmaTransactionStateInitialized when object is initialized using with 23 // Mdl/VA/Length. 24 // FxDmaTransactionStateReserved when driver calls AllocateResources until 25 // the adapter control routine returns 26 // FxDmaTransactionStateTransfer is called when the driver call Execute 27 // to start the DMA transfer. 28 // FxDmaTransactionStateTransferCompleted is when transfer is completed or 29 // aborted 30 // FxDmaTransactionStateTransferFailed is set if the framework is not able 31 // to start the transfer due to error. 32 // FxDmaTransactionStateReleased is set when the object is reinitailized for reuse 33 // FxDmaTransactionStateDeleted is set in the Dipose due to WdfObjectDelete 34 // 35 enum FxDmaTransactionState { 36 FxDmaTransactionStateInvalid = 0, 37 FxDmaTransactionStateCreated, 38 FxDmaTransactionStateReserved, 39 FxDmaTransactionStateInitialized, 40 FxDmaTransactionStateTransfer, 41 FxDmaTransactionStateTransferCompleted, 42 FxDmaTransactionStateTransferFailed, 43 FxDmaTransactionStateReleased, 44 FxDmaTransactionStateDeleted, 45 }; 46 47 // 48 // FxDmaCompletionTypeFull is used when the driver calls WdfDmaTransactionDmaComplete 49 // to indicate that last framgement has been transmitted fully and to initiate 50 // the transfer of next fragment. 51 // FxDmaCompletionTypePartial is used when the driver completes the transfer and 52 // specifies a amount of bytes it has transfered, and to initiate the next transfer 53 // from the untransmitted portion of the buffer. 54 // FxDmaCompletionTypeAbort i used when the driver calls DmaCompleteFinal to indicate 55 // that's the final transfer and not initiate anymore transfers for the remaining 56 // data. 57 // 58 enum FxDmaCompletionType { 59 FxDmaCompletionTypeFull = 1, 60 FxDmaCompletionTypePartial, 61 FxDmaCompletionTypeAbort, 62 }; 63 64 // end_wpp 65 66 // 67 // This tag is used to track whether the request pointer in the transaction 68 // has a reference taken on it. Since the pointer is guaranteed to be 69 // 4-byte aligned this can be set and cleared without destroying the pointer. 70 // 71 #define FX_STRONG_REF_TAG 0x1 72 73 // 74 // Simple set of macros to encode and decode tagged pointers. 75 // 76 #define FX_ENCODE_POINTER(T,p,tag) ((T*) ((ULONG_PTR) p | (ULONG_PTR) tag)) 77 #define FX_DECODE_POINTER(T,p,tag) ((T*) ((ULONG_PTR) p & ~(ULONG_PTR) tag)) 78 #define FX_IS_POINTER_ENCODED(p,tag) ((((ULONG_PTR) p & (ULONG_PTR) tag) == tag) ? TRUE : FALSE) 79 80 // 81 // An uninitialized value for Dma completion status 82 // 83 84 #define UNDEFINED_DMA_COMPLETION_STATUS ((DMA_COMPLETION_STATUS) -1) 85 86 class FxDmaTransactionBase : public FxNonPagedObject { 87 88 friend class FxDmaEnabler; 89 90 public: 91 92 FxDmaTransactionBase( 93 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 94 __in USHORT ObjectSize, 95 __in USHORT ExtraSize, 96 __in FxDmaEnabler *DmaEnabler 97 ); 98 99 static 100 VOID 101 _ComputeNextTransferAddress( 102 __in PMDL CurrentMdl, 103 __in size_t CurrentOffset, 104 __in ULONG Transferred, 105 __deref_out PMDL *NextMdl, 106 __out size_t *NextOffset 107 ); 108 109 _Must_inspect_result_ 110 static 111 NTSTATUS 112 _CalculateRequiredMapRegisters( 113 __in PMDL Mdl, 114 __in size_t CurrentOffset, 115 __in ULONG Length, 116 __in ULONG AvailableMapRegisters, 117 __out_opt PULONG PossibleTransferLength, 118 __out PULONG MapRegistersRequired 119 ); 120 121 virtual 122 BOOLEAN 123 Dispose( 124 VOID 125 ); 126 127 _Must_inspect_result_ 128 NTSTATUS 129 Initialize( 130 __in PFN_WDF_PROGRAM_DMA ProgramDmaFunction, 131 __in WDF_DMA_DIRECTION DmaDirection, 132 __in PMDL Mdl, 133 __in size_t Offset, 134 __in ULONG Length 135 ); 136 137 _Must_inspect_result_ 138 virtual 139 NTSTATUS 140 InitializeResources( 141 VOID 142 )=0; 143 144 _Must_inspect_result_ 145 NTSTATUS 146 Execute( 147 __in PVOID Context 148 ); 149 150 _Must_inspect_result_ 151 virtual 152 NTSTATUS 153 StartTransfer( 154 VOID 155 )=0; 156 157 _Must_inspect_result_ 158 virtual 159 NTSTATUS 160 StageTransfer( 161 VOID 162 )=0; 163 164 BOOLEAN 165 DmaCompleted( 166 __in size_t TransferredLength, 167 __out NTSTATUS * ReturnStatus, 168 __in FxDmaCompletionType CompletionType 169 ); 170 171 _Must_inspect_result_ 172 virtual 173 NTSTATUS 174 TransferCompleted( 175 VOID 176 )=0; 177 178 VOID 179 ReleaseForReuse( 180 __in BOOLEAN ForceRelease 181 ); 182 183 virtual 184 VOID 185 ReleaseResources( 186 __in BOOLEAN ForceRelease 187 )=0; 188 189 VOID 190 GetTransferInfo( 191 __out_opt ULONG *MapRegisterCount, 192 __out_opt ULONG *ScatterGatherElementCount 193 ); 194 195 __forceinline 196 size_t 197 GetBytesTransferred( 198 VOID 199 ) 200 { 201 return m_Transferred; 202 } 203 204 __forceinline 205 FxDmaEnabler * 206 GetDmaEnabler( 207 VOID 208 ) 209 { 210 return m_DmaEnabler; 211 } 212 213 __forceinline 214 FxRequest * 215 GetRequest( 216 VOID 217 ) 218 { 219 // 220 // Strip out the strong reference tag if it's set 221 // 222 return FX_DECODE_POINTER(FxRequest, 223 m_EncodedRequest, 224 FX_STRONG_REF_TAG); 225 } 226 227 __forceinline 228 BOOLEAN 229 IsRequestReferenced( 230 VOID 231 ) 232 { 233 return FX_IS_POINTER_ENCODED(m_EncodedRequest, FX_STRONG_REF_TAG); 234 } 235 236 __forceinline 237 VOID 238 SetRequest( 239 __in FxRequest* Request 240 ) 241 { 242 ASSERT(m_EncodedRequest == NULL); 243 244 // 245 // Make sure the pointer doesn't have the strong ref flag set already 246 // 247 ASSERT(FX_IS_POINTER_ENCODED(Request, FX_STRONG_REF_TAG) == FALSE); 248 249 m_EncodedRequest = Request; 250 } 251 252 __forceinline 253 VOID 254 ReferenceRequest( 255 VOID 256 ) 257 { 258 ASSERT(m_EncodedRequest != NULL); 259 ASSERT(IsRequestReferenced() == false); 260 261 // 262 // Take a reference on the irp to catch completion of request 263 // when there is a pending DMA transaction. 264 // While there is no need to take a reference on request itself, 265 // I'm keeping it to avoid regression as we are so close to 266 // shipping this. 267 // 268 m_EncodedRequest->AddIrpReference(); 269 270 // 271 // Increment reference to this Request. 272 // See complementary section in WdfDmaTransactionDelete 273 // and WdfDmaTransactionRelease. 274 // 275 m_EncodedRequest->ADDREF( this ); 276 277 m_EncodedRequest = FX_ENCODE_POINTER(FxRequest, 278 m_EncodedRequest, 279 FX_STRONG_REF_TAG); 280 } 281 282 __forceinline 283 VOID 284 ReleaseButRetainRequest( 285 VOID 286 ) 287 { 288 ASSERT(m_EncodedRequest != NULL); 289 ASSERT(IsRequestReferenced()); 290 291 // 292 // Clear the referenced bit on the encoded request. 293 // 294 m_EncodedRequest = FX_DECODE_POINTER(FxRequest, 295 m_EncodedRequest, 296 FX_STRONG_REF_TAG); 297 298 // 299 // Release this reference to the Irp and FxRequest. 300 // 301 m_EncodedRequest->ReleaseIrpReference(); 302 303 m_EncodedRequest->RELEASE( this ); 304 } 305 306 __forceinline 307 VOID 308 ClearRequest( 309 VOID 310 ) 311 { 312 if (IsRequestReferenced()) { 313 ReleaseButRetainRequest(); 314 } 315 m_EncodedRequest = NULL; 316 } 317 318 __forceinline 319 size_t 320 GetMaximumFragmentLength( 321 VOID 322 ) 323 { 324 return m_MaxFragmentLength; 325 } 326 327 __forceinline 328 VOID 329 SetMaximumFragmentLength( 330 size_t MaximumFragmentLength 331 ) 332 { 333 if (MaximumFragmentLength < m_AdapterInfo->MaximumFragmentLength) { 334 m_MaxFragmentLength = MaximumFragmentLength; 335 } 336 } 337 338 __forceinline 339 size_t 340 GetCurrentFragmentLength( 341 VOID 342 ) 343 { 344 return m_CurrentFragmentLength; 345 } 346 347 __forceinline 348 WDFDMATRANSACTION 349 GetHandle( 350 VOID 351 ) 352 { 353 return (WDFDMATRANSACTION) GetObjectHandle(); 354 } 355 356 PVOID 357 GetTransferContext( 358 VOID 359 ) 360 { 361 return m_TransferContext; 362 } 363 364 VOID 365 SetImmediateExecution( 366 __in BOOLEAN Value 367 ); 368 369 BOOLEAN 370 CancelResourceAllocation( 371 VOID 372 ); 373 374 FxDmaTransactionState 375 GetTransactionState( 376 VOID 377 ) 378 { 379 return m_State; 380 } 381 382 protected: 383 384 FxDmaTransactionState m_State; 385 386 WDF_DMA_DIRECTION m_DmaDirection; 387 388 FxDmaEnabler* m_DmaEnabler; 389 390 // 391 // Depending on the direction of the transfer, this one 392 // points to either m_ReadAdapterInfo or m_WriteAdapterInfo 393 // structure of the DMA enabler. 394 // 395 FxDmaDescription* m_AdapterInfo; 396 397 // 398 // Request associated with this transfer. Encoding uses the 399 // FX_[EN|DE]CODE_POINTER macros with the FX_STRONG_REF_TAG 400 // to indicate whether the reference has been taken or not 401 // 402 FxRequest* m_EncodedRequest; 403 404 // 405 // Callback and context for ProgramDma function 406 // 407 // The callback is overloaded to also hold the callback for 408 // Packet & System transfer's Reserve callback (to save space.) 409 // This is possible because the driver may not call execute 410 // and reserve in parallel on the same transaction. Disambiguate 411 // using the state of the transaction. 412 // 413 FxDmaTransactionProgramOrReserveDma m_DmaAcquiredFunction; 414 PVOID m_DmaAcquiredContext; 415 416 // 417 // The DMA transfer context (when using V3 DMA) 418 // 419 PVOID m_TransferContext; 420 421 // 422 // This is the first MDL of the transaction. 423 // 424 PMDL m_StartMdl; 425 426 // 427 // This is the MDL where the current transfer is being executed. 428 // If the data spans multiple MDL then this would be different 429 // from the startMDL when we stage large transfers and also 430 // if the driver performs partial transfers. 431 // 432 PMDL m_CurrentFragmentMdl; 433 434 // 435 // Starting offset in the first m_StartMdl. This might be same as 436 // m_StartMdl->StartVA. 437 // 438 size_t m_StartOffset; 439 440 // 441 // Points to address where the next transfer will begin. 442 // 443 size_t m_CurrentFragmentOffset; 444 445 // 446 // This is maximum length of transfer that can be made. This is 447 // computed based on the available map registers and driver 448 // configuration. 449 // 450 size_t m_MaxFragmentLength; 451 452 // 453 // Length of the whole transaction. 454 // 455 size_t m_TransactionLength; 456 457 // 458 // Number of bytes pending to be transfered. 459 // 460 size_t m_Remaining; 461 462 // 463 // Total number of bytes transfered. 464 // 465 size_t m_Transferred; 466 467 // 468 // Number of bytes transfered in the last transaction. 469 // 470 size_t m_CurrentFragmentLength; 471 472 // 473 // DMA flags for passing to GetScatterGatherListEx & 474 // AllocateAdapterChannelEx 475 // 476 477 ULONG m_Flags; 478 479 static 480 PVOID 481 GetStartVaFromOffset( 482 __in PMDL Mdl, 483 __in size_t Offset 484 ) 485 { 486 return ((PUCHAR) MmGetMdlVirtualAddress(Mdl)) + Offset; 487 } 488 489 virtual 490 VOID 491 Reuse( 492 VOID 493 ) 494 { 495 return; 496 } 497 498 }; 499 500 class FxDmaScatterGatherTransaction : public FxDmaTransactionBase { 501 502 public: 503 504 FxDmaScatterGatherTransaction( 505 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 506 __in USHORT ExtraSize, 507 __in FxDmaEnabler *DmaEnabler 508 ); 509 510 virtual 511 BOOLEAN 512 Dispose( 513 VOID 514 ); 515 516 _Must_inspect_result_ 517 virtual 518 NTSTATUS 519 InitializeResources( 520 VOID 521 ); 522 523 _Must_inspect_result_ 524 virtual 525 NTSTATUS 526 StartTransfer( 527 VOID 528 ); 529 530 _Must_inspect_result_ 531 virtual 532 NTSTATUS 533 TransferCompleted( 534 VOID 535 ); 536 537 _Must_inspect_result_ 538 virtual 539 NTSTATUS 540 StageTransfer( 541 VOID 542 ); 543 544 virtual 545 VOID 546 ReleaseResources( 547 __in BOOLEAN ForceRelease 548 ); 549 550 _Must_inspect_result_ 551 static 552 NTSTATUS 553 _Create( 554 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 555 __in PWDF_OBJECT_ATTRIBUTES Attributes, 556 __in FxDmaEnabler* DmaEnabler, 557 __out WDFDMATRANSACTION* Transaction 558 ); 559 560 protected: 561 562 // 563 // Scatter-Gather list provided by the system. 564 // 565 PSCATTER_GATHER_LIST m_SGList; 566 567 // 568 // Preallocated memory from lookaside buffer for the 569 // scatter-gather list when running on XP and later OSes. 570 // 571 PVOID m_LookasideBuffer; 572 573 574 private: 575 576 static 577 VOID 578 _AdapterListControl( 579 __in DEVICE_OBJECT * DeviceObject, 580 __in IRP * Irp, 581 __in SCATTER_GATHER_LIST * SgList, 582 __in VOID * Context 583 ); 584 585 _Must_inspect_result_ 586 NTSTATUS 587 GetScatterGatherList ( 588 __in PMDL Mdl, 589 __in size_t CurrentOffset, 590 __in ULONG Length, 591 __in PDRIVER_LIST_CONTROL ExecutionRoutine, 592 __in PVOID Context 593 ) 594 { 595 NTSTATUS status; 596 KIRQL irql; 597 598 KeRaiseIrql(DISPATCH_LEVEL, &irql); 599 600 if (m_DmaEnabler->UsesDmaV3()) 601 { 602 PDMA_OPERATIONS dmaOperations = 603 m_AdapterInfo->AdapterObject->DmaOperations; 604 605 status = dmaOperations->GetScatterGatherListEx( 606 m_AdapterInfo->AdapterObject, 607 m_DmaEnabler->m_FDO, 608 GetTransferContext(), 609 Mdl, 610 CurrentOffset, 611 Length, 612 m_Flags, 613 ExecutionRoutine, 614 Context, 615 (BOOLEAN) m_DmaDirection, 616 NULL, 617 NULL, 618 NULL 619 ); 620 } 621 else 622 { 623 status = m_AdapterInfo->AdapterObject->DmaOperations-> 624 GetScatterGatherList(m_AdapterInfo->AdapterObject, 625 m_DmaEnabler->m_FDO, 626 Mdl, 627 GetStartVaFromOffset(Mdl, CurrentOffset), 628 Length, 629 ExecutionRoutine, 630 Context, 631 (BOOLEAN) m_DmaDirection); 632 } 633 634 KeLowerIrql(irql); 635 636 return status; 637 } 638 639 VOID 640 PutScatterGatherList( 641 __in PSCATTER_GATHER_LIST ScatterGather 642 ) 643 { 644 KIRQL irql; 645 646 KeRaiseIrql(DISPATCH_LEVEL, &irql); 647 648 m_AdapterInfo->AdapterObject->DmaOperations-> 649 PutScatterGatherList(m_AdapterInfo->AdapterObject, 650 ScatterGather, 651 (BOOLEAN) m_DmaDirection); 652 KeLowerIrql(irql); 653 654 return; 655 } 656 657 _Must_inspect_result_ 658 NTSTATUS 659 BuildScatterGatherList( 660 __in PMDL Mdl, 661 __in size_t CurrentOffset, 662 __in ULONG Length, 663 __in PDRIVER_LIST_CONTROL ExecutionRoutine, 664 __in PVOID Context, 665 __in PVOID ScatterGatherBuffer, 666 __in ULONG ScatterGatherBufferLength 667 ) 668 { 669 NTSTATUS status; 670 KIRQL irql; 671 672 KeRaiseIrql(DISPATCH_LEVEL, &irql); 673 674 if (m_DmaEnabler->UsesDmaV3()) { 675 676 PDMA_OPERATIONS dmaOperations = 677 m_AdapterInfo->AdapterObject->DmaOperations; 678 ULONG flags = 0; 679 680 if (GetDriverGlobals()->IsVersionGreaterThanOrEqualTo(1,15)) { 681 // 682 // Though the correct behavior is to pass the m_Flags to the 683 // BuildScatterGatherListEx function, the code was not doing it 684 // for versions <= 1.13. To reduce any chance of regression, 685 // the m_Flags is honored for drivers that are 1.15 686 // or newer. 687 // 688 flags = m_Flags; 689 } 690 691 status = dmaOperations->BuildScatterGatherListEx( 692 m_AdapterInfo->AdapterObject, 693 m_DmaEnabler->m_FDO, 694 GetTransferContext(), 695 Mdl, 696 CurrentOffset, 697 Length, 698 flags, 699 ExecutionRoutine, 700 Context, 701 (BOOLEAN) m_DmaDirection, 702 ScatterGatherBuffer, 703 ScatterGatherBufferLength, 704 NULL, 705 NULL, 706 NULL 707 ); 708 } 709 else { 710 711 status = m_AdapterInfo->AdapterObject->DmaOperations-> 712 BuildScatterGatherList(m_AdapterInfo->AdapterObject, 713 m_DmaEnabler->m_FDO, 714 Mdl, 715 GetStartVaFromOffset(Mdl, CurrentOffset), 716 Length, 717 ExecutionRoutine, 718 Context, 719 (BOOLEAN) m_DmaDirection, 720 ScatterGatherBuffer, 721 ScatterGatherBufferLength); 722 } 723 724 725 KeLowerIrql(irql); 726 727 return status; 728 } 729 }; 730 731 class FxDmaPacketTransaction : public FxDmaTransactionBase { 732 733 protected: 734 FxDmaPacketTransaction( 735 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 736 __in USHORT ObjectSize, 737 __in USHORT ExtraSize, 738 __in FxDmaEnabler *DmaEnabler 739 ); 740 741 public: 742 743 _Must_inspect_result_ 744 virtual 745 NTSTATUS 746 InitializeResources( 747 VOID 748 ); 749 750 _Must_inspect_result_ 751 NTSTATUS 752 ReserveAdapter( 753 __in ULONG NumberOfMapRegisters, 754 __in WDF_DMA_DIRECTION Direction, 755 __in PFN_WDF_RESERVE_DMA Callback, 756 __in_opt PVOID Context 757 ); 758 759 VOID 760 ReleaseAdapter( 761 VOID 762 ); 763 764 _Must_inspect_result_ 765 virtual 766 NTSTATUS 767 StartTransfer( 768 VOID 769 ); 770 771 _Must_inspect_result_ 772 virtual 773 NTSTATUS 774 TransferCompleted( 775 VOID 776 ); 777 778 _Must_inspect_result_ 779 virtual 780 NTSTATUS 781 StageTransfer( 782 VOID 783 ); 784 785 virtual 786 VOID 787 ReleaseResources( 788 __in BOOLEAN ForceRelease 789 ); 790 791 _Must_inspect_result_ 792 static 793 NTSTATUS 794 _Create( 795 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 796 __in PWDF_OBJECT_ATTRIBUTES Attributes, 797 __in FxDmaEnabler* DmaEnabler, 798 __out WDFDMATRANSACTION* Transaction 799 ); 800 801 VOID 802 SetDeviceAddressOffset( 803 __in ULONG Offset 804 ) 805 { 806 m_DeviceAddressOffset = Offset; 807 } 808 809 protected: 810 811 // 812 // Number of map registers to be used in this transfer. 813 // This value is the least of the number of map registers 814 // needed to satisfy the current transfer request, and the 815 // number of available map registers returned by IoGetDmaAdapter. 816 // 817 ULONG m_MapRegistersNeeded; 818 819 // 820 // Opaque-value represents the map registers that the system has 821 // assigned for this transfer operation. We pass this value in 822 // FlushAdapterBuffers, FreeMapRegisters, and MapTransfer. 823 // 824 PVOID m_MapRegisterBase; 825 826 // 827 // TRUE when the map register base above is valid. The HAL can give 828 // us a NULL map register base when double buffering isn't required, 829 // so we can't just do a NULL test on m_MapRegisterBase to know if 830 // the DMA channel is allocated. 831 // 832 BOOLEAN m_MapRegisterBaseSet; 833 834 // 835 // For system-DMA this provides the offset from the original 836 // DeviceAddress used to compute the device register to or from which 837 // DMA should occur. 838 // 839 ULONG m_DeviceAddressOffset; 840 841 // 842 // 0 if the transaction has not reserved the enabler. Otherwise 843 // this is the number of map registers requested for the reservation. 844 // This value persists across a reuse and reinitialization of the 845 // transaction, and is only cleared when the enabler is released. 846 // 847 ULONG m_MapRegistersReserved; 848 849 // 850 // These fields are used to defer completion or staging while another 851 // thread/CPU is already staging. They are protected by the 852 // transfer state lock. 853 // 854 // These values are cleared when checked, not in InitializeResources. 855 // It's possible for a transaction being staged to complete on another 856 // CPU, get reused and reinitialized. Clearing these values in 857 // InitializeResources would destroy the state the prior call to 858 // StageTransfer depends on. 859 // 860 struct { 861 862 // 863 // Non-null when a staging operation is in progress on some CPU. 864 // When set any attempt to call the DMA completion routine or 865 // stage the transfer again (due to a call to TransferComplete) 866 // will be deferred to this thread. 867 // 868 PKTHREAD CurrentStagingThread; 869 870 // 871 // Indicates that a nested or concurrent attempt to stage 872 // the transaction was deferred. The CurrentStagingThread 873 // will restage the transaction. 874 // 875 BOOLEAN RerunStaging; 876 877 // 878 // Indicates that a nested or concurrent attempt to call the 879 // DMA completion routine occurred. The CurrentStagingThread 880 // will call the DMA completion routine when it unwinds providing 881 // the saved CompletionStatus 882 // 883 BOOLEAN RerunCompletion; 884 DMA_COMPLETION_STATUS CompletionStatus; 885 886 } m_TransferState; 887 888 // 889 // Indicates that the DMA transfer has been cancelled. 890 // Set during StopTransfer and cleared during InitializeResources 891 // Checked during StageTransfer. Stop and Initialize should never 892 // race (it's not valid for the driver to stop an uninitialized 893 // transaction). Stop and Stage can race but in that case Stop 894 // will mark the transaction context such that MapTransfer fails 895 // (and that's protected by a HAL lock). 896 // 897 // So this field does not need to be volatile or interlocked, but 898 // it needs to be sized to allow atomic writes. 899 // 900 ULONG m_IsCancelled; 901 902 903 virtual 904 VOID 905 Reuse( 906 VOID 907 ) 908 { 909 return; 910 } 911 912 protected: 913 914 inline 915 SetMapRegisterBase( 916 __in PVOID Value 917 ) 918 { 919 NT_ASSERTMSG("Map register base is already set", 920 m_MapRegisterBaseSet == FALSE); 921 922 m_MapRegisterBase = Value; 923 m_MapRegisterBaseSet = TRUE; 924 } 925 926 inline 927 ClearMapRegisterBase( 928 VOID 929 ) 930 { 931 NT_ASSERTMSG("Map register base was not set", 932 m_MapRegisterBaseSet == TRUE); 933 m_MapRegisterBaseSet = FALSE; 934 } 935 936 inline 937 IsMapRegisterBaseSet( 938 VOID 939 ) 940 { 941 return m_MapRegisterBaseSet; 942 } 943 944 inline 945 PVOID 946 GetMapRegisterBase( 947 VOID 948 ) 949 { 950 NT_ASSERTMSG("Map register base is not set", 951 m_MapRegisterBaseSet == TRUE); 952 return m_MapRegisterBase; 953 } 954 955 virtual 956 IO_ALLOCATION_ACTION 957 GetAdapterControlReturnValue( 958 VOID 959 ) 960 { 961 return DeallocateObjectKeepRegisters; 962 } 963 964 virtual 965 BOOLEAN 966 PreMapTransfer( 967 VOID 968 ) 969 { 970 return TRUE; 971 } 972 973 _Acquires_lock_(this) 974 VOID 975 __drv_raisesIRQL(DISPATCH_LEVEL) 976 #pragma prefast(suppress:__WARNING_FAILING_TO_ACQUIRE_MEDIUM_CONFIDENCE, "transferring lock name to 'this->TransferStateLock'") 977 #pragma prefast(suppress:__WARNING_FAILING_TO_RELEASE_MEDIUM_CONFIDENCE, "transferring lock name to 'this->TransferStateLock'") 978 LockTransferState( 979 __out __drv_deref(__drv_savesIRQL) KIRQL *OldIrql 980 ) 981 { 982 Lock(OldIrql); 983 } 984 985 _Requires_lock_held_(this) 986 _Releases_lock_(this) 987 VOID 988 #pragma prefast(suppress:__WARNING_FAILING_TO_RELEASE_MEDIUM_CONFIDENCE, "transferring lock name from 'this->TransferStateLock'") 989 UnlockTransferState( 990 __in __drv_restoresIRQL KIRQL OldIrql 991 ) 992 { 993 #pragma prefast(suppress:__WARNING_CALLER_FAILING_TO_HOLD, "transferring lock name from 'this->TransferStateLock'") 994 Unlock(OldIrql); 995 } 996 997 virtual 998 PDMA_COMPLETION_ROUTINE 999 GetTransferCompletionRoutine( 1000 VOID 1001 ) 1002 { 1003 return NULL; 1004 } 1005 1006 static 1007 IO_ALLOCATION_ACTION 1008 _AdapterControl( 1009 __in PDEVICE_OBJECT DeviceObject, 1010 __in PIRP Irp, 1011 __in PVOID MapRegisterBase, 1012 __in PVOID Context 1013 ); 1014 1015 _Must_inspect_result_ 1016 NTSTATUS 1017 AcquireDevice( 1018 VOID 1019 ) 1020 { 1021 if (m_DmaEnabler->UsesDmaV3() == FALSE) 1022 { 1023 return m_DmaEnabler->GetDevice()->AcquireDmaPacketTransaction(); 1024 } 1025 else 1026 { 1027 return STATUS_SUCCESS; 1028 } 1029 } 1030 1031 FORCEINLINE 1032 VOID 1033 ReleaseDevice( 1034 VOID 1035 ) 1036 { 1037 if (m_DmaEnabler->UsesDmaV3() == FALSE) 1038 { 1039 m_DmaEnabler->GetDevice()->ReleaseDmaPacketTransaction(); 1040 } 1041 } 1042 1043 _Must_inspect_result_ 1044 NTSTATUS 1045 AllocateAdapterChannel( 1046 __in BOOLEAN MapRegistersReserved 1047 ) 1048 { 1049 NTSTATUS status; 1050 KIRQL irql; 1051 1052 KeRaiseIrql(DISPATCH_LEVEL, &irql); 1053 1054 if (GetDriverGlobals()->FxVerifierOn) { 1055 1056 if (MapRegistersReserved == FALSE) { 1057 DoTraceLevelMessage( 1058 GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGDMA, 1059 "Allocating %d map registers for " 1060 "WDFDMATRANSACTION %p", 1061 m_MapRegistersNeeded, 1062 GetHandle() 1063 ); 1064 } 1065 else { 1066 DoTraceLevelMessage( 1067 GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGDMA, 1068 "Using %d reserved map registers for " 1069 "WDFDMATRANSACTION %p", 1070 m_MapRegistersNeeded, 1071 GetHandle() 1072 ); 1073 } 1074 } 1075 1076 if (m_DmaEnabler->UsesDmaV3()) { 1077 PDMA_OPERATIONS dmaOperations = 1078 m_AdapterInfo->AdapterObject->DmaOperations; 1079 1080 if (MapRegistersReserved == FALSE) 1081 { 1082 status = dmaOperations->AllocateAdapterChannelEx( 1083 m_AdapterInfo->AdapterObject, 1084 m_DmaEnabler->m_FDO, 1085 GetTransferContext(), 1086 m_MapRegistersNeeded, 1087 m_Flags, 1088 #pragma prefast(suppress: __WARNING_CLASS_MISMATCH_NONE, "This warning requires a wrapper class for the DRIVER_CONTROL type.") 1089 _AdapterControl, 1090 this, 1091 NULL 1092 ); 1093 } 1094 else { 1095 #pragma prefast(suppress:__WARNING_PASSING_FUNCTION_UNEXPECTED_NULL, "_AdapterControl does not actually use the IRP parameter."); 1096 _AdapterControl(m_DmaEnabler->m_FDO, 1097 NULL, 1098 GetMapRegisterBase(), 1099 this); 1100 status = STATUS_SUCCESS; 1101 } 1102 } 1103 else { 1104 1105 ASSERTMSG("Prereserved map registers are not compatible with DMA V2", 1106 MapRegistersReserved == FALSE); 1107 1108 status = m_AdapterInfo->AdapterObject->DmaOperations-> 1109 AllocateAdapterChannel(m_AdapterInfo->AdapterObject, 1110 m_DmaEnabler->m_FDO, 1111 m_MapRegistersNeeded, 1112 #pragma prefast(suppress: __WARNING_CLASS_MISMATCH_NONE, "This warning requires a wrapper class for the DRIVER_CONTROL type.") 1113 _AdapterControl, 1114 this); 1115 } 1116 1117 KeLowerIrql(irql); 1118 1119 if (!NT_SUCCESS(status)) 1120 { 1121 DoTraceLevelMessage( 1122 GetDriverGlobals(), TRACE_LEVEL_ERROR, TRACINGDMA, 1123 "Allocating DMA resources (%d map registers) for WDFDMATRANSACTION %p " 1124 "returned %!STATUS!", 1125 m_MapRegistersNeeded, 1126 GetHandle(), 1127 status 1128 ); 1129 } 1130 1131 return status; 1132 } 1133 1134 FORCEINLINE 1135 NTSTATUS 1136 MapTransfer( 1137 __out_bcount_opt(ScatterGatherListCb) 1138 PSCATTER_GATHER_LIST ScatterGatherList, 1139 __in ULONG ScatterGatherListCb, 1140 __in_opt PDMA_COMPLETION_ROUTINE CompletionRoutine, 1141 __in_opt PVOID CompletionContext, 1142 __out ULONG *TransferLength 1143 ) 1144 { 1145 // 1146 // Cache globals & object handle since call to MapTransferEx could 1147 // result in a DmaComplete callback before returning. 1148 // 1149 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 1150 WDFDMATRANSACTION handle = GetHandle(); 1151 #if DBG 1152 ULONG_PTR mapRegistersRequired; 1153 1154 mapRegistersRequired = ADDRESS_AND_SIZE_TO_SPAN_PAGES( 1155 GetStartVaFromOffset(m_CurrentFragmentMdl, 1156 m_CurrentFragmentOffset), 1157 m_CurrentFragmentLength 1158 ); 1159 NT_ASSERTMSG("Mapping requires too many map registers", 1160 mapRegistersRequired <= m_MapRegistersNeeded); 1161 #endif 1162 1163 NTSTATUS status; 1164 1165 // 1166 // Assume we're going to transfer the entire current fragment. 1167 // MapTransfer may say otherwise. 1168 // 1169 1170 *TransferLength = (ULONG) m_CurrentFragmentLength; 1171 1172 // 1173 // Map the transfer. 1174 // 1175 1176 if (pFxDriverGlobals->FxVerifierOn) { 1177 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1178 "Mapping transfer for WDFDMATRANSACTION %p. " 1179 "MDL %p, Offset %I64x, Length %x, MapRegisterBase %p", 1180 handle, 1181 m_CurrentFragmentMdl, 1182 m_CurrentFragmentOffset, 1183 *TransferLength, 1184 GetMapRegisterBase()); 1185 } 1186 1187 if (m_DmaEnabler->UsesDmaV3()) { 1188 1189 PDMA_OPERATIONS dmaOperations = 1190 m_AdapterInfo->AdapterObject->DmaOperations; 1191 1192 status = dmaOperations->MapTransferEx( 1193 m_AdapterInfo->AdapterObject, 1194 m_CurrentFragmentMdl, 1195 GetMapRegisterBase(), 1196 m_CurrentFragmentOffset, 1197 m_DeviceAddressOffset, 1198 TransferLength, 1199 (BOOLEAN) m_DmaDirection, 1200 ScatterGatherList, 1201 ScatterGatherListCb, 1202 CompletionRoutine, 1203 CompletionContext 1204 ); 1205 1206 NT_ASSERTMSG( 1207 "With these parameters, MapTransferEx should never fail", 1208 NT_SUCCESS(status) || status == STATUS_CANCELLED 1209 ); 1210 } 1211 else { 1212 NT_ASSERTMSG("cannot use DMA completion routine with DMAv2", 1213 CompletionRoutine == NULL); 1214 1215 NT_ASSERTMSG( 1216 "scatter gather list length must be large enough for at least one element", 1217 (ScatterGatherListCb >= (sizeof(SCATTER_GATHER_LIST) + 1218 sizeof(SCATTER_GATHER_ELEMENT))) 1219 ); 1220 1221 // 1222 // This matches the assertion above. There's no way to explain to 1223 // prefast that this code path requires the caller to provide a buffer 1224 // of sufficient size to store the SGL. The only case which doesn't 1225 // provide any buffer is system-mode DMA and that uses DMA v3 and so 1226 // won't go through this path. 1227 // 1228 1229 __assume((ScatterGatherListCb >= (sizeof(SCATTER_GATHER_LIST) + 1230 sizeof(SCATTER_GATHER_ELEMENT)))); 1231 1232 ScatterGatherList->NumberOfElements = 1; 1233 ScatterGatherList->Reserved = 0; 1234 ScatterGatherList->Elements[0].Address = 1235 m_AdapterInfo->AdapterObject->DmaOperations-> 1236 MapTransfer(m_AdapterInfo->AdapterObject, 1237 m_CurrentFragmentMdl, 1238 GetMapRegisterBase(), 1239 GetStartVaFromOffset(m_CurrentFragmentMdl, 1240 m_CurrentFragmentOffset), 1241 TransferLength, 1242 (BOOLEAN) m_DmaDirection); 1243 ScatterGatherList->Elements[0].Length = *TransferLength; 1244 1245 status = STATUS_SUCCESS; 1246 } 1247 1248 if (pFxDriverGlobals->FxVerifierOn) { 1249 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1250 "MapTransfer mapped next %d bytes of " 1251 "WDFDMATRANSACTION %p - status %!STATUS!", 1252 *TransferLength, 1253 handle, 1254 status); 1255 } 1256 1257 return status; 1258 } 1259 1260 FORCEINLINE 1261 NTSTATUS 1262 FlushAdapterBuffers( 1263 VOID 1264 ) 1265 { 1266 PDMA_OPERATIONS dmaOperations = 1267 m_AdapterInfo->AdapterObject->DmaOperations; 1268 1269 NTSTATUS status; 1270 1271 #if DBG 1272 ULONG_PTR mapRegistersRequired; 1273 1274 mapRegistersRequired = ADDRESS_AND_SIZE_TO_SPAN_PAGES( 1275 GetStartVaFromOffset(m_CurrentFragmentMdl, 1276 m_CurrentFragmentOffset), 1277 m_CurrentFragmentLength 1278 ); 1279 NT_ASSERTMSG("Mapping requires too many map registers", 1280 mapRegistersRequired <= m_MapRegistersNeeded); 1281 #endif 1282 1283 if (GetDriverGlobals()->FxVerifierOn) { 1284 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGDMA, 1285 "Flushing DMA buffers for WDFDMATRANSACTION %p. " 1286 "MDL %p, Offset %I64x, Length %I64x", 1287 GetHandle(), 1288 m_CurrentFragmentMdl, 1289 m_CurrentFragmentOffset, 1290 m_CurrentFragmentLength); 1291 } 1292 1293 if (m_DmaEnabler->UsesDmaV3()) { 1294 status = dmaOperations->FlushAdapterBuffersEx( 1295 m_AdapterInfo->AdapterObject, 1296 m_CurrentFragmentMdl, 1297 GetMapRegisterBase(), 1298 m_CurrentFragmentOffset, 1299 (ULONG) m_CurrentFragmentLength, 1300 (BOOLEAN) m_DmaDirection 1301 ); 1302 } 1303 else if (dmaOperations->FlushAdapterBuffers( 1304 m_AdapterInfo->AdapterObject, 1305 m_CurrentFragmentMdl, 1306 GetMapRegisterBase(), 1307 GetStartVaFromOffset(m_CurrentFragmentMdl, 1308 m_CurrentFragmentOffset), 1309 (ULONG) m_CurrentFragmentLength, 1310 (BOOLEAN) m_DmaDirection) == FALSE) { 1311 status = STATUS_UNSUCCESSFUL; 1312 } 1313 else { 1314 status = STATUS_SUCCESS; 1315 } 1316 1317 if (!NT_SUCCESS(status)) { 1318 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_ERROR, TRACINGDMA, 1319 "Flushing DMA buffers for WDFDMATRANSACTION %p (" 1320 "MDL %p, Offset %I64x, Length %I64x)" 1321 "completed with %!STATUS!", 1322 GetHandle(), 1323 m_CurrentFragmentMdl, 1324 m_CurrentFragmentOffset, 1325 m_CurrentFragmentLength, 1326 status); 1327 } 1328 1329 return status; 1330 } 1331 1332 virtual 1333 VOID 1334 FreeMapRegistersAndAdapter( 1335 VOID 1336 ) 1337 { 1338 KIRQL irql; 1339 1340 PVOID mapRegisterBase = GetMapRegisterBase(); 1341 1342 // 1343 // It's illegal to free a NULL map register base, even if the HAL gave it 1344 // to us. 1345 // 1346 if (mapRegisterBase == NULL) { 1347 if (GetDriverGlobals()->FxVerifierOn) { 1348 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGDMA, 1349 "Skipping free of %d map registers for WDFDMATRANSACTION %p " 1350 "because base was NULL", 1351 m_MapRegistersNeeded, 1352 GetHandle()); 1353 } 1354 1355 return; 1356 } 1357 1358 // 1359 // Free the map registers 1360 // 1361 KeRaiseIrql(DISPATCH_LEVEL, &irql); 1362 1363 if (GetDriverGlobals()->FxVerifierOn) { 1364 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGDMA, 1365 "Freeing %d map registers for WDFDMATRANSACTION %p " 1366 "(base %p)", 1367 m_MapRegistersNeeded, 1368 GetHandle(), 1369 mapRegisterBase); 1370 } 1371 1372 // 1373 // If we pre-reserved map registers then Reserved contains 1374 // the number to free. Otherwise Needed is the number allocated 1375 // for the last transaction, which is the number to free. 1376 // 1377 m_AdapterInfo->AdapterObject->DmaOperations-> 1378 FreeMapRegisters(m_AdapterInfo->AdapterObject, 1379 mapRegisterBase, 1380 (m_MapRegistersReserved > 0 ? 1381 m_MapRegistersReserved : 1382 m_MapRegistersNeeded)); 1383 KeLowerIrql(irql); 1384 1385 return; 1386 } 1387 1388 virtual 1389 VOID 1390 CallEvtDmaCompleted( 1391 __in DMA_COMPLETION_STATUS /* Status */ 1392 ) 1393 { 1394 // 1395 // Packet mode DMA doesn't support cancellation or 1396 // completion routines. So this should never run. 1397 // 1398 ASSERTMSG("EvtDmaCompleted is not a valid callback for " 1399 "a packet-mode transaction", 1400 FALSE); 1401 return; 1402 } 1403 1404 }; 1405 1406 class FxDmaSystemTransaction: public FxDmaPacketTransaction { 1407 1408 friend FxDmaPacketTransaction; 1409 1410 public: 1411 1412 FxDmaSystemTransaction( 1413 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 1414 __in USHORT ExtraSize, 1415 __in FxDmaEnabler *DmaEnabler 1416 ); 1417 1418 _Must_inspect_result_ 1419 static 1420 NTSTATUS 1421 _Create( 1422 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 1423 __in PWDF_OBJECT_ATTRIBUTES Attributes, 1424 __in FxDmaEnabler* DmaEnabler, 1425 __out WDFDMATRANSACTION* Transaction 1426 ); 1427 1428 VOID 1429 SetConfigureChannelCallback( 1430 __in_opt PFN_WDF_DMA_TRANSACTION_CONFIGURE_DMA_CHANNEL Callback, 1431 __in_opt PVOID Context 1432 ) 1433 { 1434 m_ConfigureChannelFunction.Method = Callback; 1435 m_ConfigureChannelContext = Context; 1436 } 1437 1438 VOID 1439 SetTransferCompleteCallback( 1440 __in_opt PFN_WDF_DMA_TRANSACTION_DMA_TRANSFER_COMPLETE Callback, 1441 __in_opt PVOID Context 1442 ) 1443 { 1444 m_TransferCompleteFunction.Method = Callback; 1445 m_TransferCompleteContext = Context; 1446 } 1447 1448 VOID 1449 StopTransfer( 1450 VOID 1451 ); 1452 1453 protected: 1454 1455 // 1456 // Callback and context for configure channel callback 1457 // 1458 FxDmaTransactionConfigureChannel m_ConfigureChannelFunction; 1459 PVOID m_ConfigureChannelContext; 1460 1461 // 1462 // Callback and context for DMA completion callback 1463 // 1464 FxDmaTransactionTransferComplete m_TransferCompleteFunction; 1465 PVOID m_TransferCompleteContext; 1466 1467 IO_ALLOCATION_ACTION 1468 GetAdapterControlReturnValue( 1469 VOID 1470 ) 1471 { 1472 return KeepObject; 1473 } 1474 1475 VOID 1476 FreeMapRegistersAndAdapter( 1477 VOID 1478 ) 1479 { 1480 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals(); 1481 KIRQL irql; 1482 1483 KeRaiseIrql(DISPATCH_LEVEL, &irql); 1484 1485 if (pFxDriverGlobals->FxVerifierOn) { 1486 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA, 1487 "Freeing adapter channel for WDFDMATRANSACTION %p", 1488 GetHandle()); 1489 } 1490 1491 m_AdapterInfo->AdapterObject->DmaOperations-> 1492 FreeAdapterChannel(m_AdapterInfo->AdapterObject); 1493 KeLowerIrql(irql); 1494 1495 return; 1496 } 1497 1498 BOOLEAN 1499 CancelMappedTransfer( 1500 VOID 1501 ) 1502 { 1503 NTSTATUS status; 1504 1505 ASSERT(m_DmaEnabler->UsesDmaV3()); 1506 1507 // 1508 // Cancel the transfer. if it's not yet mapped this will mark the 1509 // TC so that mapping will fail. If it's running this will invoke the 1510 // DMA completion routine. 1511 // 1512 status = 1513 m_AdapterInfo->AdapterObject->DmaOperations->CancelMappedTransfer( 1514 m_AdapterInfo->AdapterObject, 1515 GetTransferContext() 1516 ); 1517 1518 DoTraceLevelMessage(GetDriverGlobals(), TRACE_LEVEL_VERBOSE, TRACINGDMA, 1519 "Stopping WDFDMATRANSACTION %p returned status %!STATUS!", 1520 GetHandle(), 1521 status); 1522 1523 return NT_SUCCESS(status); 1524 } 1525 1526 VOID 1527 Reuse( 1528 VOID 1529 ) 1530 { 1531 __super::Reuse(); 1532 m_ConfigureChannelFunction.Method = NULL; 1533 m_ConfigureChannelContext = NULL; 1534 1535 m_TransferCompleteFunction.Method = NULL; 1536 m_TransferCompleteContext = NULL; 1537 } 1538 1539 VOID 1540 CallEvtDmaCompleted( 1541 __in DMA_COMPLETION_STATUS Status 1542 ); 1543 1544 virtual 1545 BOOLEAN 1546 PreMapTransfer( 1547 VOID 1548 ); 1549 1550 virtual 1551 PDMA_COMPLETION_ROUTINE 1552 GetTransferCompletionRoutine( 1553 VOID 1554 ); 1555 1556 static DMA_COMPLETION_ROUTINE _SystemDmaCompletion; 1557 }; 1558 1559 #endif // _FXDMATRANSACTION_HPP_ 1560