1 /*++
2
3 Copyright (c) Microsoft Corporation
4
5 Module Name:
6
7 FxDmaTransaction.cpp
8
9 Abstract:
10
11 WDF DMA Transaction Object
12
13 Environment:
14
15 Kernel mode only.
16
17 Notes:
18
19
20 Revision History:
21
22 --*/
23
24 #include "fxdmapch.hpp"
25
26 extern "C" {
27 // #include "FxDmaTransaction.tmh"
28 }
29
FxDmaTransactionBase(__in PFX_DRIVER_GLOBALS FxDriverGlobals,__in USHORT ObjectSize,__in USHORT ExtraSize,__in FxDmaEnabler * DmaEnabler)30 FxDmaTransactionBase::FxDmaTransactionBase(
31 __in PFX_DRIVER_GLOBALS FxDriverGlobals,
32 __in USHORT ObjectSize,
33 __in USHORT ExtraSize,
34 __in FxDmaEnabler *DmaEnabler
35 ) :
36 FxNonPagedObject(
37 FX_TYPE_DMA_TRANSACTION,
38 ExtraSize == 0 ? ObjectSize : COMPUTE_OBJECT_SIZE(ObjectSize, ExtraSize),
39 FxDriverGlobals)
40 {
41 m_DmaEnabler = DmaEnabler;
42 m_EncodedRequest = NULL;
43 m_MaxFragmentLength = 0;
44 m_DmaDirection = WdfDmaDirectionReadFromDevice;
45 m_DmaAcquiredContext = NULL;
46 m_CurrentFragmentMdl = NULL;
47 m_CurrentFragmentOffset = 0;
48 m_StartOffset = NULL;
49 m_StartMdl = NULL;
50 m_Remaining = 0;
51 m_CurrentFragmentLength = 0;
52 m_TransactionLength = 0;
53 m_Transferred = 0;
54 m_Flags = 0;
55
56 m_DmaAcquiredFunction.Method.ProgramDma = NULL;
57
58 m_State = FxDmaTransactionStateCreated;
59
60 if (ExtraSize == 0) {
61 m_TransferContext = NULL;
62 } else {
63 m_TransferContext = WDF_PTR_ADD_OFFSET_TYPE(
64 this,
65 COMPUTE_RAW_OBJECT_SIZE(ObjectSize),
66 PVOID
67 );
68 }
69
70 MarkDisposeOverride(ObjectDoNotLock);
71 }
72
73 BOOLEAN
Dispose(VOID)74 FxDmaTransactionBase::Dispose(
75 VOID
76 )
77 {
78 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
79
80 //
81 // Must not be in transfer state.
82 //
83 if (m_State == FxDmaTransactionStateTransfer) {
84 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
85 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
86 "is invalid", GetHandle(), m_State);
87
88 if (pFxDriverGlobals->IsVerificationEnabled(1, 9, OkForDownLevel)) {
89 FxVerifierBugCheck(pFxDriverGlobals, // globals
90 WDF_DMA_FATAL_ERROR, // type
91 (ULONG_PTR) GetObjectHandle(), // parm 2
92 (ULONG_PTR) m_State); // parm 3
93 }
94 }
95
96 m_State = FxDmaTransactionStateDeleted;
97
98 //
99 // Release resources for this Dma Transaction.
100 //
101 ReleaseResources(TRUE);
102
103 if (m_EncodedRequest != NULL) {
104 ClearRequest();
105 }
106
107 return TRUE;
108 }
109
110 _Must_inspect_result_
111 NTSTATUS
Initialize(__in PFN_WDF_PROGRAM_DMA ProgramDmaFunction,__in WDF_DMA_DIRECTION DmaDirection,__in PMDL Mdl,__in size_t Offset,__in ULONG Length)112 FxDmaTransactionBase::Initialize(
113 __in PFN_WDF_PROGRAM_DMA ProgramDmaFunction,
114 __in WDF_DMA_DIRECTION DmaDirection,
115 __in PMDL Mdl,
116 __in size_t Offset,
117 __in ULONG Length
118 )
119 {
120 NTSTATUS status;
121 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
122
123 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
124 "Enter WDFDMATRANSACTION %p", GetHandle());
125 //
126 // Must be in Reserve, Created or Released state.
127 //
128 if (m_State != FxDmaTransactionStateCreated &&
129 m_State != FxDmaTransactionStateReserved &&
130 m_State != FxDmaTransactionStateReleased) {
131
132 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
133 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
134 "is invalid", GetHandle(), m_State);
135
136 FxVerifierBugCheck(pFxDriverGlobals, // globals
137 WDF_DMA_FATAL_ERROR, // specific type
138 (ULONG_PTR) GetObjectHandle(), // parm 2
139 (ULONG_PTR) m_State); // parm 3
140 }
141
142 if (DmaDirection == WdfDmaDirectionReadFromDevice) {
143 m_AdapterInfo = m_DmaEnabler->GetReadDmaDescription();
144 } else {
145 m_AdapterInfo = m_DmaEnabler->GetWriteDmaDescription();
146 }
147
148 //
149 // Initialize the DmaTransaction object
150 //
151
152 m_MaxFragmentLength = m_AdapterInfo->MaximumFragmentLength;
153 m_DmaDirection = DmaDirection;
154 m_StartMdl = Mdl;
155 m_StartOffset = Offset;
156 m_CurrentFragmentMdl = Mdl;
157 m_CurrentFragmentOffset = Offset;
158 m_Remaining = Length;
159 m_TransactionLength = Length;
160 m_DmaAcquiredFunction.Method.ProgramDma = ProgramDmaFunction;
161
162 //
163 // If needed, initialize the transfer context.
164 //
165
166 if (m_DmaEnabler->UsesDmaV3()) {
167 m_DmaEnabler->InitializeTransferContext(GetTransferContext(),
168 m_DmaDirection);
169 }
170
171 status = InitializeResources();
172 if (NT_SUCCESS(status)) {
173 m_State = FxDmaTransactionStateInitialized;
174 } else {
175 ReleaseForReuse(FALSE);
176 }
177
178 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
179 "Exit WDFDMATRANSACTION %p, %!STATUS!",
180 GetHandle(), status);
181
182 return status;
183 }
184
185 _Must_inspect_result_
186 NTSTATUS
Execute(__in PVOID Context)187 FxDmaTransactionBase::Execute(
188 __in PVOID Context
189 )
190 {
191 NTSTATUS status;
192 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
193
194 //
195 // Must be in Initialized state.
196 //
197 if (m_State != FxDmaTransactionStateInitialized) {
198
199 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
200 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
201 "is invalid", GetHandle(), m_State);
202
203 FxVerifierBugCheck(pFxDriverGlobals, // globals
204 WDF_DMA_FATAL_ERROR, // specific type
205 (ULONG_PTR) GetObjectHandle(), // parm 2
206 (ULONG_PTR) m_State); // parm 3
207 }
208
209 //
210 // If this was initialized with a request, then reference the
211 // request now.
212 //
213 if (m_EncodedRequest != NULL) {
214 ReferenceRequest();
215 }
216
217 //
218 // Set state to Transfer.
219 // This is necessary because the Execute path complete
220 // all the way to DmaCompleted before returning to this point.
221 //
222 m_State = FxDmaTransactionStateTransfer;
223
224 //
225 // Save the caller's context
226 //
227 m_DmaAcquiredContext = Context;
228
229 ASSERT(m_Transferred == 0);
230 ASSERT(m_CurrentFragmentLength == 0);
231
232 status = StartTransfer();
233 if (!NT_SUCCESS(status)) {
234 m_State = FxDmaTransactionStateTransferFailed;
235 m_DmaAcquiredContext = NULL;
236
237 if (m_EncodedRequest != NULL) {
238 ReleaseButRetainRequest();
239 }
240 }
241
242 //
243 // StartTransfer results in a call to the EvtProgramDma routine
244 // where driver could complete and delete the object. So
245 // don't touch the object beyond this point.
246 //
247
248 return status;
249 }
250
251 BOOLEAN
DmaCompleted(__in size_t TransferredLength,__out NTSTATUS * ReturnStatus,__in FxDmaCompletionType CompletionType)252 FxDmaTransactionBase::DmaCompleted(
253 __in size_t TransferredLength,
254 __out NTSTATUS * ReturnStatus,
255 __in FxDmaCompletionType CompletionType
256 )
257 {
258 BOOLEAN hasTransitioned;
259 NTSTATUS status;
260 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
261 WDFDMATRANSACTION dmaTransaction;
262
263 //
264 // In the case of partial completion, we will start a new transfer
265 // from with in this function by calling StageTransfer. After that
266 // call, we lose ownership of the object. Since we need the handle
267 // for tracing purposes, we will save the value in a local variable and
268 // use that.
269 //
270 dmaTransaction = GetHandle();
271
272 if (pFxDriverGlobals->FxVerifierOn) {
273 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
274 "Enter WDFDMATRANSACTION %p, length %d",
275 dmaTransaction, (ULONG)TransferredLength);
276 }
277
278 //
279 // Must be in Transfer state.
280 //
281 if (m_State != FxDmaTransactionStateTransfer) {
282 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
283 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
284 "is invalid", dmaTransaction, m_State);
285
286 FxVerifierBugCheck(pFxDriverGlobals, // globals
287 WDF_DMA_FATAL_ERROR, // specific type
288 (ULONG_PTR) dmaTransaction, // parm 2
289 (ULONG_PTR) m_State); // parm 3
290 }
291
292 if (TransferredLength > m_CurrentFragmentLength) {
293 status = STATUS_INVALID_PARAMETER;
294 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
295 "WDFDMATRANSACTION %p Transfered Length %I64d can't be more "
296 "than the length asked to transfer %I64d "
297 "%!STATUS!", dmaTransaction, TransferredLength,
298 m_CurrentFragmentLength, status);
299 FxVerifierDbgBreakPoint(pFxDriverGlobals);
300 goto End;
301 }
302
303
304 if (CompletionType == FxDmaCompletionTypePartial ||
305 CompletionType == FxDmaCompletionTypeAbort) {
306 //
307 // Tally this DMA tranferred byte count into the accumulator.
308 //
309 m_Transferred += TransferredLength;
310
311 //
312 // Adjust the remaining length to account for the partial transfer.
313 //
314 m_Remaining += (m_CurrentFragmentLength - TransferredLength);
315
316 //
317 // Update CurrentDmaLength to reflect actual transfer because
318 // we need to FlushAdapterBuffers based on this value in
319 // TransferCompleted for packet based transfer.
320 //
321 m_CurrentFragmentLength = TransferredLength;
322
323 } else {
324 //
325 // Tally this DMA tranferred byte count into the accumulator.
326 //
327 m_Transferred += m_CurrentFragmentLength;
328 }
329
330 ASSERT(m_Transferred <= m_TransactionLength);
331
332 //
333 // Inform the derived object that transfer is completed so it
334 // can release resources specific to last transfer.
335 //
336 status = TransferCompleted();
337 if (!NT_SUCCESS(status)) {
338 goto End;
339 }
340
341 //
342 // If remaining DmaTransaction length is zero or if the driver wants
343 // this to be the last transfer then free the map registers and
344 // change the state to completed.
345 //
346 if (m_Remaining == 0 || CompletionType == FxDmaCompletionTypeAbort) {
347 status = STATUS_SUCCESS;
348 goto End;
349 }
350
351 //
352 // Stage the next packet for this DmaTransaction...
353 //
354 status = StageTransfer();
355
356 if (NT_SUCCESS(status)) {
357 //
358 // StageTransfer results in a call to the EvtProgramDma routine
359 // where driver could complete and delete the object. So
360 // don't touch the object beyond this point.
361 //
362 status = STATUS_MORE_PROCESSING_REQUIRED;
363 }
364 else {
365 //
366 // The error will be returned to the caller of
367 // WdfDmaTransactionDmaComplete*()
368 //
369 }
370
371 End:
372
373 if (status != STATUS_MORE_PROCESSING_REQUIRED) {
374 //
375 // Failed or succeeded. Either way free
376 // map registers and release the device.
377 //
378 if (NT_SUCCESS(status)) {
379 m_State = FxDmaTransactionStateTransferCompleted;
380 } else {
381 m_State = FxDmaTransactionStateTransferFailed;
382 }
383
384 if (pFxDriverGlobals->FxVerifierOn) {
385 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
386 "WDFDMATRANSACTION %p completed with status %!STATUS! - "
387 "releasing DMA resources",
388 GetHandle(),
389 status);
390 }
391
392 ReleaseResources(FALSE);
393
394 if (m_EncodedRequest != NULL) {
395 ReleaseButRetainRequest();
396 }
397
398 m_CurrentFragmentLength = 0;
399
400 hasTransitioned = TRUE;
401 } else {
402 hasTransitioned = FALSE;
403 }
404
405 *ReturnStatus = status;
406
407 if (pFxDriverGlobals->FxVerifierOn) {
408 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
409 "Exit WDFDMATRANSACTION %p "
410 "Transitioned(%!BOOLEAN!)",
411 dmaTransaction, hasTransitioned);
412 }
413
414 return hasTransitioned;
415 }
416
417 VOID
ReleaseForReuse(__in BOOLEAN ForceRelease)418 FxDmaTransactionBase::ReleaseForReuse(
419 __in BOOLEAN ForceRelease
420 )
421 {
422 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
423
424 if (ForceRelease == FALSE)
425 {
426 if (m_State == FxDmaTransactionStateReleased) {
427
428 //
429 // Double release is probably due to cancel during early in transaction
430 // initialization. DC2 on very slow machines shows this behavior.
431 // The double release case is rare and benign.
432 //
433 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_WARNING, TRACINGDMA,
434 "WDFDMATRANSACTION %p is already released, "
435 "%!STATUS!", GetHandle(), STATUS_SUCCESS);
436
437 return; // already released.
438 }
439
440 //
441 // Must not be in transfer state.
442 //
443 if (m_State == FxDmaTransactionStateTransfer) {
444 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
445 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
446 "is invalid (release transaction)", GetHandle(), m_State);
447
448 if (pFxDriverGlobals->IsVerificationEnabled(1, 11, OkForDownLevel)) {
449 FxVerifierBugCheck(pFxDriverGlobals, // globals
450 WDF_DMA_FATAL_ERROR, // type
451 (ULONG_PTR) GetObjectHandle(), // parm 2
452 (ULONG_PTR) m_State); // parm 3
453 }
454 }
455 }
456
457 m_State = FxDmaTransactionStateReleased;
458
459 ReleaseResources(ForceRelease);
460
461 //
462 // Except DMA enabler field and adapter info everything else should be
463 // cleared. Adapter info is cleared by ReleaseResources above.
464 //
465 m_DmaAcquiredContext = NULL;
466
467 if (m_EncodedRequest != NULL) {
468 ClearRequest();
469 }
470
471 m_StartMdl = NULL;
472 m_CurrentFragmentMdl = NULL;
473 m_StartOffset = 0;
474 m_CurrentFragmentOffset = 0;
475 m_CurrentFragmentLength = 0;
476 m_Transferred = 0;
477 m_Remaining = 0;
478 m_MaxFragmentLength = 0;
479 m_TransactionLength = 0;
480 m_Flags = 0;
481
482 m_DmaAcquiredFunction.Method.ProgramDma = NULL;
483
484 }
485
486 VOID
SetImmediateExecution(__in BOOLEAN Value)487 FxDmaTransactionBase::SetImmediateExecution(
488 __in BOOLEAN Value
489 )
490 {
491 if (m_State != FxDmaTransactionStateCreated &&
492 m_State != FxDmaTransactionStateInitialized &&
493 m_State != FxDmaTransactionStateReleased) {
494 DoTraceLevelMessage(
495 GetDriverGlobals(), TRACE_LEVEL_ERROR, TRACINGDMA,
496 "Must set immediate execution flag for WDFDMATRANSACTION "
497 "%p before calling AllocateResources or Execute (current "
498 "state is %!FxDmaTransactionState!)",
499 GetHandle(),
500 m_State
501 );
502 FxVerifierDbgBreakPoint(GetDriverGlobals());
503 }
504
505 if (Value) {
506 m_Flags |= DMA_SYNCHRONOUS_CALLBACK;
507 }
508 else {
509 m_Flags &= ~DMA_SYNCHRONOUS_CALLBACK;
510 }
511 }
512
513 BOOLEAN
CancelResourceAllocation(VOID)514 FxDmaTransactionBase::CancelResourceAllocation(
515 VOID
516 )
517 {
518 if ((m_State == FxDmaTransactionStateCreated) ||
519 (m_State == FxDmaTransactionStateReleased) ||
520 (m_State == FxDmaTransactionStateDeleted)) {
521
522 DoTraceLevelMessage(
523 GetDriverGlobals(),
524 TRACE_LEVEL_ERROR,
525 TRACINGDMA,
526 "WDFDMATRANSACTION %p cannot be cancelled in state "
527 "%!FxDmaTransactionState!",
528 GetHandle(),
529 m_State
530 );
531
532 FxVerifierBugCheck(GetDriverGlobals(),
533 WDF_DMA_FATAL_ERROR,
534 (ULONG_PTR) GetObjectHandle(),
535 (ULONG_PTR) m_State);
536 // unreachable code
537 }
538
539 PDMA_OPERATIONS dmaOperations =
540 m_AdapterInfo->AdapterObject->DmaOperations;
541
542 BOOLEAN result;
543
544 result = dmaOperations->CancelAdapterChannel(
545 m_AdapterInfo->AdapterObject,
546 m_DmaEnabler->m_FDO,
547 GetTransferContext()
548 );
549
550 if (result) {
551 m_State = FxDmaTransactionStateTransferFailed;
552
553 if (m_EncodedRequest != NULL) {
554 ReleaseButRetainRequest();
555 }
556 }
557
558 return result;
559 }
560
561 VOID
_ComputeNextTransferAddress(__in PMDL CurrentMdl,__in size_t CurrentOffset,__in ULONG Transferred,__deref_out PMDL * NextMdl,__out size_t * NextOffset)562 FxDmaTransactionBase::_ComputeNextTransferAddress(
563 __in PMDL CurrentMdl,
564 __in size_t CurrentOffset,
565 __in ULONG Transferred,
566 __deref_out PMDL *NextMdl,
567 __out size_t *NextOffset
568 )
569 /*++
570
571 Routine Description:
572
573 This function computes the next mdl and offset given the current MDL,
574 offset and bytes transfered.
575
576 Arguments:
577
578 CurrentMdl - Mdl where the transfer currently took place.
579
580 CurrentVa - Current virtual address in the buffer
581
582 Transfered - Bytes transfered or to be transfered
583
584 NextMdl - Mdl where the next transfer will take place
585
586 NextVA - Offset within NextMdl where the transfer will start
587
588 --*/
589 {
590 size_t transfered, mdlSize;
591 PMDL mdl;
592
593 mdlSize = MmGetMdlByteCount(CurrentMdl) - CurrentOffset;
594
595 if (Transferred < mdlSize) {
596 //
597 // We are still in the first MDL
598 //
599 *NextMdl = CurrentMdl;
600 *NextOffset = CurrentOffset + Transferred;
601 return;
602 }
603
604 //
605 // We have transfered the content of the first MDL.
606 // Move to the next one.
607 //
608 transfered = Transferred - mdlSize;
609 mdl = CurrentMdl->Next;
610 ASSERT(mdl != NULL);
611
612 while (transfered >= MmGetMdlByteCount(mdl)) {
613 //
614 // We have transfered the content of this MDL.
615 // Move to the next one.
616 //
617 transfered -= MmGetMdlByteCount(mdl);
618 mdl = mdl->Next;
619 ASSERT(mdl != NULL);
620 }
621
622 //
623 // This is the mdl where the last transfer occured.
624 //
625 *NextMdl = mdl;
626 *NextOffset = transfered;
627
628 return;
629 }
630
631 _Must_inspect_result_
632 NTSTATUS
_CalculateRequiredMapRegisters(__in PMDL Mdl,__in size_t CurrentOffset,__in ULONG Length,__in ULONG AvailableMapRegisters,__out_opt PULONG PossibleTransferLength,__out PULONG MapRegistersRequired)633 FxDmaTransactionBase::_CalculateRequiredMapRegisters(
634 __in PMDL Mdl,
635 __in size_t CurrentOffset,
636 __in ULONG Length,
637 __in ULONG AvailableMapRegisters,
638 __out_opt PULONG PossibleTransferLength,
639 __out PULONG MapRegistersRequired
640 )
641 /*++
642
643 Routine Description:
644
645 Used on Windows 2000 to compute number of map registered required
646 for this transfer. This is derived from HalCalculateScatterGatherListSize.
647
648 Arguments:
649
650 Mdl - Pointer to the MDL that describes the pages of memory that are being
651 read or written.
652
653 CurrentVa - Current virtual address in the buffer described by the MDL
654 that the transfer is being done to or from.
655
656 Length - Supplies the length of the transfer.
657
658 AvailableMapRegisters - Map registers available to do the transfer
659
660 PossibleTransferLength - Length that can transfered for the
661
662 MapRegistersRequired - Map registers required to the entire transfer
663
664 Return Value:
665
666 NTSTATUS
667
668 Notes:
669
670 --*/
671 {
672 PMDL tempMdl;
673 ULONG requiredMapRegisters;
674 ULONG transferLength;
675 ULONG mdlLength;
676 ULONG pageOffset;
677 ULONG possTransferLength;
678
679 //
680 // Calculate the number of required map registers.
681 //
682 tempMdl = Mdl;
683 transferLength = (ULONG) MmGetMdlByteCount(tempMdl) - (ULONG) CurrentOffset;
684 mdlLength = transferLength;
685
686 pageOffset = BYTE_OFFSET(GetStartVaFromOffset(tempMdl, CurrentOffset));
687 requiredMapRegisters = 0;
688 possTransferLength = 0;
689
690 //
691 // The virtual address should fit in the first MDL.
692 //
693
694 ASSERT(CurrentOffset <= tempMdl->ByteCount);
695
696 //
697 // Loop through chained MDLs, accumulating the required
698 // number of map registers.
699 //
700
701 while (transferLength < Length && tempMdl->Next != NULL) {
702
703 //
704 // With pageOffset and length, calculate number of pages spanned by
705 // the buffer.
706 //
707 requiredMapRegisters += (pageOffset + mdlLength + PAGE_SIZE - 1) >>
708 PAGE_SHIFT;
709
710 if (requiredMapRegisters <= AvailableMapRegisters) {
711 possTransferLength = transferLength;
712 }
713
714 tempMdl = tempMdl->Next;
715 pageOffset = tempMdl->ByteOffset;
716 mdlLength = tempMdl->ByteCount;
717 transferLength += mdlLength;
718 }
719
720 if ((transferLength + PAGE_SIZE) < (Length + pageOffset )) {
721 ASSERT(transferLength >= Length);
722 return STATUS_BUFFER_TOO_SMALL;
723 }
724
725 //
726 // Calculate the last number of map registers based on the requested
727 // length not the length of the last MDL.
728 //
729
730 ASSERT( transferLength <= mdlLength + Length );
731
732 requiredMapRegisters += (pageOffset + Length + mdlLength - transferLength +
733 PAGE_SIZE - 1) >> PAGE_SHIFT;
734
735 if (requiredMapRegisters <= AvailableMapRegisters) {
736 possTransferLength += (Length + mdlLength - transferLength);
737 }
738
739 if (PossibleTransferLength != NULL) {
740 *PossibleTransferLength = possTransferLength;
741 }
742
743 ASSERT(*PossibleTransferLength);
744
745 *MapRegistersRequired = requiredMapRegisters;
746
747 return STATUS_SUCCESS;
748 }
749
750 // ----------------------------------------------------------------------------
751 // ------------------- Scatter/Gather DMA Section -----------------------------
752 // ----------------------------------------------------------------------------
753
FxDmaScatterGatherTransaction(__in PFX_DRIVER_GLOBALS FxDriverGlobals,__in USHORT ExtraSize,__in FxDmaEnabler * DmaEnabler)754 FxDmaScatterGatherTransaction::FxDmaScatterGatherTransaction(
755 __in PFX_DRIVER_GLOBALS FxDriverGlobals,
756 __in USHORT ExtraSize,
757 __in FxDmaEnabler *DmaEnabler
758 ) :
759 FxDmaTransactionBase(FxDriverGlobals,
760 sizeof(FxDmaScatterGatherTransaction),
761 ExtraSize,
762 DmaEnabler)
763 {
764 m_LookasideBuffer = NULL;
765 m_SGList = NULL;
766 }
767
768 _Must_inspect_result_
769 NTSTATUS
_Create(__in PFX_DRIVER_GLOBALS FxDriverGlobals,__in PWDF_OBJECT_ATTRIBUTES Attributes,__in FxDmaEnabler * DmaEnabler,__out WDFDMATRANSACTION * Transaction)770 FxDmaScatterGatherTransaction::_Create(
771 __in PFX_DRIVER_GLOBALS FxDriverGlobals,
772 __in PWDF_OBJECT_ATTRIBUTES Attributes,
773 __in FxDmaEnabler* DmaEnabler,
774 __out WDFDMATRANSACTION* Transaction
775 )
776 {
777 FxDmaScatterGatherTransaction* pTransaction;
778 WDFOBJECT hTransaction;
779 NTSTATUS status;
780
781 pTransaction = new (FxDriverGlobals, Attributes, DmaEnabler->GetTransferContextSize())
782 FxDmaScatterGatherTransaction(FxDriverGlobals,
783 DmaEnabler->GetTransferContextSize(),
784 DmaEnabler);
785
786 if (pTransaction == NULL) {
787 status = STATUS_INSUFFICIENT_RESOURCES;
788 DoTraceLevelMessage(
789 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
790 "Could not allocate memory for WDFTRANSACTION, %!STATUS!", status);
791 return status;
792 }
793
794 //
795 // Commit and apply the attributes
796 //
797 status = pTransaction->Commit(Attributes, &hTransaction, DmaEnabler);
798
799 if (NT_SUCCESS(status) && DmaEnabler->m_IsSGListAllocated) {
800
801 //
802 // Allocate buffer for SGList from lookaside list.
803 //
804 pTransaction->m_LookasideBuffer = (SCATTER_GATHER_LIST *)
805 FxAllocateFromNPagedLookasideList(
806 &DmaEnabler->m_SGList.ScatterGatherProfile.Lookaside
807 );
808
809 if (pTransaction->m_LookasideBuffer == NULL) {
810 status = STATUS_INSUFFICIENT_RESOURCES;
811 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
812 "Unable to allocate memory for SG List, "
813 "WDFDMATRANSACTION %p, %!STATUS! ",
814 pTransaction->GetHandle(), status);
815 }
816 else {
817 //
818 // Take a reference on the enabler to ensure that it remains valid
819 // if the transaction's disposal is deferred.
820 //
821 DmaEnabler->ADDREF(pTransaction);
822 }
823 }
824
825 if (NT_SUCCESS(status)) {
826 *Transaction = (WDFDMATRANSACTION)hTransaction;
827 }
828 else {
829 //
830 // This will properly clean up the target's state and free it
831 //
832 pTransaction->DeleteFromFailedCreate();
833 }
834
835 return status;
836 }
837
838 BOOLEAN
Dispose(VOID)839 FxDmaScatterGatherTransaction::Dispose(
840 VOID
841 )
842 {
843 BOOLEAN ret;
844
845 ret = FxDmaTransactionBase::Dispose(); // __super call
846
847 //
848 // Free Lookaside Buffer which held SGList
849 //
850 if (m_LookasideBuffer != NULL) {
851
852 FxFreeToNPagedLookasideList(
853 &m_DmaEnabler->m_SGList.ScatterGatherProfile.Lookaside,
854 m_LookasideBuffer
855 );
856 m_LookasideBuffer = NULL;
857 m_DmaEnabler->RELEASE(this);
858 }
859
860 return ret;
861 }
862
863 _Must_inspect_result_
864 NTSTATUS
InitializeResources(VOID)865 FxDmaScatterGatherTransaction::InitializeResources(
866 VOID
867 )
868 {
869 NTSTATUS status;
870 PMDL nextMdl;
871 size_t nextOffset;
872 ULONG mapRegistersRequired;
873 size_t remLength, transferLength, transferred, possibleLength=0;
874 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
875
876 status = STATUS_SUCCESS;
877
878 //
879 // If the caller has specified a limit on the number of scatter-gather
880 // elements each transfer can support then make sure it's within the
881 // limit by breaking up the whole transfer into m_MaxFragmentLength and
882 // computing the number of map-registers required for each fragment.
883 // This check may not be valid if the driver starts to do partial
884 // transfers. So driver that do partial transfer with sg-element limit
885 // should be capable of handling STATUS_WDF_TOO_FRAGMENTED failures during
886 // dma execution.
887 //
888 remLength = m_TransactionLength;
889 transferred = 0;
890 nextMdl = m_StartMdl;
891 nextOffset = m_StartOffset;
892 transferLength = 0;
893
894 while (remLength != 0) {
895
896 _ComputeNextTransferAddress(nextMdl,
897 nextOffset,
898 (ULONG) transferLength,
899 &nextMdl,
900 &nextOffset);
901
902 transferLength = FxSizeTMin(remLength, m_MaxFragmentLength);
903
904 status = _CalculateRequiredMapRegisters(nextMdl,
905 nextOffset,
906 (ULONG) transferLength,
907 m_AdapterInfo->NumberOfMapRegisters,
908 (PULONG) &possibleLength,
909 &mapRegistersRequired
910 );
911
912 if (!NT_SUCCESS(status)) {
913 DoTraceLevelMessage(
914 pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
915 "CalculateScatterGatherList failed for "
916 "WDFDMATRANSACTION %p, %!STATUS!", GetHandle(), status);
917 FxVerifierDbgBreakPoint(pFxDriverGlobals);
918 return status;
919 }
920
921 if (mapRegistersRequired > m_DmaEnabler->m_MaxSGElements) {
922 status = STATUS_WDF_TOO_FRAGMENTED;
923 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
924 "WDFDMATRANSACTION %p for MDL %p is more fragmented (%d) "
925 "than the limit (%I64d) specified by the driver, %!STATUS! ",
926 GetHandle(), nextMdl, mapRegistersRequired,
927 m_DmaEnabler->m_MaxSGElements, status);
928 return status;
929 }
930
931 transferred += transferLength;
932 remLength -= transferLength;
933 }
934
935 return status;
936 }
937
938 VOID
ReleaseResources(__in BOOLEAN)939 FxDmaScatterGatherTransaction::ReleaseResources(
940 __in BOOLEAN /* ForceRelease */
941 )
942 {
943 if (m_SGList != NULL) {
944 PutScatterGatherList(m_SGList);
945 m_SGList = NULL;
946 }
947 m_AdapterInfo = NULL;
948 }
949
950 _Must_inspect_result_
951 NTSTATUS
StartTransfer(VOID)952 FxDmaScatterGatherTransaction::StartTransfer(
953 VOID
954 )
955 {
956 ASSERT(m_CurrentFragmentMdl == m_StartMdl);
957 ASSERT(m_CurrentFragmentOffset == m_StartOffset);
958 ASSERT(m_CurrentFragmentLength == 0);
959 ASSERT(m_Transferred == 0);
960
961 return StageTransfer();
962 }
963
964 _Must_inspect_result_
965 NTSTATUS
StageTransfer(VOID)966 FxDmaScatterGatherTransaction::StageTransfer(
967 VOID
968 )
969 {
970 NTSTATUS status;
971 ULONG mapRegistersRequired;
972 WDFDMATRANSACTION dmaTransaction;
973 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
974
975 //
976 // Use an invalid value to make the function fail if the var is not
977 // updated correctly below.
978 //
979 mapRegistersRequired = 0xFFFFFFFF;
980
981 //
982 // Client driver could complete and delete the object in
983 // EvtProgramDmaFunction. So, save the handle because we need it
984 // for tracing after we invoke the callback.
985 //
986 dmaTransaction = GetHandle();
987
988 if (pFxDriverGlobals->FxVerifierOn) {
989 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
990 "Enter WDFDMATRANSACTION %p ", GetHandle());
991 }
992
993 //
994 // Given the first MDL and the bytes transfered, find the next MDL
995 // and byteoffset within that MDL.
996 //
997 _ComputeNextTransferAddress(m_CurrentFragmentMdl,
998 m_CurrentFragmentOffset,
999 (ULONG) m_CurrentFragmentLength,
1000 &m_CurrentFragmentMdl,
1001 &m_CurrentFragmentOffset);
1002
1003 //
1004 // Get the next possible transfer size.
1005 //
1006 m_CurrentFragmentLength = FxSizeTMin(m_Remaining, m_MaxFragmentLength);
1007
1008 //
1009 // Fix m_CurrentFragmentLength to meet the map registers limit. This is done
1010 // in case the MDL is a chained MDL for an highly fragmented buffer.
1011 //
1012 status = _CalculateRequiredMapRegisters(m_CurrentFragmentMdl,
1013 m_CurrentFragmentOffset,
1014 (ULONG) m_CurrentFragmentLength,
1015 m_AdapterInfo->NumberOfMapRegisters,
1016 (PULONG)&m_CurrentFragmentLength,
1017 &mapRegistersRequired);
1018 //
1019 // We have already validated the entire transfer during initialize
1020 // to see each transfer meets the sglimit. So this call shouldn't fail.
1021 // But, if the driver does partial transfer and changes the fragment
1022 // boundaries then it's possible for the sg-elements to vary. So, check
1023 // one more time to see if we are within the bounds before building
1024 // the sglist and calling into the driver.
1025 //
1026 ASSERT(NT_SUCCESS(status));
1027
1028 if (mapRegistersRequired > m_DmaEnabler->m_MaxSGElements) {
1029 status = STATUS_WDF_TOO_FRAGMENTED;
1030 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
1031 "WDFDMATRANSACTION %p for MDL %p is more fragmented (%d) "
1032 "than the limit (%I64d) specified by the driver, %!STATUS! ",
1033 dmaTransaction, m_CurrentFragmentMdl, mapRegistersRequired,
1034 m_DmaEnabler->m_MaxSGElements, status);
1035 return status;
1036 }
1037
1038
1039 m_Remaining -= m_CurrentFragmentLength;
1040
1041 if (m_DmaEnabler->m_IsSGListAllocated) {
1042
1043 ASSERT(m_LookasideBuffer != NULL);
1044 status = BuildScatterGatherList(m_CurrentFragmentMdl,
1045 m_CurrentFragmentOffset,
1046 (ULONG) m_CurrentFragmentLength,
1047 #pragma prefast(suppress: __WARNING_CLASS_MISMATCH_NONE, "This warning requires a wrapper class for the DRIVER_LIST_CONTROL type.")
1048 _AdapterListControl,
1049 this,
1050 m_LookasideBuffer,
1051 (ULONG) m_AdapterInfo->PreallocatedSGListSize);
1052
1053 } else {
1054
1055 status = GetScatterGatherList(m_CurrentFragmentMdl,
1056 m_CurrentFragmentOffset,
1057 (ULONG) m_CurrentFragmentLength,
1058 #pragma prefast(suppress: __WARNING_CLASS_MISMATCH_NONE, "This warning requires a wrapper class for the DRIVER_LIST_CONTROL type.")
1059 _AdapterListControl,
1060 this);
1061 }
1062
1063 if (!NT_SUCCESS(status)) {
1064 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
1065 "Build or GetScatterGatherList failed for "
1066 "WDFDMATRANSACTION %p, %!STATUS!",
1067 dmaTransaction, status);
1068 //
1069 // Readjust remaining bytes transfered.
1070 //
1071 m_Remaining += m_CurrentFragmentLength;
1072 return status;
1073 }
1074
1075 //
1076 // Before GetScatterGatherList returns, _AdapterListControl can get called
1077 // on another thread and the driver could delete the transaction object.
1078 // So don't touch the object after this point.
1079 //
1080
1081 if (pFxDriverGlobals->FxVerifierOn) {
1082 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1083 "Exit WDFDMATRANSACTION %p, "
1084 "%!STATUS!", dmaTransaction, status);
1085 }
1086
1087 return status;
1088 }
1089
1090
1091 VOID
_AdapterListControl(__in PDEVICE_OBJECT DeviceObject,__in PIRP Irp,__in PSCATTER_GATHER_LIST SgList,__in PVOID Context)1092 FxDmaScatterGatherTransaction::_AdapterListControl(
1093 __in PDEVICE_OBJECT DeviceObject,
1094 __in PIRP Irp, // UNUSED
1095 __in PSCATTER_GATHER_LIST SgList,
1096 __in PVOID Context
1097 )
1098 {
1099 PFX_DRIVER_GLOBALS pFxDriverGlobals;
1100 WDFDMATRANSACTION dmaTransaction;
1101 FxDmaScatterGatherTransaction * pDmaTransaction;
1102
1103 UNREFERENCED_PARAMETER(Irp);
1104 UNREFERENCED_PARAMETER(DeviceObject);
1105
1106 pDmaTransaction = (FxDmaScatterGatherTransaction*) Context;
1107 pFxDriverGlobals = pDmaTransaction->GetDriverGlobals();
1108 dmaTransaction = pDmaTransaction->GetHandle();
1109
1110 if (pFxDriverGlobals->FxVerifierOn) {
1111 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1112 "Enter WDFDMATRANSACTION %p",
1113 dmaTransaction);
1114 }
1115
1116 ASSERT(pDmaTransaction != NULL);
1117 ASSERT(pDmaTransaction->m_DmaAcquiredFunction.Method.ProgramDma != NULL);
1118
1119 ASSERT(SgList->NumberOfElements <= pDmaTransaction->m_DmaEnabler->GetMaxSGElements());
1120
1121 pDmaTransaction->m_SGList = SgList;
1122
1123 //
1124 // We ignore the return value. The pattern we want the driver to follow is
1125 // that if it fails to program DMA transfer, it should call DmaCompletedFinal
1126 // to abort the transfer.
1127 //
1128 (VOID) pDmaTransaction->m_DmaAcquiredFunction.InvokeProgramDma(
1129 dmaTransaction,
1130 pDmaTransaction->m_DmaEnabler->m_DeviceBase->GetHandle(),
1131 pDmaTransaction->m_DmaAcquiredContext,
1132 pDmaTransaction->m_DmaDirection,
1133 SgList);
1134
1135 if (pFxDriverGlobals->FxVerifierOn) {
1136 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1137 "Exit WDFDMATRANSACTION %p",
1138 dmaTransaction);
1139 }
1140 }
1141
1142 _Must_inspect_result_
1143 NTSTATUS
TransferCompleted(VOID)1144 FxDmaScatterGatherTransaction::TransferCompleted(
1145 VOID
1146 )
1147 {
1148 //
1149 // All we have to do is release the scatter-gather list.
1150 //
1151 if (m_SGList != NULL) {
1152
1153 PutScatterGatherList(m_SGList);
1154 m_SGList = NULL;
1155 }
1156
1157 return STATUS_SUCCESS;
1158 }
1159
1160
1161 // ----------------------------------------------------------------------------
1162 // ------------------- PACKET DMA SECTION -------------------------------------
1163 // ----------------------------------------------------------------------------
1164
FxDmaPacketTransaction(__in PFX_DRIVER_GLOBALS FxDriverGlobals,__in USHORT ObjectSize,__in USHORT ExtraSize,__in FxDmaEnabler * DmaEnabler)1165 FxDmaPacketTransaction::FxDmaPacketTransaction(
1166 __in PFX_DRIVER_GLOBALS FxDriverGlobals,
1167 __in USHORT ObjectSize,
1168 __in USHORT ExtraSize,
1169 __in FxDmaEnabler *DmaEnabler
1170 ) :
1171 FxDmaTransactionBase(FxDriverGlobals, ObjectSize, ExtraSize, DmaEnabler)
1172 {
1173 m_MapRegistersNeeded = 0;
1174 m_MapRegisterBase = NULL;
1175 m_MapRegisterBaseSet = FALSE;
1176 m_DeviceAddressOffset = 0;
1177 m_MapRegistersReserved = 0;
1178
1179 m_IsCancelled = FALSE;
1180
1181 m_TransferState.CurrentStagingThread = NULL;
1182 m_TransferState.RerunStaging = FALSE;
1183 m_TransferState.RerunCompletion = FALSE;
1184 m_TransferState.CompletionStatus = UNDEFINED_DMA_COMPLETION_STATUS;
1185 }
1186
1187 _Must_inspect_result_
1188 NTSTATUS
_Create(__in PFX_DRIVER_GLOBALS FxDriverGlobals,__in PWDF_OBJECT_ATTRIBUTES Attributes,__in FxDmaEnabler * DmaEnabler,__out WDFDMATRANSACTION * Transaction)1189 FxDmaPacketTransaction::_Create(
1190 __in PFX_DRIVER_GLOBALS FxDriverGlobals,
1191 __in PWDF_OBJECT_ATTRIBUTES Attributes,
1192 __in FxDmaEnabler* DmaEnabler,
1193 __out WDFDMATRANSACTION* Transaction
1194 )
1195 {
1196 FxDmaPacketTransaction* pTransaction;
1197 WDFOBJECT hTransaction;
1198 NTSTATUS status;
1199
1200 pTransaction = new (FxDriverGlobals, Attributes, DmaEnabler->GetTransferContextSize())
1201 FxDmaPacketTransaction(FxDriverGlobals,
1202 sizeof(FxDmaPacketTransaction),
1203 DmaEnabler->GetTransferContextSize(),
1204 DmaEnabler);
1205
1206 if (pTransaction == NULL) {
1207 status = STATUS_INSUFFICIENT_RESOURCES;
1208 DoTraceLevelMessage(
1209 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
1210 "Could not allocate memory for WDFTRANSACTION, %!STATUS!", status);
1211 return status;
1212 }
1213
1214 //
1215 // Commit and apply the attributes
1216 //
1217 status = pTransaction->Commit(Attributes, &hTransaction, DmaEnabler);
1218 if (NT_SUCCESS(status)) {
1219 *Transaction = (WDFDMATRANSACTION)hTransaction;
1220 }
1221 else {
1222 //
1223 // This will properly clean up the target's state and free it
1224 //
1225 pTransaction->DeleteFromFailedCreate();
1226 }
1227
1228 return status;
1229 }
1230
1231 _Must_inspect_result_
1232 NTSTATUS
InitializeResources(VOID)1233 FxDmaPacketTransaction::InitializeResources(
1234 VOID
1235 )
1236 {
1237 KIRQL oldIrql;
1238 m_DeviceAddressOffset = 0;
1239 LockTransferState(&oldIrql);
1240 m_IsCancelled = FALSE;
1241 UnlockTransferState(oldIrql);
1242 return STATUS_SUCCESS;
1243 }
1244
1245 VOID
ReleaseResources(__in BOOLEAN ForceRelease)1246 FxDmaPacketTransaction::ReleaseResources(
1247 __in BOOLEAN ForceRelease
1248 )
1249 {
1250 //
1251 // If the map register base hasn't been assigned, then just
1252 // skip this.
1253 //
1254
1255 if (IsMapRegisterBaseSet() == FALSE) {
1256 return;
1257 }
1258
1259 //
1260 // Map registers are reserved. Unless the caller is forcing
1261 // us to free them, just return. Otherwise updated the
1262 // number of map registers that FreeMapRegistersAndAdapter
1263 // is going to look at.
1264 //
1265 if ((m_MapRegistersReserved > 0) && (ForceRelease == FALSE))
1266 {
1267 return;
1268 }
1269
1270 //
1271 // Free the map registers and release the device.
1272 //
1273 FreeMapRegistersAndAdapter();
1274
1275 ClearMapRegisterBase();
1276
1277 ReleaseDevice();
1278
1279 m_AdapterInfo = NULL;
1280 m_MapRegistersReserved = 0;
1281 }
1282
1283 _Must_inspect_result_
1284 NTSTATUS
ReserveAdapter(__in ULONG NumberOfMapRegisters,__in WDF_DMA_DIRECTION DmaDirection,__in PFN_WDF_RESERVE_DMA Callback,__in_opt PVOID Context)1285 FxDmaPacketTransaction::ReserveAdapter(
1286 __in ULONG NumberOfMapRegisters,
1287 __in WDF_DMA_DIRECTION DmaDirection,
1288 __in PFN_WDF_RESERVE_DMA Callback,
1289 __in_opt PVOID Context
1290 )
1291 {
1292 NTSTATUS status;
1293 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
1294 WDFDMATRANSACTION dmaTransaction = GetHandle();
1295
1296 if (pFxDriverGlobals->FxVerifierOn) {
1297 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1298 "Enter WDFDMATRANSACTION %p", dmaTransaction);
1299 }
1300
1301 //
1302 // If caller doesn't supply a map register count then we get the count
1303 // out of the transaction. So the transaction must be initialized.
1304 //
1305 // Otherwise the transaction can't be executing.
1306 //
1307 if (NumberOfMapRegisters == 0) {
1308 if (m_State != FxDmaTransactionStateInitialized) {
1309 status = STATUS_INVALID_PARAMETER;
1310
1311 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
1312 "RequiredMapRegisters cannot be 0 because "
1313 "WDFDMATRANSACTION %p is not initialized ("
1314 "state is %!FxDmaTransactionState!) - %!STATUS!",
1315 GetHandle(),
1316 m_State,
1317 status);
1318 FxVerifierBugCheck(pFxDriverGlobals, // globals
1319 WDF_DMA_FATAL_ERROR, // specific type
1320 (ULONG_PTR) GetObjectHandle(), // parm 2
1321 (ULONG_PTR) m_State); // parm 3
1322 }
1323 }
1324 else if (m_State != FxDmaTransactionStateCreated &&
1325 m_State != FxDmaTransactionStateInitialized &&
1326 m_State != FxDmaTransactionStateReleased) {
1327
1328 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
1329 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
1330 "is invalid", dmaTransaction, m_State);
1331
1332 FxVerifierBugCheck(pFxDriverGlobals, // globals
1333 WDF_DMA_FATAL_ERROR, // specific type
1334 (ULONG_PTR) GetObjectHandle(), // parm 2
1335 (ULONG_PTR) m_State); // parm 3
1336 }
1337
1338 //
1339 // Must not already have reserved map registers
1340 //
1341 if (m_MapRegistersReserved != 0)
1342 {
1343 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
1344 "WDFDMATRANSACTION %p already has allocated map registers.",
1345 dmaTransaction);
1346
1347 FxVerifierBugCheck(pFxDriverGlobals, // globals
1348 WDF_DMA_FATAL_ERROR, // specific type
1349 (ULONG_PTR) GetObjectHandle(), // parm 2
1350 (ULONG_PTR) m_State); // parm 3
1351 }
1352
1353 //
1354 // Get the adapter
1355 //
1356 if (DmaDirection == WdfDmaDirectionReadFromDevice) {
1357 m_AdapterInfo = m_DmaEnabler->GetReadDmaDescription();
1358 } else {
1359 m_AdapterInfo = m_DmaEnabler->GetWriteDmaDescription();
1360 }
1361
1362 //
1363 // Save the number of map registers being reserved.
1364 //
1365 if (NumberOfMapRegisters != 0) {
1366
1367 //
1368 // Use the number the caller passed us
1369 //
1370 m_MapRegistersReserved = NumberOfMapRegisters;
1371 }
1372 else if (m_DmaEnabler->IsBusMaster() == FALSE) {
1373
1374 //
1375 // For system DMA use all the map registers we have
1376 //
1377 m_MapRegistersReserved = m_AdapterInfo->NumberOfMapRegisters;
1378
1379 } else {
1380
1381 //
1382 // Compute the number of map registers required based on
1383 // the MDL and length passed in
1384 //
1385 status = _CalculateRequiredMapRegisters(
1386 m_StartMdl,
1387 m_StartOffset,
1388 (ULONG) m_TransactionLength,
1389 m_AdapterInfo->NumberOfMapRegisters,
1390 NULL,
1391 &m_MapRegistersReserved
1392 );
1393
1394 if (!NT_SUCCESS(status)) {
1395 ReleaseForReuse(TRUE);
1396 goto End;
1397 }
1398 }
1399
1400 //
1401 // Initialize the DmaTransaction object with enough data to
1402 // trick StartTransfer into allocating the adapter channel for us.
1403 //
1404 m_DmaDirection = DmaDirection;
1405 m_StartMdl = NULL;
1406 m_StartOffset = 0;
1407 m_CurrentFragmentMdl = NULL;
1408 m_CurrentFragmentOffset = 0;
1409 m_Remaining = 0;
1410 m_TransactionLength = 0;
1411
1412 //
1413 // Save the callback and context
1414 //
1415 m_DmaAcquiredFunction.Method.ReserveDma = Callback;
1416 m_DmaAcquiredContext = Context;
1417
1418 //
1419 // If needed, initialize the transfer context.
1420 //
1421 if (m_DmaEnabler->UsesDmaV3()) {
1422 m_DmaEnabler->InitializeTransferContext(GetTransferContext(),
1423 m_DmaDirection);
1424 }
1425
1426 status = InitializeResources();
1427 if (NT_SUCCESS(status)) {
1428 //
1429 // Set the state to reserved so _AdapterControl knows which
1430 // callback to invoke.
1431 //
1432 m_State = FxDmaTransactionStateReserved;
1433 } else {
1434 ReleaseForReuse(TRUE);
1435 goto End;
1436 }
1437
1438 //
1439 // Start the adapter channel allocation through StartTransfer
1440 //
1441 status = StartTransfer();
1442
1443 End:
1444 if (!NT_SUCCESS(status)) {
1445 m_State = FxDmaTransactionStateTransferFailed;
1446 m_DmaAcquiredFunction.Method.ReserveDma = NULL;
1447 m_DmaAcquiredContext = NULL;
1448 m_MapRegistersReserved = 0;
1449
1450 if (m_EncodedRequest != NULL) {
1451 ReleaseButRetainRequest();
1452 }
1453 }
1454
1455 if (pFxDriverGlobals->FxVerifierOn) {
1456 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1457 "Exit WDFDMATRANSACTION %p, %!STATUS!",
1458 dmaTransaction, status);
1459 }
1460
1461 return status;
1462 }
1463
1464 VOID
ReleaseAdapter(VOID)1465 FxDmaPacketTransaction::ReleaseAdapter(
1466 VOID
1467 )
1468 {
1469 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
1470 WDFDMATRANSACTION dmaTransaction = GetHandle();
1471
1472 if (pFxDriverGlobals->FxVerifierOn) {
1473 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1474 "Enter WDFDMATRANSACTION %p", dmaTransaction);
1475 }
1476
1477 //
1478 // Must not be in invalid, created, transfer or deleted state
1479 //
1480 if (m_State == FxDmaTransactionStateInvalid ||
1481 m_State == FxDmaTransactionStateCreated ||
1482 m_State == FxDmaTransactionStateTransfer ||
1483 m_State == FxDmaTransactionStateDeleted) {
1484
1485 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
1486 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
1487 "is invalid", dmaTransaction, m_State);
1488
1489 FxVerifierBugCheck(pFxDriverGlobals, // globals
1490 WDF_DMA_FATAL_ERROR, // specific type
1491 (ULONG_PTR) GetObjectHandle(), // parm 2
1492 (ULONG_PTR) m_State); // parm 3
1493 }
1494
1495 //
1496 // The caller wants to free the reserved map registers, so force their
1497 // release.
1498 //
1499 ReleaseForReuse(TRUE);
1500
1501 if (pFxDriverGlobals->FxVerifierOn) {
1502 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1503 "Exit WDFDMATRANSACTION %p",
1504 dmaTransaction);
1505 }
1506
1507 return;
1508 }
1509
1510 _Must_inspect_result_
1511 NTSTATUS
StartTransfer(VOID)1512 FxDmaPacketTransaction::StartTransfer(
1513 VOID
1514 )
1515 {
1516 NTSTATUS status;
1517 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
1518 WDFDMATRANSACTION dmaTransaction;
1519
1520 //
1521 // Client driver could complete and delete the object in
1522 // EvtProgramDmaFunction. So, save the handle because we need it
1523 // for tracing after we invoke the callback.
1524 //
1525 dmaTransaction = GetHandle();
1526
1527 if (pFxDriverGlobals->FxVerifierOn) {
1528 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1529 "Enter WDFDMATRANSACTION %p",
1530 dmaTransaction);
1531
1532 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1533 "Starting WDFDMATRANSACTION %p - MDL %#p, "
1534 "offset %I64x, length %I64x",
1535 dmaTransaction,
1536 m_StartMdl,
1537 m_StartOffset,
1538 m_TransactionLength);
1539 }
1540
1541 //
1542 // Reference the device when using DMA v2. For DMA v3 we can support
1543 // concurrent attempts to allocate the channel.
1544 //
1545 status = AcquireDevice();
1546 if (!NT_SUCCESS(status)) {
1547
1548 NT_ASSERTMSG("AcquireDevice should never fail when DMAv3 is in use",
1549 m_DmaEnabler->UsesDmaV3());
1550
1551 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
1552 "Only one transaction can be queued "
1553 "at one time on a packet based WDFDMAENABLER %p "
1554 "%!STATUS!", m_DmaEnabler->GetHandle(),
1555 status);
1556 FxVerifierDbgBreakPoint(pFxDriverGlobals);
1557 return status;
1558 }
1559
1560 //
1561 // Calculate the initial DMA transfer length.
1562 //
1563 m_CurrentFragmentLength = FxSizeTMin(m_Remaining, m_MaxFragmentLength);
1564
1565 m_CurrentFragmentOffset = m_StartOffset;
1566
1567 if (m_State == FxDmaTransactionStateReserved) {
1568 //
1569 // Caller is simply reserving the DMA adapter for later use. Ask for
1570 // as many map registers as the driver requested.
1571 //
1572 m_MapRegistersNeeded = m_MapRegistersReserved;
1573
1574 ASSERT(m_MapRegistersNeeded <= m_AdapterInfo->NumberOfMapRegisters);
1575
1576 status = AllocateAdapterChannel(FALSE);
1577
1578 }
1579 else {
1580
1581 if (m_DmaEnabler->IsBusMaster() == FALSE) {
1582
1583 //
1584 // Use as many map registers as we were granted.
1585 //
1586 m_MapRegistersNeeded = m_AdapterInfo->NumberOfMapRegisters;
1587 } else {
1588
1589 //
1590 // If the transfer is the size of the transaction then use the offset
1591 // to determine the number of map registers needed. If it's smaller
1592 // then use the worst-case offset to make sure we ask for enough MR's
1593 // to account for a bigger offset in one of the later transfers.
1594 //
1595 // Example:
1596 // Transaction is 8 KB and is page aligned
1597 // if max transfer is >= 8KB then this will be one transfer and only
1598 // requires two map registers. Even if the driver completes a partial
1599 // transfer and we have to do the rest in a second transfer it will
1600 // fit within two map registers becuase the overall transaction does
1601 // (and a partial transfer can't take more map registers than the
1602 // whole transaction would).
1603 //
1604 // If max transfer is 2KB then this nominally requires 4 2KB transfers.
1605 // In this case however, a partial completion of one of those transfers
1606 // would leave us attempting a second 2KB transfer starting on an
1607 // unaligned address. For example, we might transfer 2KB, then 1KB
1608 // then 2KB. Even though the first transfer was page aligned, the
1609 // 3rd transfer isn't and could cross a page boundary, requiring two
1610 // map registers rather than one.
1611 //
1612 // To account for this second case, ignore the actual MDL offset and
1613 // instead compute the maximum number of map registers than an N byte
1614 // transfer could take (with worst-case alignment).
1615 //
1616 //
1617 m_MapRegistersNeeded =
1618 (ULONG) ADDRESS_AND_SIZE_TO_SPAN_PAGES(
1619 ((m_CurrentFragmentLength == m_Remaining) ?
1620 GetStartVaFromOffset(m_CurrentFragmentMdl,
1621 m_CurrentFragmentOffset) :
1622 (PVOID)(ULONG_PTR) (PAGE_SIZE -1)),
1623 m_CurrentFragmentLength
1624 );
1625
1626
1627 ASSERT(m_MapRegistersNeeded <= m_AdapterInfo->NumberOfMapRegisters);
1628 }
1629
1630 //
1631 // NOTE: the number of map registers needed for this transfer may
1632 // exceed the number that we've reserved. StageTransfer will
1633 // take care of fragmenting the transaction accordingly.
1634 //
1635 status = AllocateAdapterChannel(m_MapRegistersReserved > 0);
1636 }
1637
1638 if (!NT_SUCCESS(status)) {
1639 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
1640 "AllocateAdapterChannel failed for "
1641 "WDFDMATRANSACTION %p, %!STATUS!",
1642 dmaTransaction, status);
1643 ReleaseDevice();
1644 }
1645
1646 //
1647 // Before AllocateAdapterChannel returns, _AdapterControl can get called
1648 // on another thread and the driver could delete the transaction object.
1649 // So don't touch the object after this point.
1650 //
1651 if (pFxDriverGlobals->FxVerifierOn) {
1652 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1653 "Exit WDFDMATRANSACTION %p, "
1654 "%!STATUS!", dmaTransaction, status);
1655 }
1656
1657 return status;
1658 }
1659
1660 IO_ALLOCATION_ACTION
_AdapterControl(__in PDEVICE_OBJECT DeviceObject,__in PIRP Irp,__in PVOID MapRegisterBase,__in PVOID Context)1661 FxDmaPacketTransaction::_AdapterControl(
1662 __in PDEVICE_OBJECT DeviceObject,
1663 __in PIRP Irp,
1664 __in PVOID MapRegisterBase,
1665 __in PVOID Context
1666 )
1667 {
1668 FxDmaPacketTransaction * pDmaTransaction;
1669 PFX_DRIVER_GLOBALS pFxDriverGlobals;
1670 IO_ALLOCATION_ACTION action;
1671 NTSTATUS status;
1672
1673 UNREFERENCED_PARAMETER(Irp);
1674 UNREFERENCED_PARAMETER(DeviceObject);
1675
1676 pDmaTransaction = (FxDmaPacketTransaction*) Context;
1677 ASSERT(pDmaTransaction);
1678
1679 pFxDriverGlobals = pDmaTransaction->GetDriverGlobals();
1680
1681 //
1682 // Cache the return value while we can still touch the transaction
1683 //
1684 action = pDmaTransaction->GetAdapterControlReturnValue();
1685
1686 //
1687 // Save the MapRegister base, unless it was previously set
1688 // during a reserve.
1689 //
1690 if (pDmaTransaction->IsMapRegisterBaseSet() == FALSE) {
1691 pDmaTransaction->SetMapRegisterBase(MapRegisterBase);
1692 }
1693 else {
1694 NT_ASSERTMSG("Caller was expected to use existing map register base",
1695 MapRegisterBase == pDmaTransaction->m_MapRegisterBase);
1696 }
1697
1698 if (pFxDriverGlobals->FxVerifierOn) {
1699 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1700 "Map registers for WDFDMATRANSACTION %p allocated "
1701 "(base %p)",
1702 pDmaTransaction->GetHandle(),
1703 MapRegisterBase);
1704 }
1705
1706 //
1707 // NOTE: KMDF used to call KeFlushIoBuffers() here to "ensure the
1708 // data buffers were flushed." However KeFlushIoBuffers did
1709 // nothing on x86 & amd64 (which are cache coherent WRT DMA)
1710 // and calling FlushAdapterBuffers() does any necessary
1711 // flushing anyway. Plus on non-cache-coherent architectures
1712 // (such as ARM) the flush operation has to be cache-line aligned
1713 // to avoid cache line tearing. So the flush is not necessary
1714 // and has been removed.
1715
1716 //
1717 // Check the state of the transaction. If it's reserve then call the
1718 // reserve callback and return. Otherwise stage the first fragment.
1719 //
1720 if (pDmaTransaction->m_State == FxDmaTransactionStateReserved)
1721 {
1722 FxDmaTransactionProgramOrReserveDma callback;
1723
1724 //
1725 // Save off and clear the callback before calling it.
1726 //
1727 callback = pDmaTransaction->m_DmaAcquiredFunction;
1728 pDmaTransaction->m_DmaAcquiredFunction.Clear();
1729
1730 ASSERTMSG("Mismatch between map register counts",
1731 (pDmaTransaction->m_MapRegistersReserved ==
1732 pDmaTransaction->m_MapRegistersNeeded));
1733
1734 //
1735 // Invoke the callback. Note that from here the driver may initialize
1736 // and execute the transaction.
1737 //
1738 callback.InvokeReserveDma(
1739 pDmaTransaction->GetHandle(),
1740 pDmaTransaction->m_DmaAcquiredContext
1741 );
1742 }
1743 else {
1744
1745 //
1746 // Stage next fragment
1747 //
1748 status = pDmaTransaction->StageTransfer();
1749
1750 if (!NT_SUCCESS(status)) {
1751
1752 DMA_COMPLETION_STATUS dmaStatus =
1753 (status == STATUS_CANCELLED ? DmaCancelled : DmaError);
1754 FxDmaSystemTransaction* systemTransaction = (FxDmaSystemTransaction*) pDmaTransaction;
1755
1756 //
1757 // Map transfer failed. There will be no DMA completion callback
1758 // and no call to EvtProgramDma. And we have no way to hand this
1759 // status back directly to the driver. Fake a DMA completion with
1760 // the appropriate status.
1761 //
1762 // This should only happen for system DMA (and there most likely
1763 // only during cancelation, though we leave the possibility that
1764 // the DMA extension may fail the transfer)
1765 //
1766 ASSERTMSG("Unexpected failure of StageTransfer for packet based "
1767 "DMA",
1768 (pDmaTransaction->GetDmaEnabler()->IsBusMaster() == false));
1769
1770 if (pFxDriverGlobals->FxVerifierOn) {
1771 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1772 "Invoking DmaCompleted callback %p (context %p) "
1773 "for WDFDMATRANSACTION %p (status %x) "
1774 "due to staging failure (%!STATUS!)",
1775 systemTransaction->m_TransferCompleteFunction.Method,
1776 systemTransaction->m_TransferCompleteContext,
1777 pDmaTransaction->GetHandle(),
1778 dmaStatus,
1779 status);
1780 }
1781
1782 pDmaTransaction->CallEvtDmaCompleted(
1783 status == STATUS_CANCELLED ? DmaCancelled : DmaError
1784 );
1785 }
1786 }
1787
1788 //
1789 // Indicate that MapRegs are to be kept
1790 //
1791 return action;
1792 }
1793
1794 _Must_inspect_result_
1795 NTSTATUS
StageTransfer(VOID)1796 FxDmaPacketTransaction::StageTransfer(
1797 VOID
1798 )
1799 {
1800 PSCATTER_GATHER_LIST sgList;
1801
1802 PFX_DRIVER_GLOBALS pFxDriverGlobals;
1803 UCHAR_MEMORY_ALIGNED sgListBuffer[sizeof(SCATTER_GATHER_LIST)
1804 + sizeof(SCATTER_GATHER_ELEMENT)];
1805 WDFDMATRANSACTION dmaTransaction;
1806
1807 KIRQL oldIrql;
1808 BOOLEAN stagingNeeded;
1809
1810 NTSTATUS status = STATUS_SUCCESS;
1811
1812 //
1813 // Client driver could complete and delete the object in
1814 // EvtProgramDmaFunction. So, save the handle because we need it
1815 // for tracing after we invoke the callback.
1816 //
1817 pFxDriverGlobals = GetDriverGlobals();
1818 dmaTransaction = GetHandle();
1819
1820 if (pFxDriverGlobals->FxVerifierOn) {
1821 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1822 "Enter WDFDMATRANSACTION %p ", dmaTransaction);
1823 }
1824
1825 //
1826 // For packet base DMA, current and startMDL will always be
1827 // same. For V2 DMA we don't support MDL chains. For V3 DMA
1828 // we use the HAL's support for MDL chains and don't walk through
1829 // the MDL chain on our own.
1830 //
1831 ASSERT(m_CurrentFragmentMdl == m_StartMdl);
1832
1833 LockTransferState(&oldIrql);
1834
1835 if (m_TransferState.CurrentStagingThread != NULL) {
1836
1837 //
1838 // Staging in progress. Indicate that another staging will
1839 // be needed.
1840 //
1841 m_TransferState.RerunStaging = TRUE;
1842
1843 stagingNeeded = FALSE;
1844
1845 if (pFxDriverGlobals->FxVerifierOn) {
1846 DoTraceLevelMessage(
1847 pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1848 "Staging next fragment of WDFDMATRANSACTION %p "
1849 "deferred",
1850 dmaTransaction
1851 );
1852 }
1853 }
1854 else {
1855 //
1856 // Staging isn't in progress anyplace else. Indicate that it's
1857 // running now so that any parallel attempt is blocked.
1858 //
1859 m_TransferState.CurrentStagingThread = KeGetCurrentThread();
1860
1861 ASSERTMSG("The thread which was staging didn't clear "
1862 "RerunStaging",
1863 (m_TransferState.RerunStaging == FALSE));
1864
1865 stagingNeeded = TRUE;
1866 }
1867
1868 UnlockTransferState(oldIrql);
1869
1870 //
1871 // Take a reference on the transaction so that we can safely
1872 // manipulate the transfer state even after it's destroyed.
1873 //
1874 AddRef(&pFxDriverGlobals);
1875
1876 //
1877 // Loop for as long as staging is required
1878 //
1879 while (stagingNeeded) {
1880
1881 //
1882 // Calculate length for this packet.
1883 //
1884 m_CurrentFragmentLength = FxSizeTMin(m_Remaining, m_MaxFragmentLength);
1885
1886 //
1887 // Calculate address for this packet.
1888 //
1889 m_CurrentFragmentOffset = m_StartOffset + m_Transferred;
1890
1891 //
1892 // Adjust the fragment length for the number of reserved map registers.
1893 //
1894 if ((m_MapRegistersReserved > 0) &&
1895 (m_MapRegistersNeeded > m_MapRegistersReserved))
1896 {
1897 size_t currentOffset = m_CurrentFragmentOffset;
1898 size_t currentPageOffset;
1899 PMDL mdl;
1900
1901 for (mdl = m_CurrentFragmentMdl; mdl != NULL; mdl = mdl->Next)
1902 {
1903 //
1904 // For packet/system transfers of chained MDLs, m_CurrentFragmentMdl
1905 // is never adjusted, and m_CurrentFragmentOFfset is the offset
1906 // into the entire chain.
1907 //
1908 // Locate the MDL which contains the current fragment.
1909 //
1910 ULONG mdlBytes = MmGetMdlByteCount(mdl);
1911 if (mdlBytes >= currentOffset)
1912 {
1913 //
1914 // This MDL is larger than the remaining offset, so it
1915 // contains the start address.
1916 //
1917 break;
1918 }
1919
1920 currentOffset -= mdlBytes;
1921 }
1922
1923 ASSERT(mdl != NULL);
1924
1925 //
1926 // Compute page offset from current MDL's initial page offset
1927 // and the offset into that MDL
1928 //
1929
1930 currentPageOffset = BYTE_OFFSET(MmGetMdlByteOffset(mdl) +
1931 currentOffset);
1932
1933 //
1934 // Compute the maximum number of bytes we can transfer with
1935 // the number of map registers we have reserved, taking into
1936 // account the offset of the first page.
1937 //
1938 size_t l = ((PAGE_SIZE * (m_MapRegistersReserved - 1)) +
1939 (PAGE_SIZE - currentPageOffset));
1940
1941 m_CurrentFragmentLength = FxSizeTMin(m_CurrentFragmentLength, l);
1942 }
1943
1944 m_Remaining -= m_CurrentFragmentLength;
1945
1946 if (pFxDriverGlobals->FxVerifierOn) {
1947 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
1948 "Initiating %s transfer for WDFDMATRANSACTION %p. "
1949 "Mdl %p, Offset %I64x, Length %I64x",
1950 m_Transferred == 0 ? "first" : "next",
1951 GetHandle(),
1952 m_CurrentFragmentMdl,
1953 m_Transferred,
1954 m_CurrentFragmentLength);
1955 }
1956
1957 //
1958 // Check for a pending cancellation. This can happen if the cancel
1959 // occurred between DMA completion and FlushAdapterBuffers -
1960 // FlushAdapterBuffers will clear the canceled bit in the transfer
1961 // context (TC), which would allow MapTransfer to succeed.
1962 //
1963 // An unprotected check of IsCancelled here is safe. A concurrent
1964 // cancel at this point would mark the TC cancelled such that
1965 // MapTransfer will fail.
1966 //
1967 if (m_IsCancelled == TRUE) {
1968 status = STATUS_CANCELLED;
1969 goto End;
1970 }
1971
1972 //
1973 // Profile specific work before mapping the transfer. if this
1974 // fails consider 'this' invalid.
1975 //
1976 if (PreMapTransfer() == FALSE) {
1977 status = STATUS_SUCCESS;
1978 goto End;
1979 }
1980
1981 //
1982 // Map this packet for transfer.
1983 //
1984
1985 //
1986 // For packet based DMA we use a single entry on-stack SGL. This
1987 // allows us to map multiple packet-based requests concurrently and
1988 // we know packet base DMA only requires a single SGL
1989 //
1990 // NOTE: It turns out the HAL doesn't handle chained MDLs in packet
1991 // mode correctly. It makes each MDL continguous, but returns
1992 // a list of SG elements - one for each MDL. That's scatter
1993 // gather DMA, not packet DMA.
1994 //
1995 // So it's actually very important in Win8 that we only use a
1996 // single entry SGL when calling MapTransferEx. This ensures
1997 // we only map the first MDL in the chain and thus get a
1998 // contiguous buffer
1999 //
2000 // For system DMA we use the SystemSGList stored in the DMA enabler
2001 // We can use a shared one in that case because we won't be mapping
2002 // multiple system DMA requests concurrently (HAL doesn't allow it)
2003 //
2004 FxDmaEnabler* enabler = GetDmaEnabler();
2005 size_t sgListSize;
2006
2007 if (enabler->IsBusMaster()) {
2008 sgList = (PSCATTER_GATHER_LIST)sgListBuffer;
2009 sgListSize = sizeof(sgListBuffer);
2010 } else {
2011 sgList = enabler->m_SGList.SystemProfile.List;
2012 sgListSize = enabler->m_SGListSize;
2013 }
2014
2015 ULONG mappedBytes;
2016
2017 status = MapTransfer(sgList,
2018 (ULONG) sgListSize,
2019 GetTransferCompletionRoutine(),
2020 this,
2021 &mappedBytes);
2022
2023 NT_ASSERTMSG("Unexpected failure of MapTransfer",
2024 ((NT_SUCCESS(status) == TRUE) ||
2025 (status == STATUS_CANCELLED)));
2026
2027 if (NT_SUCCESS(status)) {
2028
2029 NT_ASSERTMSG("unexpected number of mapped bytes",
2030 ((mappedBytes > 0) &&
2031 (mappedBytes <= m_CurrentFragmentLength)));
2032
2033 //
2034 // Adjust the remaining byte count if the HAL mapped less data than we
2035 // requested.
2036 //
2037 if (mappedBytes < m_CurrentFragmentLength) {
2038 m_Remaining += m_CurrentFragmentLength - mappedBytes;
2039 m_CurrentFragmentLength = mappedBytes;
2040 }
2041
2042 //
2043 // Do client PFN_WDF_PROGRAM_DMA callback.
2044 //
2045 if (m_DmaAcquiredFunction.Method.ProgramDma != NULL) {
2046
2047 if (pFxDriverGlobals->FxVerifierOn) {
2048 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
2049 "Invoking ProgramDma callback %p (context %p) for "
2050 "WDFDMATRANSACTION %p.",
2051 m_DmaAcquiredFunction.Method.ProgramDma,
2052 m_DmaAcquiredContext,
2053 GetHandle()
2054 );
2055 }
2056
2057 //
2058 // Call program DMA
2059 //
2060 (VOID) m_DmaAcquiredFunction.InvokeProgramDma(
2061 GetHandle(),
2062 m_DmaEnabler->m_DeviceBase->GetHandle(),
2063 m_DmaAcquiredContext,
2064 m_DmaDirection,
2065 sgList
2066 );
2067 }
2068 }
2069
2070 End:
2071 //
2072 // Process any pending completion or nested staging.
2073 //
2074 {
2075 LockTransferState(&oldIrql);
2076
2077 //
2078 // While staging we could either have deferred a call to the
2079 // completion routine or deferred another call to stage the
2080 // next fragment. We should not ever have to do both - this
2081 // would imply that the driver didn't wait for its DMA completion
2082 // routine to run when calling TransferComplete*.
2083 //
2084 ASSERTMSG("driver called TransferComplete with pending DMA "
2085 "completion callback",
2086 !((m_TransferState.RerunCompletion == TRUE) &&
2087 (m_TransferState.RerunStaging == TRUE)));
2088
2089 //
2090 // Check for pending completion. save the status away and clear it
2091 // before dropping the lock.
2092 //
2093 if (m_TransferState.RerunCompletion == TRUE) {
2094 DMA_COMPLETION_STATUS completionStatus;
2095 FxDmaSystemTransaction* systemTransaction = (FxDmaSystemTransaction*) this;
2096
2097 //
2098 // Save the completion status for when we drop the lock.
2099 //
2100 completionStatus = m_TransferState.CompletionStatus;
2101
2102 ASSERTMSG("completion needed, but status was not set or was "
2103 "already cleared",
2104 completionStatus != UNDEFINED_DMA_COMPLETION_STATUS);
2105
2106 ASSERTMSG("completion needed, but mapping failed so there shouldn't "
2107 "be any parallel work going on",
2108 NT_SUCCESS(status));
2109
2110 //
2111 // Clear the completion needed state.
2112 //
2113 m_TransferState.RerunCompletion = FALSE;
2114 m_TransferState.CompletionStatus = UNDEFINED_DMA_COMPLETION_STATUS;
2115
2116 //
2117 // Drop the lock, call the completion routine, then take the
2118 // lock again.
2119 //
2120 UnlockTransferState(oldIrql);
2121 if (pFxDriverGlobals->FxVerifierOn) {
2122 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
2123 "Invoking DmaCompleted callback %p (context %p) "
2124 "for WDFDMATRANSACTION %p (status %x) "
2125 "after deferral",
2126 systemTransaction->m_TransferCompleteFunction.Method,
2127 systemTransaction->m_TransferCompleteContext,
2128 GetHandle(),
2129 completionStatus);
2130 }
2131 CallEvtDmaCompleted(completionStatus);
2132 LockTransferState(&oldIrql);
2133
2134 //
2135 // Staging is blocked, which means we aren't starting up the
2136 // next transfer. Therefore we cannot have a queued completion.
2137 //
2138 ASSERTMSG("RerunCompletion should not be set on an unstaged "
2139 "transaction",
2140 m_TransferState.RerunCompletion == FALSE);
2141 }
2142
2143 //
2144 // Capture whether another staging is needed. If none is needed
2145 // then we can clear staging in progress.
2146 //
2147 if (m_TransferState.RerunStaging == TRUE) {
2148 stagingNeeded = TRUE;
2149 m_TransferState.RerunStaging = FALSE;
2150 }
2151 else {
2152 m_TransferState.CurrentStagingThread = NULL;
2153 stagingNeeded = FALSE;
2154 }
2155
2156 UnlockTransferState(oldIrql);
2157 }
2158
2159 #if DBG
2160 if (!NT_SUCCESS(status)) {
2161 ASSERTMSG("MapTransfer returned an error - there should not be any "
2162 "deferred work.",
2163 (stagingNeeded == FALSE));
2164 }
2165 #endif
2166
2167 } // while(stagingNeeded)
2168
2169 Release(&pFxDriverGlobals);
2170
2171 if (pFxDriverGlobals->FxVerifierOn) {
2172 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
2173 "Exit WDFDMATRANSACTION %p, "
2174 "%!STATUS!", dmaTransaction,
2175 status);
2176 }
2177
2178 return status;
2179 }
2180
2181 _Must_inspect_result_
2182 NTSTATUS
TransferCompleted(VOID)2183 FxDmaPacketTransaction::TransferCompleted(
2184 VOID
2185 )
2186 {
2187 NTSTATUS status;
2188 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
2189
2190 //
2191 // Flush the buffers
2192 //
2193 status = FlushAdapterBuffers();
2194
2195 if (!NT_SUCCESS(status)) {
2196 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
2197 "FlushAdapterBuffers on WDFDMATRANSACTION %p "
2198 "failed, %!STATUS!",
2199 GetHandle(), status);
2200 FxVerifierDbgBreakPoint(pFxDriverGlobals);
2201 }
2202
2203 return status;
2204 }
2205
2206 // ----------------------------------------------------------------------------
2207 // ------------------- SYSTEM DMA SECTION -------------------------------------
2208 // ----------------------------------------------------------------------------
2209
FxDmaSystemTransaction(__in PFX_DRIVER_GLOBALS FxDriverGlobals,__in USHORT ExtraSize,__in FxDmaEnabler * DmaEnabler)2210 FxDmaSystemTransaction::FxDmaSystemTransaction(
2211 __in PFX_DRIVER_GLOBALS FxDriverGlobals,
2212 __in USHORT ExtraSize,
2213 __in FxDmaEnabler *DmaEnabler
2214 ) :
2215 FxDmaPacketTransaction(FxDriverGlobals, sizeof(FxDmaSystemTransaction), ExtraSize, DmaEnabler)
2216 {
2217 return;
2218 }
2219
2220 _Must_inspect_result_
2221 NTSTATUS
_Create(__in PFX_DRIVER_GLOBALS FxDriverGlobals,__in PWDF_OBJECT_ATTRIBUTES Attributes,__in FxDmaEnabler * DmaEnabler,__out WDFDMATRANSACTION * Transaction)2222 FxDmaSystemTransaction::_Create(
2223 __in PFX_DRIVER_GLOBALS FxDriverGlobals,
2224 __in PWDF_OBJECT_ATTRIBUTES Attributes,
2225 __in FxDmaEnabler* DmaEnabler,
2226 __out WDFDMATRANSACTION* Transaction
2227 )
2228 {
2229 FxDmaPacketTransaction* pTransaction;
2230 WDFOBJECT hTransaction;
2231 NTSTATUS status;
2232
2233 pTransaction = new (FxDriverGlobals, Attributes, DmaEnabler->GetTransferContextSize())
2234 FxDmaSystemTransaction(FxDriverGlobals, DmaEnabler->GetTransferContextSize(), DmaEnabler);
2235
2236 if (pTransaction == NULL) {
2237 status = STATUS_INSUFFICIENT_RESOURCES;
2238 DoTraceLevelMessage(
2239 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
2240 "Could not allocate memory for WDFTRANSACTION, %!STATUS!", status);
2241 return status;
2242 }
2243
2244 //
2245 // Commit and apply the attributes
2246 //
2247 status = pTransaction->Commit(Attributes, &hTransaction, DmaEnabler);
2248 if (NT_SUCCESS(status)) {
2249 *Transaction = (WDFDMATRANSACTION)hTransaction;
2250 }
2251 else {
2252 //
2253 // This will properly clean up the target's state and free it
2254 //
2255 pTransaction->DeleteFromFailedCreate();
2256 }
2257
2258 return status;
2259 }
2260
2261 BOOLEAN
PreMapTransfer(VOID)2262 FxDmaSystemTransaction::PreMapTransfer(
2263 VOID
2264 )
2265 {
2266 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
2267 BOOLEAN result = TRUE;
2268
2269 if (m_ConfigureChannelFunction.Method != NULL) {
2270 //
2271 // Invoke the callback. If it returns false then the driver has
2272 // completed the transaction in the callback and we must abort
2273 // processing.
2274 //
2275
2276 if (pFxDriverGlobals->FxVerifierOn) {
2277 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
2278 "Invoking ConfigureChannel callback %p (context "
2279 "%p) for WDFDMATRANSACTION %p.",
2280 m_ConfigureChannelFunction.Method,
2281 m_ConfigureChannelContext,
2282 GetHandle());
2283 }
2284
2285 result = m_ConfigureChannelFunction.Invoke(
2286 GetHandle(),
2287 m_DmaEnabler->m_DeviceBase->GetHandle(),
2288 m_ConfigureChannelContext,
2289 m_CurrentFragmentMdl,
2290 m_CurrentFragmentOffset,
2291 m_CurrentFragmentLength
2292 );
2293 }
2294
2295 return result;
2296 }
2297
2298
2299 PDMA_COMPLETION_ROUTINE
GetTransferCompletionRoutine(VOID)2300 FxDmaSystemTransaction::GetTransferCompletionRoutine(
2301 VOID
2302 )
2303 {
2304 if (m_TransferCompleteFunction.Method == NULL) {
2305 return NULL;
2306 }
2307 else {
2308 return _SystemDmaCompletion;
2309 }
2310 }
2311
2312 VOID
CallEvtDmaCompleted(__in DMA_COMPLETION_STATUS Status)2313 FxDmaSystemTransaction::CallEvtDmaCompleted(
2314 __in DMA_COMPLETION_STATUS Status
2315 )
2316 {
2317 //
2318 // Call the TransferComplete callback to indicate that the
2319 // transfer was aborted.
2320 //
2321 m_TransferCompleteFunction.Invoke(
2322 GetHandle(),
2323 m_DmaEnabler->m_Device->GetHandle(),
2324 m_TransferCompleteContext,
2325 m_DmaDirection,
2326 Status
2327 );
2328 }
2329
2330 VOID
GetTransferInfo(__out_opt ULONG * MapRegisterCount,__out_opt ULONG * ScatterGatherElementCount)2331 FxDmaTransactionBase::GetTransferInfo(
2332 __out_opt ULONG *MapRegisterCount,
2333 __out_opt ULONG *ScatterGatherElementCount
2334 )
2335 {
2336 if (m_State != FxDmaTransactionStateInitialized) {
2337 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
2338
2339 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
2340 "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
2341 "is invalid", GetHandle(), m_State);
2342
2343 FxVerifierBugCheck(pFxDriverGlobals, // globals
2344 WDF_DMA_FATAL_ERROR, // specific type
2345 (ULONG_PTR) GetObjectHandle(), // parm 2
2346 (ULONG_PTR) m_State); // parm 3
2347 }
2348
2349 DMA_TRANSFER_INFO info = {0};
2350
2351
2352
2353
2354
2355
2356 if (m_DmaEnabler->UsesDmaV3()) {
2357
2358 //
2359 // Ask the HAL for information about the MDL and how many resources
2360 // it will require to transfer.
2361 //
2362 m_AdapterInfo->AdapterObject->DmaOperations->GetDmaTransferInfo(
2363 m_AdapterInfo->AdapterObject,
2364 m_StartMdl,
2365 m_StartOffset,
2366 (ULONG) this->m_TransactionLength,
2367 this->m_DmaDirection == WDF_DMA_DIRECTION::WdfDmaDirectionWriteToDevice,
2368 &info
2369 );
2370 } else {
2371 size_t offset = m_StartOffset;
2372 size_t length = m_TransactionLength;
2373
2374 //
2375 // Walk through the MDL chain and make a worst-case computation of
2376 // the number of scatter gather entries and map registers the
2377 // transaction would require.
2378 //
2379 for(PMDL mdl = m_StartMdl;
2380 mdl != NULL && length != 0;
2381 mdl = mdl->Next) {
2382
2383 size_t byteCount = MmGetMdlByteCount(mdl);
2384 if (byteCount <= offset) {
2385 offset -= byteCount;
2386 } else {
2387 ULONG_PTR startVa = (ULONG_PTR) MmGetMdlVirtualAddress(mdl);
2388
2389 startVa += offset;
2390 byteCount -= offset;
2391
2392 info.V1.MapRegisterCount +=
2393 (ULONG) ADDRESS_AND_SIZE_TO_SPAN_PAGES(
2394 startVa,
2395 min(byteCount, length)
2396 );
2397
2398 length -= min(byteCount, length);
2399 }
2400 }
2401
2402 info.V1.ScatterGatherElementCount = info.V1.MapRegisterCount;
2403 }
2404
2405 if (ARGUMENT_PRESENT(MapRegisterCount)) {
2406 *MapRegisterCount = info.V1.MapRegisterCount;
2407 }
2408
2409 if (ARGUMENT_PRESENT(ScatterGatherElementCount)) {
2410 *ScatterGatherElementCount = info.V1.ScatterGatherElementCount;
2411 }
2412
2413 return;
2414 }
2415
2416 VOID
_SystemDmaCompletion(__in PDMA_ADAPTER,__in PDEVICE_OBJECT,__in PVOID CompletionContext,__in DMA_COMPLETION_STATUS Status)2417 FxDmaSystemTransaction::_SystemDmaCompletion(
2418 __in PDMA_ADAPTER /* DmaAdapter */,
2419 __in PDEVICE_OBJECT /* DeviceObject */,
2420 __in PVOID CompletionContext,
2421 __in DMA_COMPLETION_STATUS Status
2422 )
2423 {
2424 FxDmaSystemTransaction* transaction = (FxDmaSystemTransaction*) CompletionContext;
2425 PFX_DRIVER_GLOBALS pFxDriverGlobals = transaction->GetDriverGlobals();
2426 KIRQL oldIrql;
2427 BOOLEAN completionDeferred;
2428
2429 //
2430 // Lock the transfer state so that a staging or cancelling thread
2431 // cannot change it.
2432 //
2433 transaction->LockTransferState(&oldIrql);
2434
2435 ASSERTMSG("Completion state was already set",
2436 (transaction->m_TransferState.CompletionStatus ==
2437 UNDEFINED_DMA_COMPLETION_STATUS));
2438 ASSERTMSG("Deferred completion is already pending",
2439 (transaction->m_TransferState.RerunCompletion == FALSE));
2440
2441 //
2442 // If a staging is in progress then defer the completion.
2443 //
2444 if (transaction->m_TransferState.CurrentStagingThread != NULL) {
2445 transaction->m_TransferState.CompletionStatus = Status;
2446 transaction->m_TransferState.RerunCompletion = TRUE;
2447 completionDeferred = TRUE;
2448 }
2449 else {
2450 completionDeferred = FALSE;
2451 }
2452
2453 transaction->UnlockTransferState(oldIrql);
2454
2455 //
2456 // Process the old state.
2457 //
2458 if (completionDeferred == TRUE) {
2459 //
2460 // The staging thread has not moved past EvtProgramDma. The staging thread
2461 // will detect the state change and call the completion routine.
2462 //
2463 // Nothing to do in this case.
2464 //
2465 if (pFxDriverGlobals->FxVerifierOn) {
2466 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
2467 "Deferring DmaCompleted callback for WDFDMATRANSACTION %p"
2468 "(status %x)",
2469 transaction->GetHandle(),
2470 Status);
2471 }
2472 }
2473 else {
2474 //
2475 // Completion occurred while the transfer was running or
2476 // being cancelled. Call the completion routine.
2477 //
2478 // Note: a cancel when in programming state leaves the
2479 // state as programming. that we're not in programming
2480 // means we don't need to worry about racing with
2481 // EvtProgramDma.
2482 //
2483
2484 if (pFxDriverGlobals->FxVerifierOn) {
2485 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
2486 "Invoking DmaCompleted callback %p (context %p) "
2487 "for WDFDMATRANSACTION %p (status %x)",
2488 transaction->m_TransferCompleteFunction.Method,
2489 transaction->m_TransferCompleteContext,
2490 transaction->GetHandle(),
2491 Status);
2492 }
2493 transaction->CallEvtDmaCompleted(Status);
2494 }
2495 }
2496
2497 VOID
StopTransfer(VOID)2498 FxDmaSystemTransaction::StopTransfer(
2499 VOID
2500 )
2501 {
2502 //
2503 // Mark the transfer cancelled so we have a record of it even if
2504 // a racing call to FlushAdapterBuffers clears the TC.
2505 //
2506 m_IsCancelled = TRUE;
2507
2508 //
2509 // Cancel the system DMA transfer. This arranges for one of two things
2510 // to happen:
2511 // * the next call to MapTransfer will fail
2512 // * the DMA completion routine will run
2513 //
2514 if (CancelMappedTransfer() == FALSE) {
2515
2516 //
2517 // The cancel failed. Someone has already stopped this transfer.
2518 // That's illegal.
2519 //
2520 PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
2521
2522 DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
2523 "WDFDMATRANSACTION %p has already been stopped",
2524 GetHandle());
2525
2526 if (pFxDriverGlobals->IsVerificationEnabled(1, 11, OkForDownLevel)) {
2527 FxVerifierBugCheck(pFxDriverGlobals, // globals
2528 WDF_DMA_FATAL_ERROR, // type
2529 (ULONG_PTR) GetObjectHandle(), // parm 2
2530 (ULONG_PTR) m_State); // parm 3
2531 }
2532 }
2533 }
2534