1 /*
2 * PROJECT: ReactOS nVidia nForce Ethernet Controller Driver
3 * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later)
4 * PURPOSE: Interrupt handling
5 * COPYRIGHT: Copyright 2021-2022 Dmitry Borisov <di.sean@protonmail.com>
6 */
7
8 /* INCLUDES *******************************************************************/
9
10 #include "nvnet.h"
11
12 #define NDEBUG
13 #include "debug.h"
14
15 /* FUNCTIONS ******************************************************************/
16
17 ULONG
ProcessTransmitDescriptorsLegacy(_In_ PNVNET_ADAPTER Adapter,_Inout_ PLIST_ENTRY SendReadyList)18 ProcessTransmitDescriptorsLegacy(
19 _In_ PNVNET_ADAPTER Adapter,
20 _Inout_ PLIST_ENTRY SendReadyList)
21 {
22 PNVNET_TCB Tcb = Adapter->Send.LastTcb;
23 ULONG TcbProcessed = 0;
24
25 while (Tcb != Adapter->Send.CurrentTcb)
26 {
27 NVNET_TBD Tbd = Tcb->Tbd;
28 ULONG Flags = Tbd.x32->FlagsLength;
29
30 if (Flags & NV_TX_VALID)
31 break;
32
33 NDIS_DbgPrint(MIN_TRACE, ("Packet transmitted (flags %lx)\n",
34 Flags & FLAG_MASK_V1));
35
36 if (Flags & NV_TX_ERROR)
37 {
38 ++Adapter->Statistics.TransmitErrors;
39
40 if (Flags & NV_TX_UNDERFLOW)
41 ++Adapter->Statistics.TransmitUnderrunErrors;
42 if (Flags & NV_TX_LATECOLLISION)
43 ++Adapter->Statistics.TransmitLateCollisions;
44 if (Flags & NV_TX_CARRIERLOST)
45 ++Adapter->Statistics.TransmitLostCarrierSense;
46 if (Flags & NV_TX_RETRYERROR)
47 {
48 ++Adapter->Statistics.TransmitExcessiveCollisions;
49
50 if (!(Flags & NV_TX_RETRYCOUNT_MASK))
51 {
52 NvNetBackoffReseed(Adapter);
53 }
54 }
55 }
56 else
57 {
58 ++Adapter->Statistics.TransmitOk;
59
60 if (Flags & NV_TX_DEFERRED)
61 {
62 ++Adapter->Statistics.TransmitDeferred;
63 }
64 if (!(Flags & NV_TX_RETRYCOUNT_MASK))
65 ++Adapter->Statistics.TransmitZeroRetry;
66 else if ((Flags & NV_TX_RETRYCOUNT_MASK) == NV_TX_ONE_RETRY)
67 ++Adapter->Statistics.TransmitOneRetry;
68 }
69
70 InsertTailList(SendReadyList, PACKET_ENTRY(Tcb->Packet));
71
72 NV_RELEASE_TCB(Adapter, Tcb);
73
74 ++TcbProcessed;
75
76 Tcb = NV_NEXT_TCB(Adapter, Tcb);
77 }
78
79 Adapter->Send.LastTcb = Tcb;
80
81 return TcbProcessed;
82 }
83
84 ULONG
ProcessTransmitDescriptors32(_In_ PNVNET_ADAPTER Adapter,_Inout_ PLIST_ENTRY SendReadyList)85 ProcessTransmitDescriptors32(
86 _In_ PNVNET_ADAPTER Adapter,
87 _Inout_ PLIST_ENTRY SendReadyList)
88 {
89 PNVNET_TCB Tcb = Adapter->Send.LastTcb;
90 ULONG TcbProcessed = 0;
91
92 while (Tcb != Adapter->Send.CurrentTcb)
93 {
94 NVNET_TBD Tbd = Tcb->Tbd;
95 ULONG Flags = Tbd.x32->FlagsLength;
96
97 if (Flags & NV_TX_VALID)
98 break;
99
100 NDIS_DbgPrint(MIN_TRACE, ("Packet transmitted (flags %lx)\n",
101 Flags & FLAG_MASK_V2));
102
103 if (Flags & NV_TX2_ERROR)
104 {
105 if ((Flags & NV_TX2_RETRYERROR) && !(Flags & NV_TX2_RETRYCOUNT_MASK))
106 {
107 if (Adapter->Features & DEV_HAS_GEAR_MODE)
108 NvNetBackoffReseedEx(Adapter);
109 else
110 NvNetBackoffReseed(Adapter);
111 }
112 }
113
114 InsertTailList(SendReadyList, PACKET_ENTRY(Tcb->Packet));
115
116 NV_RELEASE_TCB(Adapter, Tcb);
117
118 ++TcbProcessed;
119
120 Tcb = NV_NEXT_TCB(Adapter, Tcb);
121 }
122
123 Adapter->Send.LastTcb = Tcb;
124
125 return TcbProcessed;
126 }
127
128 ULONG
ProcessTransmitDescriptors64(_In_ PNVNET_ADAPTER Adapter,_Inout_ PLIST_ENTRY SendReadyList)129 ProcessTransmitDescriptors64(
130 _In_ PNVNET_ADAPTER Adapter,
131 _Inout_ PLIST_ENTRY SendReadyList)
132 {
133 PNVNET_TCB Tcb = Adapter->Send.LastTcb;
134 ULONG TcbProcessed = 0;
135
136 while (Tcb != Adapter->Send.CurrentTcb)
137 {
138 NVNET_TBD Tbd = Tcb->Tbd;
139 ULONG Flags = Tbd.x64->FlagsLength;
140
141 if (Flags & NV_TX_VALID)
142 break;
143
144 if (Adapter->Flags & NV_SEND_ERRATA_PRESENT)
145 {
146 PNVNET_TCB DeferredTcb;
147
148 --Adapter->Send.PacketsCount;
149
150 DeferredTcb = Adapter->Send.DeferredTcb;
151
152 if (DeferredTcb)
153 {
154 DeferredTcb->DeferredTbd.x64->FlagsLength |= NV_TX2_VALID;
155
156 ++Adapter->Send.PacketsCount;
157
158 Adapter->Send.DeferredTcb = NV_NEXT_TCB(Adapter, DeferredTcb);
159 if (Adapter->Send.DeferredTcb == Adapter->Send.CurrentTcb)
160 {
161 Adapter->Send.DeferredTcb = NULL;
162 }
163
164 NV_WRITE(Adapter, NvRegTxRxControl, Adapter->TxRxControl | NVREG_TXRXCTL_KICK);
165 }
166 }
167
168 NDIS_DbgPrint(MIN_TRACE, ("Packet transmitted (flags %lx)\n",
169 Flags & FLAG_MASK_V2));
170
171 if (Flags & NV_TX2_ERROR)
172 {
173 if ((Flags & NV_TX2_RETRYERROR) && !(Flags & NV_TX2_RETRYCOUNT_MASK))
174 {
175 if (Adapter->Features & DEV_HAS_GEAR_MODE)
176 NvNetBackoffReseedEx(Adapter);
177 else
178 NvNetBackoffReseed(Adapter);
179 }
180 }
181
182 InsertTailList(SendReadyList, PACKET_ENTRY(Tcb->Packet));
183
184 NV_RELEASE_TCB(Adapter, Tcb);
185
186 ++TcbProcessed;
187
188 Tcb = NV_NEXT_TCB(Adapter, Tcb);
189 }
190
191 Adapter->Send.LastTcb = Tcb;
192
193 return TcbProcessed;
194 }
195
196 static
197 BOOLEAN
HandleLengthError(_In_ PVOID EthHeader,_Inout_ PUSHORT Length)198 HandleLengthError(
199 _In_ PVOID EthHeader,
200 _Inout_ PUSHORT Length)
201 {
202 PUCHAR Buffer = EthHeader;
203 ULONG i;
204
205 DBG_UNREFERENCED_LOCAL_VARIABLE(Buffer);
206
207 /* TODO */
208 NDIS_DbgPrint(MAX_TRACE, ("() Length error detected (%u): \n", *Length));
209 for (i = 0; i < *Length; ++i)
210 {
211 NDIS_DbgPrint(MAX_TRACE, ("%02x ", Buffer[i]));
212 }
213 NDIS_DbgPrint(MAX_TRACE, ("\n\n*** Please report it to the team! ***\n\n"));
214
215 return FALSE;
216 }
217
218 /* TODO: This need to be rewritten. I leave it as-is for now */
219 static
220 ULONG
ProcessReceiveDescriptors(_In_ PNVNET_ADAPTER Adapter,_In_ ULONG TotalRxProcessed)221 ProcessReceiveDescriptors(
222 _In_ PNVNET_ADAPTER Adapter,
223 _In_ ULONG TotalRxProcessed)
224 {
225 ULONG i, RxProcessed = 0;
226 BOOLEAN IndicateComplete = FALSE;
227
228 for (i = 0; i < NVNET_RECEIVE_DESCRIPTORS; ++i)
229 {
230 ULONG Flags;
231 USHORT Length;
232 PUCHAR EthHeader;
233 NV_RBD NvRbd;
234
235 if (Adapter->Features & DEV_HAS_HIGH_DMA)
236 {
237 NvRbd.x64 = &Adapter->Receive.NvRbd.x64[Adapter->CurrentRx];
238 Flags = NvRbd.x64->FlagsLength;
239 }
240 else
241 {
242 NvRbd.x32 = &Adapter->Receive.NvRbd.x32[Adapter->CurrentRx];
243 Flags = NvRbd.x32->FlagsLength;
244 }
245
246 if (Flags & NV_RX_AVAIL)
247 break;
248
249 if (TotalRxProcessed + RxProcessed >= NVNET_RECEIVE_PROCESSING_LIMIT)
250 break;
251
252 if (!Adapter->PacketFilter)
253 goto NextDescriptor;
254
255 if (Adapter->Features & (DEV_HAS_HIGH_DMA | DEV_HAS_LARGEDESC))
256 {
257 if (!(Flags & NV_RX2_DESCRIPTORVALID))
258 goto NextDescriptor;
259
260 Length = Flags & LEN_MASK_V2;
261 EthHeader = &Adapter->ReceiveBuffer[Adapter->CurrentRx * NVNET_RECEIVE_BUFFER_SIZE];
262
263 if (Flags & NV_RX2_ERROR)
264 {
265 if ((Flags & NV_RX2_ERROR_MASK) == NV_RX2_ERROR4)
266 {
267 if (!HandleLengthError(EthHeader, &Length))
268 goto NextDescriptor;
269 }
270 else if ((Flags & NV_RX2_ERROR_MASK) == NV_RX2_FRAMINGERR)
271 {
272 if (Flags & NV_RX2_SUBTRACT1)
273 --Length;
274 }
275 else
276 {
277 goto NextDescriptor;
278 }
279 }
280
281 NDIS_DbgPrint(MIN_TRACE, ("Packet %d received (length %d, flags %lx)\n",
282 Adapter->CurrentRx, Length, Flags & FLAG_MASK_V2));
283 }
284 else
285 {
286 if (!(Flags & NV_RX_DESCRIPTORVALID))
287 goto NextDescriptor;
288
289 Length = Flags & LEN_MASK_V1;
290 EthHeader = &Adapter->ReceiveBuffer[Adapter->CurrentRx * NVNET_RECEIVE_BUFFER_SIZE];
291
292 if (Flags & NV_RX_ERROR)
293 {
294 if ((Flags & NV_RX_ERROR_MASK) == NV_RX_ERROR4)
295 {
296 if (!HandleLengthError(EthHeader, &Length))
297 goto NextDescriptor;
298 }
299 else if ((Flags & NV_RX_ERROR_MASK) == NV_RX_FRAMINGERR)
300 {
301 if (Flags & NV_RX_SUBTRACT1)
302 --Length;
303 }
304 else
305 {
306 ++Adapter->Statistics.ReceiveErrors;
307
308 if (Flags & NV_RX_MISSEDFRAME)
309 ++Adapter->Statistics.ReceiveNoBuffers;
310 if (Flags & NV_RX_FRAMINGERR)
311 ++Adapter->Statistics.ReceiveAlignmentErrors;
312 if (Flags & NV_RX_OVERFLOW)
313 ++Adapter->Statistics.ReceiveOverrunErrors;
314 if (Flags & NV_RX_CRCERR)
315 ++Adapter->Statistics.ReceiveCrcErrors;
316
317 goto NextDescriptor;
318 }
319 }
320 ++Adapter->Statistics.ReceiveOk;
321
322 NDIS_DbgPrint(MIN_TRACE, ("Packet %d received (length %d, flags %lx)\n",
323 Adapter->CurrentRx, Length, Flags & FLAG_MASK_V1));
324 }
325
326 NdisMEthIndicateReceive(Adapter->AdapterHandle,
327 NULL,
328 (PCHAR)EthHeader,
329 sizeof(ETH_HEADER),
330 EthHeader + sizeof(ETH_HEADER),
331 Length - sizeof(ETH_HEADER),
332 Length - sizeof(ETH_HEADER));
333 IndicateComplete = TRUE;
334
335 NextDescriptor:
336 /* Invalidate the buffer length and release the descriptor */
337 if (Adapter->Features & DEV_HAS_HIGH_DMA)
338 NvRbd.x64->FlagsLength = NV_RX2_AVAIL | NVNET_RECEIVE_BUFFER_SIZE;
339 else
340 NvRbd.x32->FlagsLength = NV_RX_AVAIL | NVNET_RECEIVE_BUFFER_SIZE;
341
342 Adapter->CurrentRx = (Adapter->CurrentRx + 1) % NVNET_RECEIVE_DESCRIPTORS;
343 ++RxProcessed;
344 }
345
346 if (IndicateComplete)
347 {
348 NdisMEthIndicateReceiveComplete(Adapter->AdapterHandle);
349 }
350
351 return RxProcessed;
352 }
353
354 static
355 inline
356 VOID
ChangeInterruptMode(_In_ PNVNET_ADAPTER Adapter,_In_ ULONG Workload)357 ChangeInterruptMode(
358 _In_ PNVNET_ADAPTER Adapter,
359 _In_ ULONG Workload)
360 {
361 if (Workload > NVNET_IM_THRESHOLD)
362 {
363 Adapter->InterruptIdleCount = 0;
364
365 /* High activity, polling based strategy */
366 Adapter->InterruptMask = NVREG_IRQMASK_CPU;
367 }
368 else
369 {
370 if (Adapter->InterruptIdleCount < NVNET_IM_MAX_IDLE)
371 {
372 ++Adapter->InterruptIdleCount;
373 }
374 else
375 {
376 /* Low activity, 1 interrupt per packet */
377 Adapter->InterruptMask = NVREG_IRQMASK_THROUGHPUT;
378 }
379 }
380 }
381
382 static
383 VOID
HandleLinkStateChange(_In_ PNVNET_ADAPTER Adapter)384 HandleLinkStateChange(
385 _In_ PNVNET_ADAPTER Adapter)
386 {
387 ULONG MiiStatus;
388 BOOLEAN Connected, Report = FALSE;
389
390 NDIS_DbgPrint(MIN_TRACE, ("()\n"));
391
392 NdisDprAcquireSpinLock(&Adapter->Lock);
393
394 MiiStatus = NV_READ(Adapter, NvRegMIIStatus);
395
396 /* Clear the link change interrupt */
397 NV_WRITE(Adapter, NvRegMIIStatus, NVREG_MIISTAT_LINKCHANGE);
398
399 if (MiiStatus & NVREG_MIISTAT_LINKCHANGE)
400 {
401 Connected = NvNetUpdateLinkSpeed(Adapter);
402 if (Adapter->Connected != Connected)
403 {
404 Adapter->Connected = Connected;
405 Report = TRUE;
406
407 if (Connected)
408 {
409 /* Link up */
410 NvNetToggleClockPowerGating(Adapter, FALSE);
411 NdisDprAcquireSpinLock(&Adapter->Receive.Lock);
412 NvNetStartReceiver(Adapter);
413 }
414 else
415 {
416 /* Link down */
417 NvNetToggleClockPowerGating(Adapter, TRUE);
418 NdisDprAcquireSpinLock(&Adapter->Receive.Lock);
419 NvNetStopReceiver(Adapter);
420 }
421
422 NdisDprReleaseSpinLock(&Adapter->Receive.Lock);
423 }
424 }
425
426 NdisDprReleaseSpinLock(&Adapter->Lock);
427
428 if (Report)
429 {
430 NdisMIndicateStatus(Adapter->AdapterHandle,
431 Connected ? NDIS_STATUS_MEDIA_CONNECT : NDIS_STATUS_MEDIA_DISCONNECT,
432 NULL,
433 0);
434 NdisMIndicateStatusComplete(Adapter->AdapterHandle);
435 }
436 }
437
438 static
439 VOID
HandleRecoverableError(_In_ PNVNET_ADAPTER Adapter)440 HandleRecoverableError(
441 _In_ PNVNET_ADAPTER Adapter)
442 {
443 /* TODO */
444 NDIS_DbgPrint(MAX_TRACE, ("() Recoverable error detected\n"));
445 }
446
447 VOID
448 NTAPI
MiniportHandleInterrupt(_In_ NDIS_HANDLE MiniportAdapterContext)449 MiniportHandleInterrupt(
450 _In_ NDIS_HANDLE MiniportAdapterContext)
451 {
452 PNVNET_ADAPTER Adapter = (PNVNET_ADAPTER)MiniportAdapterContext;
453 ULONG InterruptStatus = Adapter->InterruptStatus;
454 ULONG RxProcessed, TotalTxProcessed = 0, TotalRxProcessed = 0;
455 LIST_ENTRY SendReadyList;
456
457 NDIS_DbgPrint(MIN_TRACE, ("() Events 0x%lx\n", InterruptStatus));
458
459 if (!(Adapter->Flags & NV_ACTIVE))
460 return;
461
462 InitializeListHead(&SendReadyList);
463
464 /* Process the rings and measure network activity */
465 while (TotalRxProcessed < NVNET_RECEIVE_PROCESSING_LIMIT)
466 {
467 NdisDprAcquireSpinLock(&Adapter->Send.Lock);
468
469 TotalTxProcessed += Adapter->ProcessTransmit(Adapter, &SendReadyList);
470
471 NdisDprReleaseSpinLock(&Adapter->Send.Lock);
472
473 while (!IsListEmpty(&SendReadyList))
474 {
475 PLIST_ENTRY Entry = RemoveHeadList(&SendReadyList);
476
477 NdisMSendComplete(Adapter->AdapterHandle,
478 CONTAINING_RECORD(Entry, NDIS_PACKET, MiniportReserved),
479 NDIS_STATUS_SUCCESS);
480 }
481
482 RxProcessed = ProcessReceiveDescriptors(Adapter, TotalRxProcessed);
483 if (!RxProcessed)
484 break;
485
486 TotalRxProcessed += RxProcessed;
487 }
488
489 NDIS_DbgPrint(MIN_TRACE, ("Total TX: %d, RX: %d\n", TotalTxProcessed, TotalRxProcessed));
490
491 /* Moderate the interrupts */
492 if (Adapter->OptimizationMode == NV_OPTIMIZATION_MODE_DYNAMIC)
493 {
494 ChangeInterruptMode(Adapter, TotalTxProcessed + TotalRxProcessed);
495 }
496
497 if (InterruptStatus & NVREG_IRQ_RX_NOBUF)
498 {
499 ++Adapter->Statistics.ReceiveIrqNoBuffers;
500 }
501
502 if (InterruptStatus & NVREG_IRQ_LINK)
503 {
504 HandleLinkStateChange(Adapter);
505 }
506
507 if (InterruptStatus & NVREG_IRQ_RECOVER_ERROR)
508 {
509 HandleRecoverableError(Adapter);
510 }
511
512 /* Enable interrupts on the NIC */
513 NvNetApplyInterruptMask(Adapter);
514 }
515
516 VOID
517 NTAPI
MiniportISR(_Out_ PBOOLEAN InterruptRecognized,_Out_ PBOOLEAN QueueMiniportHandleInterrupt,_In_ NDIS_HANDLE MiniportAdapterContext)518 MiniportISR(
519 _Out_ PBOOLEAN InterruptRecognized,
520 _Out_ PBOOLEAN QueueMiniportHandleInterrupt,
521 _In_ NDIS_HANDLE MiniportAdapterContext)
522 {
523 PNVNET_ADAPTER Adapter = (PNVNET_ADAPTER)MiniportAdapterContext;
524 ULONG InterruptStatus;
525
526 NDIS_DbgPrint(MIN_TRACE, ("()\n"));
527
528 InterruptStatus = NV_READ(Adapter, NvRegIrqStatus);
529
530 /* Clear any interrupt events */
531 NV_WRITE(Adapter, NvRegIrqStatus, InterruptStatus);
532
533 if (InterruptStatus & Adapter->InterruptMask)
534 {
535 /* Disable further interrupts */
536 NvNetDisableInterrupts(Adapter);
537
538 Adapter->InterruptStatus = InterruptStatus;
539
540 *InterruptRecognized = TRUE;
541 *QueueMiniportHandleInterrupt = TRUE;
542 }
543 else
544 {
545 /* This interrupt is not ours */
546 *InterruptRecognized = FALSE;
547 *QueueMiniportHandleInterrupt = FALSE;
548 }
549 }
550