1 /*
2 * PROJECT: ReactOS nVidia nForce Ethernet Controller Driver
3 * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later)
4 * PURPOSE: Packet sending
5 * COPYRIGHT: Copyright 2021-2022 Dmitry Borisov <di.sean@protonmail.com>
6 */
7
8 /* INCLUDES *******************************************************************/
9
10 #include "nvnet.h"
11
12 #define NDEBUG
13 #include "debug.h"
14
15 /* FUNCTIONS ******************************************************************/
16
17 VOID
NvNetTransmitPacket32(_In_ PNVNET_ADAPTER Adapter,_In_ PNVNET_TCB Tcb,_In_ PSCATTER_GATHER_LIST SgList)18 NvNetTransmitPacket32(
19 _In_ PNVNET_ADAPTER Adapter,
20 _In_ PNVNET_TCB Tcb,
21 _In_ PSCATTER_GATHER_LIST SgList)
22 {
23 NVNET_TBD Tbd, LastTbd;
24 ULONG i, Flags;
25 ULONG Slots;
26
27 Flags = 0;
28 Slots = 0;
29 Tbd = Adapter->Send.CurrentTbd;
30
31 for (i = 0; i < SgList->NumberOfElements; ++i)
32 {
33 ULONG Address = NdisGetPhysicalAddressLow(SgList->Elements[i].Address);
34 ULONG Length = SgList->Elements[i].Length;
35
36 if (Length > NV_MAXIMUM_SG_SIZE)
37 {
38 ULONG ImplicitEntries = NV_IMPLICIT_ENTRIES(Length);
39
40 do
41 {
42 ++Slots;
43
44 Tbd.x32->Address = Address;
45 Tbd.x32->FlagsLength = Flags | (NV_MAXIMUM_SG_SIZE - 1);
46 LastTbd = Tbd;
47 Tbd = NV_NEXT_TBD_32(Adapter, Tbd);
48
49 Flags = NV_TX_VALID;
50
51 Length -= NV_MAXIMUM_SG_SIZE;
52 Address += NV_MAXIMUM_SG_SIZE;
53
54 --ImplicitEntries;
55 }
56 while (ImplicitEntries);
57 }
58
59 ++Slots;
60
61 Tbd.x32->Address = Address;
62 Tbd.x32->FlagsLength = Flags | (Length - 1);
63 LastTbd = Tbd;
64 Tbd = NV_NEXT_TBD_32(Adapter, Tbd);
65
66 Flags = NV_TX_VALID;
67 }
68
69 Tcb->Slots = Slots;
70 Tcb->Tbd = LastTbd;
71
72 if (Adapter->Features & DEV_HAS_LARGEDESC)
73 {
74 LastTbd.x32->FlagsLength |= NV_TX2_LASTPACKET;
75 }
76 else
77 {
78 LastTbd.x32->FlagsLength |= NV_TX_LASTPACKET;
79 }
80
81 if (Tcb->Flags & NV_TCB_LARGE_SEND)
82 {
83 Flags |= (Tcb->Mss << NV_TX2_TSO_SHIFT) | NV_TX2_TSO;
84 }
85 else
86 {
87 if (Tcb->Flags & NV_TCB_CHECKSUM_IP)
88 {
89 Flags |= NV_TX2_CHECKSUM_L3;
90 }
91 if (Tcb->Flags & (NV_TCB_CHECKSUM_TCP | NV_TCB_CHECKSUM_UDP))
92 {
93 Flags |= NV_TX2_CHECKSUM_L4;
94 }
95 }
96
97 Adapter->Send.CurrentTbd.x32->FlagsLength |= Flags;
98 Adapter->Send.CurrentTbd = Tbd;
99
100 NV_WRITE(Adapter, NvRegTxRxControl, Adapter->TxRxControl | NVREG_TXRXCTL_KICK);
101 }
102
103 VOID
NvNetTransmitPacket64(_In_ PNVNET_ADAPTER Adapter,_In_ PNVNET_TCB Tcb,_In_ PSCATTER_GATHER_LIST SgList)104 NvNetTransmitPacket64(
105 _In_ PNVNET_ADAPTER Adapter,
106 _In_ PNVNET_TCB Tcb,
107 _In_ PSCATTER_GATHER_LIST SgList)
108 {
109 NVNET_TBD Tbd, LastTbd;
110 ULONG i, Flags;
111 ULONG Slots;
112
113 Flags = 0;
114 Slots = 0;
115 Tbd = Adapter->Send.CurrentTbd;
116
117 for (i = 0; i < SgList->NumberOfElements; ++i)
118 {
119 ULONG64 Address = SgList->Elements[i].Address.QuadPart;
120 ULONG Length = SgList->Elements[i].Length;
121
122 if (Length > NV_MAXIMUM_SG_SIZE)
123 {
124 ULONG ImplicitEntries = NV_IMPLICIT_ENTRIES(Length);
125
126 do
127 {
128 ++Slots;
129
130 Tbd.x64->AddressLow = (ULONG)Address;
131 Tbd.x64->AddressHigh = Address >> 32;
132 Tbd.x64->VlanTag = 0;
133 Tbd.x64->FlagsLength = Flags | (NV_MAXIMUM_SG_SIZE - 1);
134 LastTbd = Tbd;
135 Tbd = NV_NEXT_TBD_64(Adapter, Tbd);
136
137 Flags = NV_TX2_VALID;
138
139 Length -= NV_MAXIMUM_SG_SIZE;
140 Address += NV_MAXIMUM_SG_SIZE;
141
142 --ImplicitEntries;
143 }
144 while (ImplicitEntries);
145 }
146
147 ++Slots;
148
149 Tbd.x64->AddressLow = (ULONG)Address;
150 Tbd.x64->AddressHigh = Address >> 32;
151 Tbd.x64->VlanTag = 0;
152 Tbd.x64->FlagsLength = Flags | (Length - 1);
153 LastTbd = Tbd;
154 Tbd = NV_NEXT_TBD_64(Adapter, Tbd);
155
156 Flags = NV_TX2_VALID;
157 }
158
159 Tcb->Slots = Slots;
160 Tcb->Tbd = LastTbd;
161
162 LastTbd.x64->FlagsLength |= NV_TX2_LASTPACKET;
163
164 if (Adapter->Flags & NV_SEND_ERRATA_PRESENT)
165 {
166 if (Adapter->Send.PacketsCount == NV_TX_LIMIT_COUNT)
167 {
168 Tcb->DeferredTbd = Adapter->Send.CurrentTbd;
169
170 if (!Adapter->Send.DeferredTcb)
171 {
172 Adapter->Send.DeferredTcb = Tcb;
173 }
174
175 Flags = 0;
176 }
177 else
178 {
179 ++Adapter->Send.PacketsCount;
180 }
181 }
182
183 if (Tcb->Flags & NV_TCB_LARGE_SEND)
184 {
185 Flags |= (Tcb->Mss << NV_TX2_TSO_SHIFT) | NV_TX2_TSO;
186 }
187 else
188 {
189 if (Tcb->Flags & NV_TCB_CHECKSUM_IP)
190 {
191 Flags |= NV_TX2_CHECKSUM_L3;
192 }
193 if (Tcb->Flags & (NV_TCB_CHECKSUM_TCP | NV_TCB_CHECKSUM_UDP))
194 {
195 Flags |= NV_TX2_CHECKSUM_L4;
196 }
197 }
198
199 // Adapter->Send.CurrentTbd.x64->VlanTag = NV_TX3_VLAN_TAG_PRESENT; TODO
200 Adapter->Send.CurrentTbd.x64->FlagsLength |= Flags;
201 Adapter->Send.CurrentTbd = Tbd;
202
203 NV_WRITE(Adapter, NvRegTxRxControl, Adapter->TxRxControl | NVREG_TXRXCTL_KICK);
204 }
205
206 static
207 DECLSPEC_NOINLINE
208 ULONG
NvNetQueryTcpIpHeaders(_In_ PNVNET_ADAPTER Adapter,_In_ PNDIS_PACKET Packet)209 NvNetQueryTcpIpHeaders(
210 _In_ PNVNET_ADAPTER Adapter,
211 _In_ PNDIS_PACKET Packet)
212 {
213 PNDIS_BUFFER CurrentBuffer;
214 PVOID Address;
215 UINT CurrentLength;
216 UINT PacketLength;
217 PIPv4_HEADER IpHeader;
218 PTCPv4_HEADER TcpHeader;
219 ULONG BytesCopied = 0;
220 UCHAR Buffer[136];
221
222 NdisGetFirstBufferFromPacketSafe(Packet,
223 &CurrentBuffer,
224 &Address,
225 &CurrentLength,
226 &PacketLength,
227 HighPagePriority);
228 if (!Address)
229 return 0;
230
231 while (TRUE)
232 {
233 CurrentLength = min(CurrentLength, sizeof(Buffer) - BytesCopied);
234
235 NdisMoveMemory(&Buffer[BytesCopied], Address, CurrentLength);
236 BytesCopied += CurrentLength;
237
238 if (BytesCopied >= sizeof(Buffer))
239 break;
240
241 NdisGetNextBuffer(CurrentBuffer, &CurrentBuffer);
242
243 if (!CurrentBuffer)
244 return 0;
245
246 NdisQueryBufferSafe(CurrentBuffer,
247 &Address,
248 &CurrentLength,
249 HighPagePriority);
250 }
251
252 IpHeader = (PIPv4_HEADER)&Buffer[Adapter->IpHeaderOffset];
253 TcpHeader = (PTCPv4_HEADER)((PUCHAR)IpHeader + IP_HEADER_LENGTH(IpHeader));
254
255 return IP_HEADER_LENGTH(IpHeader) + TCP_HEADER_LENGTH(TcpHeader);
256 }
257
258 static
259 BOOLEAN
NvNetCopyPacket(_In_ PNVNET_ADAPTER Adapter,_In_ PNDIS_PACKET Packet,_In_ PNVNET_TX_BUFFER Buffer)260 NvNetCopyPacket(
261 _In_ PNVNET_ADAPTER Adapter,
262 _In_ PNDIS_PACKET Packet,
263 _In_ PNVNET_TX_BUFFER Buffer)
264 {
265 PNDIS_BUFFER CurrentBuffer;
266 PVOID Address;
267 UINT CurrentLength;
268 UINT PacketLength;
269 PUCHAR Destination;
270
271 NdisGetFirstBufferFromPacketSafe(Packet,
272 &CurrentBuffer,
273 &Address,
274 &CurrentLength,
275 &PacketLength,
276 HighPagePriority);
277 if (!Address)
278 return FALSE;
279
280 Destination = Buffer->VirtualAddress;
281
282 while (TRUE)
283 {
284 NdisMoveMemory(Destination, Address, CurrentLength);
285 Destination += CurrentLength;
286
287 NdisGetNextBuffer(CurrentBuffer, &CurrentBuffer);
288
289 if (!CurrentBuffer)
290 break;
291
292 NdisQueryBufferSafe(CurrentBuffer,
293 &Address,
294 &CurrentLength,
295 HighPagePriority);
296 if (!Address)
297 return FALSE;
298 }
299
300 return TRUE;
301 }
302
303 static
304 NDIS_STATUS
NvNetSendPacketLargeSend(_In_ PNVNET_ADAPTER Adapter,_In_ PNDIS_PACKET Packet,_In_ ULONG TotalLength)305 NvNetSendPacketLargeSend(
306 _In_ PNVNET_ADAPTER Adapter,
307 _In_ PNDIS_PACKET Packet,
308 _In_ ULONG TotalLength)
309 {
310 PSCATTER_GATHER_LIST SgList;
311 ULONG Mss, Length;
312 PNVNET_TCB Tcb;
313
314 if (!Adapter->Send.TcbSlots)
315 {
316 return NDIS_STATUS_RESOURCES;
317 }
318
319 SgList = NDIS_PER_PACKET_INFO_FROM_PACKET(Packet, ScatterGatherListPacketInfo);
320
321 /* Make sure we have room to setup all fragments */
322 C_ASSERT(NVNET_TRANSMIT_DESCRIPTORS > ((NVNET_MAXIMUM_LSO_FRAME_SIZE / PAGE_SIZE) + 3));
323 ASSERT(SgList->NumberOfElements +
324 (NVNET_MAXIMUM_LSO_FRAME_SIZE / (NV_MAXIMUM_SG_SIZE + 1)) <=
325 NVNET_TRANSMIT_DESCRIPTORS);
326
327 if (SgList->NumberOfElements +
328 (NVNET_MAXIMUM_LSO_FRAME_SIZE / (NV_MAXIMUM_SG_SIZE + 1)) < Adapter->Send.TbdSlots)
329 {
330 return NDIS_STATUS_RESOURCES;
331 }
332
333 Length = NvNetQueryTcpIpHeaders(Adapter, Packet);
334 if (!Length)
335 {
336 return NDIS_STATUS_RESOURCES;
337 }
338
339 NDIS_PER_PACKET_INFO_FROM_PACKET(Packet, TcpLargeSendPacketInfo) =
340 UlongToPtr(TotalLength - Adapter->IpHeaderOffset - Length);
341
342 --Adapter->Send.TcbSlots;
343
344 Mss = PtrToUlong(NDIS_PER_PACKET_INFO_FROM_PACKET(Packet, TcpLargeSendPacketInfo));
345
346 Tcb = Adapter->Send.CurrentTcb;
347 Tcb->Mss = Mss;
348 Tcb->Packet = Packet;
349 Tcb->Flags = NV_TCB_LARGE_SEND;
350
351 Adapter->TransmitPacket(Adapter, Tcb, SgList);
352
353 ASSERT(Adapter->Send.TbdSlots >= Tcb->Slots);
354 Adapter->Send.TbdSlots -= Tcb->Slots;
355
356 Adapter->Send.CurrentTcb = NV_NEXT_TCB(Adapter, Tcb);
357
358 return NDIS_STATUS_SUCCESS;
359 }
360
361 static
362 ULONG
NvNetGetChecksumInfo(_In_ PNVNET_ADAPTER Adapter,_In_ PNDIS_PACKET Packet)363 NvNetGetChecksumInfo(
364 _In_ PNVNET_ADAPTER Adapter,
365 _In_ PNDIS_PACKET Packet)
366 {
367 ULONG Flags;
368 NDIS_TCP_IP_CHECKSUM_PACKET_INFO ChecksumInfo;
369
370 if (NDIS_GET_PACKET_PROTOCOL_TYPE(Packet) != NDIS_PROTOCOL_ID_TCP_IP)
371 return 0;
372
373 ChecksumInfo.Value = PtrToUlong(NDIS_PER_PACKET_INFO_FROM_PACKET(Packet,
374 TcpIpChecksumPacketInfo));
375
376 Flags = 0;
377 if (ChecksumInfo.Transmit.NdisPacketChecksumV4)
378 {
379 if (ChecksumInfo.Transmit.NdisPacketTcpChecksum && Adapter->Offload.SendTcpChecksum)
380 {
381 Flags |= NV_TCB_CHECKSUM_TCP;
382 }
383 if (ChecksumInfo.Transmit.NdisPacketUdpChecksum && Adapter->Offload.SendUdpChecksum)
384 {
385 Flags |= NV_TCB_CHECKSUM_UDP;
386 }
387 if (ChecksumInfo.Transmit.NdisPacketIpChecksum && Adapter->Offload.SendIpChecksum)
388 {
389 Flags |= NV_TCB_CHECKSUM_IP;
390 }
391 }
392
393 return Flags;
394 }
395
396 static
397 NDIS_STATUS
NvNetSendPacket(_In_ PNVNET_ADAPTER Adapter,_In_ PNDIS_PACKET Packet,_In_ ULONG TotalLength)398 NvNetSendPacket(
399 _In_ PNVNET_ADAPTER Adapter,
400 _In_ PNDIS_PACKET Packet,
401 _In_ ULONG TotalLength)
402 {
403 PSCATTER_GATHER_LIST SgList;
404 SCATTER_GATHER_LIST LocalSgList;
405 PNVNET_TCB Tcb;
406 ULONG Flags;
407
408 ASSERT(TotalLength <= Adapter->MaximumFrameSize);
409
410 if (!Adapter->Send.TcbSlots)
411 {
412 return NDIS_STATUS_RESOURCES;
413 }
414
415 Flags = NvNetGetChecksumInfo(Adapter, Packet);
416
417 SgList = NDIS_PER_PACKET_INFO_FROM_PACKET(Packet, ScatterGatherListPacketInfo);
418
419 if (SgList->NumberOfElements > NVNET_FRAGMENTATION_THRESHOLD)
420 {
421 if (!Adapter->Send.TbdSlots || !Adapter->Send.BufferList.Next)
422 {
423 return NDIS_STATUS_RESOURCES;
424 }
425 else
426 {
427 PNVNET_TX_BUFFER CoalesceBuffer;
428 BOOLEAN Success;
429
430 --Adapter->Send.TcbSlots;
431
432 CoalesceBuffer = (PNVNET_TX_BUFFER)PopEntryList(&Adapter->Send.BufferList);
433
434 NdisDprReleaseSpinLock(&Adapter->Send.Lock);
435
436 Success = NvNetCopyPacket(Adapter, Packet, CoalesceBuffer);
437
438 NdisDprAcquireSpinLock(&Adapter->Send.Lock);
439
440 if (!Success || !Adapter->Send.TbdSlots || !(Adapter->Flags & NV_ACTIVE))
441 {
442 PushEntryList(&Adapter->Send.BufferList, &CoalesceBuffer->Link);
443
444 ++Adapter->Send.TcbSlots;
445
446 return NDIS_STATUS_RESOURCES;
447 }
448
449 Flags |= NV_TCB_COALESCE;
450
451 LocalSgList.NumberOfElements = 1;
452 LocalSgList.Elements[0].Address = CoalesceBuffer->PhysicalAddress;
453 LocalSgList.Elements[0].Length = TotalLength;
454 SgList = &LocalSgList;
455
456 Tcb = Adapter->Send.CurrentTcb;
457 Tcb->Buffer = CoalesceBuffer;
458 }
459 }
460 else
461 {
462 if (SgList->NumberOfElements +
463 (NVNET_MAXIMUM_FRAME_SIZE_JUMBO / (NV_MAXIMUM_SG_SIZE + 1)) > Adapter->Send.TbdSlots)
464 {
465 return NDIS_STATUS_RESOURCES;
466 }
467
468 --Adapter->Send.TcbSlots;
469
470 Tcb = Adapter->Send.CurrentTcb;
471 }
472
473 Tcb->Packet = Packet;
474 Tcb->Flags = Flags;
475
476 Adapter->TransmitPacket(Adapter, Tcb, SgList);
477
478 ASSERT(Adapter->Send.TbdSlots >= Tcb->Slots);
479 Adapter->Send.TbdSlots -= Tcb->Slots;
480
481 Adapter->Send.CurrentTcb = NV_NEXT_TCB(Adapter, Tcb);
482
483 return NDIS_STATUS_PENDING;
484 }
485
486 /* FIXME: Use the proper send function (MiniportSendPackets) */
487 NDIS_STATUS
488 NTAPI
MiniportSend(_In_ NDIS_HANDLE MiniportAdapterContext,_In_ PNDIS_PACKET Packet,_In_ UINT Flags)489 MiniportSend(
490 _In_ NDIS_HANDLE MiniportAdapterContext,
491 _In_ PNDIS_PACKET Packet,
492 _In_ UINT Flags)
493 {
494 PNVNET_ADAPTER Adapter = (PNVNET_ADAPTER)MiniportAdapterContext;
495 UINT TotalLength;
496 NDIS_STATUS Status;
497
498 NDIS_DbgPrint(MIN_TRACE, ("()\n"));
499
500 NdisQueryPacketLength(Packet, &TotalLength);
501
502 NdisDprAcquireSpinLock(&Adapter->Send.Lock);
503
504 if (!(Adapter->Flags & NV_ACTIVE))
505 {
506 NdisDprReleaseSpinLock(&Adapter->Send.Lock);
507
508 return NDIS_STATUS_FAILURE;
509 }
510
511 if (Adapter->Flags & NV_SEND_LARGE_SEND &&
512 PtrToUlong(NDIS_PER_PACKET_INFO_FROM_PACKET(Packet, TcpLargeSendPacketInfo)))
513 {
514 Status = NvNetSendPacketLargeSend(Adapter, Packet, TotalLength);
515 }
516 else
517 {
518 Status = NvNetSendPacket(Adapter, Packet, TotalLength);
519 }
520
521 NdisDprReleaseSpinLock(&Adapter->Send.Lock);
522
523 return Status;
524 }
525