1 /* 2 * PROJECT: ReactOS nVidia nForce Ethernet Controller Driver 3 * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) 4 * PURPOSE: Packet sending 5 * COPYRIGHT: Copyright 2021-2022 Dmitry Borisov <di.sean@protonmail.com> 6 */ 7 8 /* INCLUDES *******************************************************************/ 9 10 #include "nvnet.h" 11 12 #define NDEBUG 13 #include "debug.h" 14 15 /* FUNCTIONS ******************************************************************/ 16 17 VOID 18 NvNetTransmitPacket32( 19 _In_ PNVNET_ADAPTER Adapter, 20 _In_ PNVNET_TCB Tcb, 21 _In_ PSCATTER_GATHER_LIST SgList) 22 { 23 NVNET_TBD Tbd, LastTbd; 24 ULONG i, Flags; 25 ULONG Slots; 26 27 Flags = 0; 28 Slots = 0; 29 Tbd = Adapter->Send.CurrentTbd; 30 31 for (i = 0; i < SgList->NumberOfElements; ++i) 32 { 33 ULONG Address = NdisGetPhysicalAddressLow(SgList->Elements[i].Address); 34 ULONG Length = SgList->Elements[i].Length; 35 36 if (Length > NV_MAXIMUM_SG_SIZE) 37 { 38 ULONG ImplicitEntries = NV_IMPLICIT_ENTRIES(Length); 39 40 do 41 { 42 ++Slots; 43 44 Tbd.x32->Address = Address; 45 Tbd.x32->FlagsLength = Flags | (NV_MAXIMUM_SG_SIZE - 1); 46 LastTbd = Tbd; 47 Tbd = NV_NEXT_TBD_32(Adapter, Tbd); 48 49 Flags = NV_TX_VALID; 50 51 Length -= NV_MAXIMUM_SG_SIZE; 52 Address += NV_MAXIMUM_SG_SIZE; 53 54 --ImplicitEntries; 55 } 56 while (ImplicitEntries); 57 } 58 59 ++Slots; 60 61 Tbd.x32->Address = Address; 62 Tbd.x32->FlagsLength = Flags | (Length - 1); 63 LastTbd = Tbd; 64 Tbd = NV_NEXT_TBD_32(Adapter, Tbd); 65 66 Flags = NV_TX_VALID; 67 } 68 69 Tcb->Slots = Slots; 70 Tcb->Tbd = LastTbd; 71 72 if (Adapter->Features & DEV_HAS_LARGEDESC) 73 { 74 LastTbd.x32->FlagsLength |= NV_TX2_LASTPACKET; 75 } 76 else 77 { 78 LastTbd.x32->FlagsLength |= NV_TX_LASTPACKET; 79 } 80 81 if (Tcb->Flags & NV_TCB_LARGE_SEND) 82 { 83 Flags |= (Tcb->Mss << NV_TX2_TSO_SHIFT) | NV_TX2_TSO; 84 } 85 else 86 { 87 if (Tcb->Flags & NV_TCB_CHECKSUM_IP) 88 { 89 Flags |= NV_TX2_CHECKSUM_L3; 90 } 91 if (Tcb->Flags & (NV_TCB_CHECKSUM_TCP | NV_TCB_CHECKSUM_UDP)) 92 { 93 Flags |= NV_TX2_CHECKSUM_L4; 94 } 95 } 96 97 Adapter->Send.CurrentTbd.x32->FlagsLength |= Flags; 98 Adapter->Send.CurrentTbd = Tbd; 99 100 NV_WRITE(Adapter, NvRegTxRxControl, Adapter->TxRxControl | NVREG_TXRXCTL_KICK); 101 } 102 103 VOID 104 NvNetTransmitPacket64( 105 _In_ PNVNET_ADAPTER Adapter, 106 _In_ PNVNET_TCB Tcb, 107 _In_ PSCATTER_GATHER_LIST SgList) 108 { 109 NVNET_TBD Tbd, LastTbd; 110 ULONG i, Flags; 111 ULONG Slots; 112 113 Flags = 0; 114 Slots = 0; 115 Tbd = Adapter->Send.CurrentTbd; 116 117 for (i = 0; i < SgList->NumberOfElements; ++i) 118 { 119 ULONG64 Address = SgList->Elements[i].Address.QuadPart; 120 ULONG Length = SgList->Elements[i].Length; 121 122 if (Length > NV_MAXIMUM_SG_SIZE) 123 { 124 ULONG ImplicitEntries = NV_IMPLICIT_ENTRIES(Length); 125 126 do 127 { 128 ++Slots; 129 130 Tbd.x64->AddressLow = (ULONG)Address; 131 Tbd.x64->AddressHigh = Address >> 32; 132 Tbd.x64->VlanTag = 0; 133 Tbd.x64->FlagsLength = Flags | (NV_MAXIMUM_SG_SIZE - 1); 134 LastTbd = Tbd; 135 Tbd = NV_NEXT_TBD_64(Adapter, Tbd); 136 137 Flags = NV_TX2_VALID; 138 139 Length -= NV_MAXIMUM_SG_SIZE; 140 Address += NV_MAXIMUM_SG_SIZE; 141 142 --ImplicitEntries; 143 } 144 while (ImplicitEntries); 145 } 146 147 ++Slots; 148 149 Tbd.x64->AddressLow = (ULONG)Address; 150 Tbd.x64->AddressHigh = Address >> 32; 151 Tbd.x64->VlanTag = 0; 152 Tbd.x64->FlagsLength = Flags | (Length - 1); 153 LastTbd = Tbd; 154 Tbd = NV_NEXT_TBD_64(Adapter, Tbd); 155 156 Flags = NV_TX2_VALID; 157 } 158 159 Tcb->Slots = Slots; 160 Tcb->Tbd = LastTbd; 161 162 LastTbd.x64->FlagsLength |= NV_TX2_LASTPACKET; 163 164 if (Adapter->Flags & NV_SEND_ERRATA_PRESENT) 165 { 166 if (Adapter->Send.PacketsCount == NV_TX_LIMIT_COUNT) 167 { 168 Tcb->DeferredTbd = Adapter->Send.CurrentTbd; 169 170 if (!Adapter->Send.DeferredTcb) 171 { 172 Adapter->Send.DeferredTcb = Tcb; 173 } 174 175 Flags = 0; 176 } 177 else 178 { 179 ++Adapter->Send.PacketsCount; 180 } 181 } 182 183 if (Tcb->Flags & NV_TCB_LARGE_SEND) 184 { 185 Flags |= (Tcb->Mss << NV_TX2_TSO_SHIFT) | NV_TX2_TSO; 186 } 187 else 188 { 189 if (Tcb->Flags & NV_TCB_CHECKSUM_IP) 190 { 191 Flags |= NV_TX2_CHECKSUM_L3; 192 } 193 if (Tcb->Flags & (NV_TCB_CHECKSUM_TCP | NV_TCB_CHECKSUM_UDP)) 194 { 195 Flags |= NV_TX2_CHECKSUM_L4; 196 } 197 } 198 199 // Adapter->Send.CurrentTbd.x64->VlanTag = NV_TX3_VLAN_TAG_PRESENT; TODO 200 Adapter->Send.CurrentTbd.x64->FlagsLength |= Flags; 201 Adapter->Send.CurrentTbd = Tbd; 202 203 NV_WRITE(Adapter, NvRegTxRxControl, Adapter->TxRxControl | NVREG_TXRXCTL_KICK); 204 } 205 206 static 207 DECLSPEC_NOINLINE 208 ULONG 209 NvNetQueryTcpIpHeaders( 210 _In_ PNVNET_ADAPTER Adapter, 211 _In_ PNDIS_PACKET Packet) 212 { 213 PNDIS_BUFFER CurrentBuffer; 214 PVOID Address; 215 UINT CurrentLength; 216 UINT PacketLength; 217 PIPv4_HEADER IpHeader; 218 PTCPv4_HEADER TcpHeader; 219 ULONG BytesCopied = 0; 220 UCHAR Buffer[136]; 221 222 NdisGetFirstBufferFromPacketSafe(Packet, 223 &CurrentBuffer, 224 &Address, 225 &CurrentLength, 226 &PacketLength, 227 HighPagePriority); 228 if (!Address) 229 return 0; 230 231 while (TRUE) 232 { 233 CurrentLength = min(CurrentLength, sizeof(Buffer) - BytesCopied); 234 235 NdisMoveMemory(&Buffer[BytesCopied], Address, CurrentLength); 236 BytesCopied += CurrentLength; 237 238 if (BytesCopied >= sizeof(Buffer)) 239 break; 240 241 NdisGetNextBuffer(CurrentBuffer, &CurrentBuffer); 242 243 if (!CurrentBuffer) 244 return 0; 245 246 NdisQueryBufferSafe(CurrentBuffer, 247 &Address, 248 &CurrentLength, 249 HighPagePriority); 250 } 251 252 IpHeader = (PIPv4_HEADER)&Buffer[Adapter->IpHeaderOffset]; 253 TcpHeader = (PTCPv4_HEADER)((PUCHAR)IpHeader + IP_HEADER_LENGTH(IpHeader)); 254 255 return IP_HEADER_LENGTH(IpHeader) + TCP_HEADER_LENGTH(TcpHeader); 256 } 257 258 static 259 BOOLEAN 260 NvNetCopyPacket( 261 _In_ PNVNET_ADAPTER Adapter, 262 _In_ PNDIS_PACKET Packet, 263 _In_ PNVNET_TX_BUFFER Buffer) 264 { 265 PNDIS_BUFFER CurrentBuffer; 266 PVOID Address; 267 UINT CurrentLength; 268 UINT PacketLength; 269 PUCHAR Destination; 270 271 NdisGetFirstBufferFromPacketSafe(Packet, 272 &CurrentBuffer, 273 &Address, 274 &CurrentLength, 275 &PacketLength, 276 HighPagePriority); 277 if (!Address) 278 return FALSE; 279 280 Destination = Buffer->VirtualAddress; 281 282 while (TRUE) 283 { 284 NdisMoveMemory(Destination, Address, CurrentLength); 285 Destination += CurrentLength; 286 287 NdisGetNextBuffer(CurrentBuffer, &CurrentBuffer); 288 289 if (!CurrentBuffer) 290 break; 291 292 NdisQueryBufferSafe(CurrentBuffer, 293 &Address, 294 &CurrentLength, 295 HighPagePriority); 296 if (!Address) 297 return FALSE; 298 } 299 300 return TRUE; 301 } 302 303 static 304 NDIS_STATUS 305 NvNetSendPacketLargeSend( 306 _In_ PNVNET_ADAPTER Adapter, 307 _In_ PNDIS_PACKET Packet, 308 _In_ ULONG TotalLength) 309 { 310 PSCATTER_GATHER_LIST SgList; 311 ULONG Mss, Length; 312 PNVNET_TCB Tcb; 313 314 if (!Adapter->Send.TcbSlots) 315 { 316 return NDIS_STATUS_RESOURCES; 317 } 318 319 SgList = NDIS_PER_PACKET_INFO_FROM_PACKET(Packet, ScatterGatherListPacketInfo); 320 321 /* Make sure we have room to setup all fragments */ 322 C_ASSERT(NVNET_TRANSMIT_DESCRIPTORS > ((NVNET_MAXIMUM_LSO_FRAME_SIZE / PAGE_SIZE) + 3)); 323 ASSERT(SgList->NumberOfElements + 324 (NVNET_MAXIMUM_LSO_FRAME_SIZE / (NV_MAXIMUM_SG_SIZE + 1)) <= 325 NVNET_TRANSMIT_DESCRIPTORS); 326 327 if (SgList->NumberOfElements + 328 (NVNET_MAXIMUM_LSO_FRAME_SIZE / (NV_MAXIMUM_SG_SIZE + 1)) < Adapter->Send.TbdSlots) 329 { 330 return NDIS_STATUS_RESOURCES; 331 } 332 333 Length = NvNetQueryTcpIpHeaders(Adapter, Packet); 334 if (!Length) 335 { 336 return NDIS_STATUS_RESOURCES; 337 } 338 339 NDIS_PER_PACKET_INFO_FROM_PACKET(Packet, TcpLargeSendPacketInfo) = 340 UlongToPtr(TotalLength - Adapter->IpHeaderOffset - Length); 341 342 --Adapter->Send.TcbSlots; 343 344 Mss = PtrToUlong(NDIS_PER_PACKET_INFO_FROM_PACKET(Packet, TcpLargeSendPacketInfo)); 345 346 Tcb = Adapter->Send.CurrentTcb; 347 Tcb->Mss = Mss; 348 Tcb->Packet = Packet; 349 Tcb->Flags = NV_TCB_LARGE_SEND; 350 351 Adapter->TransmitPacket(Adapter, Tcb, SgList); 352 353 ASSERT(Adapter->Send.TbdSlots >= Tcb->Slots); 354 Adapter->Send.TbdSlots -= Tcb->Slots; 355 356 Adapter->Send.CurrentTcb = NV_NEXT_TCB(Adapter, Tcb); 357 358 return NDIS_STATUS_SUCCESS; 359 } 360 361 static 362 ULONG 363 NvNetGetChecksumInfo( 364 _In_ PNVNET_ADAPTER Adapter, 365 _In_ PNDIS_PACKET Packet) 366 { 367 ULONG Flags; 368 NDIS_TCP_IP_CHECKSUM_PACKET_INFO ChecksumInfo; 369 370 if (NDIS_GET_PACKET_PROTOCOL_TYPE(Packet) != NDIS_PROTOCOL_ID_TCP_IP) 371 return 0; 372 373 ChecksumInfo.Value = PtrToUlong(NDIS_PER_PACKET_INFO_FROM_PACKET(Packet, 374 TcpIpChecksumPacketInfo)); 375 376 Flags = 0; 377 if (ChecksumInfo.Transmit.NdisPacketChecksumV4) 378 { 379 if (ChecksumInfo.Transmit.NdisPacketTcpChecksum && Adapter->Offload.SendTcpChecksum) 380 { 381 Flags |= NV_TCB_CHECKSUM_TCP; 382 } 383 if (ChecksumInfo.Transmit.NdisPacketUdpChecksum && Adapter->Offload.SendUdpChecksum) 384 { 385 Flags |= NV_TCB_CHECKSUM_UDP; 386 } 387 if (ChecksumInfo.Transmit.NdisPacketIpChecksum && Adapter->Offload.SendIpChecksum) 388 { 389 Flags |= NV_TCB_CHECKSUM_IP; 390 } 391 } 392 393 return Flags; 394 } 395 396 static 397 NDIS_STATUS 398 NvNetSendPacket( 399 _In_ PNVNET_ADAPTER Adapter, 400 _In_ PNDIS_PACKET Packet, 401 _In_ ULONG TotalLength) 402 { 403 PSCATTER_GATHER_LIST SgList; 404 SCATTER_GATHER_LIST LocalSgList; 405 PNVNET_TCB Tcb; 406 ULONG Flags; 407 408 ASSERT(TotalLength <= Adapter->MaximumFrameSize); 409 410 if (!Adapter->Send.TcbSlots) 411 { 412 return NDIS_STATUS_RESOURCES; 413 } 414 415 Flags = NvNetGetChecksumInfo(Adapter, Packet); 416 417 SgList = NDIS_PER_PACKET_INFO_FROM_PACKET(Packet, ScatterGatherListPacketInfo); 418 419 if (SgList->NumberOfElements > NVNET_FRAGMENTATION_THRESHOLD) 420 { 421 if (!Adapter->Send.TbdSlots || !Adapter->Send.BufferList.Next) 422 { 423 return NDIS_STATUS_RESOURCES; 424 } 425 else 426 { 427 PNVNET_TX_BUFFER CoalesceBuffer; 428 BOOLEAN Success; 429 430 --Adapter->Send.TcbSlots; 431 432 CoalesceBuffer = (PNVNET_TX_BUFFER)PopEntryList(&Adapter->Send.BufferList); 433 434 NdisDprReleaseSpinLock(&Adapter->Send.Lock); 435 436 Success = NvNetCopyPacket(Adapter, Packet, CoalesceBuffer); 437 438 NdisDprAcquireSpinLock(&Adapter->Send.Lock); 439 440 if (!Success || !Adapter->Send.TbdSlots || !(Adapter->Flags & NV_ACTIVE)) 441 { 442 PushEntryList(&Adapter->Send.BufferList, &CoalesceBuffer->Link); 443 444 ++Adapter->Send.TcbSlots; 445 446 return NDIS_STATUS_RESOURCES; 447 } 448 449 Flags |= NV_TCB_COALESCE; 450 451 LocalSgList.NumberOfElements = 1; 452 LocalSgList.Elements[0].Address = CoalesceBuffer->PhysicalAddress; 453 LocalSgList.Elements[0].Length = TotalLength; 454 SgList = &LocalSgList; 455 456 Tcb = Adapter->Send.CurrentTcb; 457 Tcb->Buffer = CoalesceBuffer; 458 } 459 } 460 else 461 { 462 if (SgList->NumberOfElements + 463 (NVNET_MAXIMUM_FRAME_SIZE_JUMBO / (NV_MAXIMUM_SG_SIZE + 1)) > Adapter->Send.TbdSlots) 464 { 465 return NDIS_STATUS_RESOURCES; 466 } 467 468 --Adapter->Send.TcbSlots; 469 470 Tcb = Adapter->Send.CurrentTcb; 471 } 472 473 Tcb->Packet = Packet; 474 Tcb->Flags = Flags; 475 476 Adapter->TransmitPacket(Adapter, Tcb, SgList); 477 478 ASSERT(Adapter->Send.TbdSlots >= Tcb->Slots); 479 Adapter->Send.TbdSlots -= Tcb->Slots; 480 481 Adapter->Send.CurrentTcb = NV_NEXT_TCB(Adapter, Tcb); 482 483 return NDIS_STATUS_PENDING; 484 } 485 486 /* FIXME: Use the proper send function (MiniportSendPackets) */ 487 NDIS_STATUS 488 NTAPI 489 MiniportSend( 490 _In_ NDIS_HANDLE MiniportAdapterContext, 491 _In_ PNDIS_PACKET Packet, 492 _In_ UINT Flags) 493 { 494 PNVNET_ADAPTER Adapter = (PNVNET_ADAPTER)MiniportAdapterContext; 495 UINT TotalLength; 496 NDIS_STATUS Status; 497 498 NDIS_DbgPrint(MIN_TRACE, ("()\n")); 499 500 NdisQueryPacketLength(Packet, &TotalLength); 501 502 NdisDprAcquireSpinLock(&Adapter->Send.Lock); 503 504 if (!(Adapter->Flags & NV_ACTIVE)) 505 { 506 NdisDprReleaseSpinLock(&Adapter->Send.Lock); 507 508 return NDIS_STATUS_FAILURE; 509 } 510 511 if (Adapter->Flags & NV_SEND_LARGE_SEND && 512 PtrToUlong(NDIS_PER_PACKET_INFO_FROM_PACKET(Packet, TcpLargeSendPacketInfo))) 513 { 514 Status = NvNetSendPacketLargeSend(Adapter, Packet, TotalLength); 515 } 516 else 517 { 518 Status = NvNetSendPacket(Adapter, Packet, TotalLength); 519 } 520 521 NdisDprReleaseSpinLock(&Adapter->Send.Lock); 522 523 return Status; 524 } 525