1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/nxge/nxge_impl.h> 29 30 /* Software LSO required header files */ 31 #include <netinet/tcp.h> 32 #include <inet/ip_impl.h> 33 #include <inet/tcp.h> 34 35 static mblk_t *nxge_lso_eliminate(mblk_t *); 36 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 37 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 38 static void nxge_hcksum_retrieve(mblk_t *, 39 uint32_t *, uint32_t *, uint32_t *, 40 uint32_t *, uint32_t *); 41 static uint32_t nxge_csgen(uint16_t *, int); 42 43 extern uint32_t nxge_reclaim_pending; 44 extern uint32_t nxge_bcopy_thresh; 45 extern uint32_t nxge_dvma_thresh; 46 extern uint32_t nxge_dma_stream_thresh; 47 extern uint32_t nxge_tx_minfree; 48 extern uint32_t nxge_tx_intr_thres; 49 extern uint32_t nxge_tx_max_gathers; 50 extern uint32_t nxge_tx_tiny_pack; 51 extern uint32_t nxge_tx_use_bcopy; 52 extern uint32_t nxge_tx_lb_policy; 53 extern uint32_t nxge_no_tx_lb; 54 extern nxge_tx_mode_t nxge_tx_scheme; 55 uint32_t nxge_lso_kick_cnt = 2; 56 57 typedef struct _mac_tx_hint { 58 uint16_t sap; 59 uint16_t vid; 60 void *hash; 61 } mac_tx_hint_t, *p_mac_tx_hint_t; 62 63 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 64 65 int 66 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 67 { 68 int status = 0; 69 p_tx_desc_t tx_desc_ring_vp; 70 npi_handle_t npi_desc_handle; 71 nxge_os_dma_handle_t tx_desc_dma_handle; 72 p_tx_desc_t tx_desc_p; 73 p_tx_msg_t tx_msg_ring; 74 p_tx_msg_t tx_msg_p; 75 tx_desc_t tx_desc, *tmp_desc_p; 76 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 77 p_tx_pkt_header_t hdrp; 78 p_tx_pkt_hdr_all_t pkthdrp; 79 uint8_t npads = 0; 80 uint64_t dma_ioaddr; 81 uint32_t dma_flags; 82 int last_bidx; 83 uint8_t *b_rptr; 84 caddr_t kaddr; 85 uint32_t nmblks; 86 uint32_t ngathers; 87 uint32_t clen; 88 int len; 89 uint32_t pkt_len, pack_len, min_len; 90 uint32_t bcopy_thresh; 91 int i, cur_index, sop_index; 92 uint16_t tail_index; 93 boolean_t tail_wrap = B_FALSE; 94 nxge_dma_common_t desc_area; 95 nxge_os_dma_handle_t dma_handle; 96 ddi_dma_cookie_t dma_cookie; 97 npi_handle_t npi_handle; 98 p_mblk_t nmp; 99 p_mblk_t t_mp; 100 uint32_t ncookies; 101 boolean_t good_packet; 102 boolean_t mark_mode = B_FALSE; 103 p_nxge_stats_t statsp; 104 p_nxge_tx_ring_stats_t tdc_stats; 105 t_uscalar_t start_offset = 0; 106 t_uscalar_t stuff_offset = 0; 107 t_uscalar_t end_offset = 0; 108 t_uscalar_t value = 0; 109 t_uscalar_t cksum_flags = 0; 110 boolean_t cksum_on = B_FALSE; 111 uint32_t boff = 0; 112 uint64_t tot_xfer_len = 0, tmp_len = 0; 113 boolean_t header_set = B_FALSE; 114 #ifdef NXGE_DEBUG 115 p_tx_desc_t tx_desc_ring_pp; 116 p_tx_desc_t tx_desc_pp; 117 tx_desc_t *save_desc_p; 118 int dump_len; 119 int sad_len; 120 uint64_t sad; 121 int xfer_len; 122 uint32_t msgsize; 123 #endif 124 p_mblk_t mp_chain = NULL; 125 boolean_t is_lso = B_FALSE; 126 boolean_t lso_again; 127 int cur_index_lso; 128 p_mblk_t nmp_lso_save; 129 uint32_t lso_ngathers; 130 boolean_t lso_tail_wrap = B_FALSE; 131 132 NXGE_DEBUG_MSG((nxgep, TX_CTL, 133 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 134 NXGE_DEBUG_MSG((nxgep, TX_CTL, 135 "==> nxge_start: Starting tdc %d desc pending %d", 136 tx_ring_p->tdc, tx_ring_p->descs_pending)); 137 138 statsp = nxgep->statsp; 139 140 if (nxgep->statsp->port_stats.lb_mode == nxge_lb_normal) { 141 if (!statsp->mac_stats.link_up) { 142 freemsg(mp); 143 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 144 "link not up or LB mode")); 145 goto nxge_start_fail1; 146 } 147 } 148 149 if (nxgep->soft_lso_enable) { 150 mp_chain = nxge_lso_eliminate(mp); 151 NXGE_DEBUG_MSG((nxgep, TX_CTL, 152 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 153 mp, mp_chain)); 154 if (mp_chain == NULL) { 155 NXGE_ERROR_MSG((nxgep, TX_CTL, 156 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 157 mp_chain, mp)); 158 goto nxge_start_fail1; 159 } 160 if (mp_chain != mp) { 161 NXGE_DEBUG_MSG((nxgep, TX_CTL, 162 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 163 mp_chain, mp)); 164 is_lso = B_TRUE; 165 mp = mp_chain; 166 mp_chain = mp_chain->b_next; 167 mp->b_next = NULL; 168 } 169 } 170 171 hcksum_retrieve(mp, NULL, NULL, &start_offset, 172 &stuff_offset, &end_offset, &value, &cksum_flags); 173 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 174 start_offset += sizeof (ether_header_t); 175 stuff_offset += sizeof (ether_header_t); 176 } else { 177 start_offset += sizeof (struct ether_vlan_header); 178 stuff_offset += sizeof (struct ether_vlan_header); 179 } 180 181 if (cksum_flags & HCK_PARTIALCKSUM) { 182 NXGE_DEBUG_MSG((nxgep, TX_CTL, 183 "==> nxge_start: mp $%p len %d " 184 "cksum_flags 0x%x (partial checksum) ", 185 mp, MBLKL(mp), cksum_flags)); 186 cksum_on = B_TRUE; 187 } 188 189 lso_again = B_FALSE; 190 lso_ngathers = 0; 191 192 MUTEX_ENTER(&tx_ring_p->lock); 193 cur_index_lso = tx_ring_p->wr_index; 194 lso_tail_wrap = tx_ring_p->wr_index_wrap; 195 start_again: 196 ngathers = 0; 197 sop_index = tx_ring_p->wr_index; 198 #ifdef NXGE_DEBUG 199 if (tx_ring_p->descs_pending) { 200 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 201 "desc pending %d ", tx_ring_p->descs_pending)); 202 } 203 204 dump_len = (int)(MBLKL(mp)); 205 dump_len = (dump_len > 128) ? 128: dump_len; 206 207 NXGE_DEBUG_MSG((nxgep, TX_CTL, 208 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 209 "(Before header reserve: ORIGINAL LEN %d)", 210 tx_ring_p->tdc, 211 mp->b_rptr, 212 dump_len)); 213 214 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 215 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 216 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 217 #endif 218 219 tdc_stats = tx_ring_p->tdc_stats; 220 mark_mode = (tx_ring_p->descs_pending && 221 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 222 < nxge_tx_minfree)); 223 224 NXGE_DEBUG_MSG((nxgep, TX_CTL, 225 "TX Descriptor ring is channel %d mark mode %d", 226 tx_ring_p->tdc, mark_mode)); 227 228 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) { 229 NXGE_DEBUG_MSG((nxgep, TX_CTL, 230 "TX Descriptor ring is full: channel %d", 231 tx_ring_p->tdc)); 232 NXGE_DEBUG_MSG((nxgep, TX_CTL, 233 "TX Descriptor ring is full: channel %d", 234 tx_ring_p->tdc)); 235 if (is_lso) { 236 /* free the current mp and mp_chain if not FULL */ 237 tdc_stats->tx_no_desc++; 238 NXGE_DEBUG_MSG((nxgep, TX_CTL, 239 "LSO packet: TX Descriptor ring is full: " 240 "channel %d", 241 tx_ring_p->tdc)); 242 goto nxge_start_fail_lso; 243 } else { 244 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 245 tdc_stats->tx_no_desc++; 246 MUTEX_EXIT(&tx_ring_p->lock); 247 if (nxgep->resched_needed && !nxgep->resched_running) { 248 nxgep->resched_running = B_TRUE; 249 ddi_trigger_softintr(nxgep->resched_id); 250 } 251 status = 1; 252 goto nxge_start_fail1; 253 } 254 } 255 256 nmp = mp; 257 i = sop_index = tx_ring_p->wr_index; 258 nmblks = 0; 259 ngathers = 0; 260 pkt_len = 0; 261 pack_len = 0; 262 clen = 0; 263 last_bidx = -1; 264 good_packet = B_TRUE; 265 266 desc_area = tx_ring_p->tdc_desc; 267 npi_handle = desc_area.npi_handle; 268 npi_desc_handle.regh = (nxge_os_acc_handle_t) 269 DMA_COMMON_ACC_HANDLE(desc_area); 270 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 271 #ifdef NXGE_DEBUG 272 tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area); 273 #endif 274 tx_desc_dma_handle = (nxge_os_dma_handle_t) 275 DMA_COMMON_HANDLE(desc_area); 276 tx_msg_ring = tx_ring_p->tx_msg_ring; 277 278 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 279 sop_index, i)); 280 281 #ifdef NXGE_DEBUG 282 msgsize = msgdsize(nmp); 283 NXGE_DEBUG_MSG((nxgep, TX_CTL, 284 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 285 sop_index, i, msgsize)); 286 #endif 287 /* 288 * The first 16 bytes of the premapped buffer are reserved 289 * for header. No padding will be used. 290 */ 291 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 292 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 293 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 294 } else { 295 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 296 } 297 while (nmp) { 298 good_packet = B_TRUE; 299 b_rptr = nmp->b_rptr; 300 len = MBLKL(nmp); 301 if (len <= 0) { 302 nmp = nmp->b_cont; 303 continue; 304 } 305 nmblks++; 306 307 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 308 "len %d pkt_len %d pack_len %d", 309 nmblks, len, pkt_len, pack_len)); 310 /* 311 * Hardware limits the transfer length to 4K for NIU and 312 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 313 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 314 * If len is longer than the limit, then we break nmp into 315 * two chunks: Make the first chunk equal to the limit and 316 * the second chunk for the remaining data. If the second 317 * chunk is still larger than the limit, then it will be 318 * broken into two in the next pass. 319 */ 320 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 321 if ((t_mp = dupb(nmp)) != NULL) { 322 nmp->b_wptr = nmp->b_rptr + 323 (TX_MAX_TRANSFER_LENGTH 324 - TX_PKT_HEADER_SIZE); 325 t_mp->b_rptr = nmp->b_wptr; 326 t_mp->b_cont = nmp->b_cont; 327 nmp->b_cont = t_mp; 328 len = MBLKL(nmp); 329 } else { 330 if (is_lso) { 331 NXGE_DEBUG_MSG((nxgep, TX_CTL, 332 "LSO packet: dupb failed: " 333 "channel %d", 334 tx_ring_p->tdc)); 335 mp = nmp; 336 goto nxge_start_fail_lso; 337 } else { 338 good_packet = B_FALSE; 339 goto nxge_start_fail2; 340 } 341 } 342 } 343 tx_desc.value = 0; 344 tx_desc_p = &tx_desc_ring_vp[i]; 345 #ifdef NXGE_DEBUG 346 tx_desc_pp = &tx_desc_ring_pp[i]; 347 #endif 348 tx_msg_p = &tx_msg_ring[i]; 349 #if defined(__i386) 350 npi_desc_handle.regp = (uint32_t)tx_desc_p; 351 #else 352 npi_desc_handle.regp = (uint64_t)tx_desc_p; 353 #endif 354 if (!header_set && 355 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 356 (len >= bcopy_thresh))) { 357 header_set = B_TRUE; 358 bcopy_thresh += TX_PKT_HEADER_SIZE; 359 boff = 0; 360 pack_len = 0; 361 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 362 hdrp = (p_tx_pkt_header_t)kaddr; 363 clen = pkt_len; 364 dma_handle = tx_msg_p->buf_dma_handle; 365 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 366 (void) ddi_dma_sync(dma_handle, 367 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 368 DDI_DMA_SYNC_FORDEV); 369 370 tx_msg_p->flags.dma_type = USE_BCOPY; 371 goto nxge_start_control_header_only; 372 } 373 374 pkt_len += len; 375 pack_len += len; 376 377 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 378 "desc entry %d " 379 "DESC IOADDR $%p " 380 "desc_vp $%p tx_desc_p $%p " 381 "desc_pp $%p tx_desc_pp $%p " 382 "len %d pkt_len %d pack_len %d", 383 i, 384 DMA_COMMON_IOADDR(desc_area), 385 tx_desc_ring_vp, tx_desc_p, 386 tx_desc_ring_pp, tx_desc_pp, 387 len, pkt_len, pack_len)); 388 389 if (len < bcopy_thresh) { 390 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 391 "USE BCOPY: ")); 392 if (nxge_tx_tiny_pack) { 393 uint32_t blst = 394 TXDMA_DESC_NEXT_INDEX(i, -1, 395 tx_ring_p->tx_wrap_mask); 396 NXGE_DEBUG_MSG((nxgep, TX_CTL, 397 "==> nxge_start(5): pack")); 398 if ((pack_len <= bcopy_thresh) && 399 (last_bidx == blst)) { 400 NXGE_DEBUG_MSG((nxgep, TX_CTL, 401 "==> nxge_start: pack(6) " 402 "(pkt_len %d pack_len %d)", 403 pkt_len, pack_len)); 404 i = blst; 405 tx_desc_p = &tx_desc_ring_vp[i]; 406 #ifdef NXGE_DEBUG 407 tx_desc_pp = &tx_desc_ring_pp[i]; 408 #endif 409 tx_msg_p = &tx_msg_ring[i]; 410 boff = pack_len - len; 411 ngathers--; 412 } else if (pack_len > bcopy_thresh && 413 header_set) { 414 pack_len = len; 415 boff = 0; 416 bcopy_thresh = nxge_bcopy_thresh; 417 NXGE_DEBUG_MSG((nxgep, TX_CTL, 418 "==> nxge_start(7): > max NEW " 419 "bcopy thresh %d " 420 "pkt_len %d pack_len %d(next)", 421 bcopy_thresh, 422 pkt_len, pack_len)); 423 } 424 last_bidx = i; 425 } 426 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 427 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 428 hdrp = (p_tx_pkt_header_t)kaddr; 429 header_set = B_TRUE; 430 NXGE_DEBUG_MSG((nxgep, TX_CTL, 431 "==> nxge_start(7_x2): " 432 "pkt_len %d pack_len %d (new hdrp $%p)", 433 pkt_len, pack_len, hdrp)); 434 } 435 tx_msg_p->flags.dma_type = USE_BCOPY; 436 kaddr += boff; 437 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 438 "USE BCOPY: before bcopy " 439 "DESC IOADDR $%p entry %d " 440 "bcopy packets %d " 441 "bcopy kaddr $%p " 442 "bcopy ioaddr (SAD) $%p " 443 "bcopy clen %d " 444 "bcopy boff %d", 445 DMA_COMMON_IOADDR(desc_area), i, 446 tdc_stats->tx_hdr_pkts, 447 kaddr, 448 dma_ioaddr, 449 clen, 450 boff)); 451 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 452 "1USE BCOPY: ")); 453 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 454 "2USE BCOPY: ")); 455 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 456 "last USE BCOPY: copy from b_rptr $%p " 457 "to KADDR $%p (len %d offset %d", 458 b_rptr, kaddr, len, boff)); 459 460 bcopy(b_rptr, kaddr, len); 461 462 #ifdef NXGE_DEBUG 463 dump_len = (len > 128) ? 128: len; 464 NXGE_DEBUG_MSG((nxgep, TX_CTL, 465 "==> nxge_start: dump packets " 466 "(After BCOPY len %d)" 467 "(b_rptr $%p): %s", len, nmp->b_rptr, 468 nxge_dump_packet((char *)nmp->b_rptr, 469 dump_len))); 470 #endif 471 472 dma_handle = tx_msg_p->buf_dma_handle; 473 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 474 (void) ddi_dma_sync(dma_handle, 475 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 476 DDI_DMA_SYNC_FORDEV); 477 clen = len + boff; 478 tdc_stats->tx_hdr_pkts++; 479 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 480 "USE BCOPY: " 481 "DESC IOADDR $%p entry %d " 482 "bcopy packets %d " 483 "bcopy kaddr $%p " 484 "bcopy ioaddr (SAD) $%p " 485 "bcopy clen %d " 486 "bcopy boff %d", 487 DMA_COMMON_IOADDR(desc_area), 488 i, 489 tdc_stats->tx_hdr_pkts, 490 kaddr, 491 dma_ioaddr, 492 clen, 493 boff)); 494 } else { 495 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 496 "USE DVMA: len %d", len)); 497 tx_msg_p->flags.dma_type = USE_DMA; 498 dma_flags = DDI_DMA_WRITE; 499 if (len < nxge_dma_stream_thresh) { 500 dma_flags |= DDI_DMA_CONSISTENT; 501 } else { 502 dma_flags |= DDI_DMA_STREAMING; 503 } 504 505 dma_handle = tx_msg_p->dma_handle; 506 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 507 (caddr_t)b_rptr, len, dma_flags, 508 DDI_DMA_DONTWAIT, NULL, 509 &dma_cookie, &ncookies); 510 if (status == DDI_DMA_MAPPED) { 511 dma_ioaddr = dma_cookie.dmac_laddress; 512 len = (int)dma_cookie.dmac_size; 513 clen = (uint32_t)dma_cookie.dmac_size; 514 NXGE_DEBUG_MSG((nxgep, TX_CTL, 515 "==> nxge_start(12_1): " 516 "USE DVMA: len %d clen %d " 517 "ngathers %d", 518 len, clen, 519 ngathers)); 520 #if defined(__i386) 521 npi_desc_handle.regp = (uint32_t)tx_desc_p; 522 #else 523 npi_desc_handle.regp = (uint64_t)tx_desc_p; 524 #endif 525 while (ncookies > 1) { 526 ngathers++; 527 /* 528 * this is the fix for multiple 529 * cookies, which are basically 530 * a descriptor entry, we don't set 531 * SOP bit as well as related fields 532 */ 533 534 (void) npi_txdma_desc_gather_set( 535 npi_desc_handle, 536 &tx_desc, 537 (ngathers -1), 538 mark_mode, 539 ngathers, 540 dma_ioaddr, 541 clen); 542 543 tx_msg_p->tx_msg_size = clen; 544 NXGE_DEBUG_MSG((nxgep, TX_CTL, 545 "==> nxge_start: DMA " 546 "ncookie %d " 547 "ngathers %d " 548 "dma_ioaddr $%p len %d" 549 "desc $%p descp $%p (%d)", 550 ncookies, 551 ngathers, 552 dma_ioaddr, clen, 553 *tx_desc_p, tx_desc_p, i)); 554 555 ddi_dma_nextcookie(dma_handle, 556 &dma_cookie); 557 dma_ioaddr = 558 dma_cookie.dmac_laddress; 559 560 len = (int)dma_cookie.dmac_size; 561 clen = (uint32_t)dma_cookie.dmac_size; 562 NXGE_DEBUG_MSG((nxgep, TX_CTL, 563 "==> nxge_start(12_2): " 564 "USE DVMA: len %d clen %d ", 565 len, clen)); 566 567 i = TXDMA_DESC_NEXT_INDEX(i, 1, 568 tx_ring_p->tx_wrap_mask); 569 tx_desc_p = &tx_desc_ring_vp[i]; 570 571 npi_desc_handle.regp = 572 #if defined(__i386) 573 (uint32_t)tx_desc_p; 574 #else 575 (uint64_t)tx_desc_p; 576 #endif 577 tx_msg_p = &tx_msg_ring[i]; 578 tx_msg_p->flags.dma_type = USE_NONE; 579 tx_desc.value = 0; 580 581 ncookies--; 582 } 583 tdc_stats->tx_ddi_pkts++; 584 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 585 "DMA: ddi packets %d", 586 tdc_stats->tx_ddi_pkts)); 587 } else { 588 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 589 "dma mapping failed for %d " 590 "bytes addr $%p flags %x (%d)", 591 len, b_rptr, status, status)); 592 good_packet = B_FALSE; 593 tdc_stats->tx_dma_bind_fail++; 594 tx_msg_p->flags.dma_type = USE_NONE; 595 if (is_lso) { 596 mp = nmp; 597 goto nxge_start_fail_lso; 598 } else { 599 goto nxge_start_fail2; 600 } 601 } 602 } /* ddi dvma */ 603 604 if (is_lso) { 605 nmp_lso_save = nmp; 606 } 607 nmp = nmp->b_cont; 608 nxge_start_control_header_only: 609 #if defined(__i386) 610 npi_desc_handle.regp = (uint32_t)tx_desc_p; 611 #else 612 npi_desc_handle.regp = (uint64_t)tx_desc_p; 613 #endif 614 ngathers++; 615 616 if (ngathers == 1) { 617 #ifdef NXGE_DEBUG 618 save_desc_p = &sop_tx_desc; 619 #endif 620 sop_tx_desc_p = &sop_tx_desc; 621 sop_tx_desc_p->value = 0; 622 sop_tx_desc_p->bits.hdw.tr_len = clen; 623 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 624 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 625 } else { 626 #ifdef NXGE_DEBUG 627 save_desc_p = &tx_desc; 628 #endif 629 tmp_desc_p = &tx_desc; 630 tmp_desc_p->value = 0; 631 tmp_desc_p->bits.hdw.tr_len = clen; 632 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 633 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 634 635 tx_desc_p->value = tmp_desc_p->value; 636 } 637 638 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 639 "Desc_entry %d ngathers %d " 640 "desc_vp $%p tx_desc_p $%p " 641 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 642 "dma_ioaddr (SAD) $%p mark %d", 643 i, ngathers, 644 tx_desc_ring_vp, tx_desc_p, 645 len, clen, pkt_len, pack_len, nmblks, 646 dma_ioaddr, mark_mode)); 647 648 #ifdef NXGE_DEBUG 649 npi_desc_handle.nxgep = nxgep; 650 npi_desc_handle.function.function = nxgep->function_num; 651 npi_desc_handle.function.instance = nxgep->instance; 652 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 653 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 654 TX_PKT_DESC_TR_LEN_SHIFT); 655 656 657 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 658 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 659 "mark %d sop %d\n", 660 save_desc_p->value, 661 sad, 662 save_desc_p->bits.hdw.tr_len, 663 xfer_len, 664 save_desc_p->bits.hdw.num_ptr, 665 save_desc_p->bits.hdw.mark, 666 save_desc_p->bits.hdw.sop)); 667 668 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 669 #endif 670 671 tx_msg_p->tx_msg_size = clen; 672 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 673 if (ngathers > nxge_tx_max_gathers) { 674 good_packet = B_FALSE; 675 hcksum_retrieve(mp, NULL, NULL, &start_offset, 676 &stuff_offset, &end_offset, &value, 677 &cksum_flags); 678 679 NXGE_DEBUG_MSG((NULL, TX_CTL, 680 "==> nxge_start(14): pull msg - " 681 "len %d pkt_len %d ngathers %d", 682 len, pkt_len, ngathers)); 683 /* Pull all message blocks from b_cont */ 684 if (is_lso) { 685 mp = nmp_lso_save; 686 goto nxge_start_fail_lso; 687 } 688 if ((msgpullup(mp, -1)) == NULL) { 689 goto nxge_start_fail2; 690 } 691 goto nxge_start_fail2; 692 } 693 } /* while (nmp) */ 694 695 tx_msg_p->tx_message = mp; 696 tx_desc_p = &tx_desc_ring_vp[sop_index]; 697 #if defined(__i386) 698 npi_desc_handle.regp = (uint32_t)tx_desc_p; 699 #else 700 npi_desc_handle.regp = (uint64_t)tx_desc_p; 701 #endif 702 703 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 704 pkthdrp->reserved = 0; 705 hdrp->value = 0; 706 (void) nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 707 (pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp); 708 709 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 710 tdc_stats->tx_jumbo_pkts++; 711 } 712 713 min_len = (nxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2)); 714 if (pkt_len < min_len) { 715 /* Assume we use bcopy to premapped buffers */ 716 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 717 NXGE_DEBUG_MSG((NULL, TX_CTL, 718 "==> nxge_start(14-1): < (msg_min + 16)" 719 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 720 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 721 bzero((kaddr + pkt_len), (min_len - pkt_len)); 722 pkt_len = tx_msg_p->tx_msg_size = min_len; 723 724 sop_tx_desc_p->bits.hdw.tr_len = min_len; 725 726 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 727 tx_desc_p->value = sop_tx_desc_p->value; 728 729 NXGE_DEBUG_MSG((NULL, TX_CTL, 730 "==> nxge_start(14-2): < msg_min - " 731 "len %d pkt_len %d min_len %d ngathers %d", 732 len, pkt_len, min_len, ngathers)); 733 } 734 735 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 736 cksum_flags)); 737 if (cksum_flags & HCK_PARTIALCKSUM) { 738 NXGE_DEBUG_MSG((nxgep, TX_CTL, 739 "==> nxge_start: cksum_flags 0x%x (partial checksum) ", 740 cksum_flags)); 741 cksum_on = B_TRUE; 742 NXGE_DEBUG_MSG((nxgep, TX_CTL, 743 "==> nxge_start: from IP cksum_flags 0x%x " 744 "(partial checksum) " 745 "start_offset %d stuff_offset %d", 746 cksum_flags, start_offset, stuff_offset)); 747 tmp_len = (uint64_t)(start_offset >> 1); 748 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT); 749 tmp_len = (uint64_t)(stuff_offset >> 1); 750 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT); 751 752 NXGE_DEBUG_MSG((nxgep, TX_CTL, 753 "==> nxge_start: from IP cksum_flags 0x%x " 754 "(partial checksum) " 755 "after SHIFT start_offset %d stuff_offset %d", 756 cksum_flags, start_offset, stuff_offset)); 757 } 758 { 759 uint64_t tmp_len; 760 761 /* pkt_len already includes 16 + paddings!! */ 762 /* Update the control header length */ 763 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 764 tmp_len = hdrp->value | 765 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 766 767 NXGE_DEBUG_MSG((nxgep, TX_CTL, 768 "==> nxge_start(15_x1): setting SOP " 769 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 770 "0x%llx hdrp->value 0x%llx", 771 tot_xfer_len, tot_xfer_len, pkt_len, 772 tmp_len, hdrp->value)); 773 #if defined(_BIG_ENDIAN) 774 hdrp->value = ddi_swap64(tmp_len); 775 #else 776 hdrp->value = tmp_len; 777 #endif 778 NXGE_DEBUG_MSG((nxgep, 779 TX_CTL, "==> nxge_start(15_x2): setting SOP " 780 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 781 "tmp_len 0x%llx hdrp->value 0x%llx", 782 tot_xfer_len, pkt_len, 783 tmp_len, hdrp->value)); 784 } 785 786 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 787 "wr_index %d " 788 "tot_xfer_len (%d) pkt_len %d npads %d", 789 sop_index, 790 tot_xfer_len, pkt_len, 791 npads)); 792 793 sop_tx_desc_p->bits.hdw.sop = 1; 794 sop_tx_desc_p->bits.hdw.mark = mark_mode; 795 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 796 797 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 798 799 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 800 801 #ifdef NXGE_DEBUG 802 npi_desc_handle.nxgep = nxgep; 803 npi_desc_handle.function.function = nxgep->function_num; 804 npi_desc_handle.function.instance = nxgep->instance; 805 806 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 807 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 808 save_desc_p->value, 809 sad, 810 save_desc_p->bits.hdw.tr_len, 811 xfer_len, 812 save_desc_p->bits.hdw.num_ptr, 813 save_desc_p->bits.hdw.mark, 814 save_desc_p->bits.hdw.sop)); 815 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 816 817 dump_len = (pkt_len > 128) ? 128: pkt_len; 818 NXGE_DEBUG_MSG((nxgep, TX_CTL, 819 "==> nxge_start: dump packets(17) (after sop set, len " 820 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 821 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 822 (char *)hdrp, 823 nxge_dump_packet((char *)hdrp, dump_len))); 824 NXGE_DEBUG_MSG((nxgep, TX_CTL, 825 "==> nxge_start(18): TX desc sync: sop_index %d", 826 sop_index)); 827 #endif 828 829 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 830 (void) ddi_dma_sync(tx_desc_dma_handle, 831 sop_index * sizeof (tx_desc_t), 832 ngathers * sizeof (tx_desc_t), 833 DDI_DMA_SYNC_FORDEV); 834 835 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 836 "cs_off = 0x%02X cs_s_off = 0x%02X " 837 "pkt_len %d ngathers %d sop_index %d\n", 838 stuff_offset, start_offset, 839 pkt_len, ngathers, sop_index)); 840 } else { /* more than one descriptor and wrap around */ 841 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 842 (void) ddi_dma_sync(tx_desc_dma_handle, 843 sop_index * sizeof (tx_desc_t), 844 nsdescs * sizeof (tx_desc_t), 845 DDI_DMA_SYNC_FORDEV); 846 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 847 "cs_off = 0x%02X cs_s_off = 0x%02X " 848 "pkt_len %d ngathers %d sop_index %d\n", 849 stuff_offset, start_offset, 850 pkt_len, ngathers, sop_index)); 851 852 (void) ddi_dma_sync(tx_desc_dma_handle, 853 0, 854 (ngathers - nsdescs) * sizeof (tx_desc_t), 855 DDI_DMA_SYNC_FORDEV); 856 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 857 "cs_off = 0x%02X cs_s_off = 0x%02X " 858 "pkt_len %d ngathers %d sop_index %d\n", 859 stuff_offset, start_offset, 860 pkt_len, ngathers, sop_index)); 861 } 862 863 tail_index = tx_ring_p->wr_index; 864 tail_wrap = tx_ring_p->wr_index_wrap; 865 866 tx_ring_p->wr_index = i; 867 if (tx_ring_p->wr_index <= tail_index) { 868 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 869 B_FALSE : B_TRUE); 870 } 871 872 tx_ring_p->descs_pending += ngathers; 873 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 874 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 875 tx_ring_p->tdc, 876 tx_ring_p->wr_index, 877 tx_ring_p->wr_index_wrap, 878 ngathers, 879 tx_ring_p->descs_pending)); 880 881 if (is_lso) { 882 lso_ngathers += ngathers; 883 if (mp_chain != NULL) { 884 mp = mp_chain; 885 mp_chain = mp_chain->b_next; 886 mp->b_next = NULL; 887 if (nxge_lso_kick_cnt == lso_ngathers) { 888 { 889 tx_ring_kick_t kick; 890 891 kick.value = 0; 892 kick.bits.ldw.wrap = 893 tx_ring_p->wr_index_wrap; 894 kick.bits.ldw.tail = 895 (uint16_t)tx_ring_p->wr_index; 896 897 /* Kick the Transmit kick register */ 898 TXDMA_REG_WRITE64( 899 NXGE_DEV_NPI_HANDLE(nxgep), 900 TX_RING_KICK_REG, 901 (uint8_t)tx_ring_p->tdc, 902 kick.value); 903 tdc_stats->tx_starts++; 904 NXGE_DEBUG_MSG((nxgep, TX_CTL, 905 "==> nxge_start: more LSO: " 906 "LSO_CNT %d", 907 lso_gathers)); 908 } 909 lso_ngathers = 0; 910 ngathers = 0; 911 cur_index_lso = sop_index = tx_ring_p->wr_index; 912 lso_tail_wrap = tx_ring_p->wr_index_wrap; 913 } 914 NXGE_DEBUG_MSG((nxgep, TX_CTL, 915 "==> nxge_start: lso again: " 916 "lso_gathers %d ngathers %d cur_index_lso %d " 917 "wr_index %d sop_index %d", 918 lso_ngathers, ngathers, cur_index_lso, 919 tx_ring_p->wr_index, sop_index)); 920 921 NXGE_DEBUG_MSG((nxgep, TX_CTL, 922 "==> nxge_start: next : count %d", 923 lso_gathers)); 924 lso_again = B_TRUE; 925 goto start_again; 926 } 927 } 928 929 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 930 931 { 932 tx_ring_kick_t kick; 933 934 kick.value = 0; 935 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 936 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 937 938 /* Kick start the Transmit kick register */ 939 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 940 TX_RING_KICK_REG, 941 (uint8_t)tx_ring_p->tdc, 942 kick.value); 943 } 944 945 tdc_stats->tx_starts++; 946 947 MUTEX_EXIT(&tx_ring_p->lock); 948 949 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 950 951 return (status); 952 953 nxge_start_fail_lso: 954 status = 0; 955 good_packet = B_FALSE; 956 if (mp != NULL) { 957 freemsg(mp); 958 } 959 if (mp_chain != NULL) { 960 freemsg(mp_chain); 961 } 962 if (!lso_again && !ngathers) { 963 MUTEX_EXIT(&tx_ring_p->lock); 964 NXGE_DEBUG_MSG((nxgep, TX_CTL, 965 "==> nxge_start: lso exit (nothing changed)")); 966 goto nxge_start_fail1; 967 } 968 969 NXGE_DEBUG_MSG((nxgep, TX_CTL, 970 "==> nxge_start (channel %d): before lso " 971 "lso_gathers %d ngathers %d cur_index_lso %d " 972 "wr_index %d sop_index %d lso_again %d", 973 tx_ring_p->tdc, 974 lso_ngathers, ngathers, cur_index_lso, 975 tx_ring_p->wr_index, sop_index, lso_again)); 976 977 if (lso_again) { 978 lso_ngathers += ngathers; 979 ngathers = lso_ngathers; 980 sop_index = cur_index_lso; 981 tx_ring_p->wr_index = sop_index; 982 tx_ring_p->wr_index_wrap = lso_tail_wrap; 983 } 984 985 NXGE_DEBUG_MSG((nxgep, TX_CTL, 986 "==> nxge_start (channel %d): after lso " 987 "lso_gathers %d ngathers %d cur_index_lso %d " 988 "wr_index %d sop_index %d lso_again %d", 989 tx_ring_p->tdc, 990 lso_ngathers, ngathers, cur_index_lso, 991 tx_ring_p->wr_index, sop_index, lso_again)); 992 993 nxge_start_fail2: 994 if (good_packet == B_FALSE) { 995 cur_index = sop_index; 996 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 997 for (i = 0; i < ngathers; i++) { 998 tx_desc_p = &tx_desc_ring_vp[cur_index]; 999 #if defined(__i386) 1000 npi_handle.regp = (uint32_t)tx_desc_p; 1001 #else 1002 npi_handle.regp = (uint64_t)tx_desc_p; 1003 #endif 1004 tx_msg_p = &tx_msg_ring[cur_index]; 1005 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1006 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1007 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1008 "tx_desc_p = %X index = %d", 1009 tx_desc_p, tx_ring_p->rd_index)); 1010 (void) dvma_unload(tx_msg_p->dvma_handle, 1011 0, -1); 1012 tx_msg_p->dvma_handle = NULL; 1013 if (tx_ring_p->dvma_wr_index == 1014 tx_ring_p->dvma_wrap_mask) 1015 tx_ring_p->dvma_wr_index = 0; 1016 else 1017 tx_ring_p->dvma_wr_index++; 1018 tx_ring_p->dvma_pending--; 1019 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1020 if (ddi_dma_unbind_handle( 1021 tx_msg_p->dma_handle)) { 1022 cmn_err(CE_WARN, "!nxge_start: " 1023 "ddi_dma_unbind_handle failed"); 1024 } 1025 } 1026 tx_msg_p->flags.dma_type = USE_NONE; 1027 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1028 tx_ring_p->tx_wrap_mask); 1029 1030 } 1031 1032 nxgep->resched_needed = B_TRUE; 1033 } 1034 1035 MUTEX_EXIT(&tx_ring_p->lock); 1036 1037 nxge_start_fail1: 1038 /* Add FMA to check the access handle nxge_hregh */ 1039 1040 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1041 1042 return (status); 1043 } 1044 1045 int 1046 nxge_serial_tx(mblk_t *mp, void *arg) 1047 { 1048 p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 1049 p_nxge_t nxgep = tx_ring_p->nxgep; 1050 1051 return (nxge_start(nxgep, tx_ring_p, mp)); 1052 } 1053 1054 boolean_t 1055 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 1056 { 1057 p_tx_ring_t *tx_rings; 1058 uint8_t ring_index; 1059 p_tx_ring_t tx_ring_p; 1060 1061 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 1062 1063 ASSERT(mp->b_next == NULL); 1064 1065 ring_index = nxge_tx_lb_ring_1(mp, nxgep->max_tdcs, hp); 1066 tx_rings = nxgep->tx_rings->rings; 1067 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: tx_rings $%p", 1068 tx_rings)); 1069 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: max_tdcs %d " 1070 "ring_index %d", nxgep->max_tdcs, ring_index)); 1071 1072 switch (nxge_tx_scheme) { 1073 case NXGE_USE_START: 1074 if (nxge_start(nxgep, tx_rings[ring_index], mp)) { 1075 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 1076 "ring index %d", ring_index)); 1077 return (B_FALSE); 1078 } 1079 break; 1080 1081 case NXGE_USE_SERIAL: 1082 default: 1083 tx_ring_p = tx_rings[ring_index]; 1084 nxge_serialize_enter(tx_ring_p->serial, mp); 1085 break; 1086 } 1087 1088 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 1089 ring_index)); 1090 1091 return (B_TRUE); 1092 } 1093 1094 /* 1095 * nxge_m_tx() - send a chain of packets 1096 */ 1097 mblk_t * 1098 nxge_m_tx(void *arg, mblk_t *mp) 1099 { 1100 p_nxge_t nxgep = (p_nxge_t)arg; 1101 mblk_t *next; 1102 mac_tx_hint_t hint; 1103 1104 if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { 1105 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1106 "==> nxge_m_tx: hardware not initialized")); 1107 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1108 "<== nxge_m_tx")); 1109 return (mp); 1110 } 1111 1112 hint.hash = NULL; 1113 hint.vid = 0; 1114 hint.sap = 0; 1115 1116 while (mp != NULL) { 1117 next = mp->b_next; 1118 mp->b_next = NULL; 1119 1120 /* 1121 * Until Nemo tx resource works, the mac driver 1122 * does the load balancing based on TCP port, 1123 * or CPU. For debugging, we use a system 1124 * configurable parameter. 1125 */ 1126 if (!nxge_send(nxgep, mp, &hint)) { 1127 mp->b_next = next; 1128 break; 1129 } 1130 1131 mp = next; 1132 1133 NXGE_DEBUG_MSG((NULL, TX_CTL, 1134 "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 1135 mp, next)); 1136 } 1137 1138 return (mp); 1139 } 1140 1141 int 1142 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 1143 { 1144 uint8_t ring_index = 0; 1145 uint8_t *tcp_port; 1146 p_mblk_t nmp; 1147 size_t mblk_len; 1148 size_t iph_len; 1149 size_t hdrs_size; 1150 uint8_t hdrs_buf[sizeof (struct ether_header) + 1151 IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 1152 /* 1153 * allocate space big enough to cover 1154 * the max ip header length and the first 1155 * 4 bytes of the TCP/IP header. 1156 */ 1157 1158 boolean_t qos = B_FALSE; 1159 1160 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 1161 1162 if (hp->vid) { 1163 qos = B_TRUE; 1164 } 1165 switch (nxge_tx_lb_policy) { 1166 case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 1167 default: 1168 tcp_port = mp->b_rptr; 1169 if (!nxge_no_tx_lb && !qos && 1170 (ntohs(((p_ether_header_t)tcp_port)->ether_type) 1171 == ETHERTYPE_IP)) { 1172 nmp = mp; 1173 mblk_len = MBLKL(nmp); 1174 tcp_port = NULL; 1175 if (mblk_len > sizeof (struct ether_header) + 1176 sizeof (uint8_t)) { 1177 tcp_port = nmp->b_rptr + 1178 sizeof (struct ether_header); 1179 mblk_len -= sizeof (struct ether_header); 1180 iph_len = ((*tcp_port) & 0x0f) << 2; 1181 if (mblk_len > (iph_len + sizeof (uint32_t))) { 1182 tcp_port = nmp->b_rptr; 1183 } else { 1184 tcp_port = NULL; 1185 } 1186 } 1187 if (tcp_port == NULL) { 1188 hdrs_size = 0; 1189 ((p_ether_header_t)hdrs_buf)->ether_type = 0; 1190 while ((nmp) && (hdrs_size < 1191 sizeof (hdrs_buf))) { 1192 mblk_len = MBLKL(nmp); 1193 if (mblk_len >= 1194 (sizeof (hdrs_buf) - hdrs_size)) 1195 mblk_len = sizeof (hdrs_buf) - 1196 hdrs_size; 1197 bcopy(nmp->b_rptr, 1198 &hdrs_buf[hdrs_size], mblk_len); 1199 hdrs_size += mblk_len; 1200 nmp = nmp->b_cont; 1201 } 1202 tcp_port = hdrs_buf; 1203 } 1204 tcp_port += sizeof (ether_header_t); 1205 if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1206 switch (tcp_port[9]) { 1207 case IPPROTO_TCP: 1208 case IPPROTO_UDP: 1209 case IPPROTO_ESP: 1210 tcp_port += ((*tcp_port) & 0x0f) << 2; 1211 ring_index = 1212 ((tcp_port[0] ^ 1213 tcp_port[1] ^ 1214 tcp_port[2] ^ 1215 tcp_port[3]) % maxtdcs); 1216 break; 1217 1218 case IPPROTO_AH: 1219 /* SPI starts at the 4th byte */ 1220 tcp_port += ((*tcp_port) & 0x0f) << 2; 1221 ring_index = 1222 ((tcp_port[4] ^ 1223 tcp_port[5] ^ 1224 tcp_port[6] ^ 1225 tcp_port[7]) % maxtdcs); 1226 break; 1227 1228 default: 1229 ring_index = tcp_port[19] % maxtdcs; 1230 break; 1231 } 1232 } else { /* fragmented packet */ 1233 ring_index = tcp_port[19] % maxtdcs; 1234 } 1235 } else { 1236 ring_index = mp->b_band % maxtdcs; 1237 } 1238 break; 1239 1240 case NXGE_TX_LB_HASH: 1241 if (hp->hash) { 1242 #if defined(__i386) 1243 ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1244 #else 1245 ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1246 #endif 1247 } else { 1248 ring_index = mp->b_band % maxtdcs; 1249 } 1250 break; 1251 1252 case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 1253 tcp_port = mp->b_rptr; 1254 ring_index = tcp_port[5] % maxtdcs; 1255 break; 1256 } 1257 1258 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 1259 1260 return (ring_index); 1261 } 1262 1263 uint_t 1264 nxge_reschedule(caddr_t arg) 1265 { 1266 p_nxge_t nxgep; 1267 1268 nxgep = (p_nxge_t)arg; 1269 1270 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 1271 1272 if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1273 nxgep->resched_needed) { 1274 mac_tx_update(nxgep->mach); 1275 nxgep->resched_needed = B_FALSE; 1276 nxgep->resched_running = B_FALSE; 1277 } 1278 1279 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 1280 return (DDI_INTR_CLAIMED); 1281 } 1282 1283 1284 /* Software LSO starts here */ 1285 static void 1286 nxge_hcksum_retrieve(mblk_t *mp, 1287 uint32_t *start, uint32_t *stuff, uint32_t *end, 1288 uint32_t *value, uint32_t *flags) 1289 { 1290 if (mp->b_datap->db_type == M_DATA) { 1291 if (flags != NULL) { 1292 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1293 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1294 HCK_FULLCKSUM_OK); 1295 if ((*flags & (HCK_PARTIALCKSUM | 1296 HCK_FULLCKSUM)) != 0) { 1297 if (value != NULL) 1298 *value = (uint32_t)DB_CKSUM16(mp); 1299 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1300 if (start != NULL) 1301 *start = 1302 (uint32_t)DB_CKSUMSTART(mp); 1303 if (stuff != NULL) 1304 *stuff = 1305 (uint32_t)DB_CKSUMSTUFF(mp); 1306 if (end != NULL) 1307 *end = 1308 (uint32_t)DB_CKSUMEND(mp); 1309 } 1310 } 1311 } 1312 } 1313 } 1314 1315 static void 1316 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1317 { 1318 ASSERT(DB_TYPE(mp) == M_DATA); 1319 1320 *mss = 0; 1321 if (flags != NULL) { 1322 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1323 if ((*flags != 0) && (mss != NULL)) { 1324 *mss = (uint32_t)DB_LSOMSS(mp); 1325 } 1326 NXGE_DEBUG_MSG((NULL, TX_CTL, 1327 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1328 *mss, *flags)); 1329 } 1330 1331 NXGE_DEBUG_MSG((NULL, TX_CTL, 1332 "<== nxge_lso_info_get: mss %d", *mss)); 1333 } 1334 1335 /* 1336 * Do Soft LSO on the oversized packet. 1337 * 1338 * 1. Create a chain of message for headers. 1339 * 2. Fill up header messages with proper information. 1340 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1341 * each new message with necessary adjustments. 1342 * * Unchange the ethernet header for DIX frames. (by default) 1343 * * IP Total Length field is updated to MSS or less(only for the last one). 1344 * * IP Identification value is incremented by one for each packet. 1345 * * TCP sequence Number is recalculated according to the payload length. 1346 * * Set FIN and/or PSH flags for the *last* packet if applied. 1347 * * TCP partial Checksum 1348 * 4. Update LSO information in the first message header. 1349 * 5. Release the original message header. 1350 */ 1351 static mblk_t * 1352 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1353 { 1354 uint32_t hckflags; 1355 int pktlen; 1356 int hdrlen; 1357 int segnum; 1358 int i; 1359 struct ether_vlan_header *evh; 1360 int ehlen, iphlen, tcphlen; 1361 struct ip *oiph, *niph; 1362 struct tcphdr *otcph, *ntcph; 1363 int available, len, left; 1364 uint16_t ip_id; 1365 uint32_t tcp_seq; 1366 #ifdef __sparc 1367 uint32_t tcp_seq_tmp; 1368 #endif 1369 mblk_t *datamp; 1370 uchar_t *rptr; 1371 mblk_t *nmp; 1372 mblk_t *cmp; 1373 mblk_t *mp_chain; 1374 boolean_t do_cleanup = B_FALSE; 1375 t_uscalar_t start_offset = 0; 1376 t_uscalar_t stuff_offset = 0; 1377 t_uscalar_t value = 0; 1378 uint16_t l4_len; 1379 ipaddr_t src, dst; 1380 uint32_t cksum, sum, l4cksum; 1381 1382 NXGE_DEBUG_MSG((NULL, TX_CTL, 1383 "==> nxge_do_softlso")); 1384 /* 1385 * check the length of LSO packet payload and calculate the number of 1386 * segments to be generated. 1387 */ 1388 pktlen = msgsize(mp); 1389 evh = (struct ether_vlan_header *)mp->b_rptr; 1390 1391 /* VLAN? */ 1392 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1393 ehlen = sizeof (struct ether_vlan_header); 1394 else 1395 ehlen = sizeof (struct ether_header); 1396 oiph = (struct ip *)(mp->b_rptr + ehlen); 1397 iphlen = oiph->ip_hl * 4; 1398 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1399 tcphlen = otcph->th_off * 4; 1400 1401 l4_len = pktlen - ehlen - iphlen; 1402 1403 NXGE_DEBUG_MSG((NULL, TX_CTL, 1404 "==> nxge_do_softlso: mss %d oiph $%p " 1405 "original ip_sum oiph->ip_sum 0x%x " 1406 "original tcp_sum otcph->th_sum 0x%x " 1407 "oiph->ip_len %d pktlen %d ehlen %d " 1408 "l4_len %d (0x%x) ip_len - iphlen %d ", 1409 mss, 1410 oiph, 1411 oiph->ip_sum, 1412 otcph->th_sum, 1413 ntohs(oiph->ip_len), pktlen, 1414 ehlen, 1415 l4_len, 1416 l4_len, 1417 ntohs(oiph->ip_len) - iphlen)); 1418 1419 /* IPv4 + TCP */ 1420 if (!(oiph->ip_v == IPV4_VERSION)) { 1421 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1422 "<== nxge_do_softlso: not IPV4 " 1423 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1424 ntohs(oiph->ip_len), pktlen, ehlen, 1425 tcphlen)); 1426 freemsg(mp); 1427 return (NULL); 1428 } 1429 1430 if (!(oiph->ip_p == IPPROTO_TCP)) { 1431 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1432 "<== nxge_do_softlso: not TCP " 1433 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1434 ntohs(oiph->ip_len), pktlen, ehlen, 1435 tcphlen)); 1436 freemsg(mp); 1437 return (NULL); 1438 } 1439 1440 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1441 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1442 "<== nxge_do_softlso: len not matched " 1443 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1444 ntohs(oiph->ip_len), pktlen, ehlen, 1445 tcphlen)); 1446 freemsg(mp); 1447 return (NULL); 1448 } 1449 1450 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1451 tcphlen = otcph->th_off * 4; 1452 1453 /* TCP flags can not include URG, RST, or SYN */ 1454 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1455 1456 hdrlen = ehlen + iphlen + tcphlen; 1457 1458 VERIFY(MBLKL(mp) >= hdrlen); 1459 1460 if (MBLKL(mp) > hdrlen) { 1461 datamp = mp; 1462 rptr = mp->b_rptr + hdrlen; 1463 } else { /* = */ 1464 datamp = mp->b_cont; 1465 rptr = datamp->b_rptr; 1466 } 1467 1468 NXGE_DEBUG_MSG((NULL, TX_CTL, 1469 "nxge_do_softlso: otcph $%p pktlen: %d, " 1470 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1471 "mblkl(mp): %d, mblkl(datamp): %d", 1472 otcph, 1473 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1474 (int)MBLKL(mp), (int)MBLKL(datamp))); 1475 1476 hckflags = 0; 1477 nxge_hcksum_retrieve(mp, 1478 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1479 1480 dst = oiph->ip_dst.s_addr; 1481 src = oiph->ip_src.s_addr; 1482 1483 cksum = (dst >> 16) + (dst & 0xFFFF) + 1484 (src >> 16) + (src & 0xFFFF); 1485 l4cksum = cksum + IP_TCP_CSUM_COMP; 1486 1487 sum = l4_len + l4cksum; 1488 sum = (sum & 0xFFFF) + (sum >> 16); 1489 1490 NXGE_DEBUG_MSG((NULL, TX_CTL, 1491 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1492 "hckflags 0x%x start_offset %d stuff_offset %d " 1493 "value (original) 0x%x th_sum 0x%x " 1494 "pktlen %d l4_len %d (0x%x) " 1495 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1496 dst, src, 1497 (sum & 0xffff), (~sum & 0xffff), 1498 hckflags, start_offset, stuff_offset, 1499 value, otcph->th_sum, 1500 pktlen, 1501 l4_len, 1502 l4_len, 1503 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1504 (int)MBLKL(datamp), 1505 nxge_dump_packet((char *)evh, 12))); 1506 1507 /* 1508 * Start to process. 1509 */ 1510 available = pktlen - hdrlen; 1511 segnum = (available - 1) / mss + 1; 1512 1513 NXGE_DEBUG_MSG((NULL, TX_CTL, 1514 "==> nxge_do_softlso: pktlen %d " 1515 "MBLKL(mp): %d, MBLKL(datamp): %d " 1516 "available %d mss %d segnum %d", 1517 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1518 available, 1519 mss, 1520 segnum)); 1521 1522 VERIFY(segnum >= 2); 1523 1524 /* 1525 * Try to pre-allocate all header messages 1526 */ 1527 mp_chain = NULL; 1528 for (i = 0; i < segnum; i++) { 1529 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1530 /* Clean up the mp_chain */ 1531 while (mp_chain != NULL) { 1532 nmp = mp_chain; 1533 mp_chain = mp_chain->b_next; 1534 freemsg(nmp); 1535 } 1536 NXGE_DEBUG_MSG((NULL, TX_CTL, 1537 "<== nxge_do_softlso: " 1538 "Could not allocate enough messages for headers!")); 1539 freemsg(mp); 1540 return (NULL); 1541 } 1542 nmp->b_next = mp_chain; 1543 mp_chain = nmp; 1544 1545 NXGE_DEBUG_MSG((NULL, TX_CTL, 1546 "==> nxge_do_softlso: " 1547 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1548 mp, nmp, mp_chain, mp_chain->b_next)); 1549 } 1550 1551 NXGE_DEBUG_MSG((NULL, TX_CTL, 1552 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1553 mp, nmp, mp_chain)); 1554 1555 /* 1556 * Associate payload with new packets 1557 */ 1558 cmp = mp_chain; 1559 left = available; 1560 while (cmp != NULL) { 1561 nmp = dupb(datamp); 1562 if (nmp == NULL) { 1563 do_cleanup = B_TRUE; 1564 NXGE_DEBUG_MSG((NULL, TX_CTL, 1565 "==>nxge_do_softlso: " 1566 "Can not dupb(datamp), have to do clean up")); 1567 goto cleanup_allocated_msgs; 1568 } 1569 1570 NXGE_DEBUG_MSG((NULL, TX_CTL, 1571 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1572 "dupb nmp $%p len %d left %d msd %d ", 1573 mp, cmp, nmp, len, left, mss)); 1574 1575 cmp->b_cont = nmp; 1576 nmp->b_rptr = rptr; 1577 len = (left < mss) ? left : mss; 1578 left -= len; 1579 1580 NXGE_DEBUG_MSG((NULL, TX_CTL, 1581 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1582 "dupb nmp $%p len %d left %d mss %d ", 1583 mp, cmp, nmp, len, left, mss)); 1584 NXGE_DEBUG_MSG((NULL, TX_CTL, 1585 "nxge_do_softlso: before available: %d, " 1586 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1587 available, left, len, segnum, (int)MBLKL(nmp))); 1588 1589 len -= MBLKL(nmp); 1590 NXGE_DEBUG_MSG((NULL, TX_CTL, 1591 "nxge_do_softlso: after available: %d, " 1592 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1593 available, left, len, segnum, (int)MBLKL(nmp))); 1594 1595 while (len > 0) { 1596 mblk_t *mmp = NULL; 1597 1598 NXGE_DEBUG_MSG((NULL, TX_CTL, 1599 "nxge_do_softlso: (4) len > 0 available: %d, " 1600 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1601 available, left, len, segnum, (int)MBLKL(nmp))); 1602 1603 if (datamp->b_cont != NULL) { 1604 datamp = datamp->b_cont; 1605 rptr = datamp->b_rptr; 1606 mmp = dupb(datamp); 1607 if (mmp == NULL) { 1608 do_cleanup = B_TRUE; 1609 NXGE_DEBUG_MSG((NULL, TX_CTL, 1610 "==> nxge_do_softlso: " 1611 "Can not dupb(datamp) (1), : 1612 "have to do clean up")); 1613 NXGE_DEBUG_MSG((NULL, TX_CTL, 1614 "==> nxge_do_softlso: " 1615 "available: %d, left: %d, " 1616 "len: %d, MBLKL(nmp): %d", 1617 available, left, len, 1618 (int)MBLKL(nmp))); 1619 goto cleanup_allocated_msgs; 1620 } 1621 } else { 1622 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1623 "==> nxge_do_softlso: " 1624 "(1)available: %d, left: %d, " 1625 "len: %d, MBLKL(nmp): %d", 1626 available, left, len, 1627 (int)MBLKL(nmp))); 1628 cmn_err(CE_PANIC, 1629 "==> nxge_do_softlso: " 1630 "Pointers must have been corrupted!\n" 1631 "datamp: $%p, nmp: $%p, rptr: $%p", 1632 (void *)datamp, 1633 (void *)nmp, 1634 (void *)rptr); 1635 } 1636 nmp->b_cont = mmp; 1637 nmp = mmp; 1638 len -= MBLKL(nmp); 1639 } 1640 if (len < 0) { 1641 nmp->b_wptr += len; 1642 rptr = nmp->b_wptr; 1643 NXGE_DEBUG_MSG((NULL, TX_CTL, 1644 "(5) len < 0 (less than 0)" 1645 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1646 available, left, len, (int)MBLKL(nmp))); 1647 1648 } else if (len == 0) { 1649 if (datamp->b_cont != NULL) { 1650 NXGE_DEBUG_MSG((NULL, TX_CTL, 1651 "(5) len == 0" 1652 "available: %d, left: %d, len: %d, " 1653 "MBLKL(nmp): %d", 1654 available, left, len, (int)MBLKL(nmp))); 1655 datamp = datamp->b_cont; 1656 rptr = datamp->b_rptr; 1657 } else { 1658 NXGE_DEBUG_MSG((NULL, TX_CTL, 1659 "(6)available b_cont == NULL : %d, " 1660 "left: %d, len: %d, MBLKL(nmp): %d", 1661 available, left, len, (int)MBLKL(nmp))); 1662 1663 VERIFY(cmp->b_next == NULL); 1664 VERIFY(left == 0); 1665 break; /* Done! */ 1666 } 1667 } 1668 cmp = cmp->b_next; 1669 1670 NXGE_DEBUG_MSG((NULL, TX_CTL, 1671 "(7) do_softlso: " 1672 "next mp in mp_chain available len != 0 : %d, " 1673 "left: %d, len: %d, MBLKL(nmp): %d", 1674 available, left, len, (int)MBLKL(nmp))); 1675 } 1676 1677 /* 1678 * From now, start to fill up all headers for the first message 1679 * Hardware checksum flags need to be updated separately for FULLCKSUM 1680 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1681 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1682 * required fields need to be updated properly. 1683 */ 1684 nmp = mp_chain; 1685 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1686 nmp->b_wptr = nmp->b_rptr + hdrlen; 1687 niph = (struct ip *)(nmp->b_rptr + ehlen); 1688 niph->ip_len = htons(mss + iphlen + tcphlen); 1689 ip_id = ntohs(niph->ip_id); 1690 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1691 #ifdef __sparc 1692 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1693 tcp_seq = ntohl(tcp_seq_tmp); 1694 #else 1695 tcp_seq = ntohl(ntcph->th_seq); 1696 #endif 1697 1698 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1699 1700 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1701 DB_CKSUMSTART(nmp) = start_offset; 1702 DB_CKSUMSTUFF(nmp) = stuff_offset; 1703 1704 /* calculate IP checksum and TCP pseudo header checksum */ 1705 niph->ip_sum = 0; 1706 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1707 1708 l4_len = mss + tcphlen; 1709 sum = htons(l4_len) + l4cksum; 1710 sum = (sum & 0xFFFF) + (sum >> 16); 1711 ntcph->th_sum = (sum & 0xffff); 1712 1713 NXGE_DEBUG_MSG((NULL, TX_CTL, 1714 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1715 "mss %d pktlen %d l4_len %d (0x%x) " 1716 "MBLKL(mp): %d, MBLKL(datamp): %d " 1717 "ip_sum 0x%x " 1718 "th_sum 0x%x sum 0x%x ) " 1719 "dump first ip->tcp %s", 1720 nmp, mp_chain, 1721 mss, 1722 pktlen, 1723 l4_len, 1724 l4_len, 1725 (int)MBLKL(mp), (int)MBLKL(datamp), 1726 niph->ip_sum, 1727 ntcph->th_sum, 1728 sum, 1729 nxge_dump_packet((char *)niph, 52))); 1730 1731 cmp = nmp; 1732 while ((nmp = nmp->b_next)->b_next != NULL) { 1733 NXGE_DEBUG_MSG((NULL, TX_CTL, 1734 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1735 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1736 nmp->b_wptr = nmp->b_rptr + hdrlen; 1737 niph = (struct ip *)(nmp->b_rptr + ehlen); 1738 niph->ip_id = htons(++ip_id); 1739 niph->ip_len = htons(mss + iphlen + tcphlen); 1740 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1741 tcp_seq += mss; 1742 1743 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1744 1745 #ifdef __sparc 1746 tcp_seq_tmp = htonl(tcp_seq); 1747 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1748 #else 1749 ntcph->th_seq = htonl(tcp_seq); 1750 #endif 1751 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1752 DB_CKSUMSTART(nmp) = start_offset; 1753 DB_CKSUMSTUFF(nmp) = stuff_offset; 1754 1755 /* calculate IP checksum and TCP pseudo header checksum */ 1756 niph->ip_sum = 0; 1757 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1758 ntcph->th_sum = (sum & 0xffff); 1759 1760 NXGE_DEBUG_MSG((NULL, TX_CTL, 1761 "==> nxge_do_softlso: middle ip_sum 0x%x " 1762 "th_sum 0x%x " 1763 " mp $%p (mp_chain $%p) pktlen %d " 1764 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1765 niph->ip_sum, 1766 ntcph->th_sum, 1767 nmp, mp_chain, 1768 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1769 } 1770 1771 /* Last segment */ 1772 /* 1773 * Set FIN and/or PSH flags if present only in the last packet. 1774 * The ip_len could be different from prior packets. 1775 */ 1776 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1777 nmp->b_wptr = nmp->b_rptr + hdrlen; 1778 niph = (struct ip *)(nmp->b_rptr + ehlen); 1779 niph->ip_id = htons(++ip_id); 1780 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1781 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1782 tcp_seq += mss; 1783 #ifdef __sparc 1784 tcp_seq_tmp = htonl(tcp_seq); 1785 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1786 #else 1787 ntcph->th_seq = htonl(tcp_seq); 1788 #endif 1789 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1790 1791 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1792 DB_CKSUMSTART(nmp) = start_offset; 1793 DB_CKSUMSTUFF(nmp) = stuff_offset; 1794 1795 /* calculate IP checksum and TCP pseudo header checksum */ 1796 niph->ip_sum = 0; 1797 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1798 1799 l4_len = ntohs(niph->ip_len) - iphlen; 1800 sum = htons(l4_len) + l4cksum; 1801 sum = (sum & 0xFFFF) + (sum >> 16); 1802 ntcph->th_sum = (sum & 0xffff); 1803 1804 NXGE_DEBUG_MSG((NULL, TX_CTL, 1805 "==> nxge_do_softlso: last next " 1806 "niph->ip_sum 0x%x " 1807 "ntcph->th_sum 0x%x sum 0x%x " 1808 "dump last ip->tcp %s " 1809 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1810 "l4_len %d (0x%x) " 1811 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1812 niph->ip_sum, 1813 ntcph->th_sum, sum, 1814 nxge_dump_packet((char *)niph, 52), 1815 cmp, nmp, mp_chain, 1816 pktlen, pktlen, 1817 l4_len, 1818 l4_len, 1819 (int)MBLKL(mp), (int)MBLKL(datamp))); 1820 1821 cleanup_allocated_msgs: 1822 if (do_cleanup) { 1823 NXGE_DEBUG_MSG((NULL, TX_CTL, 1824 "==> nxge_do_softlso: " 1825 "Failed allocating messages, " 1826 "have to clean up and fail!")); 1827 while (mp_chain != NULL) { 1828 nmp = mp_chain; 1829 mp_chain = mp_chain->b_next; 1830 freemsg(nmp); 1831 } 1832 } 1833 /* 1834 * We're done here, so just free the original message and return the 1835 * new message chain, that could be NULL if failed, back to the caller. 1836 */ 1837 freemsg(mp); 1838 1839 NXGE_DEBUG_MSG((NULL, TX_CTL, 1840 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1841 return (mp_chain); 1842 } 1843 1844 /* 1845 * Will be called before NIC driver do further operation on the message. 1846 * The input message may include LSO information, if so, go to softlso logic 1847 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1848 * The return could be the same non-LSO message or a message chain for LSO case. 1849 * 1850 * The driver needs to call this function per packet and process the whole chain 1851 * if applied. 1852 */ 1853 static mblk_t * 1854 nxge_lso_eliminate(mblk_t *mp) 1855 { 1856 uint32_t lsoflags; 1857 uint32_t mss; 1858 1859 NXGE_DEBUG_MSG((NULL, TX_CTL, 1860 "==>nxge_lso_eliminate:")); 1861 nxge_lso_info_get(mp, &mss, &lsoflags); 1862 1863 if (lsoflags & HW_LSO) { 1864 mblk_t *nmp; 1865 1866 NXGE_DEBUG_MSG((NULL, TX_CTL, 1867 "==>nxge_lso_eliminate:" 1868 "HW_LSO:mss %d mp $%p", 1869 mss, mp)); 1870 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1871 NXGE_DEBUG_MSG((NULL, TX_CTL, 1872 "<== nxge_lso_eliminate: " 1873 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 1874 nmp, mss, mp)); 1875 return (nmp); 1876 } else { 1877 NXGE_DEBUG_MSG((NULL, TX_CTL, 1878 "<== nxge_lso_eliminate_ " 1879 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 1880 nmp, mss, mp)); 1881 return (NULL); 1882 } 1883 } 1884 1885 NXGE_DEBUG_MSG((NULL, TX_CTL, 1886 "<== nxge_lso_eliminate")); 1887 return (mp); 1888 } 1889 1890 static uint32_t 1891 nxge_csgen(uint16_t *adr, int len) 1892 { 1893 int i, odd; 1894 uint32_t sum = 0; 1895 uint32_t c = 0; 1896 1897 odd = len % 2; 1898 for (i = 0; i < (len / 2); i++) { 1899 sum += (adr[i] & 0xffff); 1900 } 1901 if (odd) { 1902 sum += adr[len / 2] & 0xff00; 1903 } 1904 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 1905 sum &= 0xffff; 1906 sum += c; 1907 } 1908 return (~sum & 0xffff); 1909 } 1910