1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Emulex. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Source file containing the implementation of the Transmit 29 * Path 30 */ 31 32 #include <oce_impl.h> 33 34 static void oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed); 35 static int oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, 36 mblk_t *mp); 37 static int oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp, 38 uint32_t pkt_len); 39 static void oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd); 40 static int oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq, 41 size_t size, int flags); 42 static oce_wq_bdesc_t *oce_wqb_alloc(struct oce_wq *wq); 43 static void oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd); 44 45 static void oce_wqmd_free(struct oce_wq *wq, oce_wqe_desc_t *wqed); 46 static void oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd); 47 static inline oce_wq_mdesc_t *oce_wqm_alloc(struct oce_wq *wq); 48 static int oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq); 49 static void oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd); 50 static void oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed); 51 static void oce_remove_vtag(mblk_t *mp); 52 static void oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag); 53 static inline int oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm); 54 55 56 static ddi_dma_attr_t tx_map_dma_attr = { 57 DMA_ATTR_V0, /* version number */ 58 0x0000000000000000ull, /* low address */ 59 0xFFFFFFFFFFFFFFFFull, /* high address */ 60 0x0000000000010000ull, /* dma counter max */ 61 OCE_TXMAP_ALIGN, /* alignment */ 62 0x7FF, /* burst sizes */ 63 0x00000001, /* minimum transfer size */ 64 0x00000000FFFFFFFFull, /* maximum transfer size */ 65 0xFFFFFFFFFFFFFFFFull, /* maximum segment size */ 66 OCE_MAX_TXDMA_COOKIES, /* scatter/gather list length */ 67 0x00000001, /* granularity */ 68 0 /* DMA flags */ 69 }; 70 71 /* 72 * WQ map handle destructor 73 * 74 * wq - Pointer to WQ structure 75 * wqmd - pointer to WQE mapping handle descriptor 76 * 77 * return none 78 */ 79 80 static void 81 oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd) 82 { 83 _NOTE(ARGUNUSED(wq)); 84 /* Free the DMA handle */ 85 if (wqmd->dma_handle != NULL) 86 (void) ddi_dma_free_handle(&(wqmd->dma_handle)); 87 wqmd->dma_handle = NULL; 88 } /* oce_wqm_dtor */ 89 90 /* 91 * WQ map handles contructor 92 * 93 * wqmd - pointer to WQE mapping handle descriptor 94 * wq - Pointer to WQ structure 95 * 96 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 97 */ 98 static int 99 oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq) 100 { 101 struct oce_dev *dev; 102 int ret; 103 104 dev = wq->parent; 105 /* Allocate DMA handle */ 106 ret = ddi_dma_alloc_handle(dev->dip, &tx_map_dma_attr, 107 DDI_DMA_DONTWAIT, NULL, &wqmd->dma_handle); 108 109 return (ret); 110 } /* oce_wqm_ctor */ 111 112 /* 113 * function to create WQ mapping handles cache 114 * 115 * wq - pointer to WQ structure 116 * 117 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 118 */ 119 int 120 oce_wqm_cache_create(struct oce_wq *wq) 121 { 122 struct oce_dev *dev = wq->parent; 123 int size; 124 int cnt; 125 int ret; 126 127 size = wq->cfg.nhdl * sizeof (oce_wq_mdesc_t); 128 wq->wq_mdesc_array = kmem_zalloc(size, KM_NOSLEEP); 129 if (wq->wq_mdesc_array == NULL) { 130 return (DDI_FAILURE); 131 } 132 133 /* Create the free buffer list */ 134 OCE_LIST_CREATE(&wq->wq_mdesc_list, DDI_INTR_PRI(dev->intr_pri)); 135 136 for (cnt = 0; cnt < wq->cfg.nhdl; cnt++) { 137 ret = oce_wqm_ctor(&wq->wq_mdesc_array[cnt], wq); 138 if (ret != DDI_SUCCESS) { 139 goto wqm_fail; 140 } 141 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list, 142 &wq->wq_mdesc_array[cnt]); 143 } 144 return (DDI_SUCCESS); 145 146 wqm_fail: 147 oce_wqm_cache_destroy(wq); 148 return (DDI_FAILURE); 149 } 150 151 /* 152 * function to destroy WQ mapping handles cache 153 * 154 * wq - pointer to WQ structure 155 * 156 * return none 157 */ 158 void 159 oce_wqm_cache_destroy(struct oce_wq *wq) 160 { 161 oce_wq_mdesc_t *wqmd; 162 163 while ((wqmd = OCE_LIST_REM_HEAD(&wq->wq_mdesc_list)) != NULL) { 164 oce_wqm_dtor(wq, wqmd); 165 } 166 167 kmem_free(wq->wq_mdesc_array, 168 wq->cfg.nhdl * sizeof (oce_wq_mdesc_t)); 169 170 OCE_LIST_DESTROY(&wq->wq_mdesc_list); 171 } 172 173 /* 174 * function to create WQ buffer cache 175 * 176 * wq - pointer to WQ structure 177 * buf_size - size of the buffer 178 * 179 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 180 */ 181 int 182 oce_wqb_cache_create(struct oce_wq *wq, size_t buf_size) 183 { 184 struct oce_dev *dev = wq->parent; 185 int size; 186 int cnt; 187 int ret; 188 189 size = wq->cfg.nbufs * sizeof (oce_wq_bdesc_t); 190 wq->wq_bdesc_array = kmem_zalloc(size, KM_NOSLEEP); 191 if (wq->wq_bdesc_array == NULL) { 192 return (DDI_FAILURE); 193 } 194 195 /* Create the free buffer list */ 196 OCE_LIST_CREATE(&wq->wq_buf_list, DDI_INTR_PRI(dev->intr_pri)); 197 198 for (cnt = 0; cnt < wq->cfg.nbufs; cnt++) { 199 ret = oce_wqb_ctor(&wq->wq_bdesc_array[cnt], 200 wq, buf_size, DDI_DMA_STREAMING); 201 if (ret != DDI_SUCCESS) { 202 goto wqb_fail; 203 } 204 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list, 205 &wq->wq_bdesc_array[cnt]); 206 } 207 return (DDI_SUCCESS); 208 209 wqb_fail: 210 oce_wqb_cache_destroy(wq); 211 return (DDI_FAILURE); 212 } 213 214 /* 215 * function to destroy WQ buffer cache 216 * 217 * wq - pointer to WQ structure 218 * 219 * return none 220 */ 221 void 222 oce_wqb_cache_destroy(struct oce_wq *wq) 223 { 224 oce_wq_bdesc_t *wqbd; 225 while ((wqbd = OCE_LIST_REM_HEAD(&wq->wq_buf_list)) != NULL) { 226 oce_wqb_dtor(wq, wqbd); 227 } 228 kmem_free(wq->wq_bdesc_array, 229 wq->cfg.nbufs * sizeof (oce_wq_bdesc_t)); 230 OCE_LIST_DESTROY(&wq->wq_buf_list); 231 } 232 233 /* 234 * WQ buffer constructor 235 * 236 * wqbd - pointer to WQ buffer descriptor 237 * wq - pointer to WQ structure 238 * size - size of the buffer 239 * flags - KM_SLEEP or KM_NOSLEEP 240 * 241 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 242 */ 243 static int 244 oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq, size_t size, int flags) 245 { 246 struct oce_dev *dev; 247 dev = wq->parent; 248 wqbd->wqb = oce_alloc_dma_buffer(dev, size, flags); 249 if (wqbd->wqb == NULL) { 250 return (DDI_FAILURE); 251 } 252 wqbd->frag_addr.dw.addr_lo = ADDR_LO(wqbd->wqb->addr); 253 wqbd->frag_addr.dw.addr_hi = ADDR_HI(wqbd->wqb->addr); 254 return (DDI_SUCCESS); 255 } 256 257 /* 258 * WQ buffer destructor 259 * 260 * wq - pointer to WQ structure 261 * wqbd - pointer to WQ buffer descriptor 262 * 263 * return none 264 */ 265 static void 266 oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd) 267 { 268 oce_free_dma_buffer(wq->parent, wqbd->wqb); 269 } 270 271 /* 272 * function to alloc WQE buffer descriptor 273 * 274 * wq - pointer to WQ structure 275 * 276 * return pointer to WQE buffer descriptor 277 */ 278 static inline oce_wq_bdesc_t * 279 oce_wqb_alloc(struct oce_wq *wq) 280 { 281 oce_wq_bdesc_t *wqbd; 282 wqbd = OCE_LIST_REM_HEAD(&wq->wq_buf_list); 283 return (wqbd); 284 } 285 286 /* 287 * function to free WQE buffer descriptor 288 * 289 * wq - pointer to WQ structure 290 * wqbd - pointer to WQ buffer descriptor 291 * 292 * return none 293 */ 294 static inline void 295 oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd) 296 { 297 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list, wqbd); 298 } /* oce_wqb_free */ 299 300 /* 301 * function to allocate WQE mapping descriptor 302 * 303 * wq - pointer to WQ structure 304 * 305 * return pointer to WQE mapping descriptor 306 */ 307 static inline oce_wq_mdesc_t * 308 oce_wqm_alloc(struct oce_wq *wq) 309 { 310 oce_wq_mdesc_t *wqmd; 311 wqmd = OCE_LIST_REM_HEAD(&wq->wq_mdesc_list); 312 return (wqmd); 313 314 } /* oce_wqm_alloc */ 315 316 /* 317 * function to insert WQE mapping descriptor to the list 318 * 319 * wq - pointer to WQ structure 320 * wqmd - Pointer to WQ mapping descriptor 321 * 322 * return none 323 */ 324 static inline void 325 oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd) 326 { 327 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list, wqmd); 328 } 329 330 /* 331 * function to free WQE mapping descriptor 332 * 333 * wq - pointer to WQ structure 334 * wqmd - Pointer to WQ mapping descriptor 335 * 336 * return none 337 */ 338 static void 339 oce_wqmd_free(struct oce_wq *wq, oce_wqe_desc_t *wqed) 340 { 341 int ndesc; 342 oce_wq_mdesc_t *wqmd; 343 344 if (wqed == NULL) { 345 return; 346 } 347 for (ndesc = 0; ndesc < wqed->nhdl; ndesc++) { 348 wqmd = wqed->hdesc[ndesc].hdl; 349 (void) ddi_dma_unbind_handle(wqmd->dma_handle); 350 oce_wqm_free(wq, wqmd); 351 } 352 } 353 354 /* 355 * WQED kmem_cache constructor 356 * 357 * buf - pointer to WQE descriptor 358 * 359 * return DDI_SUCCESS 360 */ 361 int 362 oce_wqe_desc_ctor(void *buf, void *arg, int kmflags) 363 { 364 oce_wqe_desc_t *wqed = (oce_wqe_desc_t *)buf; 365 366 _NOTE(ARGUNUSED(arg)); 367 _NOTE(ARGUNUSED(kmflags)); 368 369 bzero(wqed, sizeof (oce_wqe_desc_t)); 370 return (DDI_SUCCESS); 371 } 372 373 /* 374 * WQED kmem_cache destructor 375 * 376 * buf - pointer to WQE descriptor 377 * 378 * return none 379 */ 380 void 381 oce_wqe_desc_dtor(void *buf, void *arg) 382 { 383 _NOTE(ARGUNUSED(buf)); 384 _NOTE(ARGUNUSED(arg)); 385 } 386 387 /* 388 * function to choose a WQ given a mblk depending on priority, flowID etc. 389 * 390 * dev - software handle to device 391 * pkt - the mblk to send 392 * 393 * return pointer to the WQ selected 394 */ 395 struct oce_wq * 396 oce_get_wq(struct oce_dev *dev, mblk_t *pkt) 397 { 398 _NOTE(ARGUNUSED(pkt)); 399 /* for the time being hardcode */ 400 return (dev->wq[0]); 401 } /* oce_get_wq */ 402 403 /* 404 * function to populate the single WQE 405 * 406 * wq - pointer to wq 407 * wqed - pointer to WQ entry descriptor 408 * 409 * return none 410 */ 411 #pragma inline(oce_fill_ring_descs) 412 static void 413 oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed) 414 { 415 416 struct oce_nic_frag_wqe *wqe; 417 int i; 418 /* Copy the precreate WQE descs to the ring desc */ 419 for (i = 0; i < wqed->wqe_cnt; i++) { 420 wqe = RING_GET_PRODUCER_ITEM_VA(wq->ring, 421 struct oce_nic_frag_wqe); 422 423 bcopy(&wqed->frag[i], wqe, NIC_WQE_SIZE); 424 RING_PUT(wq->ring, 1); 425 } 426 } /* oce_fill_ring_descs */ 427 428 /* 429 * function to copy the packet to preallocated Tx buffer 430 * 431 * wq - pointer to WQ 432 * wqed - Pointer to WQE descriptor 433 * mp - Pointer to packet chain 434 * pktlen - Size of the packet 435 * 436 * return 0=>success, error code otherwise 437 */ 438 static int 439 oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp, 440 uint32_t pkt_len) 441 { 442 oce_wq_bdesc_t *wqbd; 443 caddr_t buf_va; 444 struct oce_dev *dev = wq->parent; 445 446 wqbd = oce_wqb_alloc(wq); 447 if (wqbd == NULL) { 448 atomic_inc_32(&dev->tx_noxmtbuf); 449 oce_log(dev, CE_WARN, MOD_TX, "%s", 450 "wqb pool empty"); 451 return (ENOMEM); 452 } 453 454 /* create a fragment wqe for the packet */ 455 wqed->frag[1].u0.s.frag_pa_hi = wqbd->frag_addr.dw.addr_hi; 456 wqed->frag[1].u0.s.frag_pa_lo = wqbd->frag_addr.dw.addr_lo; 457 buf_va = DBUF_VA(wqbd->wqb); 458 459 /* copy pkt into buffer */ 460 for (; mp != NULL; mp = mp->b_cont) { 461 bcopy(mp->b_rptr, buf_va, MBLKL(mp)); 462 buf_va += MBLKL(mp); 463 } 464 465 wqed->frag[1].u0.s.frag_len = pkt_len; 466 wqed->hdesc[0].hdl = (void *)(wqbd); 467 468 (void) ddi_dma_sync(DBUF_DHDL(wqbd->wqb), 0, pkt_len, 469 DDI_DMA_SYNC_FORDEV); 470 471 if (oce_fm_check_dma_handle(dev, DBUF_DHDL(wqbd->wqb))) { 472 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED); 473 /* Free the buffer */ 474 oce_wqb_free(wq, wqbd); 475 return (EIO); 476 } 477 wqed->frag_cnt = 2; 478 wqed->nhdl = 1; 479 wqed->type = COPY_WQE; 480 return (0); 481 } /* oce_bcopy_wqe */ 482 483 /* 484 * function to copy the packet or dma map on the fly depending on size 485 * 486 * wq - pointer to WQ 487 * wqed - Pointer to WQE descriptor 488 * mp - Pointer to packet chain 489 * 490 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 491 */ 492 static int 493 oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp) 494 { 495 ddi_dma_cookie_t cookie; 496 oce_wq_mdesc_t *wqmd; 497 int32_t nfrag = 1; 498 uint32_t ncookies; 499 int ret; 500 uint32_t len; 501 struct oce_dev *dev = wq->parent; 502 503 wqed->nhdl = 0; 504 wqed->mp = mp; 505 506 for (; mp != NULL; mp = mp->b_cont) { 507 len = MBLKL(mp); 508 if (len == 0) { 509 oce_log(dev, CE_NOTE, MOD_TX, "%s", 510 "Zero len MBLK "); 511 continue; 512 } 513 514 wqmd = oce_wqm_alloc(wq); 515 if (wqmd == NULL) { 516 oce_log(dev, CE_WARN, MOD_TX, "%s", 517 "wqm pool empty"); 518 ret = ENOMEM; 519 goto map_fail; 520 } 521 522 ret = ddi_dma_addr_bind_handle(wqmd->dma_handle, 523 (struct as *)0, (caddr_t)mp->b_rptr, 524 len, DDI_DMA_WRITE | DDI_DMA_STREAMING, 525 DDI_DMA_DONTWAIT, NULL, &cookie, &ncookies); 526 if (ret != DDI_DMA_MAPPED) { 527 oce_log(dev, CE_WARN, MOD_TX, "%s", 528 "Failed to Map SGL"); 529 /* free the last one */ 530 oce_wqm_free(wq, wqmd); 531 goto map_fail; 532 } 533 534 do { 535 wqed->frag[nfrag].u0.s.frag_pa_hi = 536 ADDR_HI(cookie.dmac_laddress); 537 wqed->frag[nfrag].u0.s.frag_pa_lo = 538 ADDR_LO(cookie.dmac_laddress); 539 wqed->frag[nfrag].u0.s.frag_len = 540 (uint32_t)cookie.dmac_size; 541 nfrag++; 542 if (--ncookies > 0) 543 ddi_dma_nextcookie(wqmd->dma_handle, 544 &cookie); 545 else break; 546 } while (ncookies > 0); 547 548 wqed->hdesc[wqed->nhdl].hdl = (void *)wqmd; 549 wqed->nhdl++; 550 551 (void) ddi_dma_sync(wqmd->dma_handle, 552 0, 0, DDI_DMA_SYNC_FORDEV); 553 } 554 wqed->frag_cnt = nfrag; 555 wqed->type = MAPPED_WQE; 556 return (0); 557 558 map_fail: 559 wqed->mp = NULL; 560 oce_wqmd_free(wq, wqed); 561 return (ret); 562 } /* oce_map_wqe */ 563 564 static inline int 565 oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm) 566 { 567 struct oce_nic_tx_cqe *cqe; 568 uint16_t num_cqe = 0; 569 struct oce_cq *cq; 570 oce_wqe_desc_t *wqed; 571 int wqe_freed = 0; 572 struct oce_dev *dev; 573 574 cq = wq->cq; 575 dev = wq->parent; 576 (void) ddi_dma_sync(cq->ring->dbuf->dma_handle, 0, 0, 577 DDI_DMA_SYNC_FORKERNEL); 578 579 mutex_enter(&wq->txc_lock); 580 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 581 while (WQ_CQE_VALID(cqe)) { 582 583 DW_SWAP(u32ptr(cqe), sizeof (struct oce_nic_tx_cqe)); 584 585 /* update stats */ 586 if (cqe->u0.s.status != 0) { 587 atomic_inc_32(&dev->tx_errors); 588 } 589 590 /* complete the WQEs */ 591 wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list); 592 593 wqe_freed = wqed->wqe_cnt; 594 oce_free_wqed(wq, wqed); 595 RING_GET(wq->ring, wqe_freed); 596 atomic_add_32(&wq->wq_free, wqe_freed); 597 /* clear the valid bit and progress cqe */ 598 WQ_CQE_INVALIDATE(cqe); 599 RING_GET(cq->ring, 1); 600 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, 601 struct oce_nic_tx_cqe); 602 num_cqe++; 603 } /* for all valid CQE */ 604 mutex_exit(&wq->txc_lock); 605 if (num_cqe) 606 oce_arm_cq(wq->parent, cq->cq_id, num_cqe, rearm); 607 return (num_cqe); 608 } /* oce_process_tx_completion */ 609 610 /* 611 * function to drain a TxCQ and process its CQEs 612 * 613 * dev - software handle to the device 614 * cq - pointer to the cq to drain 615 * 616 * return the number of CQEs processed 617 */ 618 uint16_t 619 oce_drain_wq_cq(void *arg) 620 { 621 uint16_t num_cqe = 0; 622 struct oce_dev *dev; 623 struct oce_wq *wq; 624 625 wq = (struct oce_wq *)arg; 626 dev = wq->parent; 627 628 /* do while we do not reach a cqe that is not valid */ 629 num_cqe = oce_process_tx_compl(wq, B_FALSE); 630 631 /* check if we need to restart Tx */ 632 if (wq->resched && num_cqe) { 633 wq->resched = B_FALSE; 634 mac_tx_update(dev->mac_handle); 635 } 636 637 return (num_cqe); 638 } /* oce_process_wq_cqe */ 639 640 /* 641 * function to insert vtag to packet 642 * 643 * mp - mblk pointer 644 * vlan_tag - tag to be inserted 645 * 646 * return none 647 */ 648 static inline void 649 oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag) 650 { 651 struct ether_vlan_header *evh; 652 (void) memmove(mp->b_rptr - VLAN_TAGSZ, 653 mp->b_rptr, 2 * ETHERADDRL); 654 mp->b_rptr -= VLAN_TAGSZ; 655 evh = (struct ether_vlan_header *)(void *)mp->b_rptr; 656 evh->ether_tpid = htons(VLAN_TPID); 657 evh->ether_tci = htons(vlan_tag); 658 } 659 660 /* 661 * function to strip vtag from packet 662 * 663 * mp - mblk pointer 664 * 665 * return none 666 */ 667 668 static inline void 669 oce_remove_vtag(mblk_t *mp) 670 { 671 (void) memmove(mp->b_rptr + VLAN_TAGSZ, mp->b_rptr, 672 ETHERADDRL * 2); 673 mp->b_rptr += VLAN_TAGSZ; 674 } 675 676 /* 677 * function to xmit Single packet over the wire 678 * 679 * wq - pointer to WQ 680 * mp - Pointer to packet chain 681 * 682 * return pointer to the packet 683 */ 684 mblk_t * 685 oce_send_packet(struct oce_wq *wq, mblk_t *mp) 686 { 687 688 struct oce_nic_hdr_wqe *wqeh; 689 struct oce_dev *dev; 690 struct ether_header *eh; 691 struct ether_vlan_header *evh; 692 int32_t num_wqes; 693 uint16_t etype; 694 uint32_t ip_offset; 695 uint32_t csum_flags = 0; 696 boolean_t use_copy = B_FALSE; 697 boolean_t tagged = B_FALSE; 698 uint16_t vlan_tag; 699 uint32_t reg_value = 0; 700 oce_wqe_desc_t *wqed = NULL; 701 mblk_t *nmp = NULL; 702 mblk_t *tmp = NULL; 703 uint32_t pkt_len = 0; 704 int num_mblks = 0; 705 int ret = 0; 706 uint32_t flags = 0; 707 uint32_t mss = 0; 708 709 /* retrieve the adap priv struct ptr */ 710 dev = wq->parent; 711 712 /* check if we have enough free slots */ 713 if (wq->wq_free < wq->cfg.q_len/2) { 714 (void) oce_process_tx_compl(wq, B_FALSE); 715 } 716 if (wq->wq_free < OCE_MAX_TX_HDL) { 717 return (mp); 718 } 719 720 /* check if we should copy */ 721 for (tmp = mp; tmp != NULL; tmp = tmp->b_cont) { 722 pkt_len += MBLKL(tmp); 723 num_mblks++; 724 } 725 726 /* Retrieve LSO info */ 727 mac_lso_get(mp, &mss, &flags); 728 729 /* get the offload flags */ 730 mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &csum_flags); 731 732 /* Limit should be always less than Tx Buffer Size */ 733 if (pkt_len < dev->tx_bcopy_limit) { 734 use_copy = B_TRUE; 735 } else { 736 /* restrict the mapped segment to wat we support */ 737 if (num_mblks > OCE_MAX_TX_HDL) { 738 nmp = msgpullup(mp, -1); 739 if (nmp == NULL) { 740 atomic_inc_32(&wq->pkt_drops); 741 freemsg(mp); 742 return (NULL); 743 } 744 /* Reset it to new collapsed mp */ 745 freemsg(mp); 746 mp = nmp; 747 } 748 } 749 750 /* Get the packet descriptor for Tx */ 751 wqed = kmem_cache_alloc(wq->wqed_cache, KM_NOSLEEP); 752 if (wqed == NULL) { 753 atomic_inc_32(&wq->pkt_drops); 754 freemsg(mp); 755 return (NULL); 756 } 757 eh = (struct ether_header *)(void *)mp->b_rptr; 758 if (ntohs(eh->ether_type) == VLAN_TPID) { 759 evh = (struct ether_vlan_header *)(void *)mp->b_rptr; 760 tagged = B_TRUE; 761 etype = ntohs(evh->ether_type); 762 ip_offset = sizeof (struct ether_vlan_header); 763 pkt_len -= VLAN_TAGSZ; 764 vlan_tag = ntohs(evh->ether_tci); 765 oce_remove_vtag(mp); 766 } else { 767 etype = ntohs(eh->ether_type); 768 ip_offset = sizeof (struct ether_header); 769 } 770 bzero(wqed, sizeof (oce_wqe_desc_t)); 771 772 /* Save the WQ pointer */ 773 wqed->wq = wq; 774 if (use_copy == B_TRUE) { 775 ret = oce_bcopy_wqe(wq, wqed, mp, pkt_len); 776 } else { 777 ret = oce_map_wqe(wq, wqed, mp); 778 } 779 780 /* 781 * Any failure other than insufficient Q entries 782 * drop the packet 783 */ 784 if (ret != 0) { 785 kmem_cache_free(wq->wqed_cache, wqed); 786 /* drop the packet */ 787 atomic_inc_32(&wq->pkt_drops); 788 freemsg(mp); 789 return (NULL); 790 } 791 792 /* increment pending wqed to be scheduled */ 793 wqeh = (struct oce_nic_hdr_wqe *)&wqed->frag[0]; 794 795 /* fill rest of wqe header fields based on packet */ 796 if (flags & HW_LSO) { 797 wqeh->u0.s.lso = B_TRUE; 798 wqeh->u0.s.lso_mss = mss; 799 } 800 if (csum_flags & HCK_FULLCKSUM) { 801 uint8_t *proto; 802 if (etype == ETHERTYPE_IP) { 803 proto = (uint8_t *)(void *) 804 (mp->b_rptr + ip_offset); 805 if (proto[9] == 6) 806 /* IPPROTO_TCP */ 807 wqeh->u0.s.tcpcs = B_TRUE; 808 else if (proto[9] == 17) 809 /* IPPROTO_UDP */ 810 wqeh->u0.s.udpcs = B_TRUE; 811 } 812 } 813 814 if (csum_flags & HCK_IPV4_HDRCKSUM) 815 wqeh->u0.s.ipcs = B_TRUE; 816 if (tagged) { 817 wqeh->u0.s.vlan = B_TRUE; 818 wqeh->u0.s.vlan_tag = vlan_tag; 819 } 820 821 wqeh->u0.s.complete = B_TRUE; 822 wqeh->u0.s.event = B_TRUE; 823 wqeh->u0.s.crc = B_TRUE; 824 wqeh->u0.s.total_length = pkt_len; 825 826 /* frag count + header wqe */ 827 num_wqes = wqed->frag_cnt; 828 829 /* h/w expects even no. of WQEs */ 830 if (num_wqes & 0x1) 831 num_wqes++; 832 wqed->wqe_cnt = (uint16_t)num_wqes; 833 wqeh->u0.s.num_wqe = num_wqes; 834 DW_SWAP(u32ptr(&wqed->frag[0]), (wqed->wqe_cnt * NIC_WQE_SIZE)); 835 836 mutex_enter(&wq->tx_lock); 837 if (num_wqes > wq->wq_free) { 838 atomic_inc_32(&wq->tx_deferd); 839 mutex_exit(&wq->tx_lock); 840 goto wqe_fail; 841 } 842 atomic_add_32(&wq->wq_free, -num_wqes); 843 wqed->wq_start_idx = wq->ring->pidx; 844 845 /* fill the wq for adapter */ 846 oce_fill_ring_descs(wq, wqed); 847 848 /* Add the packet desc to list to be retrieved during cmpl */ 849 OCE_LIST_INSERT_TAIL(&wq->wqe_desc_list, wqed); 850 (void) ddi_dma_sync(wq->ring->dbuf->dma_handle, 0, 0, 851 DDI_DMA_SYNC_FORDEV); 852 853 /* ring tx doorbell */ 854 reg_value = (num_wqes << 16) | wq->wq_id; 855 /* Ring the door bell */ 856 OCE_DB_WRITE32(dev, PD_TXULP_DB, reg_value); 857 if (oce_fm_check_acc_handle(dev, dev->db_handle) != DDI_FM_OK) { 858 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED); 859 } 860 mutex_exit(&wq->tx_lock); 861 862 /* free mp if copied or packet chain collapsed */ 863 if (use_copy == B_TRUE) { 864 freemsg(mp); 865 } 866 return (NULL); 867 868 wqe_fail: 869 870 if (tagged) { 871 oce_insert_vtag(mp, vlan_tag); 872 } 873 874 /* set it to null in case map_wqe has set it */ 875 wqed->mp = NULL; 876 oce_free_wqed(wq, wqed); 877 return (mp); 878 } /* oce_send_packet */ 879 880 /* 881 * function to free the WQE descriptor 882 * 883 * wq - pointer to WQ 884 * wqed - Pointer to WQE descriptor 885 * 886 * return none 887 */ 888 #pragma inline(oce_free_wqed) 889 static void 890 oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed) 891 { 892 if (wqed == NULL) { 893 return; 894 } 895 896 if (wqed->type == COPY_WQE) { 897 oce_wqb_free(wq, wqed->hdesc[0].hdl); 898 } else if (wqed->type == MAPPED_WQE) { 899 oce_wqmd_free(wq, wqed); 900 } else ASSERT(0); 901 902 if (wqed->mp) 903 freemsg(wqed->mp); 904 kmem_cache_free(wq->wqed_cache, wqed); 905 } /* oce_free_wqed */ 906 907 /* 908 * function to start the WQ 909 * 910 * wq - pointer to WQ 911 * 912 * return DDI_SUCCESS 913 */ 914 915 int 916 oce_start_wq(struct oce_wq *wq) 917 { 918 _NOTE(ARGUNUSED(wq)); 919 return (DDI_SUCCESS); 920 } /* oce_start_wq */ 921 922 /* 923 * function to stop the WQ 924 * 925 * wq - pointer to WQ 926 * 927 * return none 928 */ 929 void 930 oce_clean_wq(struct oce_wq *wq) 931 { 932 oce_wqe_desc_t *wqed; 933 int ti; 934 935 /* Wait for already posted Tx to complete */ 936 937 for (ti = 0; ti < DEFAULT_DRAIN_TIME; ti++) { 938 (void) oce_process_tx_compl(wq, B_FALSE); 939 OCE_MSDELAY(1); 940 } 941 942 /* Free the remaining descriptors */ 943 while ((wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list)) != NULL) { 944 atomic_add_32(&wq->wq_free, wqed->wqe_cnt); 945 oce_free_wqed(wq, wqed); 946 } 947 oce_drain_eq(wq->cq->eq); 948 } /* oce_stop_wq */ 949 950 /* 951 * function to set the tx mapping handle fma attr 952 * 953 * fm_caps - capability flags 954 * 955 * return none 956 */ 957 958 void 959 oce_set_tx_map_dma_fma_flags(int fm_caps) 960 { 961 if (fm_caps == DDI_FM_NOT_CAPABLE) { 962 return; 963 } 964 965 if (DDI_FM_DMA_ERR_CAP(fm_caps)) { 966 tx_map_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 967 } else { 968 tx_map_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR; 969 } 970 } /* oce_set_tx_map_dma_fma_flags */ 971