1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (C) 2013 Emulex 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Emulex Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 * Contact Information: 34 * freebsd-drivers@emulex.com 35 * 36 * Emulex 37 * 3333 Susan Street 38 * Costa Mesa, CA 92626 39 */ 40 41 /* $FreeBSD$ */ 42 43 #include "opt_inet6.h" 44 #include "opt_inet.h" 45 46 #include "oce_if.h" 47 #include "oce_user.h" 48 49 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO) 50 51 /* UE Status Low CSR */ 52 static char *ue_status_low_desc[] = { 53 "CEV", 54 "CTX", 55 "DBUF", 56 "ERX", 57 "Host", 58 "MPU", 59 "NDMA", 60 "PTC ", 61 "RDMA ", 62 "RXF ", 63 "RXIPS ", 64 "RXULP0 ", 65 "RXULP1 ", 66 "RXULP2 ", 67 "TIM ", 68 "TPOST ", 69 "TPRE ", 70 "TXIPS ", 71 "TXULP0 ", 72 "TXULP1 ", 73 "UC ", 74 "WDMA ", 75 "TXULP2 ", 76 "HOST1 ", 77 "P0_OB_LINK ", 78 "P1_OB_LINK ", 79 "HOST_GPIO ", 80 "MBOX ", 81 "AXGMAC0", 82 "AXGMAC1", 83 "JTAG", 84 "MPU_INTPEND" 85 }; 86 87 /* UE Status High CSR */ 88 static char *ue_status_hi_desc[] = { 89 "LPCMEMHOST", 90 "MGMT_MAC", 91 "PCS0ONLINE", 92 "MPU_IRAM", 93 "PCS1ONLINE", 94 "PCTL0", 95 "PCTL1", 96 "PMEM", 97 "RR", 98 "TXPB", 99 "RXPP", 100 "XAUI", 101 "TXP", 102 "ARM", 103 "IPC", 104 "HOST2", 105 "HOST3", 106 "HOST4", 107 "HOST5", 108 "HOST6", 109 "HOST7", 110 "HOST8", 111 "HOST9", 112 "NETC", 113 "Unknown", 114 "Unknown", 115 "Unknown", 116 "Unknown", 117 "Unknown", 118 "Unknown", 119 "Unknown", 120 "Unknown" 121 }; 122 123 struct oce_common_cqe_info{ 124 uint8_t vtp:1; 125 uint8_t l4_cksum_pass:1; 126 uint8_t ip_cksum_pass:1; 127 uint8_t ipv6_frame:1; 128 uint8_t qnq:1; 129 uint8_t rsvd:3; 130 uint8_t num_frags; 131 uint16_t pkt_size; 132 uint16_t vtag; 133 }; 134 135 136 /* Driver entry points prototypes */ 137 static int oce_probe(device_t dev); 138 static int oce_attach(device_t dev); 139 static int oce_detach(device_t dev); 140 static int oce_shutdown(device_t dev); 141 static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data); 142 static void oce_init(void *xsc); 143 static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m); 144 static void oce_multiq_flush(struct ifnet *ifp); 145 146 /* Driver interrupt routines protypes */ 147 static void oce_intr(void *arg, int pending); 148 static int oce_setup_intr(POCE_SOFTC sc); 149 static int oce_fast_isr(void *arg); 150 static int oce_alloc_intr(POCE_SOFTC sc, int vector, 151 void (*isr) (void *arg, int pending)); 152 153 /* Media callbacks prototypes */ 154 static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req); 155 static int oce_media_change(struct ifnet *ifp); 156 157 /* Transmit routines prototypes */ 158 static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index); 159 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq); 160 static void oce_process_tx_completion(struct oce_wq *wq); 161 static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, 162 struct oce_wq *wq); 163 164 /* Receive routines prototypes */ 165 static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 166 static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 167 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe); 168 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq); 169 static uint16_t oce_rq_handler_lro(void *arg); 170 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2); 171 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2); 172 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m); 173 174 /* Helper function prototypes in this file */ 175 static int oce_attach_ifp(POCE_SOFTC sc); 176 static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 177 static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 178 static int oce_vid_config(POCE_SOFTC sc); 179 static void oce_mac_addr_set(POCE_SOFTC sc); 180 static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data); 181 static void oce_local_timer(void *arg); 182 static void oce_if_deactivate(POCE_SOFTC sc); 183 static void oce_if_activate(POCE_SOFTC sc); 184 static void setup_max_queues_want(POCE_SOFTC sc); 185 static void update_queues_got(POCE_SOFTC sc); 186 static void process_link_state(POCE_SOFTC sc, 187 struct oce_async_cqe_link_state *acqe); 188 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m); 189 static void oce_get_config(POCE_SOFTC sc); 190 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete); 191 static void oce_read_env_variables(POCE_SOFTC sc); 192 193 194 /* IP specific */ 195 #if defined(INET6) || defined(INET) 196 static int oce_init_lro(POCE_SOFTC sc); 197 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp); 198 #endif 199 200 static device_method_t oce_dispatch[] = { 201 DEVMETHOD(device_probe, oce_probe), 202 DEVMETHOD(device_attach, oce_attach), 203 DEVMETHOD(device_detach, oce_detach), 204 DEVMETHOD(device_shutdown, oce_shutdown), 205 206 DEVMETHOD_END 207 }; 208 209 static driver_t oce_driver = { 210 "oce", 211 oce_dispatch, 212 sizeof(OCE_SOFTC) 213 }; 214 static devclass_t oce_devclass; 215 216 217 /* global vars */ 218 const char component_revision[32] = {"///" COMPONENT_REVISION "///"}; 219 220 /* Module capabilites and parameters */ 221 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED; 222 uint32_t oce_enable_rss = OCE_MODCAP_RSS; 223 uint32_t oce_rq_buf_size = 2048; 224 225 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled); 226 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss); 227 228 229 /* Supported devices table */ 230 static uint32_t supportedDevices[] = { 231 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2, 232 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3, 233 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3, 234 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201, 235 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF, 236 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH 237 }; 238 239 240 DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0); 241 MODULE_PNP_INFO("W32:vendor/device", pci, oce, supportedDevices, 242 nitems(supportedDevices)); 243 MODULE_DEPEND(oce, pci, 1, 1, 1); 244 MODULE_DEPEND(oce, ether, 1, 1, 1); 245 MODULE_VERSION(oce, 1); 246 247 248 POCE_SOFTC softc_head = NULL; 249 POCE_SOFTC softc_tail = NULL; 250 251 struct oce_rdma_if *oce_rdma_if = NULL; 252 253 /***************************************************************************** 254 * Driver entry points functions * 255 *****************************************************************************/ 256 257 static int 258 oce_probe(device_t dev) 259 { 260 uint16_t vendor = 0; 261 uint16_t device = 0; 262 int i = 0; 263 char str[256] = {0}; 264 POCE_SOFTC sc; 265 266 sc = device_get_softc(dev); 267 bzero(sc, sizeof(OCE_SOFTC)); 268 sc->dev = dev; 269 270 vendor = pci_get_vendor(dev); 271 device = pci_get_device(dev); 272 273 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) { 274 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) { 275 if (device == (supportedDevices[i] & 0xffff)) { 276 sprintf(str, "%s:%s", "Emulex CNA NIC function", 277 component_revision); 278 device_set_desc_copy(dev, str); 279 280 switch (device) { 281 case PCI_PRODUCT_BE2: 282 sc->flags |= OCE_FLAGS_BE2; 283 break; 284 case PCI_PRODUCT_BE3: 285 sc->flags |= OCE_FLAGS_BE3; 286 break; 287 case PCI_PRODUCT_XE201: 288 case PCI_PRODUCT_XE201_VF: 289 sc->flags |= OCE_FLAGS_XE201; 290 break; 291 case PCI_PRODUCT_SH: 292 sc->flags |= OCE_FLAGS_SH; 293 break; 294 default: 295 return ENXIO; 296 } 297 return BUS_PROBE_DEFAULT; 298 } 299 } 300 } 301 302 return ENXIO; 303 } 304 305 306 static int 307 oce_attach(device_t dev) 308 { 309 POCE_SOFTC sc; 310 int rc = 0; 311 312 sc = device_get_softc(dev); 313 314 rc = oce_hw_pci_alloc(sc); 315 if (rc) 316 return rc; 317 318 sc->tx_ring_size = OCE_TX_RING_SIZE; 319 sc->rx_ring_size = OCE_RX_RING_SIZE; 320 /* receive fragment size should be multiple of 2K */ 321 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048); 322 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL; 323 sc->promisc = OCE_DEFAULT_PROMISCUOUS; 324 325 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock"); 326 LOCK_CREATE(&sc->dev_lock, "Device_lock"); 327 328 /* initialise the hardware */ 329 rc = oce_hw_init(sc); 330 if (rc) 331 goto pci_res_free; 332 333 oce_read_env_variables(sc); 334 335 oce_get_config(sc); 336 337 setup_max_queues_want(sc); 338 339 rc = oce_setup_intr(sc); 340 if (rc) 341 goto mbox_free; 342 343 rc = oce_queue_init_all(sc); 344 if (rc) 345 goto intr_free; 346 347 rc = oce_attach_ifp(sc); 348 if (rc) 349 goto queues_free; 350 351 #if defined(INET6) || defined(INET) 352 rc = oce_init_lro(sc); 353 if (rc) 354 goto ifp_free; 355 #endif 356 357 rc = oce_hw_start(sc); 358 if (rc) 359 goto lro_free; 360 361 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 362 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST); 363 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 364 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST); 365 366 rc = oce_stats_init(sc); 367 if (rc) 368 goto vlan_free; 369 370 oce_add_sysctls(sc); 371 372 callout_init(&sc->timer, CALLOUT_MPSAFE); 373 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc); 374 if (rc) 375 goto stats_free; 376 377 sc->next =NULL; 378 if (softc_tail != NULL) { 379 softc_tail->next = sc; 380 } else { 381 softc_head = sc; 382 } 383 softc_tail = sc; 384 385 return 0; 386 387 stats_free: 388 callout_drain(&sc->timer); 389 oce_stats_free(sc); 390 vlan_free: 391 if (sc->vlan_attach) 392 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 393 if (sc->vlan_detach) 394 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 395 oce_hw_intr_disable(sc); 396 lro_free: 397 #if defined(INET6) || defined(INET) 398 oce_free_lro(sc); 399 ifp_free: 400 #endif 401 ether_ifdetach(sc->ifp); 402 if_free(sc->ifp); 403 queues_free: 404 oce_queue_release_all(sc); 405 intr_free: 406 oce_intr_free(sc); 407 mbox_free: 408 oce_dma_free(sc, &sc->bsmbx); 409 pci_res_free: 410 oce_hw_pci_free(sc); 411 LOCK_DESTROY(&sc->dev_lock); 412 LOCK_DESTROY(&sc->bmbx_lock); 413 return rc; 414 415 } 416 417 418 static int 419 oce_detach(device_t dev) 420 { 421 POCE_SOFTC sc = device_get_softc(dev); 422 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL; 423 424 poce_sc_tmp = softc_head; 425 ppoce_sc_tmp1 = &softc_head; 426 while (poce_sc_tmp != NULL) { 427 if (poce_sc_tmp == sc) { 428 *ppoce_sc_tmp1 = sc->next; 429 if (sc->next == NULL) { 430 softc_tail = poce_sc_tmp2; 431 } 432 break; 433 } 434 poce_sc_tmp2 = poce_sc_tmp; 435 ppoce_sc_tmp1 = &poce_sc_tmp->next; 436 poce_sc_tmp = poce_sc_tmp->next; 437 } 438 439 LOCK(&sc->dev_lock); 440 oce_if_deactivate(sc); 441 UNLOCK(&sc->dev_lock); 442 443 callout_drain(&sc->timer); 444 445 if (sc->vlan_attach != NULL) 446 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 447 if (sc->vlan_detach != NULL) 448 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 449 450 ether_ifdetach(sc->ifp); 451 452 if_free(sc->ifp); 453 454 oce_hw_shutdown(sc); 455 456 bus_generic_detach(dev); 457 458 return 0; 459 } 460 461 462 static int 463 oce_shutdown(device_t dev) 464 { 465 int rc; 466 467 rc = oce_detach(dev); 468 469 return rc; 470 } 471 472 473 static int 474 oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 475 { 476 struct ifreq *ifr = (struct ifreq *)data; 477 POCE_SOFTC sc = ifp->if_softc; 478 int rc = 0; 479 uint32_t u; 480 481 switch (command) { 482 483 case SIOCGIFMEDIA: 484 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command); 485 break; 486 487 case SIOCSIFMTU: 488 if (ifr->ifr_mtu > OCE_MAX_MTU) 489 rc = EINVAL; 490 else 491 ifp->if_mtu = ifr->ifr_mtu; 492 break; 493 494 case SIOCSIFFLAGS: 495 if (ifp->if_flags & IFF_UP) { 496 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 497 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 498 oce_init(sc); 499 } 500 device_printf(sc->dev, "Interface Up\n"); 501 } else { 502 LOCK(&sc->dev_lock); 503 504 sc->ifp->if_drv_flags &= 505 ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 506 oce_if_deactivate(sc); 507 508 UNLOCK(&sc->dev_lock); 509 510 device_printf(sc->dev, "Interface Down\n"); 511 } 512 513 if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) { 514 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1)))) 515 sc->promisc = TRUE; 516 } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) { 517 if (!oce_rxf_set_promiscuous(sc, 0)) 518 sc->promisc = FALSE; 519 } 520 521 break; 522 523 case SIOCADDMULTI: 524 case SIOCDELMULTI: 525 rc = oce_hw_update_multicast(sc); 526 if (rc) 527 device_printf(sc->dev, 528 "Update multicast address failed\n"); 529 break; 530 531 case SIOCSIFCAP: 532 u = ifr->ifr_reqcap ^ ifp->if_capenable; 533 534 if (u & IFCAP_TXCSUM) { 535 ifp->if_capenable ^= IFCAP_TXCSUM; 536 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 537 538 if (IFCAP_TSO & ifp->if_capenable && 539 !(IFCAP_TXCSUM & ifp->if_capenable)) { 540 ifp->if_capenable &= ~IFCAP_TSO; 541 ifp->if_hwassist &= ~CSUM_TSO; 542 if_printf(ifp, 543 "TSO disabled due to -txcsum.\n"); 544 } 545 } 546 547 if (u & IFCAP_RXCSUM) 548 ifp->if_capenable ^= IFCAP_RXCSUM; 549 550 if (u & IFCAP_TSO4) { 551 ifp->if_capenable ^= IFCAP_TSO4; 552 553 if (IFCAP_TSO & ifp->if_capenable) { 554 if (IFCAP_TXCSUM & ifp->if_capenable) 555 ifp->if_hwassist |= CSUM_TSO; 556 else { 557 ifp->if_capenable &= ~IFCAP_TSO; 558 ifp->if_hwassist &= ~CSUM_TSO; 559 if_printf(ifp, 560 "Enable txcsum first.\n"); 561 rc = EAGAIN; 562 } 563 } else 564 ifp->if_hwassist &= ~CSUM_TSO; 565 } 566 567 if (u & IFCAP_VLAN_HWTAGGING) 568 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 569 570 if (u & IFCAP_VLAN_HWFILTER) { 571 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 572 oce_vid_config(sc); 573 } 574 #if defined(INET6) || defined(INET) 575 if (u & IFCAP_LRO) { 576 ifp->if_capenable ^= IFCAP_LRO; 577 if(sc->enable_hwlro) { 578 if(ifp->if_capenable & IFCAP_LRO) { 579 rc = oce_mbox_nic_set_iface_lro_config(sc, 1); 580 }else { 581 rc = oce_mbox_nic_set_iface_lro_config(sc, 0); 582 } 583 } 584 } 585 #endif 586 587 break; 588 589 case SIOCGPRIVATE_0: 590 rc = oce_handle_passthrough(ifp, data); 591 break; 592 default: 593 rc = ether_ioctl(ifp, command, data); 594 break; 595 } 596 597 return rc; 598 } 599 600 601 static void 602 oce_init(void *arg) 603 { 604 POCE_SOFTC sc = arg; 605 606 LOCK(&sc->dev_lock); 607 608 if (sc->ifp->if_flags & IFF_UP) { 609 oce_if_deactivate(sc); 610 oce_if_activate(sc); 611 } 612 613 UNLOCK(&sc->dev_lock); 614 615 } 616 617 618 static int 619 oce_multiq_start(struct ifnet *ifp, struct mbuf *m) 620 { 621 POCE_SOFTC sc = ifp->if_softc; 622 struct oce_wq *wq = NULL; 623 int queue_index = 0; 624 int status = 0; 625 626 if (!sc->link_status) 627 return ENXIO; 628 629 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 630 queue_index = m->m_pkthdr.flowid % sc->nwqs; 631 632 wq = sc->wq[queue_index]; 633 634 LOCK(&wq->tx_lock); 635 status = oce_multiq_transmit(ifp, m, wq); 636 UNLOCK(&wq->tx_lock); 637 638 return status; 639 640 } 641 642 643 static void 644 oce_multiq_flush(struct ifnet *ifp) 645 { 646 POCE_SOFTC sc = ifp->if_softc; 647 struct mbuf *m; 648 int i = 0; 649 650 for (i = 0; i < sc->nwqs; i++) { 651 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL) 652 m_freem(m); 653 } 654 if_qflush(ifp); 655 } 656 657 658 659 /***************************************************************************** 660 * Driver interrupt routines functions * 661 *****************************************************************************/ 662 663 static void 664 oce_intr(void *arg, int pending) 665 { 666 667 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 668 POCE_SOFTC sc = ii->sc; 669 struct oce_eq *eq = ii->eq; 670 struct oce_eqe *eqe; 671 struct oce_cq *cq = NULL; 672 int i, num_eqes = 0; 673 674 675 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 676 BUS_DMASYNC_POSTWRITE); 677 do { 678 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe); 679 if (eqe->evnt == 0) 680 break; 681 eqe->evnt = 0; 682 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 683 BUS_DMASYNC_POSTWRITE); 684 RING_GET(eq->ring, 1); 685 num_eqes++; 686 687 } while (TRUE); 688 689 if (!num_eqes) 690 goto eq_arm; /* Spurious */ 691 692 /* Clear EQ entries, but dont arm */ 693 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE); 694 695 /* Process TX, RX and MCC. But dont arm CQ*/ 696 for (i = 0; i < eq->cq_valid; i++) { 697 cq = eq->cq[i]; 698 (*cq->cq_handler)(cq->cb_arg); 699 } 700 701 /* Arm all cqs connected to this EQ */ 702 for (i = 0; i < eq->cq_valid; i++) { 703 cq = eq->cq[i]; 704 oce_arm_cq(sc, cq->cq_id, 0, TRUE); 705 } 706 707 eq_arm: 708 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 709 710 return; 711 } 712 713 714 static int 715 oce_setup_intr(POCE_SOFTC sc) 716 { 717 int rc = 0, use_intx = 0; 718 int vector = 0, req_vectors = 0; 719 int tot_req_vectors, tot_vectors; 720 721 if (is_rss_enabled(sc)) 722 req_vectors = MAX((sc->nrqs - 1), sc->nwqs); 723 else 724 req_vectors = 1; 725 726 tot_req_vectors = req_vectors; 727 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 728 if (req_vectors > 1) { 729 tot_req_vectors += OCE_RDMA_VECTORS; 730 sc->roce_intr_count = OCE_RDMA_VECTORS; 731 } 732 } 733 734 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) { 735 sc->intr_count = req_vectors; 736 tot_vectors = tot_req_vectors; 737 rc = pci_alloc_msix(sc->dev, &tot_vectors); 738 if (rc != 0) { 739 use_intx = 1; 740 pci_release_msi(sc->dev); 741 } else { 742 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 743 if (tot_vectors < tot_req_vectors) { 744 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) { 745 sc->roce_intr_count = (tot_vectors / 2); 746 } 747 sc->intr_count = tot_vectors - sc->roce_intr_count; 748 } 749 } else { 750 sc->intr_count = tot_vectors; 751 } 752 sc->flags |= OCE_FLAGS_USING_MSIX; 753 } 754 } else 755 use_intx = 1; 756 757 if (use_intx) 758 sc->intr_count = 1; 759 760 /* Scale number of queues based on intr we got */ 761 update_queues_got(sc); 762 763 if (use_intx) { 764 device_printf(sc->dev, "Using legacy interrupt\n"); 765 rc = oce_alloc_intr(sc, vector, oce_intr); 766 if (rc) 767 goto error; 768 } else { 769 for (; vector < sc->intr_count; vector++) { 770 rc = oce_alloc_intr(sc, vector, oce_intr); 771 if (rc) 772 goto error; 773 } 774 } 775 776 return 0; 777 error: 778 oce_intr_free(sc); 779 return rc; 780 } 781 782 783 static int 784 oce_fast_isr(void *arg) 785 { 786 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 787 POCE_SOFTC sc = ii->sc; 788 789 if (ii->eq == NULL) 790 return FILTER_STRAY; 791 792 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE); 793 794 taskqueue_enqueue(ii->tq, &ii->task); 795 796 ii->eq->intr++; 797 798 return FILTER_HANDLED; 799 } 800 801 802 static int 803 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending)) 804 { 805 POCE_INTR_INFO ii = &sc->intrs[vector]; 806 int rc = 0, rr; 807 808 if (vector >= OCE_MAX_EQ) 809 return (EINVAL); 810 811 /* Set the resource id for the interrupt. 812 * MSIx is vector + 1 for the resource id, 813 * INTx is 0 for the resource id. 814 */ 815 if (sc->flags & OCE_FLAGS_USING_MSIX) 816 rr = vector + 1; 817 else 818 rr = 0; 819 ii->intr_res = bus_alloc_resource_any(sc->dev, 820 SYS_RES_IRQ, 821 &rr, RF_ACTIVE|RF_SHAREABLE); 822 ii->irq_rr = rr; 823 if (ii->intr_res == NULL) { 824 device_printf(sc->dev, 825 "Could not allocate interrupt\n"); 826 rc = ENXIO; 827 return rc; 828 } 829 830 TASK_INIT(&ii->task, 0, isr, ii); 831 ii->vector = vector; 832 sprintf(ii->task_name, "oce_task[%d]", ii->vector); 833 ii->tq = taskqueue_create_fast(ii->task_name, 834 M_NOWAIT, 835 taskqueue_thread_enqueue, 836 &ii->tq); 837 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq", 838 device_get_nameunit(sc->dev)); 839 840 ii->sc = sc; 841 rc = bus_setup_intr(sc->dev, 842 ii->intr_res, 843 INTR_TYPE_NET, 844 oce_fast_isr, NULL, ii, &ii->tag); 845 return rc; 846 847 } 848 849 850 void 851 oce_intr_free(POCE_SOFTC sc) 852 { 853 int i = 0; 854 855 for (i = 0; i < sc->intr_count; i++) { 856 857 if (sc->intrs[i].tag != NULL) 858 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res, 859 sc->intrs[i].tag); 860 if (sc->intrs[i].tq != NULL) 861 taskqueue_free(sc->intrs[i].tq); 862 863 if (sc->intrs[i].intr_res != NULL) 864 bus_release_resource(sc->dev, SYS_RES_IRQ, 865 sc->intrs[i].irq_rr, 866 sc->intrs[i].intr_res); 867 sc->intrs[i].tag = NULL; 868 sc->intrs[i].intr_res = NULL; 869 } 870 871 if (sc->flags & OCE_FLAGS_USING_MSIX) 872 pci_release_msi(sc->dev); 873 874 } 875 876 877 878 /****************************************************************************** 879 * Media callbacks functions * 880 ******************************************************************************/ 881 882 static void 883 oce_media_status(struct ifnet *ifp, struct ifmediareq *req) 884 { 885 POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc; 886 887 888 req->ifm_status = IFM_AVALID; 889 req->ifm_active = IFM_ETHER; 890 891 if (sc->link_status == 1) 892 req->ifm_status |= IFM_ACTIVE; 893 else 894 return; 895 896 switch (sc->link_speed) { 897 case 1: /* 10 Mbps */ 898 req->ifm_active |= IFM_10_T | IFM_FDX; 899 sc->speed = 10; 900 break; 901 case 2: /* 100 Mbps */ 902 req->ifm_active |= IFM_100_TX | IFM_FDX; 903 sc->speed = 100; 904 break; 905 case 3: /* 1 Gbps */ 906 req->ifm_active |= IFM_1000_T | IFM_FDX; 907 sc->speed = 1000; 908 break; 909 case 4: /* 10 Gbps */ 910 req->ifm_active |= IFM_10G_SR | IFM_FDX; 911 sc->speed = 10000; 912 break; 913 case 5: /* 20 Gbps */ 914 req->ifm_active |= IFM_10G_SR | IFM_FDX; 915 sc->speed = 20000; 916 break; 917 case 6: /* 25 Gbps */ 918 req->ifm_active |= IFM_10G_SR | IFM_FDX; 919 sc->speed = 25000; 920 break; 921 case 7: /* 40 Gbps */ 922 req->ifm_active |= IFM_40G_SR4 | IFM_FDX; 923 sc->speed = 40000; 924 break; 925 default: 926 sc->speed = 0; 927 break; 928 } 929 930 return; 931 } 932 933 934 int 935 oce_media_change(struct ifnet *ifp) 936 { 937 return 0; 938 } 939 940 941 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc, 942 struct mbuf *m, boolean_t *os2bmc, 943 struct mbuf **m_new) 944 { 945 struct ether_header *eh = NULL; 946 947 eh = mtod(m, struct ether_header *); 948 949 if (!is_os2bmc_enabled(sc) || *os2bmc) { 950 *os2bmc = FALSE; 951 goto done; 952 } 953 if (!ETHER_IS_MULTICAST(eh->ether_dhost)) 954 goto done; 955 956 if (is_mc_allowed_on_bmc(sc, eh) || 957 is_bc_allowed_on_bmc(sc, eh) || 958 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) { 959 *os2bmc = TRUE; 960 goto done; 961 } 962 963 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) { 964 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 965 uint8_t nexthdr = ip6->ip6_nxt; 966 if (nexthdr == IPPROTO_ICMPV6) { 967 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1); 968 switch (icmp6->icmp6_type) { 969 case ND_ROUTER_ADVERT: 970 *os2bmc = is_ipv6_ra_filt_enabled(sc); 971 goto done; 972 case ND_NEIGHBOR_ADVERT: 973 *os2bmc = is_ipv6_na_filt_enabled(sc); 974 goto done; 975 default: 976 break; 977 } 978 } 979 } 980 981 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) { 982 struct ip *ip = mtod(m, struct ip *); 983 int iphlen = ip->ip_hl << 2; 984 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen); 985 switch (uh->uh_dport) { 986 case DHCP_CLIENT_PORT: 987 *os2bmc = is_dhcp_client_filt_enabled(sc); 988 goto done; 989 case DHCP_SERVER_PORT: 990 *os2bmc = is_dhcp_srvr_filt_enabled(sc); 991 goto done; 992 case NET_BIOS_PORT1: 993 case NET_BIOS_PORT2: 994 *os2bmc = is_nbios_filt_enabled(sc); 995 goto done; 996 case DHCPV6_RAS_PORT: 997 *os2bmc = is_ipv6_ras_filt_enabled(sc); 998 goto done; 999 default: 1000 break; 1001 } 1002 } 1003 done: 1004 if (*os2bmc) { 1005 *m_new = m_dup(m, M_NOWAIT); 1006 if (!*m_new) { 1007 *os2bmc = FALSE; 1008 return; 1009 } 1010 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL); 1011 } 1012 } 1013 1014 1015 1016 /***************************************************************************** 1017 * Transmit routines functions * 1018 *****************************************************************************/ 1019 1020 static int 1021 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index) 1022 { 1023 int rc = 0, i, retry_cnt = 0; 1024 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS]; 1025 struct mbuf *m, *m_temp, *m_new = NULL; 1026 struct oce_wq *wq = sc->wq[wq_index]; 1027 struct oce_packet_desc *pd; 1028 struct oce_nic_hdr_wqe *nichdr; 1029 struct oce_nic_frag_wqe *nicfrag; 1030 struct ether_header *eh = NULL; 1031 int num_wqes; 1032 uint32_t reg_value; 1033 boolean_t complete = TRUE; 1034 boolean_t os2bmc = FALSE; 1035 1036 m = *mpp; 1037 if (!m) 1038 return EINVAL; 1039 1040 if (!(m->m_flags & M_PKTHDR)) { 1041 rc = ENXIO; 1042 goto free_ret; 1043 } 1044 1045 /* Don't allow non-TSO packets longer than MTU */ 1046 if (!is_tso_pkt(m)) { 1047 eh = mtod(m, struct ether_header *); 1048 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE)) 1049 goto free_ret; 1050 } 1051 1052 if(oce_tx_asic_stall_verify(sc, m)) { 1053 m = oce_insert_vlan_tag(sc, m, &complete); 1054 if(!m) { 1055 device_printf(sc->dev, "Insertion unsuccessful\n"); 1056 return 0; 1057 } 1058 1059 } 1060 1061 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less 1062 * may cause a transmit stall on that port. So the work-around is to 1063 * pad short packets (<= 32 bytes) to a 36-byte length. 1064 */ 1065 if(IS_SH(sc) || IS_XE201(sc) ) { 1066 if(m->m_pkthdr.len <= 32) { 1067 char buf[36]; 1068 bzero((void *)buf, 36); 1069 m_append(m, (36 - m->m_pkthdr.len), buf); 1070 } 1071 } 1072 1073 tx_start: 1074 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1075 /* consolidate packet buffers for TSO/LSO segment offload */ 1076 #if defined(INET6) || defined(INET) 1077 m = oce_tso_setup(sc, mpp); 1078 #else 1079 m = NULL; 1080 #endif 1081 if (m == NULL) { 1082 rc = ENXIO; 1083 goto free_ret; 1084 } 1085 } 1086 1087 1088 pd = &wq->pckts[wq->pkt_desc_head]; 1089 1090 retry: 1091 rc = bus_dmamap_load_mbuf_sg(wq->tag, 1092 pd->map, 1093 m, segs, &pd->nsegs, BUS_DMA_NOWAIT); 1094 if (rc == 0) { 1095 num_wqes = pd->nsegs + 1; 1096 if (IS_BE(sc) || IS_SH(sc)) { 1097 /*Dummy required only for BE3.*/ 1098 if (num_wqes & 1) 1099 num_wqes++; 1100 } 1101 if (num_wqes >= RING_NUM_FREE(wq->ring)) { 1102 bus_dmamap_unload(wq->tag, pd->map); 1103 return EBUSY; 1104 } 1105 atomic_store_rel_int(&wq->pkt_desc_head, 1106 (wq->pkt_desc_head + 1) % \ 1107 OCE_WQ_PACKET_ARRAY_SIZE); 1108 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE); 1109 pd->mbuf = m; 1110 1111 nichdr = 1112 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe); 1113 nichdr->u0.dw[0] = 0; 1114 nichdr->u0.dw[1] = 0; 1115 nichdr->u0.dw[2] = 0; 1116 nichdr->u0.dw[3] = 0; 1117 1118 nichdr->u0.s.complete = complete; 1119 nichdr->u0.s.mgmt = os2bmc; 1120 nichdr->u0.s.event = 1; 1121 nichdr->u0.s.crc = 1; 1122 nichdr->u0.s.forward = 0; 1123 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0; 1124 nichdr->u0.s.udpcs = 1125 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0; 1126 nichdr->u0.s.tcpcs = 1127 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0; 1128 nichdr->u0.s.num_wqe = num_wqes; 1129 nichdr->u0.s.total_length = m->m_pkthdr.len; 1130 1131 if (m->m_flags & M_VLANTAG) { 1132 nichdr->u0.s.vlan = 1; /*Vlan present*/ 1133 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag; 1134 } 1135 1136 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1137 if (m->m_pkthdr.tso_segsz) { 1138 nichdr->u0.s.lso = 1; 1139 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz; 1140 } 1141 if (!IS_BE(sc) || !IS_SH(sc)) 1142 nichdr->u0.s.ipcs = 1; 1143 } 1144 1145 RING_PUT(wq->ring, 1); 1146 atomic_add_int(&wq->ring->num_used, 1); 1147 1148 for (i = 0; i < pd->nsegs; i++) { 1149 nicfrag = 1150 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1151 struct oce_nic_frag_wqe); 1152 nicfrag->u0.s.rsvd0 = 0; 1153 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr); 1154 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr); 1155 nicfrag->u0.s.frag_len = segs[i].ds_len; 1156 pd->wqe_idx = wq->ring->pidx; 1157 RING_PUT(wq->ring, 1); 1158 atomic_add_int(&wq->ring->num_used, 1); 1159 } 1160 if (num_wqes > (pd->nsegs + 1)) { 1161 nicfrag = 1162 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1163 struct oce_nic_frag_wqe); 1164 nicfrag->u0.dw[0] = 0; 1165 nicfrag->u0.dw[1] = 0; 1166 nicfrag->u0.dw[2] = 0; 1167 nicfrag->u0.dw[3] = 0; 1168 pd->wqe_idx = wq->ring->pidx; 1169 RING_PUT(wq->ring, 1); 1170 atomic_add_int(&wq->ring->num_used, 1); 1171 pd->nsegs++; 1172 } 1173 1174 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 1175 wq->tx_stats.tx_reqs++; 1176 wq->tx_stats.tx_wrbs += num_wqes; 1177 wq->tx_stats.tx_bytes += m->m_pkthdr.len; 1178 wq->tx_stats.tx_pkts++; 1179 1180 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map, 1181 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1182 reg_value = (num_wqes << 16) | wq->wq_id; 1183 1184 /* if os2bmc is not enabled or if the pkt is already tagged as 1185 bmc, do nothing 1186 */ 1187 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new); 1188 1189 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value); 1190 1191 } else if (rc == EFBIG) { 1192 if (retry_cnt == 0) { 1193 m_temp = m_defrag(m, M_NOWAIT); 1194 if (m_temp == NULL) 1195 goto free_ret; 1196 m = m_temp; 1197 *mpp = m_temp; 1198 retry_cnt = retry_cnt + 1; 1199 goto retry; 1200 } else 1201 goto free_ret; 1202 } else if (rc == ENOMEM) 1203 return rc; 1204 else 1205 goto free_ret; 1206 1207 if (os2bmc) { 1208 m = m_new; 1209 goto tx_start; 1210 } 1211 1212 return 0; 1213 1214 free_ret: 1215 m_freem(*mpp); 1216 *mpp = NULL; 1217 return rc; 1218 } 1219 1220 1221 static void 1222 oce_process_tx_completion(struct oce_wq *wq) 1223 { 1224 struct oce_packet_desc *pd; 1225 POCE_SOFTC sc = (POCE_SOFTC) wq->parent; 1226 struct mbuf *m; 1227 1228 pd = &wq->pckts[wq->pkt_desc_tail]; 1229 atomic_store_rel_int(&wq->pkt_desc_tail, 1230 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE); 1231 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1); 1232 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1233 bus_dmamap_unload(wq->tag, pd->map); 1234 1235 m = pd->mbuf; 1236 m_freem(m); 1237 pd->mbuf = NULL; 1238 1239 1240 if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) { 1241 if (wq->ring->num_used < (wq->ring->num_items / 2)) { 1242 sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE); 1243 oce_tx_restart(sc, wq); 1244 } 1245 } 1246 } 1247 1248 1249 static void 1250 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq) 1251 { 1252 1253 if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) 1254 return; 1255 1256 #if __FreeBSD_version >= 800000 1257 if (!drbr_empty(sc->ifp, wq->br)) 1258 #else 1259 if (!IFQ_DRV_IS_EMPTY(&sc->ifp->if_snd)) 1260 #endif 1261 taskqueue_enqueue(taskqueue_swi, &wq->txtask); 1262 1263 } 1264 1265 1266 #if defined(INET6) || defined(INET) 1267 static struct mbuf * 1268 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp) 1269 { 1270 struct mbuf *m; 1271 #ifdef INET 1272 struct ip *ip; 1273 #endif 1274 #ifdef INET6 1275 struct ip6_hdr *ip6; 1276 #endif 1277 struct ether_vlan_header *eh; 1278 struct tcphdr *th; 1279 uint16_t etype; 1280 int total_len = 0, ehdrlen = 0; 1281 1282 m = *mpp; 1283 1284 if (M_WRITABLE(m) == 0) { 1285 m = m_dup(*mpp, M_NOWAIT); 1286 if (!m) 1287 return NULL; 1288 m_freem(*mpp); 1289 *mpp = m; 1290 } 1291 1292 eh = mtod(m, struct ether_vlan_header *); 1293 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1294 etype = ntohs(eh->evl_proto); 1295 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1296 } else { 1297 etype = ntohs(eh->evl_encap_proto); 1298 ehdrlen = ETHER_HDR_LEN; 1299 } 1300 1301 switch (etype) { 1302 #ifdef INET 1303 case ETHERTYPE_IP: 1304 ip = (struct ip *)(m->m_data + ehdrlen); 1305 if (ip->ip_p != IPPROTO_TCP) 1306 return NULL; 1307 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); 1308 1309 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2); 1310 break; 1311 #endif 1312 #ifdef INET6 1313 case ETHERTYPE_IPV6: 1314 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen); 1315 if (ip6->ip6_nxt != IPPROTO_TCP) 1316 return NULL; 1317 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr)); 1318 1319 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2); 1320 break; 1321 #endif 1322 default: 1323 return NULL; 1324 } 1325 1326 m = m_pullup(m, total_len); 1327 if (!m) 1328 return NULL; 1329 *mpp = m; 1330 return m; 1331 1332 } 1333 #endif /* INET6 || INET */ 1334 1335 void 1336 oce_tx_task(void *arg, int npending) 1337 { 1338 struct oce_wq *wq = arg; 1339 POCE_SOFTC sc = wq->parent; 1340 struct ifnet *ifp = sc->ifp; 1341 int rc = 0; 1342 1343 #if __FreeBSD_version >= 800000 1344 LOCK(&wq->tx_lock); 1345 rc = oce_multiq_transmit(ifp, NULL, wq); 1346 if (rc) { 1347 device_printf(sc->dev, 1348 "TX[%d] restart failed\n", wq->queue_index); 1349 } 1350 UNLOCK(&wq->tx_lock); 1351 #else 1352 oce_start(ifp); 1353 #endif 1354 1355 } 1356 1357 1358 void 1359 oce_start(struct ifnet *ifp) 1360 { 1361 POCE_SOFTC sc = ifp->if_softc; 1362 struct mbuf *m; 1363 int rc = 0; 1364 int def_q = 0; /* Defualt tx queue is 0*/ 1365 1366 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1367 IFF_DRV_RUNNING) 1368 return; 1369 1370 if (!sc->link_status) 1371 return; 1372 1373 do { 1374 IF_DEQUEUE(&sc->ifp->if_snd, m); 1375 if (m == NULL) 1376 break; 1377 1378 LOCK(&sc->wq[def_q]->tx_lock); 1379 rc = oce_tx(sc, &m, def_q); 1380 UNLOCK(&sc->wq[def_q]->tx_lock); 1381 if (rc) { 1382 if (m != NULL) { 1383 sc->wq[def_q]->tx_stats.tx_stops ++; 1384 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1385 IFQ_DRV_PREPEND(&ifp->if_snd, m); 1386 m = NULL; 1387 } 1388 break; 1389 } 1390 if (m != NULL) 1391 ETHER_BPF_MTAP(ifp, m); 1392 1393 } while (TRUE); 1394 1395 return; 1396 } 1397 1398 1399 /* Handle the Completion Queue for transmit */ 1400 uint16_t 1401 oce_wq_handler(void *arg) 1402 { 1403 struct oce_wq *wq = (struct oce_wq *)arg; 1404 POCE_SOFTC sc = wq->parent; 1405 struct oce_cq *cq = wq->cq; 1406 struct oce_nic_tx_cqe *cqe; 1407 int num_cqes = 0; 1408 1409 LOCK(&wq->tx_compl_lock); 1410 bus_dmamap_sync(cq->ring->dma.tag, 1411 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1412 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1413 while (cqe->u0.dw[3]) { 1414 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe)); 1415 1416 wq->ring->cidx = cqe->u0.s.wqe_index + 1; 1417 if (wq->ring->cidx >= wq->ring->num_items) 1418 wq->ring->cidx -= wq->ring->num_items; 1419 1420 oce_process_tx_completion(wq); 1421 wq->tx_stats.tx_compl++; 1422 cqe->u0.dw[3] = 0; 1423 RING_GET(cq->ring, 1); 1424 bus_dmamap_sync(cq->ring->dma.tag, 1425 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1426 cqe = 1427 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1428 num_cqes++; 1429 } 1430 1431 if (num_cqes) 1432 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 1433 1434 UNLOCK(&wq->tx_compl_lock); 1435 return num_cqes; 1436 } 1437 1438 1439 static int 1440 oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq) 1441 { 1442 POCE_SOFTC sc = ifp->if_softc; 1443 int status = 0, queue_index = 0; 1444 struct mbuf *next = NULL; 1445 struct buf_ring *br = NULL; 1446 1447 br = wq->br; 1448 queue_index = wq->queue_index; 1449 1450 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1451 IFF_DRV_RUNNING) { 1452 if (m != NULL) 1453 status = drbr_enqueue(ifp, br, m); 1454 return status; 1455 } 1456 1457 if (m != NULL) { 1458 if ((status = drbr_enqueue(ifp, br, m)) != 0) 1459 return status; 1460 } 1461 while ((next = drbr_peek(ifp, br)) != NULL) { 1462 if (oce_tx(sc, &next, queue_index)) { 1463 if (next == NULL) { 1464 drbr_advance(ifp, br); 1465 } else { 1466 drbr_putback(ifp, br, next); 1467 wq->tx_stats.tx_stops ++; 1468 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1469 } 1470 break; 1471 } 1472 drbr_advance(ifp, br); 1473 if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len); 1474 if (next->m_flags & M_MCAST) 1475 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 1476 ETHER_BPF_MTAP(ifp, next); 1477 } 1478 1479 return 0; 1480 } 1481 1482 1483 1484 1485 /***************************************************************************** 1486 * Receive routines functions * 1487 *****************************************************************************/ 1488 1489 static void 1490 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2) 1491 { 1492 uint32_t *p; 1493 struct ether_header *eh = NULL; 1494 struct tcphdr *tcp_hdr = NULL; 1495 struct ip *ip4_hdr = NULL; 1496 struct ip6_hdr *ip6 = NULL; 1497 uint32_t payload_len = 0; 1498 1499 eh = mtod(m, struct ether_header *); 1500 /* correct IP header */ 1501 if(!cqe2->ipv6_frame) { 1502 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header)); 1503 ip4_hdr->ip_ttl = cqe2->frame_lifespan; 1504 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header)); 1505 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip)); 1506 }else { 1507 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header)); 1508 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan; 1509 payload_len = cqe2->coalesced_size - sizeof(struct ether_header) 1510 - sizeof(struct ip6_hdr); 1511 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len); 1512 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr)); 1513 } 1514 1515 /* correct tcp header */ 1516 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num); 1517 if(cqe2->push) { 1518 tcp_hdr->th_flags |= TH_PUSH; 1519 } 1520 tcp_hdr->th_win = htons(cqe2->tcp_window); 1521 tcp_hdr->th_sum = 0xffff; 1522 if(cqe2->ts_opt) { 1523 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2); 1524 *p = cqe1->tcp_timestamp_val; 1525 *(p+1) = cqe1->tcp_timestamp_ecr; 1526 } 1527 1528 return; 1529 } 1530 1531 static void 1532 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m) 1533 { 1534 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1535 uint32_t i = 0, frag_len = 0; 1536 uint32_t len = cqe_info->pkt_size; 1537 struct oce_packet_desc *pd; 1538 struct mbuf *tail = NULL; 1539 1540 for (i = 0; i < cqe_info->num_frags; i++) { 1541 if (rq->ring->cidx == rq->ring->pidx) { 1542 device_printf(sc->dev, 1543 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n"); 1544 return; 1545 } 1546 pd = &rq->pckts[rq->ring->cidx]; 1547 1548 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1549 bus_dmamap_unload(rq->tag, pd->map); 1550 RING_GET(rq->ring, 1); 1551 rq->pending--; 1552 1553 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len; 1554 pd->mbuf->m_len = frag_len; 1555 1556 if (tail != NULL) { 1557 /* additional fragments */ 1558 pd->mbuf->m_flags &= ~M_PKTHDR; 1559 tail->m_next = pd->mbuf; 1560 if(rq->islro) 1561 tail->m_nextpkt = NULL; 1562 tail = pd->mbuf; 1563 } else { 1564 /* first fragment, fill out much of the packet header */ 1565 pd->mbuf->m_pkthdr.len = len; 1566 if(rq->islro) 1567 pd->mbuf->m_nextpkt = NULL; 1568 pd->mbuf->m_pkthdr.csum_flags = 0; 1569 if (IF_CSUM_ENABLED(sc)) { 1570 if (cqe_info->l4_cksum_pass) { 1571 if(!cqe_info->ipv6_frame) { /* IPV4 */ 1572 pd->mbuf->m_pkthdr.csum_flags |= 1573 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1574 }else { /* IPV6 frame */ 1575 if(rq->islro) { 1576 pd->mbuf->m_pkthdr.csum_flags |= 1577 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1578 } 1579 } 1580 pd->mbuf->m_pkthdr.csum_data = 0xffff; 1581 } 1582 if (cqe_info->ip_cksum_pass) { 1583 pd->mbuf->m_pkthdr.csum_flags |= 1584 (CSUM_IP_CHECKED|CSUM_IP_VALID); 1585 } 1586 } 1587 *m = tail = pd->mbuf; 1588 } 1589 pd->mbuf = NULL; 1590 len -= frag_len; 1591 } 1592 1593 return; 1594 } 1595 1596 static void 1597 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2) 1598 { 1599 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1600 struct nic_hwlro_cqe_part1 *cqe1 = NULL; 1601 struct mbuf *m = NULL; 1602 struct oce_common_cqe_info cq_info; 1603 1604 /* parse cqe */ 1605 if(cqe2 == NULL) { 1606 cq_info.pkt_size = cqe->pkt_size; 1607 cq_info.vtag = cqe->vlan_tag; 1608 cq_info.l4_cksum_pass = cqe->l4_cksum_pass; 1609 cq_info.ip_cksum_pass = cqe->ip_cksum_pass; 1610 cq_info.ipv6_frame = cqe->ipv6_frame; 1611 cq_info.vtp = cqe->vtp; 1612 cq_info.qnq = cqe->qnq; 1613 }else { 1614 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe; 1615 cq_info.pkt_size = cqe2->coalesced_size; 1616 cq_info.vtag = cqe2->vlan_tag; 1617 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass; 1618 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass; 1619 cq_info.ipv6_frame = cqe2->ipv6_frame; 1620 cq_info.vtp = cqe2->vtp; 1621 cq_info.qnq = cqe1->qnq; 1622 } 1623 1624 cq_info.vtag = BSWAP_16(cq_info.vtag); 1625 1626 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size; 1627 if(cq_info.pkt_size % rq->cfg.frag_size) 1628 cq_info.num_frags++; 1629 1630 oce_rx_mbuf_chain(rq, &cq_info, &m); 1631 1632 if (m) { 1633 if(cqe2) { 1634 //assert(cqe2->valid != 0); 1635 1636 //assert(cqe2->cqe_type != 2); 1637 oce_correct_header(m, cqe1, cqe2); 1638 } 1639 1640 m->m_pkthdr.rcvif = sc->ifp; 1641 #if __FreeBSD_version >= 800000 1642 if (rq->queue_index) 1643 m->m_pkthdr.flowid = (rq->queue_index - 1); 1644 else 1645 m->m_pkthdr.flowid = rq->queue_index; 1646 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1647 #endif 1648 /* This deternies if vlan tag is Valid */ 1649 if (cq_info.vtp) { 1650 if (sc->function_mode & FNM_FLEX10_MODE) { 1651 /* FLEX10. If QnQ is not set, neglect VLAN */ 1652 if (cq_info.qnq) { 1653 m->m_pkthdr.ether_vtag = cq_info.vtag; 1654 m->m_flags |= M_VLANTAG; 1655 } 1656 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) { 1657 /* In UMC mode generally pvid will be striped by 1658 hw. But in some cases we have seen it comes 1659 with pvid. So if pvid == vlan, neglect vlan. 1660 */ 1661 m->m_pkthdr.ether_vtag = cq_info.vtag; 1662 m->m_flags |= M_VLANTAG; 1663 } 1664 } 1665 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1666 1667 (*sc->ifp->if_input) (sc->ifp, m); 1668 1669 /* Update rx stats per queue */ 1670 rq->rx_stats.rx_pkts++; 1671 rq->rx_stats.rx_bytes += cq_info.pkt_size; 1672 rq->rx_stats.rx_frags += cq_info.num_frags; 1673 rq->rx_stats.rx_ucast_pkts++; 1674 } 1675 return; 1676 } 1677 1678 static void 1679 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe) 1680 { 1681 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1682 int len; 1683 struct mbuf *m = NULL; 1684 struct oce_common_cqe_info cq_info; 1685 uint16_t vtag = 0; 1686 1687 /* Is it a flush compl that has no data */ 1688 if(!cqe->u0.s.num_fragments) 1689 goto exit; 1690 1691 len = cqe->u0.s.pkt_size; 1692 if (!len) { 1693 /*partial DMA workaround for Lancer*/ 1694 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1695 goto exit; 1696 } 1697 1698 if (!oce_cqe_portid_valid(sc, cqe)) { 1699 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1700 goto exit; 1701 } 1702 1703 /* Get vlan_tag value */ 1704 if(IS_BE(sc) || IS_SH(sc)) 1705 vtag = BSWAP_16(cqe->u0.s.vlan_tag); 1706 else 1707 vtag = cqe->u0.s.vlan_tag; 1708 1709 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass; 1710 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass; 1711 cq_info.ipv6_frame = cqe->u0.s.ip_ver; 1712 cq_info.num_frags = cqe->u0.s.num_fragments; 1713 cq_info.pkt_size = cqe->u0.s.pkt_size; 1714 1715 oce_rx_mbuf_chain(rq, &cq_info, &m); 1716 1717 if (m) { 1718 m->m_pkthdr.rcvif = sc->ifp; 1719 #if __FreeBSD_version >= 800000 1720 if (rq->queue_index) 1721 m->m_pkthdr.flowid = (rq->queue_index - 1); 1722 else 1723 m->m_pkthdr.flowid = rq->queue_index; 1724 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1725 #endif 1726 /* This deternies if vlan tag is Valid */ 1727 if (oce_cqe_vtp_valid(sc, cqe)) { 1728 if (sc->function_mode & FNM_FLEX10_MODE) { 1729 /* FLEX10. If QnQ is not set, neglect VLAN */ 1730 if (cqe->u0.s.qnq) { 1731 m->m_pkthdr.ether_vtag = vtag; 1732 m->m_flags |= M_VLANTAG; 1733 } 1734 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) { 1735 /* In UMC mode generally pvid will be striped by 1736 hw. But in some cases we have seen it comes 1737 with pvid. So if pvid == vlan, neglect vlan. 1738 */ 1739 m->m_pkthdr.ether_vtag = vtag; 1740 m->m_flags |= M_VLANTAG; 1741 } 1742 } 1743 1744 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1745 #if defined(INET6) || defined(INET) 1746 /* Try to queue to LRO */ 1747 if (IF_LRO_ENABLED(sc) && 1748 (cqe->u0.s.ip_cksum_pass) && 1749 (cqe->u0.s.l4_cksum_pass) && 1750 (!cqe->u0.s.ip_ver) && 1751 (rq->lro.lro_cnt != 0)) { 1752 1753 if (tcp_lro_rx(&rq->lro, m, 0) == 0) { 1754 rq->lro_pkts_queued ++; 1755 goto post_done; 1756 } 1757 /* If LRO posting fails then try to post to STACK */ 1758 } 1759 #endif 1760 1761 (*sc->ifp->if_input) (sc->ifp, m); 1762 #if defined(INET6) || defined(INET) 1763 post_done: 1764 #endif 1765 /* Update rx stats per queue */ 1766 rq->rx_stats.rx_pkts++; 1767 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size; 1768 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments; 1769 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET) 1770 rq->rx_stats.rx_mcast_pkts++; 1771 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET) 1772 rq->rx_stats.rx_ucast_pkts++; 1773 } 1774 exit: 1775 return; 1776 } 1777 1778 1779 void 1780 oce_discard_rx_comp(struct oce_rq *rq, int num_frags) 1781 { 1782 uint32_t i = 0; 1783 struct oce_packet_desc *pd; 1784 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1785 1786 for (i = 0; i < num_frags; i++) { 1787 if (rq->ring->cidx == rq->ring->pidx) { 1788 device_printf(sc->dev, 1789 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n"); 1790 return; 1791 } 1792 pd = &rq->pckts[rq->ring->cidx]; 1793 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1794 bus_dmamap_unload(rq->tag, pd->map); 1795 if (pd->mbuf != NULL) { 1796 m_freem(pd->mbuf); 1797 pd->mbuf = NULL; 1798 } 1799 1800 RING_GET(rq->ring, 1); 1801 rq->pending--; 1802 } 1803 } 1804 1805 1806 static int 1807 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1808 { 1809 struct oce_nic_rx_cqe_v1 *cqe_v1; 1810 int vtp = 0; 1811 1812 if (sc->be3_native) { 1813 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1814 vtp = cqe_v1->u0.s.vlan_tag_present; 1815 } else 1816 vtp = cqe->u0.s.vlan_tag_present; 1817 1818 return vtp; 1819 1820 } 1821 1822 1823 static int 1824 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1825 { 1826 struct oce_nic_rx_cqe_v1 *cqe_v1; 1827 int port_id = 0; 1828 1829 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) { 1830 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1831 port_id = cqe_v1->u0.s.port; 1832 if (sc->port_id != port_id) 1833 return 0; 1834 } else 1835 ;/* For BE3 legacy and Lancer this is dummy */ 1836 1837 return 1; 1838 1839 } 1840 1841 #if defined(INET6) || defined(INET) 1842 void 1843 oce_rx_flush_lro(struct oce_rq *rq) 1844 { 1845 struct lro_ctrl *lro = &rq->lro; 1846 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1847 1848 if (!IF_LRO_ENABLED(sc)) 1849 return; 1850 1851 tcp_lro_flush_all(lro); 1852 rq->lro_pkts_queued = 0; 1853 1854 return; 1855 } 1856 1857 1858 static int 1859 oce_init_lro(POCE_SOFTC sc) 1860 { 1861 struct lro_ctrl *lro = NULL; 1862 int i = 0, rc = 0; 1863 1864 for (i = 0; i < sc->nrqs; i++) { 1865 lro = &sc->rq[i]->lro; 1866 rc = tcp_lro_init(lro); 1867 if (rc != 0) { 1868 device_printf(sc->dev, "LRO init failed\n"); 1869 return rc; 1870 } 1871 lro->ifp = sc->ifp; 1872 } 1873 1874 return rc; 1875 } 1876 1877 1878 void 1879 oce_free_lro(POCE_SOFTC sc) 1880 { 1881 struct lro_ctrl *lro = NULL; 1882 int i = 0; 1883 1884 for (i = 0; i < sc->nrqs; i++) { 1885 lro = &sc->rq[i]->lro; 1886 if (lro) 1887 tcp_lro_free(lro); 1888 } 1889 } 1890 #endif 1891 1892 int 1893 oce_alloc_rx_bufs(struct oce_rq *rq, int count) 1894 { 1895 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1896 int i, in, rc; 1897 struct oce_packet_desc *pd; 1898 bus_dma_segment_t segs[6]; 1899 int nsegs, added = 0; 1900 struct oce_nic_rqe *rqe; 1901 pd_rxulp_db_t rxdb_reg; 1902 uint32_t val = 0; 1903 uint32_t oce_max_rq_posts = 64; 1904 1905 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t)); 1906 for (i = 0; i < count; i++) { 1907 in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE; 1908 1909 pd = &rq->pckts[rq->ring->pidx]; 1910 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size); 1911 if (pd->mbuf == NULL) { 1912 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size); 1913 break; 1914 } 1915 pd->mbuf->m_nextpkt = NULL; 1916 1917 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size; 1918 1919 rc = bus_dmamap_load_mbuf_sg(rq->tag, 1920 pd->map, 1921 pd->mbuf, 1922 segs, &nsegs, BUS_DMA_NOWAIT); 1923 if (rc) { 1924 m_free(pd->mbuf); 1925 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc); 1926 break; 1927 } 1928 1929 if (nsegs != 1) { 1930 i--; 1931 continue; 1932 } 1933 1934 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD); 1935 1936 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe); 1937 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr); 1938 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr); 1939 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe)); 1940 RING_PUT(rq->ring, 1); 1941 added++; 1942 rq->pending++; 1943 } 1944 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS; 1945 if (added != 0) { 1946 for (i = added / oce_max_rq_posts; i > 0; i--) { 1947 rxdb_reg.bits.num_posted = oce_max_rq_posts; 1948 rxdb_reg.bits.qid = rq->rq_id; 1949 if(rq->islro) { 1950 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1951 val |= oce_max_rq_posts << 16; 1952 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1953 }else { 1954 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1955 } 1956 added -= oce_max_rq_posts; 1957 } 1958 if (added > 0) { 1959 rxdb_reg.bits.qid = rq->rq_id; 1960 rxdb_reg.bits.num_posted = added; 1961 if(rq->islro) { 1962 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1963 val |= added << 16; 1964 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1965 }else { 1966 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1967 } 1968 } 1969 } 1970 1971 return 0; 1972 } 1973 1974 static void 1975 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq) 1976 { 1977 if (num_cqes) { 1978 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE); 1979 if(!sc->enable_hwlro) { 1980 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1) 1981 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1)); 1982 }else { 1983 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64) 1984 oce_alloc_rx_bufs(rq, 64); 1985 } 1986 } 1987 1988 return; 1989 } 1990 1991 uint16_t 1992 oce_rq_handler_lro(void *arg) 1993 { 1994 struct oce_rq *rq = (struct oce_rq *)arg; 1995 struct oce_cq *cq = rq->cq; 1996 POCE_SOFTC sc = rq->parent; 1997 struct nic_hwlro_singleton_cqe *cqe; 1998 struct nic_hwlro_cqe_part2 *cqe2; 1999 int num_cqes = 0; 2000 2001 LOCK(&rq->rx_lock); 2002 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2003 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2004 while (cqe->valid) { 2005 if(cqe->cqe_type == 0) { /* singleton cqe */ 2006 /* we should not get singleton cqe after cqe1 on same rq */ 2007 if(rq->cqe_firstpart != NULL) { 2008 device_printf(sc->dev, "Got singleton cqe after cqe1 \n"); 2009 goto exit_rq_handler_lro; 2010 } 2011 if(cqe->error != 0) { 2012 rq->rx_stats.rxcp_err++; 2013 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2014 } 2015 oce_rx_lro(rq, cqe, NULL); 2016 rq->rx_stats.rx_compl++; 2017 cqe->valid = 0; 2018 RING_GET(cq->ring, 1); 2019 num_cqes++; 2020 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2021 break; 2022 }else if(cqe->cqe_type == 0x1) { /* first part */ 2023 /* we should not get cqe1 after cqe1 on same rq */ 2024 if(rq->cqe_firstpart != NULL) { 2025 device_printf(sc->dev, "Got cqe1 after cqe1 \n"); 2026 goto exit_rq_handler_lro; 2027 } 2028 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe; 2029 RING_GET(cq->ring, 1); 2030 }else if(cqe->cqe_type == 0x2) { /* second part */ 2031 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe; 2032 if(cqe2->error != 0) { 2033 rq->rx_stats.rxcp_err++; 2034 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2035 } 2036 /* We should not get cqe2 without cqe1 */ 2037 if(rq->cqe_firstpart == NULL) { 2038 device_printf(sc->dev, "Got cqe2 without cqe1 \n"); 2039 goto exit_rq_handler_lro; 2040 } 2041 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2); 2042 2043 rq->rx_stats.rx_compl++; 2044 rq->cqe_firstpart->valid = 0; 2045 cqe2->valid = 0; 2046 rq->cqe_firstpart = NULL; 2047 2048 RING_GET(cq->ring, 1); 2049 num_cqes += 2; 2050 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2051 break; 2052 } 2053 2054 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2055 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2056 } 2057 oce_check_rx_bufs(sc, num_cqes, rq); 2058 exit_rq_handler_lro: 2059 UNLOCK(&rq->rx_lock); 2060 return 0; 2061 } 2062 2063 /* Handle the Completion Queue for receive */ 2064 uint16_t 2065 oce_rq_handler(void *arg) 2066 { 2067 struct oce_rq *rq = (struct oce_rq *)arg; 2068 struct oce_cq *cq = rq->cq; 2069 POCE_SOFTC sc = rq->parent; 2070 struct oce_nic_rx_cqe *cqe; 2071 int num_cqes = 0; 2072 2073 if(rq->islro) { 2074 oce_rq_handler_lro(arg); 2075 return 0; 2076 } 2077 LOCK(&rq->rx_lock); 2078 bus_dmamap_sync(cq->ring->dma.tag, 2079 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2080 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2081 while (cqe->u0.dw[2]) { 2082 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe)); 2083 2084 if (cqe->u0.s.error == 0) { 2085 oce_rx(rq, cqe); 2086 } else { 2087 rq->rx_stats.rxcp_err++; 2088 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2089 /* Post L3/L4 errors to stack.*/ 2090 oce_rx(rq, cqe); 2091 } 2092 rq->rx_stats.rx_compl++; 2093 cqe->u0.dw[2] = 0; 2094 2095 #if defined(INET6) || defined(INET) 2096 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) { 2097 oce_rx_flush_lro(rq); 2098 } 2099 #endif 2100 2101 RING_GET(cq->ring, 1); 2102 bus_dmamap_sync(cq->ring->dma.tag, 2103 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2104 cqe = 2105 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2106 num_cqes++; 2107 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2108 break; 2109 } 2110 2111 #if defined(INET6) || defined(INET) 2112 if (IF_LRO_ENABLED(sc)) 2113 oce_rx_flush_lro(rq); 2114 #endif 2115 2116 oce_check_rx_bufs(sc, num_cqes, rq); 2117 UNLOCK(&rq->rx_lock); 2118 return 0; 2119 2120 } 2121 2122 2123 2124 2125 /***************************************************************************** 2126 * Helper function prototypes in this file * 2127 *****************************************************************************/ 2128 2129 static int 2130 oce_attach_ifp(POCE_SOFTC sc) 2131 { 2132 2133 sc->ifp = if_alloc(IFT_ETHER); 2134 if (!sc->ifp) 2135 return ENOMEM; 2136 2137 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status); 2138 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2139 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 2140 2141 sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST; 2142 sc->ifp->if_ioctl = oce_ioctl; 2143 sc->ifp->if_start = oce_start; 2144 sc->ifp->if_init = oce_init; 2145 sc->ifp->if_mtu = ETHERMTU; 2146 sc->ifp->if_softc = sc; 2147 #if __FreeBSD_version >= 800000 2148 sc->ifp->if_transmit = oce_multiq_start; 2149 sc->ifp->if_qflush = oce_multiq_flush; 2150 #endif 2151 2152 if_initname(sc->ifp, 2153 device_get_name(sc->dev), device_get_unit(sc->dev)); 2154 2155 sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1; 2156 IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen); 2157 IFQ_SET_READY(&sc->ifp->if_snd); 2158 2159 sc->ifp->if_hwassist = OCE_IF_HWASSIST; 2160 sc->ifp->if_hwassist |= CSUM_TSO; 2161 sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP); 2162 2163 sc->ifp->if_capabilities = OCE_IF_CAPABILITIES; 2164 sc->ifp->if_capabilities |= IFCAP_HWCSUM; 2165 sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2166 2167 #if defined(INET6) || defined(INET) 2168 sc->ifp->if_capabilities |= IFCAP_TSO; 2169 sc->ifp->if_capabilities |= IFCAP_LRO; 2170 sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 2171 #endif 2172 2173 sc->ifp->if_capenable = sc->ifp->if_capabilities; 2174 sc->ifp->if_baudrate = IF_Gbps(10); 2175 2176 #if __FreeBSD_version >= 1000000 2177 sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 2178 sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS; 2179 sc->ifp->if_hw_tsomaxsegsize = 4096; 2180 #endif 2181 2182 ether_ifattach(sc->ifp, sc->macaddr.mac_addr); 2183 2184 return 0; 2185 } 2186 2187 2188 static void 2189 oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2190 { 2191 POCE_SOFTC sc = ifp->if_softc; 2192 2193 if (ifp->if_softc != arg) 2194 return; 2195 if ((vtag == 0) || (vtag > 4095)) 2196 return; 2197 2198 sc->vlan_tag[vtag] = 1; 2199 sc->vlans_added++; 2200 if (sc->vlans_added <= (sc->max_vlans + 1)) 2201 oce_vid_config(sc); 2202 } 2203 2204 2205 static void 2206 oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2207 { 2208 POCE_SOFTC sc = ifp->if_softc; 2209 2210 if (ifp->if_softc != arg) 2211 return; 2212 if ((vtag == 0) || (vtag > 4095)) 2213 return; 2214 2215 sc->vlan_tag[vtag] = 0; 2216 sc->vlans_added--; 2217 oce_vid_config(sc); 2218 } 2219 2220 2221 /* 2222 * A max of 64 vlans can be configured in BE. If the user configures 2223 * more, place the card in vlan promiscuous mode. 2224 */ 2225 static int 2226 oce_vid_config(POCE_SOFTC sc) 2227 { 2228 struct normal_vlan vtags[MAX_VLANFILTER_SIZE]; 2229 uint16_t ntags = 0, i; 2230 int status = 0; 2231 2232 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) && 2233 (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) { 2234 for (i = 0; i < MAX_VLANS; i++) { 2235 if (sc->vlan_tag[i]) { 2236 vtags[ntags].vtag = i; 2237 ntags++; 2238 } 2239 } 2240 if (ntags) 2241 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2242 vtags, ntags, 1, 0); 2243 } else 2244 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2245 NULL, 0, 1, 1); 2246 return status; 2247 } 2248 2249 2250 static void 2251 oce_mac_addr_set(POCE_SOFTC sc) 2252 { 2253 uint32_t old_pmac_id = sc->pmac_id; 2254 int status = 0; 2255 2256 2257 status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2258 sc->macaddr.size_of_struct); 2259 if (!status) 2260 return; 2261 2262 status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)), 2263 sc->if_id, &sc->pmac_id); 2264 if (!status) { 2265 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id); 2266 bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2267 sc->macaddr.size_of_struct); 2268 } 2269 if (status) 2270 device_printf(sc->dev, "Failed update macaddress\n"); 2271 2272 } 2273 2274 2275 static int 2276 oce_handle_passthrough(struct ifnet *ifp, caddr_t data) 2277 { 2278 POCE_SOFTC sc = ifp->if_softc; 2279 struct ifreq *ifr = (struct ifreq *)data; 2280 int rc = ENXIO; 2281 char cookie[32] = {0}; 2282 void *priv_data = ifr_data_get_ptr(ifr); 2283 void *ioctl_ptr; 2284 uint32_t req_size; 2285 struct mbx_hdr req; 2286 OCE_DMA_MEM dma_mem; 2287 struct mbx_common_get_cntl_attr *fw_cmd; 2288 2289 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE))) 2290 return EFAULT; 2291 2292 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE))) 2293 return EINVAL; 2294 2295 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE); 2296 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr))) 2297 return EFAULT; 2298 2299 req_size = le32toh(req.u0.req.request_length); 2300 if (req_size > 65536) 2301 return EINVAL; 2302 2303 req_size += sizeof(struct mbx_hdr); 2304 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0); 2305 if (rc) 2306 return ENOMEM; 2307 2308 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) { 2309 rc = EFAULT; 2310 goto dma_free; 2311 } 2312 2313 rc = oce_pass_through_mbox(sc, &dma_mem, req_size); 2314 if (rc) { 2315 rc = EIO; 2316 goto dma_free; 2317 } 2318 2319 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) 2320 rc = EFAULT; 2321 2322 /* 2323 firmware is filling all the attributes for this ioctl except 2324 the driver version..so fill it 2325 */ 2326 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) { 2327 fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr; 2328 strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str, 2329 COMPONENT_REVISION, strlen(COMPONENT_REVISION)); 2330 } 2331 2332 dma_free: 2333 oce_dma_free(sc, &dma_mem); 2334 return rc; 2335 2336 } 2337 2338 static void 2339 oce_eqd_set_periodic(POCE_SOFTC sc) 2340 { 2341 struct oce_set_eqd set_eqd[OCE_MAX_EQ]; 2342 struct oce_aic_obj *aic; 2343 struct oce_eq *eqo; 2344 uint64_t now = 0, delta; 2345 int eqd, i, num = 0; 2346 uint32_t tx_reqs = 0, rxpkts = 0, pps; 2347 struct oce_wq *wq; 2348 struct oce_rq *rq; 2349 2350 #define ticks_to_msecs(t) (1000 * (t) / hz) 2351 2352 for (i = 0 ; i < sc->neqs; i++) { 2353 eqo = sc->eq[i]; 2354 aic = &sc->aic_obj[i]; 2355 /* When setting the static eq delay from the user space */ 2356 if (!aic->enable) { 2357 if (aic->ticks) 2358 aic->ticks = 0; 2359 eqd = aic->et_eqd; 2360 goto modify_eqd; 2361 } 2362 2363 rq = sc->rq[i]; 2364 rxpkts = rq->rx_stats.rx_pkts; 2365 wq = sc->wq[i]; 2366 tx_reqs = wq->tx_stats.tx_reqs; 2367 now = ticks; 2368 2369 if (!aic->ticks || now < aic->ticks || 2370 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) { 2371 aic->prev_rxpkts = rxpkts; 2372 aic->prev_txreqs = tx_reqs; 2373 aic->ticks = now; 2374 continue; 2375 } 2376 2377 delta = ticks_to_msecs(now - aic->ticks); 2378 2379 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) + 2380 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta); 2381 eqd = (pps / 15000) << 2; 2382 if (eqd < 8) 2383 eqd = 0; 2384 2385 /* Make sure that the eq delay is in the known range */ 2386 eqd = min(eqd, aic->max_eqd); 2387 eqd = max(eqd, aic->min_eqd); 2388 2389 aic->prev_rxpkts = rxpkts; 2390 aic->prev_txreqs = tx_reqs; 2391 aic->ticks = now; 2392 2393 modify_eqd: 2394 if (eqd != aic->cur_eqd) { 2395 set_eqd[num].delay_multiplier = (eqd * 65)/100; 2396 set_eqd[num].eq_id = eqo->eq_id; 2397 aic->cur_eqd = eqd; 2398 num++; 2399 } 2400 } 2401 2402 /* Is there atleast one eq that needs to be modified? */ 2403 for(i = 0; i < num; i += 8) { 2404 if((num - i) >=8 ) 2405 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8); 2406 else 2407 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i)); 2408 } 2409 2410 } 2411 2412 static void oce_detect_hw_error(POCE_SOFTC sc) 2413 { 2414 2415 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0; 2416 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 2417 uint32_t i; 2418 2419 if (sc->hw_error) 2420 return; 2421 2422 if (IS_XE201(sc)) { 2423 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET); 2424 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2425 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET); 2426 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET); 2427 } 2428 } else { 2429 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW); 2430 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH); 2431 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK); 2432 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK); 2433 2434 ue_low = (ue_low & ~ue_low_mask); 2435 ue_high = (ue_high & ~ue_high_mask); 2436 } 2437 2438 /* On certain platforms BE hardware can indicate spurious UEs. 2439 * Allow the h/w to stop working completely in case of a real UE. 2440 * Hence not setting the hw_error for UE detection. 2441 */ 2442 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2443 sc->hw_error = TRUE; 2444 device_printf(sc->dev, "Error detected in the card\n"); 2445 } 2446 2447 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2448 device_printf(sc->dev, 2449 "ERR: sliport status 0x%x\n", sliport_status); 2450 device_printf(sc->dev, 2451 "ERR: sliport error1 0x%x\n", sliport_err1); 2452 device_printf(sc->dev, 2453 "ERR: sliport error2 0x%x\n", sliport_err2); 2454 } 2455 2456 if (ue_low) { 2457 for (i = 0; ue_low; ue_low >>= 1, i++) { 2458 if (ue_low & 1) 2459 device_printf(sc->dev, "UE: %s bit set\n", 2460 ue_status_low_desc[i]); 2461 } 2462 } 2463 2464 if (ue_high) { 2465 for (i = 0; ue_high; ue_high >>= 1, i++) { 2466 if (ue_high & 1) 2467 device_printf(sc->dev, "UE: %s bit set\n", 2468 ue_status_hi_desc[i]); 2469 } 2470 } 2471 2472 } 2473 2474 2475 static void 2476 oce_local_timer(void *arg) 2477 { 2478 POCE_SOFTC sc = arg; 2479 int i = 0; 2480 2481 oce_detect_hw_error(sc); 2482 oce_refresh_nic_stats(sc); 2483 oce_refresh_queue_stats(sc); 2484 oce_mac_addr_set(sc); 2485 2486 /* TX Watch Dog*/ 2487 for (i = 0; i < sc->nwqs; i++) 2488 oce_tx_restart(sc, sc->wq[i]); 2489 2490 /* calculate and set the eq delay for optimal interrupt rate */ 2491 if (IS_BE(sc) || IS_SH(sc)) 2492 oce_eqd_set_periodic(sc); 2493 2494 callout_reset(&sc->timer, hz, oce_local_timer, sc); 2495 } 2496 2497 static void 2498 oce_tx_compl_clean(POCE_SOFTC sc) 2499 { 2500 struct oce_wq *wq; 2501 int i = 0, timeo = 0, num_wqes = 0; 2502 int pending_txqs = sc->nwqs; 2503 2504 /* Stop polling for compls when HW has been silent for 10ms or 2505 * hw_error or no outstanding completions expected 2506 */ 2507 do { 2508 pending_txqs = sc->nwqs; 2509 2510 for_all_wq_queues(sc, wq, i) { 2511 num_wqes = oce_wq_handler(wq); 2512 2513 if(num_wqes) 2514 timeo = 0; 2515 2516 if(!wq->ring->num_used) 2517 pending_txqs--; 2518 } 2519 2520 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error) 2521 break; 2522 2523 DELAY(1000); 2524 } while (TRUE); 2525 2526 for_all_wq_queues(sc, wq, i) { 2527 while(wq->ring->num_used) { 2528 LOCK(&wq->tx_compl_lock); 2529 oce_process_tx_completion(wq); 2530 UNLOCK(&wq->tx_compl_lock); 2531 } 2532 } 2533 2534 } 2535 2536 /* NOTE : This should only be called holding 2537 * DEVICE_LOCK. 2538 */ 2539 static void 2540 oce_if_deactivate(POCE_SOFTC sc) 2541 { 2542 int i; 2543 struct oce_rq *rq; 2544 struct oce_wq *wq; 2545 struct oce_eq *eq; 2546 2547 sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2548 2549 oce_tx_compl_clean(sc); 2550 2551 /* Stop intrs and finish any bottom halves pending */ 2552 oce_hw_intr_disable(sc); 2553 2554 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire 2555 any other lock. So unlock device lock and require after 2556 completing taskqueue_drain. 2557 */ 2558 UNLOCK(&sc->dev_lock); 2559 for (i = 0; i < sc->intr_count; i++) { 2560 if (sc->intrs[i].tq != NULL) { 2561 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task); 2562 } 2563 } 2564 LOCK(&sc->dev_lock); 2565 2566 /* Delete RX queue in card with flush param */ 2567 oce_stop_rx(sc); 2568 2569 /* Invalidate any pending cq and eq entries*/ 2570 for_all_evnt_queues(sc, eq, i) 2571 oce_drain_eq(eq); 2572 for_all_rq_queues(sc, rq, i) 2573 oce_drain_rq_cq(rq); 2574 for_all_wq_queues(sc, wq, i) 2575 oce_drain_wq_cq(wq); 2576 2577 /* But still we need to get MCC aync events. 2578 So enable intrs and also arm first EQ 2579 */ 2580 oce_hw_intr_enable(sc); 2581 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE); 2582 2583 DELAY(10); 2584 } 2585 2586 2587 static void 2588 oce_if_activate(POCE_SOFTC sc) 2589 { 2590 struct oce_eq *eq; 2591 struct oce_rq *rq; 2592 struct oce_wq *wq; 2593 int i, rc = 0; 2594 2595 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 2596 2597 oce_hw_intr_disable(sc); 2598 2599 oce_start_rx(sc); 2600 2601 for_all_rq_queues(sc, rq, i) { 2602 rc = oce_start_rq(rq); 2603 if (rc) 2604 device_printf(sc->dev, "Unable to start RX\n"); 2605 } 2606 2607 for_all_wq_queues(sc, wq, i) { 2608 rc = oce_start_wq(wq); 2609 if (rc) 2610 device_printf(sc->dev, "Unable to start TX\n"); 2611 } 2612 2613 2614 for_all_evnt_queues(sc, eq, i) 2615 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 2616 2617 oce_hw_intr_enable(sc); 2618 2619 } 2620 2621 static void 2622 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe) 2623 { 2624 /* Update Link status */ 2625 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) == 2626 ASYNC_EVENT_LINK_UP) { 2627 sc->link_status = ASYNC_EVENT_LINK_UP; 2628 if_link_state_change(sc->ifp, LINK_STATE_UP); 2629 } else { 2630 sc->link_status = ASYNC_EVENT_LINK_DOWN; 2631 if_link_state_change(sc->ifp, LINK_STATE_DOWN); 2632 } 2633 } 2634 2635 2636 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc, 2637 struct oce_async_evt_grp5_os2bmc *evt) 2638 { 2639 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc)); 2640 if (evt->u.s.mgmt_enable) 2641 sc->flags |= OCE_FLAGS_OS2BMC; 2642 else 2643 return; 2644 2645 sc->bmc_filt_mask = evt->u.s.arp_filter; 2646 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1); 2647 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2); 2648 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3); 2649 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4); 2650 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5); 2651 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6); 2652 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7); 2653 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8); 2654 } 2655 2656 2657 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe) 2658 { 2659 struct oce_async_event_grp5_pvid_state *gcqe; 2660 struct oce_async_evt_grp5_os2bmc *bmccqe; 2661 2662 switch (cqe->u0.s.async_type) { 2663 case ASYNC_EVENT_PVID_STATE: 2664 /* GRP5 PVID */ 2665 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe; 2666 if (gcqe->enabled) 2667 sc->pvid = gcqe->tag & VLAN_VID_MASK; 2668 else 2669 sc->pvid = 0; 2670 break; 2671 case ASYNC_EVENT_OS2BMC: 2672 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe; 2673 oce_async_grp5_osbmc_process(sc, bmccqe); 2674 break; 2675 default: 2676 break; 2677 } 2678 } 2679 2680 /* Handle the Completion Queue for the Mailbox/Async notifications */ 2681 uint16_t 2682 oce_mq_handler(void *arg) 2683 { 2684 struct oce_mq *mq = (struct oce_mq *)arg; 2685 POCE_SOFTC sc = mq->parent; 2686 struct oce_cq *cq = mq->cq; 2687 int num_cqes = 0, evt_type = 0, optype = 0; 2688 struct oce_mq_cqe *cqe; 2689 struct oce_async_cqe_link_state *acqe; 2690 struct oce_async_event_qnq *dbgcqe; 2691 2692 2693 bus_dmamap_sync(cq->ring->dma.tag, 2694 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2695 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2696 2697 while (cqe->u0.dw[3]) { 2698 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe)); 2699 if (cqe->u0.s.async_event) { 2700 evt_type = cqe->u0.s.event_type; 2701 optype = cqe->u0.s.async_type; 2702 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) { 2703 /* Link status evt */ 2704 acqe = (struct oce_async_cqe_link_state *)cqe; 2705 process_link_state(sc, acqe); 2706 } else if (evt_type == ASYNC_EVENT_GRP5) { 2707 oce_process_grp5_events(sc, cqe); 2708 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG && 2709 optype == ASYNC_EVENT_DEBUG_QNQ) { 2710 dbgcqe = (struct oce_async_event_qnq *)cqe; 2711 if(dbgcqe->valid) 2712 sc->qnqid = dbgcqe->vlan_tag; 2713 sc->qnq_debug_event = TRUE; 2714 } 2715 } 2716 cqe->u0.dw[3] = 0; 2717 RING_GET(cq->ring, 1); 2718 bus_dmamap_sync(cq->ring->dma.tag, 2719 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2720 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2721 num_cqes++; 2722 } 2723 2724 if (num_cqes) 2725 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 2726 2727 return 0; 2728 } 2729 2730 2731 static void 2732 setup_max_queues_want(POCE_SOFTC sc) 2733 { 2734 /* Check if it is FLEX machine. Is so dont use RSS */ 2735 if ((sc->function_mode & FNM_FLEX10_MODE) || 2736 (sc->function_mode & FNM_UMC_MODE) || 2737 (sc->function_mode & FNM_VNIC_MODE) || 2738 (!is_rss_enabled(sc)) || 2739 IS_BE2(sc)) { 2740 sc->nrqs = 1; 2741 sc->nwqs = 1; 2742 } else { 2743 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2744 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs); 2745 } 2746 2747 if (IS_BE2(sc) && is_rss_enabled(sc)) 2748 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2749 } 2750 2751 2752 static void 2753 update_queues_got(POCE_SOFTC sc) 2754 { 2755 if (is_rss_enabled(sc)) { 2756 sc->nrqs = sc->intr_count + 1; 2757 sc->nwqs = sc->intr_count; 2758 } else { 2759 sc->nrqs = 1; 2760 sc->nwqs = 1; 2761 } 2762 2763 if (IS_BE2(sc)) 2764 sc->nwqs = 1; 2765 } 2766 2767 static int 2768 oce_check_ipv6_ext_hdr(struct mbuf *m) 2769 { 2770 struct ether_header *eh = mtod(m, struct ether_header *); 2771 caddr_t m_datatemp = m->m_data; 2772 2773 if (eh->ether_type == htons(ETHERTYPE_IPV6)) { 2774 m->m_data += sizeof(struct ether_header); 2775 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 2776 2777 if((ip6->ip6_nxt != IPPROTO_TCP) && \ 2778 (ip6->ip6_nxt != IPPROTO_UDP)){ 2779 struct ip6_ext *ip6e = NULL; 2780 m->m_data += sizeof(struct ip6_hdr); 2781 2782 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *); 2783 if(ip6e->ip6e_len == 0xff) { 2784 m->m_data = m_datatemp; 2785 return TRUE; 2786 } 2787 } 2788 m->m_data = m_datatemp; 2789 } 2790 return FALSE; 2791 } 2792 2793 static int 2794 is_be3_a1(POCE_SOFTC sc) 2795 { 2796 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) { 2797 return TRUE; 2798 } 2799 return FALSE; 2800 } 2801 2802 static struct mbuf * 2803 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete) 2804 { 2805 uint16_t vlan_tag = 0; 2806 2807 if(!M_WRITABLE(m)) 2808 return NULL; 2809 2810 /* Embed vlan tag in the packet if it is not part of it */ 2811 if(m->m_flags & M_VLANTAG) { 2812 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag); 2813 m->m_flags &= ~M_VLANTAG; 2814 } 2815 2816 /* if UMC, ignore vlan tag insertion and instead insert pvid */ 2817 if(sc->pvid) { 2818 if(!vlan_tag) 2819 vlan_tag = sc->pvid; 2820 if (complete) 2821 *complete = FALSE; 2822 } 2823 2824 if(vlan_tag) { 2825 m = ether_vlanencap(m, vlan_tag); 2826 } 2827 2828 if(sc->qnqid) { 2829 m = ether_vlanencap(m, sc->qnqid); 2830 2831 if (complete) 2832 *complete = FALSE; 2833 } 2834 return m; 2835 } 2836 2837 static int 2838 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m) 2839 { 2840 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \ 2841 oce_check_ipv6_ext_hdr(m)) { 2842 return TRUE; 2843 } 2844 return FALSE; 2845 } 2846 2847 static void 2848 oce_get_config(POCE_SOFTC sc) 2849 { 2850 int rc = 0; 2851 uint32_t max_rss = 0; 2852 2853 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native)) 2854 max_rss = OCE_LEGACY_MODE_RSS; 2855 else 2856 max_rss = OCE_MAX_RSS; 2857 2858 if (!IS_BE(sc)) { 2859 rc = oce_get_profile_config(sc, max_rss); 2860 if (rc) { 2861 sc->nwqs = OCE_MAX_WQ; 2862 sc->nrssqs = max_rss; 2863 sc->nrqs = sc->nrssqs + 1; 2864 } 2865 } 2866 else { /* For BE3 don't rely on fw for determining the resources */ 2867 sc->nrssqs = max_rss; 2868 sc->nrqs = sc->nrssqs + 1; 2869 sc->nwqs = OCE_MAX_WQ; 2870 sc->max_vlans = MAX_VLANFILTER_SIZE; 2871 } 2872 } 2873 2874 static void 2875 oce_rdma_close(void) 2876 { 2877 if (oce_rdma_if != NULL) { 2878 oce_rdma_if = NULL; 2879 } 2880 } 2881 2882 static void 2883 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr) 2884 { 2885 memcpy(macaddr, sc->macaddr.mac_addr, 6); 2886 } 2887 2888 int 2889 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if) 2890 { 2891 POCE_SOFTC sc; 2892 struct oce_dev_info di; 2893 int i; 2894 2895 if ((rdma_info == NULL) || (rdma_if == NULL)) { 2896 return -EINVAL; 2897 } 2898 2899 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) || 2900 (rdma_if->size != OCE_RDMA_IF_SIZE)) { 2901 return -ENXIO; 2902 } 2903 2904 rdma_info->close = oce_rdma_close; 2905 rdma_info->mbox_post = oce_mbox_post; 2906 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init; 2907 rdma_info->get_mac_addr = oce_get_mac_addr; 2908 2909 oce_rdma_if = rdma_if; 2910 2911 sc = softc_head; 2912 while (sc != NULL) { 2913 if (oce_rdma_if->announce != NULL) { 2914 memset(&di, 0, sizeof(di)); 2915 di.dev = sc->dev; 2916 di.softc = sc; 2917 di.ifp = sc->ifp; 2918 di.db_bhandle = sc->db_bhandle; 2919 di.db_btag = sc->db_btag; 2920 di.db_page_size = 4096; 2921 if (sc->flags & OCE_FLAGS_USING_MSIX) { 2922 di.intr_mode = OCE_INTERRUPT_MODE_MSIX; 2923 } else if (sc->flags & OCE_FLAGS_USING_MSI) { 2924 di.intr_mode = OCE_INTERRUPT_MODE_MSI; 2925 } else { 2926 di.intr_mode = OCE_INTERRUPT_MODE_INTX; 2927 } 2928 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk 2929 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) { 2930 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count; 2931 di.msix.start_vector = sc->intr_count; 2932 for (i=0; i<di.msix.num_vectors; i++) { 2933 di.msix.vector_list[i] = sc->intrs[i].vector; 2934 } 2935 } else { 2936 } 2937 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6); 2938 di.vendor_id = pci_get_vendor(sc->dev); 2939 di.dev_id = pci_get_device(sc->dev); 2940 2941 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 2942 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED; 2943 } 2944 2945 rdma_if->announce(&di); 2946 sc = sc->next; 2947 } 2948 } 2949 2950 return 0; 2951 } 2952 2953 static void 2954 oce_read_env_variables( POCE_SOFTC sc ) 2955 { 2956 char *value = NULL; 2957 int rc = 0; 2958 2959 /* read if user wants to enable hwlro or swlro */ 2960 //value = getenv("oce_enable_hwlro"); 2961 if(value && IS_SH(sc)) { 2962 sc->enable_hwlro = strtol(value, NULL, 10); 2963 if(sc->enable_hwlro) { 2964 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL); 2965 if(rc) { 2966 device_printf(sc->dev, "no hardware lro support\n"); 2967 device_printf(sc->dev, "software lro enabled\n"); 2968 sc->enable_hwlro = 0; 2969 }else { 2970 device_printf(sc->dev, "hardware lro enabled\n"); 2971 oce_max_rsp_handled = 32; 2972 } 2973 }else { 2974 device_printf(sc->dev, "software lro enabled\n"); 2975 } 2976 }else { 2977 sc->enable_hwlro = 0; 2978 } 2979 2980 /* read mbuf size */ 2981 //value = getenv("oce_rq_buf_size"); 2982 if(value && IS_SH(sc)) { 2983 oce_rq_buf_size = strtol(value, NULL, 10); 2984 switch(oce_rq_buf_size) { 2985 case 2048: 2986 case 4096: 2987 case 9216: 2988 case 16384: 2989 break; 2990 2991 default: 2992 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n"); 2993 oce_rq_buf_size = 2048; 2994 } 2995 } 2996 2997 return; 2998 } 2999