1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (C) 2013 Emulex 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Emulex Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 * Contact Information: 34 * freebsd-drivers@emulex.com 35 * 36 * Emulex 37 * 3333 Susan Street 38 * Costa Mesa, CA 92626 39 */ 40 41 /* $FreeBSD$ */ 42 43 #include "opt_inet6.h" 44 #include "opt_inet.h" 45 46 #include "oce_if.h" 47 #include "oce_user.h" 48 49 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO) 50 51 /* UE Status Low CSR */ 52 static char *ue_status_low_desc[] = { 53 "CEV", 54 "CTX", 55 "DBUF", 56 "ERX", 57 "Host", 58 "MPU", 59 "NDMA", 60 "PTC ", 61 "RDMA ", 62 "RXF ", 63 "RXIPS ", 64 "RXULP0 ", 65 "RXULP1 ", 66 "RXULP2 ", 67 "TIM ", 68 "TPOST ", 69 "TPRE ", 70 "TXIPS ", 71 "TXULP0 ", 72 "TXULP1 ", 73 "UC ", 74 "WDMA ", 75 "TXULP2 ", 76 "HOST1 ", 77 "P0_OB_LINK ", 78 "P1_OB_LINK ", 79 "HOST_GPIO ", 80 "MBOX ", 81 "AXGMAC0", 82 "AXGMAC1", 83 "JTAG", 84 "MPU_INTPEND" 85 }; 86 87 /* UE Status High CSR */ 88 static char *ue_status_hi_desc[] = { 89 "LPCMEMHOST", 90 "MGMT_MAC", 91 "PCS0ONLINE", 92 "MPU_IRAM", 93 "PCS1ONLINE", 94 "PCTL0", 95 "PCTL1", 96 "PMEM", 97 "RR", 98 "TXPB", 99 "RXPP", 100 "XAUI", 101 "TXP", 102 "ARM", 103 "IPC", 104 "HOST2", 105 "HOST3", 106 "HOST4", 107 "HOST5", 108 "HOST6", 109 "HOST7", 110 "HOST8", 111 "HOST9", 112 "NETC", 113 "Unknown", 114 "Unknown", 115 "Unknown", 116 "Unknown", 117 "Unknown", 118 "Unknown", 119 "Unknown", 120 "Unknown" 121 }; 122 123 struct oce_common_cqe_info{ 124 uint8_t vtp:1; 125 uint8_t l4_cksum_pass:1; 126 uint8_t ip_cksum_pass:1; 127 uint8_t ipv6_frame:1; 128 uint8_t qnq:1; 129 uint8_t rsvd:3; 130 uint8_t num_frags; 131 uint16_t pkt_size; 132 uint16_t vtag; 133 }; 134 135 136 /* Driver entry points prototypes */ 137 static int oce_probe(device_t dev); 138 static int oce_attach(device_t dev); 139 static int oce_detach(device_t dev); 140 static int oce_shutdown(device_t dev); 141 static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data); 142 static void oce_init(void *xsc); 143 static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m); 144 static void oce_multiq_flush(struct ifnet *ifp); 145 146 /* Driver interrupt routines protypes */ 147 static void oce_intr(void *arg, int pending); 148 static int oce_setup_intr(POCE_SOFTC sc); 149 static int oce_fast_isr(void *arg); 150 static int oce_alloc_intr(POCE_SOFTC sc, int vector, 151 void (*isr) (void *arg, int pending)); 152 153 /* Media callbacks prototypes */ 154 static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req); 155 static int oce_media_change(struct ifnet *ifp); 156 157 /* Transmit routines prototypes */ 158 static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index); 159 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq); 160 static void oce_process_tx_completion(struct oce_wq *wq); 161 static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, 162 struct oce_wq *wq); 163 164 /* Receive routines prototypes */ 165 static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 166 static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 167 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe); 168 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq); 169 static uint16_t oce_rq_handler_lro(void *arg); 170 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2); 171 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2); 172 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m); 173 174 /* Helper function prototypes in this file */ 175 static int oce_attach_ifp(POCE_SOFTC sc); 176 static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 177 static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 178 static int oce_vid_config(POCE_SOFTC sc); 179 static void oce_mac_addr_set(POCE_SOFTC sc); 180 static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data); 181 static void oce_local_timer(void *arg); 182 static void oce_if_deactivate(POCE_SOFTC sc); 183 static void oce_if_activate(POCE_SOFTC sc); 184 static void setup_max_queues_want(POCE_SOFTC sc); 185 static void update_queues_got(POCE_SOFTC sc); 186 static void process_link_state(POCE_SOFTC sc, 187 struct oce_async_cqe_link_state *acqe); 188 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m); 189 static void oce_get_config(POCE_SOFTC sc); 190 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete); 191 static void oce_read_env_variables(POCE_SOFTC sc); 192 193 194 /* IP specific */ 195 #if defined(INET6) || defined(INET) 196 static int oce_init_lro(POCE_SOFTC sc); 197 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp); 198 #endif 199 200 static device_method_t oce_dispatch[] = { 201 DEVMETHOD(device_probe, oce_probe), 202 DEVMETHOD(device_attach, oce_attach), 203 DEVMETHOD(device_detach, oce_detach), 204 DEVMETHOD(device_shutdown, oce_shutdown), 205 206 DEVMETHOD_END 207 }; 208 209 static driver_t oce_driver = { 210 "oce", 211 oce_dispatch, 212 sizeof(OCE_SOFTC) 213 }; 214 static devclass_t oce_devclass; 215 216 217 /* global vars */ 218 const char component_revision[32] = {"///" COMPONENT_REVISION "///"}; 219 220 /* Module capabilites and parameters */ 221 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED; 222 uint32_t oce_enable_rss = OCE_MODCAP_RSS; 223 uint32_t oce_rq_buf_size = 2048; 224 225 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled); 226 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss); 227 228 229 /* Supported devices table */ 230 static uint32_t supportedDevices[] = { 231 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2, 232 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3, 233 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3, 234 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201, 235 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF, 236 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH 237 }; 238 239 240 DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0); 241 MODULE_PNP_INFO("W32:vendor/device", pci, oce, supportedDevices, 242 nitems(supportedDevices)); 243 MODULE_DEPEND(oce, pci, 1, 1, 1); 244 MODULE_DEPEND(oce, ether, 1, 1, 1); 245 MODULE_VERSION(oce, 1); 246 247 248 POCE_SOFTC softc_head = NULL; 249 POCE_SOFTC softc_tail = NULL; 250 251 struct oce_rdma_if *oce_rdma_if = NULL; 252 253 /***************************************************************************** 254 * Driver entry points functions * 255 *****************************************************************************/ 256 257 static int 258 oce_probe(device_t dev) 259 { 260 uint16_t vendor = 0; 261 uint16_t device = 0; 262 int i = 0; 263 char str[256] = {0}; 264 POCE_SOFTC sc; 265 266 sc = device_get_softc(dev); 267 bzero(sc, sizeof(OCE_SOFTC)); 268 sc->dev = dev; 269 270 vendor = pci_get_vendor(dev); 271 device = pci_get_device(dev); 272 273 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) { 274 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) { 275 if (device == (supportedDevices[i] & 0xffff)) { 276 sprintf(str, "%s:%s", "Emulex CNA NIC function", 277 component_revision); 278 device_set_desc_copy(dev, str); 279 280 switch (device) { 281 case PCI_PRODUCT_BE2: 282 sc->flags |= OCE_FLAGS_BE2; 283 break; 284 case PCI_PRODUCT_BE3: 285 sc->flags |= OCE_FLAGS_BE3; 286 break; 287 case PCI_PRODUCT_XE201: 288 case PCI_PRODUCT_XE201_VF: 289 sc->flags |= OCE_FLAGS_XE201; 290 break; 291 case PCI_PRODUCT_SH: 292 sc->flags |= OCE_FLAGS_SH; 293 break; 294 default: 295 return ENXIO; 296 } 297 return BUS_PROBE_DEFAULT; 298 } 299 } 300 } 301 302 return ENXIO; 303 } 304 305 306 static int 307 oce_attach(device_t dev) 308 { 309 POCE_SOFTC sc; 310 int rc = 0; 311 312 sc = device_get_softc(dev); 313 314 rc = oce_hw_pci_alloc(sc); 315 if (rc) 316 return rc; 317 318 sc->tx_ring_size = OCE_TX_RING_SIZE; 319 sc->rx_ring_size = OCE_RX_RING_SIZE; 320 /* receive fragment size should be multiple of 2K */ 321 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048); 322 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL; 323 sc->promisc = OCE_DEFAULT_PROMISCUOUS; 324 325 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock"); 326 LOCK_CREATE(&sc->dev_lock, "Device_lock"); 327 328 /* initialise the hardware */ 329 rc = oce_hw_init(sc); 330 if (rc) 331 goto pci_res_free; 332 333 oce_read_env_variables(sc); 334 335 oce_get_config(sc); 336 337 setup_max_queues_want(sc); 338 339 rc = oce_setup_intr(sc); 340 if (rc) 341 goto mbox_free; 342 343 rc = oce_queue_init_all(sc); 344 if (rc) 345 goto intr_free; 346 347 rc = oce_attach_ifp(sc); 348 if (rc) 349 goto queues_free; 350 351 #if defined(INET6) || defined(INET) 352 rc = oce_init_lro(sc); 353 if (rc) 354 goto ifp_free; 355 #endif 356 357 rc = oce_hw_start(sc); 358 if (rc) 359 goto lro_free; 360 361 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 362 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST); 363 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 364 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST); 365 366 rc = oce_stats_init(sc); 367 if (rc) 368 goto vlan_free; 369 370 oce_add_sysctls(sc); 371 372 callout_init(&sc->timer, CALLOUT_MPSAFE); 373 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc); 374 if (rc) 375 goto stats_free; 376 377 sc->next =NULL; 378 if (softc_tail != NULL) { 379 softc_tail->next = sc; 380 } else { 381 softc_head = sc; 382 } 383 softc_tail = sc; 384 385 return 0; 386 387 stats_free: 388 callout_drain(&sc->timer); 389 oce_stats_free(sc); 390 vlan_free: 391 if (sc->vlan_attach) 392 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 393 if (sc->vlan_detach) 394 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 395 oce_hw_intr_disable(sc); 396 lro_free: 397 #if defined(INET6) || defined(INET) 398 oce_free_lro(sc); 399 ifp_free: 400 #endif 401 ether_ifdetach(sc->ifp); 402 if_free(sc->ifp); 403 queues_free: 404 oce_queue_release_all(sc); 405 intr_free: 406 oce_intr_free(sc); 407 mbox_free: 408 oce_dma_free(sc, &sc->bsmbx); 409 pci_res_free: 410 oce_hw_pci_free(sc); 411 LOCK_DESTROY(&sc->dev_lock); 412 LOCK_DESTROY(&sc->bmbx_lock); 413 return rc; 414 415 } 416 417 418 static int 419 oce_detach(device_t dev) 420 { 421 POCE_SOFTC sc = device_get_softc(dev); 422 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL; 423 424 poce_sc_tmp = softc_head; 425 ppoce_sc_tmp1 = &softc_head; 426 while (poce_sc_tmp != NULL) { 427 if (poce_sc_tmp == sc) { 428 *ppoce_sc_tmp1 = sc->next; 429 if (sc->next == NULL) { 430 softc_tail = poce_sc_tmp2; 431 } 432 break; 433 } 434 poce_sc_tmp2 = poce_sc_tmp; 435 ppoce_sc_tmp1 = &poce_sc_tmp->next; 436 poce_sc_tmp = poce_sc_tmp->next; 437 } 438 439 LOCK(&sc->dev_lock); 440 oce_if_deactivate(sc); 441 UNLOCK(&sc->dev_lock); 442 443 callout_drain(&sc->timer); 444 445 if (sc->vlan_attach != NULL) 446 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 447 if (sc->vlan_detach != NULL) 448 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 449 450 ether_ifdetach(sc->ifp); 451 452 if_free(sc->ifp); 453 454 oce_hw_shutdown(sc); 455 456 bus_generic_detach(dev); 457 458 return 0; 459 } 460 461 462 static int 463 oce_shutdown(device_t dev) 464 { 465 int rc; 466 467 rc = oce_detach(dev); 468 469 return rc; 470 } 471 472 473 static int 474 oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 475 { 476 struct ifreq *ifr = (struct ifreq *)data; 477 POCE_SOFTC sc = ifp->if_softc; 478 struct ifi2creq i2c; 479 uint8_t offset = 0; 480 int rc = 0; 481 uint32_t u; 482 483 switch (command) { 484 485 case SIOCGIFMEDIA: 486 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command); 487 break; 488 489 case SIOCSIFMTU: 490 if (ifr->ifr_mtu > OCE_MAX_MTU) 491 rc = EINVAL; 492 else 493 ifp->if_mtu = ifr->ifr_mtu; 494 break; 495 496 case SIOCSIFFLAGS: 497 if (ifp->if_flags & IFF_UP) { 498 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 499 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 500 oce_init(sc); 501 } 502 device_printf(sc->dev, "Interface Up\n"); 503 } else { 504 LOCK(&sc->dev_lock); 505 506 sc->ifp->if_drv_flags &= 507 ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 508 oce_if_deactivate(sc); 509 510 UNLOCK(&sc->dev_lock); 511 512 device_printf(sc->dev, "Interface Down\n"); 513 } 514 515 if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) { 516 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1)))) 517 sc->promisc = TRUE; 518 } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) { 519 if (!oce_rxf_set_promiscuous(sc, 0)) 520 sc->promisc = FALSE; 521 } 522 523 break; 524 525 case SIOCADDMULTI: 526 case SIOCDELMULTI: 527 rc = oce_hw_update_multicast(sc); 528 if (rc) 529 device_printf(sc->dev, 530 "Update multicast address failed\n"); 531 break; 532 533 case SIOCSIFCAP: 534 u = ifr->ifr_reqcap ^ ifp->if_capenable; 535 536 if (u & IFCAP_TXCSUM) { 537 ifp->if_capenable ^= IFCAP_TXCSUM; 538 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 539 540 if (IFCAP_TSO & ifp->if_capenable && 541 !(IFCAP_TXCSUM & ifp->if_capenable)) { 542 ifp->if_capenable &= ~IFCAP_TSO; 543 ifp->if_hwassist &= ~CSUM_TSO; 544 if_printf(ifp, 545 "TSO disabled due to -txcsum.\n"); 546 } 547 } 548 549 if (u & IFCAP_RXCSUM) 550 ifp->if_capenable ^= IFCAP_RXCSUM; 551 552 if (u & IFCAP_TSO4) { 553 ifp->if_capenable ^= IFCAP_TSO4; 554 555 if (IFCAP_TSO & ifp->if_capenable) { 556 if (IFCAP_TXCSUM & ifp->if_capenable) 557 ifp->if_hwassist |= CSUM_TSO; 558 else { 559 ifp->if_capenable &= ~IFCAP_TSO; 560 ifp->if_hwassist &= ~CSUM_TSO; 561 if_printf(ifp, 562 "Enable txcsum first.\n"); 563 rc = EAGAIN; 564 } 565 } else 566 ifp->if_hwassist &= ~CSUM_TSO; 567 } 568 569 if (u & IFCAP_VLAN_HWTAGGING) 570 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 571 572 if (u & IFCAP_VLAN_HWFILTER) { 573 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 574 oce_vid_config(sc); 575 } 576 #if defined(INET6) || defined(INET) 577 if (u & IFCAP_LRO) { 578 ifp->if_capenable ^= IFCAP_LRO; 579 if(sc->enable_hwlro) { 580 if(ifp->if_capenable & IFCAP_LRO) { 581 rc = oce_mbox_nic_set_iface_lro_config(sc, 1); 582 }else { 583 rc = oce_mbox_nic_set_iface_lro_config(sc, 0); 584 } 585 } 586 } 587 #endif 588 589 break; 590 591 case SIOCGI2C: 592 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c)); 593 if (rc) 594 break; 595 596 if (i2c.dev_addr == PAGE_NUM_A0) { 597 offset = i2c.offset; 598 } else if (i2c.dev_addr == PAGE_NUM_A2) { 599 offset = TRANSCEIVER_A0_SIZE + i2c.offset; 600 } else { 601 rc = EINVAL; 602 break; 603 } 604 605 if (i2c.len > sizeof(i2c.data) || 606 i2c.len + offset > sizeof(sfp_vpd_dump_buffer)) { 607 rc = EINVAL; 608 break; 609 } 610 611 rc = oce_mbox_read_transrecv_data(sc, i2c.dev_addr); 612 if (rc) { 613 rc = -rc; 614 break; 615 } 616 617 memcpy(&i2c.data[0], &sfp_vpd_dump_buffer[offset], i2c.len); 618 619 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c)); 620 break; 621 622 case SIOCGPRIVATE_0: 623 rc = priv_check(curthread, PRIV_DRIVER); 624 if (rc != 0) 625 break; 626 rc = oce_handle_passthrough(ifp, data); 627 break; 628 default: 629 rc = ether_ioctl(ifp, command, data); 630 break; 631 } 632 633 return rc; 634 } 635 636 637 static void 638 oce_init(void *arg) 639 { 640 POCE_SOFTC sc = arg; 641 642 LOCK(&sc->dev_lock); 643 644 if (sc->ifp->if_flags & IFF_UP) { 645 oce_if_deactivate(sc); 646 oce_if_activate(sc); 647 } 648 649 UNLOCK(&sc->dev_lock); 650 651 } 652 653 654 static int 655 oce_multiq_start(struct ifnet *ifp, struct mbuf *m) 656 { 657 POCE_SOFTC sc = ifp->if_softc; 658 struct oce_wq *wq = NULL; 659 int queue_index = 0; 660 int status = 0; 661 662 if (!sc->link_status) 663 return ENXIO; 664 665 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 666 queue_index = m->m_pkthdr.flowid % sc->nwqs; 667 668 wq = sc->wq[queue_index]; 669 670 LOCK(&wq->tx_lock); 671 status = oce_multiq_transmit(ifp, m, wq); 672 UNLOCK(&wq->tx_lock); 673 674 return status; 675 676 } 677 678 679 static void 680 oce_multiq_flush(struct ifnet *ifp) 681 { 682 POCE_SOFTC sc = ifp->if_softc; 683 struct mbuf *m; 684 int i = 0; 685 686 for (i = 0; i < sc->nwqs; i++) { 687 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL) 688 m_freem(m); 689 } 690 if_qflush(ifp); 691 } 692 693 694 695 /***************************************************************************** 696 * Driver interrupt routines functions * 697 *****************************************************************************/ 698 699 static void 700 oce_intr(void *arg, int pending) 701 { 702 703 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 704 POCE_SOFTC sc = ii->sc; 705 struct oce_eq *eq = ii->eq; 706 struct oce_eqe *eqe; 707 struct oce_cq *cq = NULL; 708 int i, num_eqes = 0; 709 710 711 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 712 BUS_DMASYNC_POSTWRITE); 713 do { 714 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe); 715 if (eqe->evnt == 0) 716 break; 717 eqe->evnt = 0; 718 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 719 BUS_DMASYNC_POSTWRITE); 720 RING_GET(eq->ring, 1); 721 num_eqes++; 722 723 } while (TRUE); 724 725 if (!num_eqes) 726 goto eq_arm; /* Spurious */ 727 728 /* Clear EQ entries, but dont arm */ 729 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE); 730 731 /* Process TX, RX and MCC. But dont arm CQ*/ 732 for (i = 0; i < eq->cq_valid; i++) { 733 cq = eq->cq[i]; 734 (*cq->cq_handler)(cq->cb_arg); 735 } 736 737 /* Arm all cqs connected to this EQ */ 738 for (i = 0; i < eq->cq_valid; i++) { 739 cq = eq->cq[i]; 740 oce_arm_cq(sc, cq->cq_id, 0, TRUE); 741 } 742 743 eq_arm: 744 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 745 746 return; 747 } 748 749 750 static int 751 oce_setup_intr(POCE_SOFTC sc) 752 { 753 int rc = 0, use_intx = 0; 754 int vector = 0, req_vectors = 0; 755 int tot_req_vectors, tot_vectors; 756 757 if (is_rss_enabled(sc)) 758 req_vectors = MAX((sc->nrqs - 1), sc->nwqs); 759 else 760 req_vectors = 1; 761 762 tot_req_vectors = req_vectors; 763 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 764 if (req_vectors > 1) { 765 tot_req_vectors += OCE_RDMA_VECTORS; 766 sc->roce_intr_count = OCE_RDMA_VECTORS; 767 } 768 } 769 770 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) { 771 sc->intr_count = req_vectors; 772 tot_vectors = tot_req_vectors; 773 rc = pci_alloc_msix(sc->dev, &tot_vectors); 774 if (rc != 0) { 775 use_intx = 1; 776 pci_release_msi(sc->dev); 777 } else { 778 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 779 if (tot_vectors < tot_req_vectors) { 780 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) { 781 sc->roce_intr_count = (tot_vectors / 2); 782 } 783 sc->intr_count = tot_vectors - sc->roce_intr_count; 784 } 785 } else { 786 sc->intr_count = tot_vectors; 787 } 788 sc->flags |= OCE_FLAGS_USING_MSIX; 789 } 790 } else 791 use_intx = 1; 792 793 if (use_intx) 794 sc->intr_count = 1; 795 796 /* Scale number of queues based on intr we got */ 797 update_queues_got(sc); 798 799 if (use_intx) { 800 device_printf(sc->dev, "Using legacy interrupt\n"); 801 rc = oce_alloc_intr(sc, vector, oce_intr); 802 if (rc) 803 goto error; 804 } else { 805 for (; vector < sc->intr_count; vector++) { 806 rc = oce_alloc_intr(sc, vector, oce_intr); 807 if (rc) 808 goto error; 809 } 810 } 811 812 return 0; 813 error: 814 oce_intr_free(sc); 815 return rc; 816 } 817 818 819 static int 820 oce_fast_isr(void *arg) 821 { 822 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 823 POCE_SOFTC sc = ii->sc; 824 825 if (ii->eq == NULL) 826 return FILTER_STRAY; 827 828 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE); 829 830 taskqueue_enqueue(ii->tq, &ii->task); 831 832 ii->eq->intr++; 833 834 return FILTER_HANDLED; 835 } 836 837 838 static int 839 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending)) 840 { 841 POCE_INTR_INFO ii; 842 int rc = 0, rr; 843 844 if (vector >= OCE_MAX_EQ) 845 return (EINVAL); 846 847 ii = &sc->intrs[vector]; 848 849 /* Set the resource id for the interrupt. 850 * MSIx is vector + 1 for the resource id, 851 * INTx is 0 for the resource id. 852 */ 853 if (sc->flags & OCE_FLAGS_USING_MSIX) 854 rr = vector + 1; 855 else 856 rr = 0; 857 ii->intr_res = bus_alloc_resource_any(sc->dev, 858 SYS_RES_IRQ, 859 &rr, RF_ACTIVE|RF_SHAREABLE); 860 ii->irq_rr = rr; 861 if (ii->intr_res == NULL) { 862 device_printf(sc->dev, 863 "Could not allocate interrupt\n"); 864 rc = ENXIO; 865 return rc; 866 } 867 868 TASK_INIT(&ii->task, 0, isr, ii); 869 ii->vector = vector; 870 sprintf(ii->task_name, "oce_task[%d]", ii->vector); 871 ii->tq = taskqueue_create_fast(ii->task_name, 872 M_NOWAIT, 873 taskqueue_thread_enqueue, 874 &ii->tq); 875 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq", 876 device_get_nameunit(sc->dev)); 877 878 ii->sc = sc; 879 rc = bus_setup_intr(sc->dev, 880 ii->intr_res, 881 INTR_TYPE_NET, 882 oce_fast_isr, NULL, ii, &ii->tag); 883 return rc; 884 885 } 886 887 888 void 889 oce_intr_free(POCE_SOFTC sc) 890 { 891 int i = 0; 892 893 for (i = 0; i < sc->intr_count; i++) { 894 895 if (sc->intrs[i].tag != NULL) 896 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res, 897 sc->intrs[i].tag); 898 if (sc->intrs[i].tq != NULL) 899 taskqueue_free(sc->intrs[i].tq); 900 901 if (sc->intrs[i].intr_res != NULL) 902 bus_release_resource(sc->dev, SYS_RES_IRQ, 903 sc->intrs[i].irq_rr, 904 sc->intrs[i].intr_res); 905 sc->intrs[i].tag = NULL; 906 sc->intrs[i].intr_res = NULL; 907 } 908 909 if (sc->flags & OCE_FLAGS_USING_MSIX) 910 pci_release_msi(sc->dev); 911 912 } 913 914 915 916 /****************************************************************************** 917 * Media callbacks functions * 918 ******************************************************************************/ 919 920 static void 921 oce_media_status(struct ifnet *ifp, struct ifmediareq *req) 922 { 923 POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc; 924 925 926 req->ifm_status = IFM_AVALID; 927 req->ifm_active = IFM_ETHER; 928 929 if (sc->link_status == 1) 930 req->ifm_status |= IFM_ACTIVE; 931 else 932 return; 933 934 switch (sc->link_speed) { 935 case 1: /* 10 Mbps */ 936 req->ifm_active |= IFM_10_T | IFM_FDX; 937 sc->speed = 10; 938 break; 939 case 2: /* 100 Mbps */ 940 req->ifm_active |= IFM_100_TX | IFM_FDX; 941 sc->speed = 100; 942 break; 943 case 3: /* 1 Gbps */ 944 req->ifm_active |= IFM_1000_T | IFM_FDX; 945 sc->speed = 1000; 946 break; 947 case 4: /* 10 Gbps */ 948 req->ifm_active |= IFM_10G_SR | IFM_FDX; 949 sc->speed = 10000; 950 break; 951 case 5: /* 20 Gbps */ 952 req->ifm_active |= IFM_10G_SR | IFM_FDX; 953 sc->speed = 20000; 954 break; 955 case 6: /* 25 Gbps */ 956 req->ifm_active |= IFM_10G_SR | IFM_FDX; 957 sc->speed = 25000; 958 break; 959 case 7: /* 40 Gbps */ 960 req->ifm_active |= IFM_40G_SR4 | IFM_FDX; 961 sc->speed = 40000; 962 break; 963 default: 964 sc->speed = 0; 965 break; 966 } 967 968 return; 969 } 970 971 972 int 973 oce_media_change(struct ifnet *ifp) 974 { 975 return 0; 976 } 977 978 979 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc, 980 struct mbuf *m, boolean_t *os2bmc, 981 struct mbuf **m_new) 982 { 983 struct ether_header *eh = NULL; 984 985 eh = mtod(m, struct ether_header *); 986 987 if (!is_os2bmc_enabled(sc) || *os2bmc) { 988 *os2bmc = FALSE; 989 goto done; 990 } 991 if (!ETHER_IS_MULTICAST(eh->ether_dhost)) 992 goto done; 993 994 if (is_mc_allowed_on_bmc(sc, eh) || 995 is_bc_allowed_on_bmc(sc, eh) || 996 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) { 997 *os2bmc = TRUE; 998 goto done; 999 } 1000 1001 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) { 1002 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 1003 uint8_t nexthdr = ip6->ip6_nxt; 1004 if (nexthdr == IPPROTO_ICMPV6) { 1005 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1); 1006 switch (icmp6->icmp6_type) { 1007 case ND_ROUTER_ADVERT: 1008 *os2bmc = is_ipv6_ra_filt_enabled(sc); 1009 goto done; 1010 case ND_NEIGHBOR_ADVERT: 1011 *os2bmc = is_ipv6_na_filt_enabled(sc); 1012 goto done; 1013 default: 1014 break; 1015 } 1016 } 1017 } 1018 1019 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) { 1020 struct ip *ip = mtod(m, struct ip *); 1021 int iphlen = ip->ip_hl << 2; 1022 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen); 1023 switch (uh->uh_dport) { 1024 case DHCP_CLIENT_PORT: 1025 *os2bmc = is_dhcp_client_filt_enabled(sc); 1026 goto done; 1027 case DHCP_SERVER_PORT: 1028 *os2bmc = is_dhcp_srvr_filt_enabled(sc); 1029 goto done; 1030 case NET_BIOS_PORT1: 1031 case NET_BIOS_PORT2: 1032 *os2bmc = is_nbios_filt_enabled(sc); 1033 goto done; 1034 case DHCPV6_RAS_PORT: 1035 *os2bmc = is_ipv6_ras_filt_enabled(sc); 1036 goto done; 1037 default: 1038 break; 1039 } 1040 } 1041 done: 1042 if (*os2bmc) { 1043 *m_new = m_dup(m, M_NOWAIT); 1044 if (!*m_new) { 1045 *os2bmc = FALSE; 1046 return; 1047 } 1048 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL); 1049 } 1050 } 1051 1052 1053 1054 /***************************************************************************** 1055 * Transmit routines functions * 1056 *****************************************************************************/ 1057 1058 static int 1059 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index) 1060 { 1061 int rc = 0, i, retry_cnt = 0; 1062 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS]; 1063 struct mbuf *m, *m_temp, *m_new = NULL; 1064 struct oce_wq *wq = sc->wq[wq_index]; 1065 struct oce_packet_desc *pd; 1066 struct oce_nic_hdr_wqe *nichdr; 1067 struct oce_nic_frag_wqe *nicfrag; 1068 struct ether_header *eh = NULL; 1069 int num_wqes; 1070 uint32_t reg_value; 1071 boolean_t complete = TRUE; 1072 boolean_t os2bmc = FALSE; 1073 1074 m = *mpp; 1075 if (!m) 1076 return EINVAL; 1077 1078 if (!(m->m_flags & M_PKTHDR)) { 1079 rc = ENXIO; 1080 goto free_ret; 1081 } 1082 1083 /* Don't allow non-TSO packets longer than MTU */ 1084 if (!is_tso_pkt(m)) { 1085 eh = mtod(m, struct ether_header *); 1086 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE)) 1087 goto free_ret; 1088 } 1089 1090 if(oce_tx_asic_stall_verify(sc, m)) { 1091 m = oce_insert_vlan_tag(sc, m, &complete); 1092 if(!m) { 1093 device_printf(sc->dev, "Insertion unsuccessful\n"); 1094 return 0; 1095 } 1096 1097 } 1098 1099 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less 1100 * may cause a transmit stall on that port. So the work-around is to 1101 * pad short packets (<= 32 bytes) to a 36-byte length. 1102 */ 1103 if(IS_SH(sc) || IS_XE201(sc) ) { 1104 if(m->m_pkthdr.len <= 32) { 1105 char buf[36]; 1106 bzero((void *)buf, 36); 1107 m_append(m, (36 - m->m_pkthdr.len), buf); 1108 } 1109 } 1110 1111 tx_start: 1112 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1113 /* consolidate packet buffers for TSO/LSO segment offload */ 1114 #if defined(INET6) || defined(INET) 1115 m = oce_tso_setup(sc, mpp); 1116 #else 1117 m = NULL; 1118 #endif 1119 if (m == NULL) { 1120 rc = ENXIO; 1121 goto free_ret; 1122 } 1123 } 1124 1125 1126 pd = &wq->pckts[wq->pkt_desc_head]; 1127 1128 retry: 1129 rc = bus_dmamap_load_mbuf_sg(wq->tag, 1130 pd->map, 1131 m, segs, &pd->nsegs, BUS_DMA_NOWAIT); 1132 if (rc == 0) { 1133 num_wqes = pd->nsegs + 1; 1134 if (IS_BE(sc) || IS_SH(sc)) { 1135 /*Dummy required only for BE3.*/ 1136 if (num_wqes & 1) 1137 num_wqes++; 1138 } 1139 if (num_wqes >= RING_NUM_FREE(wq->ring)) { 1140 bus_dmamap_unload(wq->tag, pd->map); 1141 return EBUSY; 1142 } 1143 atomic_store_rel_int(&wq->pkt_desc_head, 1144 (wq->pkt_desc_head + 1) % \ 1145 OCE_WQ_PACKET_ARRAY_SIZE); 1146 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE); 1147 pd->mbuf = m; 1148 1149 nichdr = 1150 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe); 1151 nichdr->u0.dw[0] = 0; 1152 nichdr->u0.dw[1] = 0; 1153 nichdr->u0.dw[2] = 0; 1154 nichdr->u0.dw[3] = 0; 1155 1156 nichdr->u0.s.complete = complete; 1157 nichdr->u0.s.mgmt = os2bmc; 1158 nichdr->u0.s.event = 1; 1159 nichdr->u0.s.crc = 1; 1160 nichdr->u0.s.forward = 0; 1161 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0; 1162 nichdr->u0.s.udpcs = 1163 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0; 1164 nichdr->u0.s.tcpcs = 1165 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0; 1166 nichdr->u0.s.num_wqe = num_wqes; 1167 nichdr->u0.s.total_length = m->m_pkthdr.len; 1168 1169 if (m->m_flags & M_VLANTAG) { 1170 nichdr->u0.s.vlan = 1; /*Vlan present*/ 1171 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag; 1172 } 1173 1174 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1175 if (m->m_pkthdr.tso_segsz) { 1176 nichdr->u0.s.lso = 1; 1177 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz; 1178 } 1179 if (!IS_BE(sc) || !IS_SH(sc)) 1180 nichdr->u0.s.ipcs = 1; 1181 } 1182 1183 RING_PUT(wq->ring, 1); 1184 atomic_add_int(&wq->ring->num_used, 1); 1185 1186 for (i = 0; i < pd->nsegs; i++) { 1187 nicfrag = 1188 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1189 struct oce_nic_frag_wqe); 1190 nicfrag->u0.s.rsvd0 = 0; 1191 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr); 1192 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr); 1193 nicfrag->u0.s.frag_len = segs[i].ds_len; 1194 pd->wqe_idx = wq->ring->pidx; 1195 RING_PUT(wq->ring, 1); 1196 atomic_add_int(&wq->ring->num_used, 1); 1197 } 1198 if (num_wqes > (pd->nsegs + 1)) { 1199 nicfrag = 1200 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1201 struct oce_nic_frag_wqe); 1202 nicfrag->u0.dw[0] = 0; 1203 nicfrag->u0.dw[1] = 0; 1204 nicfrag->u0.dw[2] = 0; 1205 nicfrag->u0.dw[3] = 0; 1206 pd->wqe_idx = wq->ring->pidx; 1207 RING_PUT(wq->ring, 1); 1208 atomic_add_int(&wq->ring->num_used, 1); 1209 pd->nsegs++; 1210 } 1211 1212 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 1213 wq->tx_stats.tx_reqs++; 1214 wq->tx_stats.tx_wrbs += num_wqes; 1215 wq->tx_stats.tx_bytes += m->m_pkthdr.len; 1216 wq->tx_stats.tx_pkts++; 1217 1218 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map, 1219 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1220 reg_value = (num_wqes << 16) | wq->wq_id; 1221 1222 /* if os2bmc is not enabled or if the pkt is already tagged as 1223 bmc, do nothing 1224 */ 1225 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new); 1226 1227 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value); 1228 1229 } else if (rc == EFBIG) { 1230 if (retry_cnt == 0) { 1231 m_temp = m_defrag(m, M_NOWAIT); 1232 if (m_temp == NULL) 1233 goto free_ret; 1234 m = m_temp; 1235 *mpp = m_temp; 1236 retry_cnt = retry_cnt + 1; 1237 goto retry; 1238 } else 1239 goto free_ret; 1240 } else if (rc == ENOMEM) 1241 return rc; 1242 else 1243 goto free_ret; 1244 1245 if (os2bmc) { 1246 m = m_new; 1247 goto tx_start; 1248 } 1249 1250 return 0; 1251 1252 free_ret: 1253 m_freem(*mpp); 1254 *mpp = NULL; 1255 return rc; 1256 } 1257 1258 1259 static void 1260 oce_process_tx_completion(struct oce_wq *wq) 1261 { 1262 struct oce_packet_desc *pd; 1263 POCE_SOFTC sc = (POCE_SOFTC) wq->parent; 1264 struct mbuf *m; 1265 1266 pd = &wq->pckts[wq->pkt_desc_tail]; 1267 atomic_store_rel_int(&wq->pkt_desc_tail, 1268 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE); 1269 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1); 1270 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1271 bus_dmamap_unload(wq->tag, pd->map); 1272 1273 m = pd->mbuf; 1274 m_freem(m); 1275 pd->mbuf = NULL; 1276 1277 1278 if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) { 1279 if (wq->ring->num_used < (wq->ring->num_items / 2)) { 1280 sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE); 1281 oce_tx_restart(sc, wq); 1282 } 1283 } 1284 } 1285 1286 1287 static void 1288 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq) 1289 { 1290 1291 if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) 1292 return; 1293 1294 if (!drbr_empty(sc->ifp, wq->br)) 1295 taskqueue_enqueue(taskqueue_swi, &wq->txtask); 1296 1297 } 1298 1299 1300 #if defined(INET6) || defined(INET) 1301 static struct mbuf * 1302 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp) 1303 { 1304 struct mbuf *m; 1305 #ifdef INET 1306 struct ip *ip; 1307 #endif 1308 #ifdef INET6 1309 struct ip6_hdr *ip6; 1310 #endif 1311 struct ether_vlan_header *eh; 1312 struct tcphdr *th; 1313 uint16_t etype; 1314 int total_len = 0, ehdrlen = 0; 1315 1316 m = *mpp; 1317 1318 if (M_WRITABLE(m) == 0) { 1319 m = m_dup(*mpp, M_NOWAIT); 1320 if (!m) 1321 return NULL; 1322 m_freem(*mpp); 1323 *mpp = m; 1324 } 1325 1326 eh = mtod(m, struct ether_vlan_header *); 1327 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1328 etype = ntohs(eh->evl_proto); 1329 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1330 } else { 1331 etype = ntohs(eh->evl_encap_proto); 1332 ehdrlen = ETHER_HDR_LEN; 1333 } 1334 1335 switch (etype) { 1336 #ifdef INET 1337 case ETHERTYPE_IP: 1338 ip = (struct ip *)(m->m_data + ehdrlen); 1339 if (ip->ip_p != IPPROTO_TCP) 1340 return NULL; 1341 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); 1342 1343 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2); 1344 break; 1345 #endif 1346 #ifdef INET6 1347 case ETHERTYPE_IPV6: 1348 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen); 1349 if (ip6->ip6_nxt != IPPROTO_TCP) 1350 return NULL; 1351 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr)); 1352 1353 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2); 1354 break; 1355 #endif 1356 default: 1357 return NULL; 1358 } 1359 1360 m = m_pullup(m, total_len); 1361 if (!m) 1362 return NULL; 1363 *mpp = m; 1364 return m; 1365 1366 } 1367 #endif /* INET6 || INET */ 1368 1369 void 1370 oce_tx_task(void *arg, int npending) 1371 { 1372 struct oce_wq *wq = arg; 1373 POCE_SOFTC sc = wq->parent; 1374 struct ifnet *ifp = sc->ifp; 1375 int rc = 0; 1376 1377 LOCK(&wq->tx_lock); 1378 rc = oce_multiq_transmit(ifp, NULL, wq); 1379 if (rc) { 1380 device_printf(sc->dev, 1381 "TX[%d] restart failed\n", wq->queue_index); 1382 } 1383 UNLOCK(&wq->tx_lock); 1384 } 1385 1386 1387 void 1388 oce_start(struct ifnet *ifp) 1389 { 1390 POCE_SOFTC sc = ifp->if_softc; 1391 struct mbuf *m; 1392 int rc = 0; 1393 int def_q = 0; /* Defualt tx queue is 0*/ 1394 1395 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1396 IFF_DRV_RUNNING) 1397 return; 1398 1399 if (!sc->link_status) 1400 return; 1401 1402 do { 1403 IF_DEQUEUE(&sc->ifp->if_snd, m); 1404 if (m == NULL) 1405 break; 1406 1407 LOCK(&sc->wq[def_q]->tx_lock); 1408 rc = oce_tx(sc, &m, def_q); 1409 UNLOCK(&sc->wq[def_q]->tx_lock); 1410 if (rc) { 1411 if (m != NULL) { 1412 sc->wq[def_q]->tx_stats.tx_stops ++; 1413 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1414 IFQ_DRV_PREPEND(&ifp->if_snd, m); 1415 m = NULL; 1416 } 1417 break; 1418 } 1419 if (m != NULL) 1420 ETHER_BPF_MTAP(ifp, m); 1421 1422 } while (TRUE); 1423 1424 return; 1425 } 1426 1427 1428 /* Handle the Completion Queue for transmit */ 1429 uint16_t 1430 oce_wq_handler(void *arg) 1431 { 1432 struct oce_wq *wq = (struct oce_wq *)arg; 1433 POCE_SOFTC sc = wq->parent; 1434 struct oce_cq *cq = wq->cq; 1435 struct oce_nic_tx_cqe *cqe; 1436 int num_cqes = 0; 1437 1438 LOCK(&wq->tx_compl_lock); 1439 bus_dmamap_sync(cq->ring->dma.tag, 1440 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1441 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1442 while (cqe->u0.dw[3]) { 1443 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe)); 1444 1445 wq->ring->cidx = cqe->u0.s.wqe_index + 1; 1446 if (wq->ring->cidx >= wq->ring->num_items) 1447 wq->ring->cidx -= wq->ring->num_items; 1448 1449 oce_process_tx_completion(wq); 1450 wq->tx_stats.tx_compl++; 1451 cqe->u0.dw[3] = 0; 1452 RING_GET(cq->ring, 1); 1453 bus_dmamap_sync(cq->ring->dma.tag, 1454 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1455 cqe = 1456 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1457 num_cqes++; 1458 } 1459 1460 if (num_cqes) 1461 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 1462 1463 UNLOCK(&wq->tx_compl_lock); 1464 return num_cqes; 1465 } 1466 1467 1468 static int 1469 oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq) 1470 { 1471 POCE_SOFTC sc = ifp->if_softc; 1472 int status = 0, queue_index = 0; 1473 struct mbuf *next = NULL; 1474 struct buf_ring *br = NULL; 1475 1476 br = wq->br; 1477 queue_index = wq->queue_index; 1478 1479 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1480 IFF_DRV_RUNNING) { 1481 if (m != NULL) 1482 status = drbr_enqueue(ifp, br, m); 1483 return status; 1484 } 1485 1486 if (m != NULL) { 1487 if ((status = drbr_enqueue(ifp, br, m)) != 0) 1488 return status; 1489 } 1490 while ((next = drbr_peek(ifp, br)) != NULL) { 1491 if (oce_tx(sc, &next, queue_index)) { 1492 if (next == NULL) { 1493 drbr_advance(ifp, br); 1494 } else { 1495 drbr_putback(ifp, br, next); 1496 wq->tx_stats.tx_stops ++; 1497 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1498 } 1499 break; 1500 } 1501 drbr_advance(ifp, br); 1502 if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len); 1503 if (next->m_flags & M_MCAST) 1504 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 1505 ETHER_BPF_MTAP(ifp, next); 1506 } 1507 1508 return 0; 1509 } 1510 1511 1512 1513 1514 /***************************************************************************** 1515 * Receive routines functions * 1516 *****************************************************************************/ 1517 1518 static void 1519 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2) 1520 { 1521 uint32_t *p; 1522 struct ether_header *eh = NULL; 1523 struct tcphdr *tcp_hdr = NULL; 1524 struct ip *ip4_hdr = NULL; 1525 struct ip6_hdr *ip6 = NULL; 1526 uint32_t payload_len = 0; 1527 1528 eh = mtod(m, struct ether_header *); 1529 /* correct IP header */ 1530 if(!cqe2->ipv6_frame) { 1531 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header)); 1532 ip4_hdr->ip_ttl = cqe2->frame_lifespan; 1533 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header)); 1534 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip)); 1535 }else { 1536 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header)); 1537 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan; 1538 payload_len = cqe2->coalesced_size - sizeof(struct ether_header) 1539 - sizeof(struct ip6_hdr); 1540 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len); 1541 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr)); 1542 } 1543 1544 /* correct tcp header */ 1545 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num); 1546 if(cqe2->push) { 1547 tcp_hdr->th_flags |= TH_PUSH; 1548 } 1549 tcp_hdr->th_win = htons(cqe2->tcp_window); 1550 tcp_hdr->th_sum = 0xffff; 1551 if(cqe2->ts_opt) { 1552 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2); 1553 *p = cqe1->tcp_timestamp_val; 1554 *(p+1) = cqe1->tcp_timestamp_ecr; 1555 } 1556 1557 return; 1558 } 1559 1560 static void 1561 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m) 1562 { 1563 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1564 uint32_t i = 0, frag_len = 0; 1565 uint32_t len = cqe_info->pkt_size; 1566 struct oce_packet_desc *pd; 1567 struct mbuf *tail = NULL; 1568 1569 for (i = 0; i < cqe_info->num_frags; i++) { 1570 if (rq->ring->cidx == rq->ring->pidx) { 1571 device_printf(sc->dev, 1572 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n"); 1573 return; 1574 } 1575 pd = &rq->pckts[rq->ring->cidx]; 1576 1577 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1578 bus_dmamap_unload(rq->tag, pd->map); 1579 RING_GET(rq->ring, 1); 1580 rq->pending--; 1581 1582 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len; 1583 pd->mbuf->m_len = frag_len; 1584 1585 if (tail != NULL) { 1586 /* additional fragments */ 1587 pd->mbuf->m_flags &= ~M_PKTHDR; 1588 tail->m_next = pd->mbuf; 1589 if(rq->islro) 1590 tail->m_nextpkt = NULL; 1591 tail = pd->mbuf; 1592 } else { 1593 /* first fragment, fill out much of the packet header */ 1594 pd->mbuf->m_pkthdr.len = len; 1595 if(rq->islro) 1596 pd->mbuf->m_nextpkt = NULL; 1597 pd->mbuf->m_pkthdr.csum_flags = 0; 1598 if (IF_CSUM_ENABLED(sc)) { 1599 if (cqe_info->l4_cksum_pass) { 1600 if(!cqe_info->ipv6_frame) { /* IPV4 */ 1601 pd->mbuf->m_pkthdr.csum_flags |= 1602 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1603 }else { /* IPV6 frame */ 1604 if(rq->islro) { 1605 pd->mbuf->m_pkthdr.csum_flags |= 1606 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1607 } 1608 } 1609 pd->mbuf->m_pkthdr.csum_data = 0xffff; 1610 } 1611 if (cqe_info->ip_cksum_pass) { 1612 pd->mbuf->m_pkthdr.csum_flags |= 1613 (CSUM_IP_CHECKED|CSUM_IP_VALID); 1614 } 1615 } 1616 *m = tail = pd->mbuf; 1617 } 1618 pd->mbuf = NULL; 1619 len -= frag_len; 1620 } 1621 1622 return; 1623 } 1624 1625 static void 1626 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2) 1627 { 1628 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1629 struct nic_hwlro_cqe_part1 *cqe1 = NULL; 1630 struct mbuf *m = NULL; 1631 struct oce_common_cqe_info cq_info; 1632 1633 /* parse cqe */ 1634 if(cqe2 == NULL) { 1635 cq_info.pkt_size = cqe->pkt_size; 1636 cq_info.vtag = cqe->vlan_tag; 1637 cq_info.l4_cksum_pass = cqe->l4_cksum_pass; 1638 cq_info.ip_cksum_pass = cqe->ip_cksum_pass; 1639 cq_info.ipv6_frame = cqe->ipv6_frame; 1640 cq_info.vtp = cqe->vtp; 1641 cq_info.qnq = cqe->qnq; 1642 }else { 1643 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe; 1644 cq_info.pkt_size = cqe2->coalesced_size; 1645 cq_info.vtag = cqe2->vlan_tag; 1646 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass; 1647 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass; 1648 cq_info.ipv6_frame = cqe2->ipv6_frame; 1649 cq_info.vtp = cqe2->vtp; 1650 cq_info.qnq = cqe1->qnq; 1651 } 1652 1653 cq_info.vtag = BSWAP_16(cq_info.vtag); 1654 1655 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size; 1656 if(cq_info.pkt_size % rq->cfg.frag_size) 1657 cq_info.num_frags++; 1658 1659 oce_rx_mbuf_chain(rq, &cq_info, &m); 1660 1661 if (m) { 1662 if(cqe2) { 1663 //assert(cqe2->valid != 0); 1664 1665 //assert(cqe2->cqe_type != 2); 1666 oce_correct_header(m, cqe1, cqe2); 1667 } 1668 1669 m->m_pkthdr.rcvif = sc->ifp; 1670 if (rq->queue_index) 1671 m->m_pkthdr.flowid = (rq->queue_index - 1); 1672 else 1673 m->m_pkthdr.flowid = rq->queue_index; 1674 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1675 1676 /* This deternies if vlan tag is Valid */ 1677 if (cq_info.vtp) { 1678 if (sc->function_mode & FNM_FLEX10_MODE) { 1679 /* FLEX10. If QnQ is not set, neglect VLAN */ 1680 if (cq_info.qnq) { 1681 m->m_pkthdr.ether_vtag = cq_info.vtag; 1682 m->m_flags |= M_VLANTAG; 1683 } 1684 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) { 1685 /* In UMC mode generally pvid will be striped by 1686 hw. But in some cases we have seen it comes 1687 with pvid. So if pvid == vlan, neglect vlan. 1688 */ 1689 m->m_pkthdr.ether_vtag = cq_info.vtag; 1690 m->m_flags |= M_VLANTAG; 1691 } 1692 } 1693 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1694 1695 (*sc->ifp->if_input) (sc->ifp, m); 1696 1697 /* Update rx stats per queue */ 1698 rq->rx_stats.rx_pkts++; 1699 rq->rx_stats.rx_bytes += cq_info.pkt_size; 1700 rq->rx_stats.rx_frags += cq_info.num_frags; 1701 rq->rx_stats.rx_ucast_pkts++; 1702 } 1703 return; 1704 } 1705 1706 static void 1707 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe) 1708 { 1709 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1710 int len; 1711 struct mbuf *m = NULL; 1712 struct oce_common_cqe_info cq_info; 1713 uint16_t vtag = 0; 1714 1715 /* Is it a flush compl that has no data */ 1716 if(!cqe->u0.s.num_fragments) 1717 goto exit; 1718 1719 len = cqe->u0.s.pkt_size; 1720 if (!len) { 1721 /*partial DMA workaround for Lancer*/ 1722 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1723 goto exit; 1724 } 1725 1726 if (!oce_cqe_portid_valid(sc, cqe)) { 1727 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1728 goto exit; 1729 } 1730 1731 /* Get vlan_tag value */ 1732 if(IS_BE(sc) || IS_SH(sc)) 1733 vtag = BSWAP_16(cqe->u0.s.vlan_tag); 1734 else 1735 vtag = cqe->u0.s.vlan_tag; 1736 1737 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass; 1738 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass; 1739 cq_info.ipv6_frame = cqe->u0.s.ip_ver; 1740 cq_info.num_frags = cqe->u0.s.num_fragments; 1741 cq_info.pkt_size = cqe->u0.s.pkt_size; 1742 1743 oce_rx_mbuf_chain(rq, &cq_info, &m); 1744 1745 if (m) { 1746 m->m_pkthdr.rcvif = sc->ifp; 1747 if (rq->queue_index) 1748 m->m_pkthdr.flowid = (rq->queue_index - 1); 1749 else 1750 m->m_pkthdr.flowid = rq->queue_index; 1751 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1752 1753 /* This deternies if vlan tag is Valid */ 1754 if (oce_cqe_vtp_valid(sc, cqe)) { 1755 if (sc->function_mode & FNM_FLEX10_MODE) { 1756 /* FLEX10. If QnQ is not set, neglect VLAN */ 1757 if (cqe->u0.s.qnq) { 1758 m->m_pkthdr.ether_vtag = vtag; 1759 m->m_flags |= M_VLANTAG; 1760 } 1761 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) { 1762 /* In UMC mode generally pvid will be striped by 1763 hw. But in some cases we have seen it comes 1764 with pvid. So if pvid == vlan, neglect vlan. 1765 */ 1766 m->m_pkthdr.ether_vtag = vtag; 1767 m->m_flags |= M_VLANTAG; 1768 } 1769 } 1770 1771 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1772 #if defined(INET6) || defined(INET) 1773 /* Try to queue to LRO */ 1774 if (IF_LRO_ENABLED(sc) && 1775 (cqe->u0.s.ip_cksum_pass) && 1776 (cqe->u0.s.l4_cksum_pass) && 1777 (!cqe->u0.s.ip_ver) && 1778 (rq->lro.lro_cnt != 0)) { 1779 1780 if (tcp_lro_rx(&rq->lro, m, 0) == 0) { 1781 rq->lro_pkts_queued ++; 1782 goto post_done; 1783 } 1784 /* If LRO posting fails then try to post to STACK */ 1785 } 1786 #endif 1787 1788 (*sc->ifp->if_input) (sc->ifp, m); 1789 #if defined(INET6) || defined(INET) 1790 post_done: 1791 #endif 1792 /* Update rx stats per queue */ 1793 rq->rx_stats.rx_pkts++; 1794 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size; 1795 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments; 1796 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET) 1797 rq->rx_stats.rx_mcast_pkts++; 1798 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET) 1799 rq->rx_stats.rx_ucast_pkts++; 1800 } 1801 exit: 1802 return; 1803 } 1804 1805 1806 void 1807 oce_discard_rx_comp(struct oce_rq *rq, int num_frags) 1808 { 1809 uint32_t i = 0; 1810 struct oce_packet_desc *pd; 1811 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1812 1813 for (i = 0; i < num_frags; i++) { 1814 if (rq->ring->cidx == rq->ring->pidx) { 1815 device_printf(sc->dev, 1816 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n"); 1817 return; 1818 } 1819 pd = &rq->pckts[rq->ring->cidx]; 1820 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1821 bus_dmamap_unload(rq->tag, pd->map); 1822 if (pd->mbuf != NULL) { 1823 m_freem(pd->mbuf); 1824 pd->mbuf = NULL; 1825 } 1826 1827 RING_GET(rq->ring, 1); 1828 rq->pending--; 1829 } 1830 } 1831 1832 1833 static int 1834 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1835 { 1836 struct oce_nic_rx_cqe_v1 *cqe_v1; 1837 int vtp = 0; 1838 1839 if (sc->be3_native) { 1840 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1841 vtp = cqe_v1->u0.s.vlan_tag_present; 1842 } else 1843 vtp = cqe->u0.s.vlan_tag_present; 1844 1845 return vtp; 1846 1847 } 1848 1849 1850 static int 1851 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1852 { 1853 struct oce_nic_rx_cqe_v1 *cqe_v1; 1854 int port_id = 0; 1855 1856 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) { 1857 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1858 port_id = cqe_v1->u0.s.port; 1859 if (sc->port_id != port_id) 1860 return 0; 1861 } else 1862 ;/* For BE3 legacy and Lancer this is dummy */ 1863 1864 return 1; 1865 1866 } 1867 1868 #if defined(INET6) || defined(INET) 1869 void 1870 oce_rx_flush_lro(struct oce_rq *rq) 1871 { 1872 struct lro_ctrl *lro = &rq->lro; 1873 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1874 1875 if (!IF_LRO_ENABLED(sc)) 1876 return; 1877 1878 tcp_lro_flush_all(lro); 1879 rq->lro_pkts_queued = 0; 1880 1881 return; 1882 } 1883 1884 1885 static int 1886 oce_init_lro(POCE_SOFTC sc) 1887 { 1888 struct lro_ctrl *lro = NULL; 1889 int i = 0, rc = 0; 1890 1891 for (i = 0; i < sc->nrqs; i++) { 1892 lro = &sc->rq[i]->lro; 1893 rc = tcp_lro_init(lro); 1894 if (rc != 0) { 1895 device_printf(sc->dev, "LRO init failed\n"); 1896 return rc; 1897 } 1898 lro->ifp = sc->ifp; 1899 } 1900 1901 return rc; 1902 } 1903 1904 1905 void 1906 oce_free_lro(POCE_SOFTC sc) 1907 { 1908 struct lro_ctrl *lro = NULL; 1909 int i = 0; 1910 1911 for (i = 0; i < sc->nrqs; i++) { 1912 lro = &sc->rq[i]->lro; 1913 if (lro) 1914 tcp_lro_free(lro); 1915 } 1916 } 1917 #endif 1918 1919 int 1920 oce_alloc_rx_bufs(struct oce_rq *rq, int count) 1921 { 1922 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1923 int i, in, rc; 1924 struct oce_packet_desc *pd; 1925 bus_dma_segment_t segs[6]; 1926 int nsegs, added = 0; 1927 struct oce_nic_rqe *rqe; 1928 pd_rxulp_db_t rxdb_reg; 1929 uint32_t val = 0; 1930 uint32_t oce_max_rq_posts = 64; 1931 1932 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t)); 1933 for (i = 0; i < count; i++) { 1934 in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE; 1935 1936 pd = &rq->pckts[rq->ring->pidx]; 1937 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size); 1938 if (pd->mbuf == NULL) { 1939 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size); 1940 break; 1941 } 1942 pd->mbuf->m_nextpkt = NULL; 1943 1944 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size; 1945 1946 rc = bus_dmamap_load_mbuf_sg(rq->tag, 1947 pd->map, 1948 pd->mbuf, 1949 segs, &nsegs, BUS_DMA_NOWAIT); 1950 if (rc) { 1951 m_free(pd->mbuf); 1952 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc); 1953 break; 1954 } 1955 1956 if (nsegs != 1) { 1957 i--; 1958 continue; 1959 } 1960 1961 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD); 1962 1963 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe); 1964 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr); 1965 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr); 1966 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe)); 1967 RING_PUT(rq->ring, 1); 1968 added++; 1969 rq->pending++; 1970 } 1971 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS; 1972 if (added != 0) { 1973 for (i = added / oce_max_rq_posts; i > 0; i--) { 1974 rxdb_reg.bits.num_posted = oce_max_rq_posts; 1975 rxdb_reg.bits.qid = rq->rq_id; 1976 if(rq->islro) { 1977 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1978 val |= oce_max_rq_posts << 16; 1979 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1980 }else { 1981 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1982 } 1983 added -= oce_max_rq_posts; 1984 } 1985 if (added > 0) { 1986 rxdb_reg.bits.qid = rq->rq_id; 1987 rxdb_reg.bits.num_posted = added; 1988 if(rq->islro) { 1989 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1990 val |= added << 16; 1991 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1992 }else { 1993 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1994 } 1995 } 1996 } 1997 1998 return 0; 1999 } 2000 2001 static void 2002 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq) 2003 { 2004 if (num_cqes) { 2005 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE); 2006 if(!sc->enable_hwlro) { 2007 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1) 2008 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1)); 2009 }else { 2010 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64) 2011 oce_alloc_rx_bufs(rq, 64); 2012 } 2013 } 2014 2015 return; 2016 } 2017 2018 uint16_t 2019 oce_rq_handler_lro(void *arg) 2020 { 2021 struct oce_rq *rq = (struct oce_rq *)arg; 2022 struct oce_cq *cq = rq->cq; 2023 POCE_SOFTC sc = rq->parent; 2024 struct nic_hwlro_singleton_cqe *cqe; 2025 struct nic_hwlro_cqe_part2 *cqe2; 2026 int num_cqes = 0; 2027 2028 LOCK(&rq->rx_lock); 2029 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2030 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2031 while (cqe->valid) { 2032 if(cqe->cqe_type == 0) { /* singleton cqe */ 2033 /* we should not get singleton cqe after cqe1 on same rq */ 2034 if(rq->cqe_firstpart != NULL) { 2035 device_printf(sc->dev, "Got singleton cqe after cqe1 \n"); 2036 goto exit_rq_handler_lro; 2037 } 2038 if(cqe->error != 0) { 2039 rq->rx_stats.rxcp_err++; 2040 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2041 } 2042 oce_rx_lro(rq, cqe, NULL); 2043 rq->rx_stats.rx_compl++; 2044 cqe->valid = 0; 2045 RING_GET(cq->ring, 1); 2046 num_cqes++; 2047 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2048 break; 2049 }else if(cqe->cqe_type == 0x1) { /* first part */ 2050 /* we should not get cqe1 after cqe1 on same rq */ 2051 if(rq->cqe_firstpart != NULL) { 2052 device_printf(sc->dev, "Got cqe1 after cqe1 \n"); 2053 goto exit_rq_handler_lro; 2054 } 2055 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe; 2056 RING_GET(cq->ring, 1); 2057 }else if(cqe->cqe_type == 0x2) { /* second part */ 2058 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe; 2059 if(cqe2->error != 0) { 2060 rq->rx_stats.rxcp_err++; 2061 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2062 } 2063 /* We should not get cqe2 without cqe1 */ 2064 if(rq->cqe_firstpart == NULL) { 2065 device_printf(sc->dev, "Got cqe2 without cqe1 \n"); 2066 goto exit_rq_handler_lro; 2067 } 2068 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2); 2069 2070 rq->rx_stats.rx_compl++; 2071 rq->cqe_firstpart->valid = 0; 2072 cqe2->valid = 0; 2073 rq->cqe_firstpart = NULL; 2074 2075 RING_GET(cq->ring, 1); 2076 num_cqes += 2; 2077 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2078 break; 2079 } 2080 2081 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2082 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2083 } 2084 oce_check_rx_bufs(sc, num_cqes, rq); 2085 exit_rq_handler_lro: 2086 UNLOCK(&rq->rx_lock); 2087 return 0; 2088 } 2089 2090 /* Handle the Completion Queue for receive */ 2091 uint16_t 2092 oce_rq_handler(void *arg) 2093 { 2094 struct oce_rq *rq = (struct oce_rq *)arg; 2095 struct oce_cq *cq = rq->cq; 2096 POCE_SOFTC sc = rq->parent; 2097 struct oce_nic_rx_cqe *cqe; 2098 int num_cqes = 0; 2099 2100 if(rq->islro) { 2101 oce_rq_handler_lro(arg); 2102 return 0; 2103 } 2104 LOCK(&rq->rx_lock); 2105 bus_dmamap_sync(cq->ring->dma.tag, 2106 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2107 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2108 while (cqe->u0.dw[2]) { 2109 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe)); 2110 2111 if (cqe->u0.s.error == 0) { 2112 oce_rx(rq, cqe); 2113 } else { 2114 rq->rx_stats.rxcp_err++; 2115 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2116 /* Post L3/L4 errors to stack.*/ 2117 oce_rx(rq, cqe); 2118 } 2119 rq->rx_stats.rx_compl++; 2120 cqe->u0.dw[2] = 0; 2121 2122 #if defined(INET6) || defined(INET) 2123 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) { 2124 oce_rx_flush_lro(rq); 2125 } 2126 #endif 2127 2128 RING_GET(cq->ring, 1); 2129 bus_dmamap_sync(cq->ring->dma.tag, 2130 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2131 cqe = 2132 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2133 num_cqes++; 2134 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2135 break; 2136 } 2137 2138 #if defined(INET6) || defined(INET) 2139 if (IF_LRO_ENABLED(sc)) 2140 oce_rx_flush_lro(rq); 2141 #endif 2142 2143 oce_check_rx_bufs(sc, num_cqes, rq); 2144 UNLOCK(&rq->rx_lock); 2145 return 0; 2146 2147 } 2148 2149 2150 2151 2152 /***************************************************************************** 2153 * Helper function prototypes in this file * 2154 *****************************************************************************/ 2155 2156 static int 2157 oce_attach_ifp(POCE_SOFTC sc) 2158 { 2159 2160 sc->ifp = if_alloc(IFT_ETHER); 2161 if (!sc->ifp) 2162 return ENOMEM; 2163 2164 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status); 2165 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2166 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 2167 2168 sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST; 2169 sc->ifp->if_ioctl = oce_ioctl; 2170 sc->ifp->if_start = oce_start; 2171 sc->ifp->if_init = oce_init; 2172 sc->ifp->if_mtu = ETHERMTU; 2173 sc->ifp->if_softc = sc; 2174 sc->ifp->if_transmit = oce_multiq_start; 2175 sc->ifp->if_qflush = oce_multiq_flush; 2176 2177 if_initname(sc->ifp, 2178 device_get_name(sc->dev), device_get_unit(sc->dev)); 2179 2180 sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1; 2181 IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen); 2182 IFQ_SET_READY(&sc->ifp->if_snd); 2183 2184 sc->ifp->if_hwassist = OCE_IF_HWASSIST; 2185 sc->ifp->if_hwassist |= CSUM_TSO; 2186 sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP); 2187 2188 sc->ifp->if_capabilities = OCE_IF_CAPABILITIES; 2189 sc->ifp->if_capabilities |= IFCAP_HWCSUM; 2190 sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2191 2192 #if defined(INET6) || defined(INET) 2193 sc->ifp->if_capabilities |= IFCAP_TSO; 2194 sc->ifp->if_capabilities |= IFCAP_LRO; 2195 sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 2196 #endif 2197 2198 sc->ifp->if_capenable = sc->ifp->if_capabilities; 2199 sc->ifp->if_baudrate = IF_Gbps(10); 2200 2201 sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 2202 sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS; 2203 sc->ifp->if_hw_tsomaxsegsize = 4096; 2204 2205 ether_ifattach(sc->ifp, sc->macaddr.mac_addr); 2206 2207 return 0; 2208 } 2209 2210 2211 static void 2212 oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2213 { 2214 POCE_SOFTC sc = ifp->if_softc; 2215 2216 if (ifp->if_softc != arg) 2217 return; 2218 if ((vtag == 0) || (vtag > 4095)) 2219 return; 2220 2221 sc->vlan_tag[vtag] = 1; 2222 sc->vlans_added++; 2223 if (sc->vlans_added <= (sc->max_vlans + 1)) 2224 oce_vid_config(sc); 2225 } 2226 2227 2228 static void 2229 oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2230 { 2231 POCE_SOFTC sc = ifp->if_softc; 2232 2233 if (ifp->if_softc != arg) 2234 return; 2235 if ((vtag == 0) || (vtag > 4095)) 2236 return; 2237 2238 sc->vlan_tag[vtag] = 0; 2239 sc->vlans_added--; 2240 oce_vid_config(sc); 2241 } 2242 2243 2244 /* 2245 * A max of 64 vlans can be configured in BE. If the user configures 2246 * more, place the card in vlan promiscuous mode. 2247 */ 2248 static int 2249 oce_vid_config(POCE_SOFTC sc) 2250 { 2251 struct normal_vlan vtags[MAX_VLANFILTER_SIZE]; 2252 uint16_t ntags = 0, i; 2253 int status = 0; 2254 2255 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) && 2256 (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) { 2257 for (i = 0; i < MAX_VLANS; i++) { 2258 if (sc->vlan_tag[i]) { 2259 vtags[ntags].vtag = i; 2260 ntags++; 2261 } 2262 } 2263 if (ntags) 2264 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2265 vtags, ntags, 1, 0); 2266 } else 2267 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2268 NULL, 0, 1, 1); 2269 return status; 2270 } 2271 2272 2273 static void 2274 oce_mac_addr_set(POCE_SOFTC sc) 2275 { 2276 uint32_t old_pmac_id = sc->pmac_id; 2277 int status = 0; 2278 2279 2280 status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2281 sc->macaddr.size_of_struct); 2282 if (!status) 2283 return; 2284 2285 status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)), 2286 sc->if_id, &sc->pmac_id); 2287 if (!status) { 2288 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id); 2289 bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2290 sc->macaddr.size_of_struct); 2291 } 2292 if (status) 2293 device_printf(sc->dev, "Failed update macaddress\n"); 2294 2295 } 2296 2297 2298 static int 2299 oce_handle_passthrough(struct ifnet *ifp, caddr_t data) 2300 { 2301 POCE_SOFTC sc = ifp->if_softc; 2302 struct ifreq *ifr = (struct ifreq *)data; 2303 int rc = ENXIO; 2304 char cookie[32] = {0}; 2305 void *priv_data = ifr_data_get_ptr(ifr); 2306 void *ioctl_ptr; 2307 uint32_t req_size; 2308 struct mbx_hdr req; 2309 OCE_DMA_MEM dma_mem; 2310 struct mbx_common_get_cntl_attr *fw_cmd; 2311 2312 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE))) 2313 return EFAULT; 2314 2315 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE))) 2316 return EINVAL; 2317 2318 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE); 2319 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr))) 2320 return EFAULT; 2321 2322 req_size = le32toh(req.u0.req.request_length); 2323 if (req_size > 65536) 2324 return EINVAL; 2325 2326 req_size += sizeof(struct mbx_hdr); 2327 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0); 2328 if (rc) 2329 return ENOMEM; 2330 2331 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) { 2332 rc = EFAULT; 2333 goto dma_free; 2334 } 2335 2336 rc = oce_pass_through_mbox(sc, &dma_mem, req_size); 2337 if (rc) { 2338 rc = EIO; 2339 goto dma_free; 2340 } 2341 2342 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) 2343 rc = EFAULT; 2344 2345 /* 2346 firmware is filling all the attributes for this ioctl except 2347 the driver version..so fill it 2348 */ 2349 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) { 2350 fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr; 2351 strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str, 2352 COMPONENT_REVISION, strlen(COMPONENT_REVISION)); 2353 } 2354 2355 dma_free: 2356 oce_dma_free(sc, &dma_mem); 2357 return rc; 2358 2359 } 2360 2361 static void 2362 oce_eqd_set_periodic(POCE_SOFTC sc) 2363 { 2364 struct oce_set_eqd set_eqd[OCE_MAX_EQ]; 2365 struct oce_aic_obj *aic; 2366 struct oce_eq *eqo; 2367 uint64_t now = 0, delta; 2368 int eqd, i, num = 0; 2369 uint32_t tx_reqs = 0, rxpkts = 0, pps; 2370 struct oce_wq *wq; 2371 struct oce_rq *rq; 2372 2373 #define ticks_to_msecs(t) (1000 * (t) / hz) 2374 2375 for (i = 0 ; i < sc->neqs; i++) { 2376 eqo = sc->eq[i]; 2377 aic = &sc->aic_obj[i]; 2378 /* When setting the static eq delay from the user space */ 2379 if (!aic->enable) { 2380 if (aic->ticks) 2381 aic->ticks = 0; 2382 eqd = aic->et_eqd; 2383 goto modify_eqd; 2384 } 2385 2386 if (i == 0) { 2387 rq = sc->rq[0]; 2388 rxpkts = rq->rx_stats.rx_pkts; 2389 } else 2390 rxpkts = 0; 2391 if (i + 1 < sc->nrqs) { 2392 rq = sc->rq[i + 1]; 2393 rxpkts += rq->rx_stats.rx_pkts; 2394 } 2395 if (i < sc->nwqs) { 2396 wq = sc->wq[i]; 2397 tx_reqs = wq->tx_stats.tx_reqs; 2398 } else 2399 tx_reqs = 0; 2400 now = ticks; 2401 2402 if (!aic->ticks || now < aic->ticks || 2403 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) { 2404 aic->prev_rxpkts = rxpkts; 2405 aic->prev_txreqs = tx_reqs; 2406 aic->ticks = now; 2407 continue; 2408 } 2409 2410 delta = ticks_to_msecs(now - aic->ticks); 2411 2412 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) + 2413 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta); 2414 eqd = (pps / 15000) << 2; 2415 if (eqd < 8) 2416 eqd = 0; 2417 2418 /* Make sure that the eq delay is in the known range */ 2419 eqd = min(eqd, aic->max_eqd); 2420 eqd = max(eqd, aic->min_eqd); 2421 2422 aic->prev_rxpkts = rxpkts; 2423 aic->prev_txreqs = tx_reqs; 2424 aic->ticks = now; 2425 2426 modify_eqd: 2427 if (eqd != aic->cur_eqd) { 2428 set_eqd[num].delay_multiplier = (eqd * 65)/100; 2429 set_eqd[num].eq_id = eqo->eq_id; 2430 aic->cur_eqd = eqd; 2431 num++; 2432 } 2433 } 2434 2435 /* Is there atleast one eq that needs to be modified? */ 2436 for(i = 0; i < num; i += 8) { 2437 if((num - i) >=8 ) 2438 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8); 2439 else 2440 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i)); 2441 } 2442 2443 } 2444 2445 static void oce_detect_hw_error(POCE_SOFTC sc) 2446 { 2447 2448 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0; 2449 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 2450 uint32_t i; 2451 2452 if (sc->hw_error) 2453 return; 2454 2455 if (IS_XE201(sc)) { 2456 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET); 2457 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2458 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET); 2459 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET); 2460 } 2461 } else { 2462 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW); 2463 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH); 2464 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK); 2465 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK); 2466 2467 ue_low = (ue_low & ~ue_low_mask); 2468 ue_high = (ue_high & ~ue_high_mask); 2469 } 2470 2471 /* On certain platforms BE hardware can indicate spurious UEs. 2472 * Allow the h/w to stop working completely in case of a real UE. 2473 * Hence not setting the hw_error for UE detection. 2474 */ 2475 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2476 sc->hw_error = TRUE; 2477 device_printf(sc->dev, "Error detected in the card\n"); 2478 } 2479 2480 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2481 device_printf(sc->dev, 2482 "ERR: sliport status 0x%x\n", sliport_status); 2483 device_printf(sc->dev, 2484 "ERR: sliport error1 0x%x\n", sliport_err1); 2485 device_printf(sc->dev, 2486 "ERR: sliport error2 0x%x\n", sliport_err2); 2487 } 2488 2489 if (ue_low) { 2490 for (i = 0; ue_low; ue_low >>= 1, i++) { 2491 if (ue_low & 1) 2492 device_printf(sc->dev, "UE: %s bit set\n", 2493 ue_status_low_desc[i]); 2494 } 2495 } 2496 2497 if (ue_high) { 2498 for (i = 0; ue_high; ue_high >>= 1, i++) { 2499 if (ue_high & 1) 2500 device_printf(sc->dev, "UE: %s bit set\n", 2501 ue_status_hi_desc[i]); 2502 } 2503 } 2504 2505 } 2506 2507 2508 static void 2509 oce_local_timer(void *arg) 2510 { 2511 POCE_SOFTC sc = arg; 2512 int i = 0; 2513 2514 oce_detect_hw_error(sc); 2515 oce_refresh_nic_stats(sc); 2516 oce_refresh_queue_stats(sc); 2517 oce_mac_addr_set(sc); 2518 2519 /* TX Watch Dog*/ 2520 for (i = 0; i < sc->nwqs; i++) 2521 oce_tx_restart(sc, sc->wq[i]); 2522 2523 /* calculate and set the eq delay for optimal interrupt rate */ 2524 if (IS_BE(sc) || IS_SH(sc)) 2525 oce_eqd_set_periodic(sc); 2526 2527 callout_reset(&sc->timer, hz, oce_local_timer, sc); 2528 } 2529 2530 static void 2531 oce_tx_compl_clean(POCE_SOFTC sc) 2532 { 2533 struct oce_wq *wq; 2534 int i = 0, timeo = 0, num_wqes = 0; 2535 int pending_txqs = sc->nwqs; 2536 2537 /* Stop polling for compls when HW has been silent for 10ms or 2538 * hw_error or no outstanding completions expected 2539 */ 2540 do { 2541 pending_txqs = sc->nwqs; 2542 2543 for_all_wq_queues(sc, wq, i) { 2544 num_wqes = oce_wq_handler(wq); 2545 2546 if(num_wqes) 2547 timeo = 0; 2548 2549 if(!wq->ring->num_used) 2550 pending_txqs--; 2551 } 2552 2553 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error) 2554 break; 2555 2556 DELAY(1000); 2557 } while (TRUE); 2558 2559 for_all_wq_queues(sc, wq, i) { 2560 while(wq->ring->num_used) { 2561 LOCK(&wq->tx_compl_lock); 2562 oce_process_tx_completion(wq); 2563 UNLOCK(&wq->tx_compl_lock); 2564 } 2565 } 2566 2567 } 2568 2569 /* NOTE : This should only be called holding 2570 * DEVICE_LOCK. 2571 */ 2572 static void 2573 oce_if_deactivate(POCE_SOFTC sc) 2574 { 2575 int i; 2576 struct oce_rq *rq; 2577 struct oce_wq *wq; 2578 struct oce_eq *eq; 2579 2580 sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2581 2582 oce_tx_compl_clean(sc); 2583 2584 /* Stop intrs and finish any bottom halves pending */ 2585 oce_hw_intr_disable(sc); 2586 2587 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire 2588 any other lock. So unlock device lock and require after 2589 completing taskqueue_drain. 2590 */ 2591 UNLOCK(&sc->dev_lock); 2592 for (i = 0; i < sc->intr_count; i++) { 2593 if (sc->intrs[i].tq != NULL) { 2594 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task); 2595 } 2596 } 2597 LOCK(&sc->dev_lock); 2598 2599 /* Delete RX queue in card with flush param */ 2600 oce_stop_rx(sc); 2601 2602 /* Invalidate any pending cq and eq entries*/ 2603 for_all_evnt_queues(sc, eq, i) 2604 oce_drain_eq(eq); 2605 for_all_rq_queues(sc, rq, i) 2606 oce_drain_rq_cq(rq); 2607 for_all_wq_queues(sc, wq, i) 2608 oce_drain_wq_cq(wq); 2609 2610 /* But still we need to get MCC aync events. 2611 So enable intrs and also arm first EQ 2612 */ 2613 oce_hw_intr_enable(sc); 2614 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE); 2615 2616 DELAY(10); 2617 } 2618 2619 2620 static void 2621 oce_if_activate(POCE_SOFTC sc) 2622 { 2623 struct oce_eq *eq; 2624 struct oce_rq *rq; 2625 struct oce_wq *wq; 2626 int i, rc = 0; 2627 2628 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 2629 2630 oce_hw_intr_disable(sc); 2631 2632 oce_start_rx(sc); 2633 2634 for_all_rq_queues(sc, rq, i) { 2635 rc = oce_start_rq(rq); 2636 if (rc) 2637 device_printf(sc->dev, "Unable to start RX\n"); 2638 } 2639 2640 for_all_wq_queues(sc, wq, i) { 2641 rc = oce_start_wq(wq); 2642 if (rc) 2643 device_printf(sc->dev, "Unable to start TX\n"); 2644 } 2645 2646 2647 for_all_evnt_queues(sc, eq, i) 2648 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 2649 2650 oce_hw_intr_enable(sc); 2651 2652 } 2653 2654 static void 2655 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe) 2656 { 2657 /* Update Link status */ 2658 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) == 2659 ASYNC_EVENT_LINK_UP) { 2660 sc->link_status = ASYNC_EVENT_LINK_UP; 2661 if_link_state_change(sc->ifp, LINK_STATE_UP); 2662 } else { 2663 sc->link_status = ASYNC_EVENT_LINK_DOWN; 2664 if_link_state_change(sc->ifp, LINK_STATE_DOWN); 2665 } 2666 } 2667 2668 2669 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc, 2670 struct oce_async_evt_grp5_os2bmc *evt) 2671 { 2672 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc)); 2673 if (evt->u.s.mgmt_enable) 2674 sc->flags |= OCE_FLAGS_OS2BMC; 2675 else 2676 return; 2677 2678 sc->bmc_filt_mask = evt->u.s.arp_filter; 2679 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1); 2680 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2); 2681 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3); 2682 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4); 2683 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5); 2684 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6); 2685 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7); 2686 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8); 2687 } 2688 2689 2690 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe) 2691 { 2692 struct oce_async_event_grp5_pvid_state *gcqe; 2693 struct oce_async_evt_grp5_os2bmc *bmccqe; 2694 2695 switch (cqe->u0.s.async_type) { 2696 case ASYNC_EVENT_PVID_STATE: 2697 /* GRP5 PVID */ 2698 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe; 2699 if (gcqe->enabled) 2700 sc->pvid = gcqe->tag & VLAN_VID_MASK; 2701 else 2702 sc->pvid = 0; 2703 break; 2704 case ASYNC_EVENT_OS2BMC: 2705 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe; 2706 oce_async_grp5_osbmc_process(sc, bmccqe); 2707 break; 2708 default: 2709 break; 2710 } 2711 } 2712 2713 /* Handle the Completion Queue for the Mailbox/Async notifications */ 2714 uint16_t 2715 oce_mq_handler(void *arg) 2716 { 2717 struct oce_mq *mq = (struct oce_mq *)arg; 2718 POCE_SOFTC sc = mq->parent; 2719 struct oce_cq *cq = mq->cq; 2720 int num_cqes = 0, evt_type = 0, optype = 0; 2721 struct oce_mq_cqe *cqe; 2722 struct oce_async_cqe_link_state *acqe; 2723 struct oce_async_event_qnq *dbgcqe; 2724 2725 2726 bus_dmamap_sync(cq->ring->dma.tag, 2727 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2728 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2729 2730 while (cqe->u0.dw[3]) { 2731 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe)); 2732 if (cqe->u0.s.async_event) { 2733 evt_type = cqe->u0.s.event_type; 2734 optype = cqe->u0.s.async_type; 2735 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) { 2736 /* Link status evt */ 2737 acqe = (struct oce_async_cqe_link_state *)cqe; 2738 process_link_state(sc, acqe); 2739 } else if (evt_type == ASYNC_EVENT_GRP5) { 2740 oce_process_grp5_events(sc, cqe); 2741 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG && 2742 optype == ASYNC_EVENT_DEBUG_QNQ) { 2743 dbgcqe = (struct oce_async_event_qnq *)cqe; 2744 if(dbgcqe->valid) 2745 sc->qnqid = dbgcqe->vlan_tag; 2746 sc->qnq_debug_event = TRUE; 2747 } 2748 } 2749 cqe->u0.dw[3] = 0; 2750 RING_GET(cq->ring, 1); 2751 bus_dmamap_sync(cq->ring->dma.tag, 2752 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2753 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2754 num_cqes++; 2755 } 2756 2757 if (num_cqes) 2758 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 2759 2760 return 0; 2761 } 2762 2763 2764 static void 2765 setup_max_queues_want(POCE_SOFTC sc) 2766 { 2767 /* Check if it is FLEX machine. Is so dont use RSS */ 2768 if ((sc->function_mode & FNM_FLEX10_MODE) || 2769 (sc->function_mode & FNM_UMC_MODE) || 2770 (sc->function_mode & FNM_VNIC_MODE) || 2771 (!is_rss_enabled(sc)) || 2772 IS_BE2(sc)) { 2773 sc->nrqs = 1; 2774 sc->nwqs = 1; 2775 } else { 2776 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2777 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs); 2778 } 2779 2780 if (IS_BE2(sc) && is_rss_enabled(sc)) 2781 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2782 } 2783 2784 2785 static void 2786 update_queues_got(POCE_SOFTC sc) 2787 { 2788 if (is_rss_enabled(sc)) { 2789 sc->nrqs = sc->intr_count + 1; 2790 sc->nwqs = sc->intr_count; 2791 } else { 2792 sc->nrqs = 1; 2793 sc->nwqs = 1; 2794 } 2795 2796 if (IS_BE2(sc)) 2797 sc->nwqs = 1; 2798 } 2799 2800 static int 2801 oce_check_ipv6_ext_hdr(struct mbuf *m) 2802 { 2803 struct ether_header *eh = mtod(m, struct ether_header *); 2804 caddr_t m_datatemp = m->m_data; 2805 2806 if (eh->ether_type == htons(ETHERTYPE_IPV6)) { 2807 m->m_data += sizeof(struct ether_header); 2808 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 2809 2810 if((ip6->ip6_nxt != IPPROTO_TCP) && \ 2811 (ip6->ip6_nxt != IPPROTO_UDP)){ 2812 struct ip6_ext *ip6e = NULL; 2813 m->m_data += sizeof(struct ip6_hdr); 2814 2815 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *); 2816 if(ip6e->ip6e_len == 0xff) { 2817 m->m_data = m_datatemp; 2818 return TRUE; 2819 } 2820 } 2821 m->m_data = m_datatemp; 2822 } 2823 return FALSE; 2824 } 2825 2826 static int 2827 is_be3_a1(POCE_SOFTC sc) 2828 { 2829 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) { 2830 return TRUE; 2831 } 2832 return FALSE; 2833 } 2834 2835 static struct mbuf * 2836 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete) 2837 { 2838 uint16_t vlan_tag = 0; 2839 2840 if(!M_WRITABLE(m)) 2841 return NULL; 2842 2843 /* Embed vlan tag in the packet if it is not part of it */ 2844 if(m->m_flags & M_VLANTAG) { 2845 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag); 2846 m->m_flags &= ~M_VLANTAG; 2847 } 2848 2849 /* if UMC, ignore vlan tag insertion and instead insert pvid */ 2850 if(sc->pvid) { 2851 if(!vlan_tag) 2852 vlan_tag = sc->pvid; 2853 if (complete) 2854 *complete = FALSE; 2855 } 2856 2857 if(vlan_tag) { 2858 m = ether_vlanencap(m, vlan_tag); 2859 } 2860 2861 if(sc->qnqid) { 2862 m = ether_vlanencap(m, sc->qnqid); 2863 2864 if (complete) 2865 *complete = FALSE; 2866 } 2867 return m; 2868 } 2869 2870 static int 2871 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m) 2872 { 2873 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \ 2874 oce_check_ipv6_ext_hdr(m)) { 2875 return TRUE; 2876 } 2877 return FALSE; 2878 } 2879 2880 static void 2881 oce_get_config(POCE_SOFTC sc) 2882 { 2883 int rc = 0; 2884 uint32_t max_rss = 0; 2885 2886 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native)) 2887 max_rss = OCE_LEGACY_MODE_RSS; 2888 else 2889 max_rss = OCE_MAX_RSS; 2890 2891 if (!IS_BE(sc)) { 2892 rc = oce_get_profile_config(sc, max_rss); 2893 if (rc) { 2894 sc->nwqs = OCE_MAX_WQ; 2895 sc->nrssqs = max_rss; 2896 sc->nrqs = sc->nrssqs + 1; 2897 } 2898 } 2899 else { /* For BE3 don't rely on fw for determining the resources */ 2900 sc->nrssqs = max_rss; 2901 sc->nrqs = sc->nrssqs + 1; 2902 sc->nwqs = OCE_MAX_WQ; 2903 sc->max_vlans = MAX_VLANFILTER_SIZE; 2904 } 2905 } 2906 2907 static void 2908 oce_rdma_close(void) 2909 { 2910 if (oce_rdma_if != NULL) { 2911 oce_rdma_if = NULL; 2912 } 2913 } 2914 2915 static void 2916 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr) 2917 { 2918 memcpy(macaddr, sc->macaddr.mac_addr, 6); 2919 } 2920 2921 int 2922 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if) 2923 { 2924 POCE_SOFTC sc; 2925 struct oce_dev_info di; 2926 int i; 2927 2928 if ((rdma_info == NULL) || (rdma_if == NULL)) { 2929 return -EINVAL; 2930 } 2931 2932 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) || 2933 (rdma_if->size != OCE_RDMA_IF_SIZE)) { 2934 return -ENXIO; 2935 } 2936 2937 rdma_info->close = oce_rdma_close; 2938 rdma_info->mbox_post = oce_mbox_post; 2939 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init; 2940 rdma_info->get_mac_addr = oce_get_mac_addr; 2941 2942 oce_rdma_if = rdma_if; 2943 2944 sc = softc_head; 2945 while (sc != NULL) { 2946 if (oce_rdma_if->announce != NULL) { 2947 memset(&di, 0, sizeof(di)); 2948 di.dev = sc->dev; 2949 di.softc = sc; 2950 di.ifp = sc->ifp; 2951 di.db_bhandle = sc->db_bhandle; 2952 di.db_btag = sc->db_btag; 2953 di.db_page_size = 4096; 2954 if (sc->flags & OCE_FLAGS_USING_MSIX) { 2955 di.intr_mode = OCE_INTERRUPT_MODE_MSIX; 2956 } else if (sc->flags & OCE_FLAGS_USING_MSI) { 2957 di.intr_mode = OCE_INTERRUPT_MODE_MSI; 2958 } else { 2959 di.intr_mode = OCE_INTERRUPT_MODE_INTX; 2960 } 2961 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk 2962 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) { 2963 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count; 2964 di.msix.start_vector = sc->intr_count; 2965 for (i=0; i<di.msix.num_vectors; i++) { 2966 di.msix.vector_list[i] = sc->intrs[i].vector; 2967 } 2968 } else { 2969 } 2970 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6); 2971 di.vendor_id = pci_get_vendor(sc->dev); 2972 di.dev_id = pci_get_device(sc->dev); 2973 2974 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 2975 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED; 2976 } 2977 2978 rdma_if->announce(&di); 2979 sc = sc->next; 2980 } 2981 } 2982 2983 return 0; 2984 } 2985 2986 static void 2987 oce_read_env_variables( POCE_SOFTC sc ) 2988 { 2989 char *value = NULL; 2990 int rc = 0; 2991 2992 /* read if user wants to enable hwlro or swlro */ 2993 //value = getenv("oce_enable_hwlro"); 2994 if(value && IS_SH(sc)) { 2995 sc->enable_hwlro = strtol(value, NULL, 10); 2996 if(sc->enable_hwlro) { 2997 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL); 2998 if(rc) { 2999 device_printf(sc->dev, "no hardware lro support\n"); 3000 device_printf(sc->dev, "software lro enabled\n"); 3001 sc->enable_hwlro = 0; 3002 }else { 3003 device_printf(sc->dev, "hardware lro enabled\n"); 3004 oce_max_rsp_handled = 32; 3005 } 3006 }else { 3007 device_printf(sc->dev, "software lro enabled\n"); 3008 } 3009 }else { 3010 sc->enable_hwlro = 0; 3011 } 3012 3013 /* read mbuf size */ 3014 //value = getenv("oce_rq_buf_size"); 3015 if(value && IS_SH(sc)) { 3016 oce_rq_buf_size = strtol(value, NULL, 10); 3017 switch(oce_rq_buf_size) { 3018 case 2048: 3019 case 4096: 3020 case 9216: 3021 case 16384: 3022 break; 3023 3024 default: 3025 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n"); 3026 oce_rq_buf_size = 2048; 3027 } 3028 } 3029 3030 return; 3031 } 3032