1 /* 2 * Copyright (c) 2001-2013, Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the Intel Corporation nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include "opt_ix.h" 33 34 #include <sys/param.h> 35 #include <sys/bus.h> 36 #include <sys/endian.h> 37 #include <sys/interrupt.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/proc.h> 42 #include <sys/rman.h> 43 #include <sys/serialize.h> 44 #include <sys/serialize2.h> 45 #include <sys/socket.h> 46 #include <sys/sockio.h> 47 #include <sys/sysctl.h> 48 #include <sys/systm.h> 49 50 #include <net/bpf.h> 51 #include <net/ethernet.h> 52 #include <net/if.h> 53 #include <net/if_arp.h> 54 #include <net/if_dl.h> 55 #include <net/if_media.h> 56 #include <net/ifq_var.h> 57 #include <net/toeplitz.h> 58 #include <net/toeplitz2.h> 59 #include <net/vlan/if_vlan_var.h> 60 #include <net/vlan/if_vlan_ether.h> 61 #include <net/if_poll.h> 62 63 #include <netinet/in_systm.h> 64 #include <netinet/in.h> 65 #include <netinet/ip.h> 66 67 #include <bus/pci/pcivar.h> 68 #include <bus/pci/pcireg.h> 69 70 #include <dev/netif/ix/ixgbe_api.h> 71 #include <dev/netif/ix/if_ix.h> 72 73 #ifdef IX_RSS_DEBUG 74 #define IX_RSS_DPRINTF(sc, lvl, fmt, ...) \ 75 do { \ 76 if (sc->rss_debug >= lvl) \ 77 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \ 78 } while (0) 79 #else /* !IX_RSS_DEBUG */ 80 #define IX_RSS_DPRINTF(sc, lvl, fmt, ...) ((void)0) 81 #endif /* IX_RSS_DEBUG */ 82 83 #define IX_NAME "Intel(R) PRO/10GbE " 84 #define IX_DEVICE(id) \ 85 { IXGBE_VENDOR_ID, IXGBE_DEV_ID_##id, IX_NAME #id } 86 #define IX_DEVICE_NULL { 0, 0, NULL } 87 88 static struct ix_device { 89 uint16_t vid; 90 uint16_t did; 91 const char *desc; 92 } ix_devices[] = { 93 IX_DEVICE(82598AF_DUAL_PORT), 94 IX_DEVICE(82598AF_SINGLE_PORT), 95 IX_DEVICE(82598EB_CX4), 96 IX_DEVICE(82598AT), 97 IX_DEVICE(82598AT2), 98 IX_DEVICE(82598), 99 IX_DEVICE(82598_DA_DUAL_PORT), 100 IX_DEVICE(82598_CX4_DUAL_PORT), 101 IX_DEVICE(82598EB_XF_LR), 102 IX_DEVICE(82598_SR_DUAL_PORT_EM), 103 IX_DEVICE(82598EB_SFP_LOM), 104 IX_DEVICE(82599_KX4), 105 IX_DEVICE(82599_KX4_MEZZ), 106 IX_DEVICE(82599_SFP), 107 IX_DEVICE(82599_XAUI_LOM), 108 IX_DEVICE(82599_CX4), 109 IX_DEVICE(82599_T3_LOM), 110 IX_DEVICE(82599_COMBO_BACKPLANE), 111 IX_DEVICE(82599_BACKPLANE_FCOE), 112 IX_DEVICE(82599_SFP_SF2), 113 IX_DEVICE(82599_SFP_FCOE), 114 IX_DEVICE(82599EN_SFP), 115 IX_DEVICE(82599_SFP_SF_QP), 116 IX_DEVICE(X540T), 117 118 /* required last entry */ 119 IX_DEVICE_NULL 120 }; 121 122 static int ix_probe(device_t); 123 static int ix_attach(device_t); 124 static int ix_detach(device_t); 125 static int ix_shutdown(device_t); 126 127 static void ix_serialize(struct ifnet *, enum ifnet_serialize); 128 static void ix_deserialize(struct ifnet *, enum ifnet_serialize); 129 static int ix_tryserialize(struct ifnet *, enum ifnet_serialize); 130 #ifdef INVARIANTS 131 static void ix_serialize_assert(struct ifnet *, enum ifnet_serialize, 132 boolean_t); 133 #endif 134 static void ix_start(struct ifnet *, struct ifaltq_subque *); 135 static void ix_watchdog(struct ifaltq_subque *); 136 static int ix_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 137 static void ix_init(void *); 138 static void ix_stop(struct ix_softc *); 139 static void ix_media_status(struct ifnet *, struct ifmediareq *); 140 static int ix_media_change(struct ifnet *); 141 static void ix_timer(void *); 142 143 static void ix_add_sysctl(struct ix_softc *); 144 static int ix_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS); 145 static int ix_sysctl_rx_wreg_nsegs(SYSCTL_HANDLER_ARGS); 146 static int ix_sysctl_txd(SYSCTL_HANDLER_ARGS); 147 static int ix_sysctl_rxd(SYSCTL_HANDLER_ARGS); 148 static int ix_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS); 149 static int ix_sysctl_intr_rate(SYSCTL_HANDLER_ARGS); 150 static int ix_sysctl_flowctrl(SYSCTL_HANDLER_ARGS); 151 #ifdef foo 152 static int ix_sysctl_advspeed(SYSCTL_HANDLER_ARGS); 153 #endif 154 #if 0 155 static void ix_add_hw_stats(struct ix_softc *); 156 #endif 157 158 static void ix_slot_info(struct ix_softc *); 159 static int ix_alloc_rings(struct ix_softc *); 160 static void ix_free_rings(struct ix_softc *); 161 static void ix_setup_ifp(struct ix_softc *); 162 static void ix_setup_serialize(struct ix_softc *); 163 static void ix_set_ring_inuse(struct ix_softc *, boolean_t); 164 static void ix_set_timer_cpuid(struct ix_softc *, boolean_t); 165 static void ix_update_stats(struct ix_softc *); 166 167 static void ix_set_promisc(struct ix_softc *); 168 static void ix_set_multi(struct ix_softc *); 169 static void ix_set_vlan(struct ix_softc *); 170 static uint8_t *ix_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 171 172 static int ix_get_txring_inuse(const struct ix_softc *, boolean_t); 173 static void ix_init_tx_ring(struct ix_tx_ring *); 174 static void ix_free_tx_ring(struct ix_tx_ring *); 175 static int ix_create_tx_ring(struct ix_tx_ring *); 176 static void ix_destroy_tx_ring(struct ix_tx_ring *, int); 177 static void ix_init_tx_unit(struct ix_softc *); 178 static int ix_encap(struct ix_tx_ring *, struct mbuf **, 179 uint16_t *, int *); 180 static int ix_tx_ctx_setup(struct ix_tx_ring *, 181 const struct mbuf *, uint32_t *, uint32_t *); 182 static int ix_tso_ctx_setup(struct ix_tx_ring *, 183 const struct mbuf *, uint32_t *, uint32_t *); 184 static void ix_txeof(struct ix_tx_ring *); 185 186 static int ix_get_rxring_inuse(const struct ix_softc *, boolean_t); 187 static int ix_init_rx_ring(struct ix_rx_ring *); 188 static void ix_free_rx_ring(struct ix_rx_ring *); 189 static int ix_create_rx_ring(struct ix_rx_ring *); 190 static void ix_destroy_rx_ring(struct ix_rx_ring *, int); 191 static void ix_init_rx_unit(struct ix_softc *); 192 #if 0 193 static void ix_setup_hw_rsc(struct ix_rx_ring *); 194 #endif 195 static int ix_newbuf(struct ix_rx_ring *, int, boolean_t); 196 static void ix_rxeof(struct ix_rx_ring *); 197 static void ix_rx_discard(struct ix_rx_ring *, int, boolean_t); 198 static void ix_enable_rx_drop(struct ix_softc *); 199 static void ix_disable_rx_drop(struct ix_softc *); 200 201 static int ix_alloc_intr(struct ix_softc *); 202 static void ix_free_intr(struct ix_softc *); 203 static int ix_setup_intr(struct ix_softc *); 204 static void ix_teardown_intr(struct ix_softc *, int); 205 static void ix_enable_intr(struct ix_softc *); 206 static void ix_disable_intr(struct ix_softc *); 207 static void ix_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t); 208 #if 0 209 static void ix_configure_ivars(struct ix_softc *); 210 #endif 211 static void ix_set_eitr(struct ix_softc *, int, int); 212 static void ix_intr(void *); 213 214 static void ix_config_link(struct ix_softc *); 215 static boolean_t ix_sfp_probe(struct ix_softc *); 216 static boolean_t ix_is_sfp(const struct ixgbe_hw *); 217 static void ix_setup_optics(struct ix_softc *); 218 static void ix_update_link_status(struct ix_softc *); 219 static void ix_handle_link(struct ix_softc *); 220 static void ix_handle_mod(struct ix_softc *); 221 static void ix_handle_msf(struct ix_softc *); 222 223 #if 0 224 static void ix_msix_que(void *); 225 static void ix_msix_link(void *); 226 static int ix_allocate_msix(struct ix_softc *); 227 static int ix_setup_msix(struct ix_softc *); 228 static void ix_handle_que(void *, int); 229 #endif 230 231 /* XXX Shared code structure requires this for the moment */ 232 extern void ixgbe_stop_mac_link_on_d3_82599(struct ixgbe_hw *); 233 234 static device_method_t ix_methods[] = { 235 /* Device interface */ 236 DEVMETHOD(device_probe, ix_probe), 237 DEVMETHOD(device_attach, ix_attach), 238 DEVMETHOD(device_detach, ix_detach), 239 DEVMETHOD(device_shutdown, ix_shutdown), 240 DEVMETHOD_END 241 }; 242 243 static driver_t ix_driver = { 244 "ix", 245 ix_methods, 246 sizeof(struct ix_softc) 247 }; 248 249 static devclass_t ix_devclass; 250 251 DECLARE_DUMMY_MODULE(if_ix); 252 DRIVER_MODULE(if_ix, pci, ix_driver, ix_devclass, NULL, NULL); 253 254 static int ix_msi_enable = 1; 255 static int ix_rxr = 0; 256 static int ix_txd = IX_PERF_TXD; 257 static int ix_rxd = IX_PERF_RXD; 258 static int ix_unsupported_sfp = 0; 259 260 TUNABLE_INT("hw.ix.msi.enable", &ix_msi_enable); 261 TUNABLE_INT("hw.ix.rxr", &ix_rxr); 262 TUNABLE_INT("hw.ix.txd", &ix_txd); 263 TUNABLE_INT("hw.ix.rxd", &ix_rxd); 264 TUNABLE_INT("hw.ix.unsupported_sfp", &ix_unsupported_sfp); 265 266 /* 267 * Smart speed setting, default to on. This only works 268 * as a compile option right now as its during attach, 269 * set this to 'ixgbe_smart_speed_off' to disable. 270 */ 271 static const enum ixgbe_smart_speed ix_smart_speed = 272 ixgbe_smart_speed_on; 273 274 static int 275 ix_probe(device_t dev) 276 { 277 const struct ix_device *d; 278 uint16_t vid, did; 279 280 vid = pci_get_vendor(dev); 281 did = pci_get_device(dev); 282 283 for (d = ix_devices; d->desc != NULL; ++d) { 284 if (vid == d->vid && did == d->did) { 285 device_set_desc(dev, d->desc); 286 return 0; 287 } 288 } 289 return ENXIO; 290 } 291 292 static int 293 ix_attach(device_t dev) 294 { 295 struct ix_softc *sc = device_get_softc(dev); 296 struct ixgbe_hw *hw; 297 int error; 298 uint16_t csum; 299 uint32_t ctrl_ext; 300 301 sc->dev = sc->osdep.dev = dev; 302 hw = &sc->hw; 303 304 if_initname(&sc->arpcom.ac_if, device_get_name(dev), 305 device_get_unit(dev)); 306 ifmedia_init(&sc->media, IFM_IMASK, 307 ix_media_change, ix_media_status); 308 309 /* Save frame size */ 310 sc->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN; 311 312 callout_init_mp(&sc->timer); 313 lwkt_serialize_init(&sc->main_serialize); 314 315 /* 316 * Save off the information about this board 317 */ 318 hw->vendor_id = pci_get_vendor(dev); 319 hw->device_id = pci_get_device(dev); 320 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); 321 hw->subsystem_vendor_id = pci_read_config(dev, PCIR_SUBVEND_0, 2); 322 hw->subsystem_device_id = pci_read_config(dev, PCIR_SUBDEV_0, 2); 323 324 ixgbe_set_mac_type(hw); 325 326 /* Pick up the 82599 and VF settings */ 327 if (hw->mac.type != ixgbe_mac_82598EB) 328 hw->phy.smart_speed = ix_smart_speed; 329 330 /* Enable bus mastering */ 331 pci_enable_busmaster(dev); 332 333 /* 334 * Allocate IO memory 335 */ 336 sc->mem_rid = PCIR_BAR(0); 337 sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 338 &sc->mem_rid, RF_ACTIVE); 339 if (sc->mem_res == NULL) { 340 device_printf(dev, "Unable to allocate bus resource: memory\n"); 341 error = ENXIO; 342 goto failed; 343 } 344 345 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->mem_res); 346 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->mem_res); 347 348 sc->hw.hw_addr = (uint8_t *)&sc->osdep.mem_bus_space_handle; 349 sc->hw.back = &sc->osdep; 350 351 /* 352 * Configure total supported RX/TX ring count 353 */ 354 sc->rx_ring_cnt = device_getenv_int(dev, "rxr", ix_rxr); 355 sc->rx_ring_cnt = if_ring_count2(sc->rx_ring_cnt, IX_MAX_RXRING); 356 sc->rx_ring_inuse = sc->rx_ring_cnt; 357 358 sc->tx_ring_cnt = 1; 359 sc->tx_ring_inuse = sc->tx_ring_cnt; 360 361 /* Allocate TX/RX rings */ 362 error = ix_alloc_rings(sc); 363 if (error) 364 goto failed; 365 366 /* Allocate interrupt */ 367 error = ix_alloc_intr(sc); 368 if (error) 369 goto failed; 370 371 /* Setup serializes */ 372 ix_setup_serialize(sc); 373 374 /* Allocate multicast array memory. */ 375 sc->mta = kmalloc(IXGBE_ETH_LENGTH_OF_ADDRESS * IX_MAX_MCASTADDR, 376 M_DEVBUF, M_WAITOK); 377 378 /* Initialize the shared code */ 379 hw->allow_unsupported_sfp = ix_unsupported_sfp; 380 error = ixgbe_init_shared_code(hw); 381 if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 382 /* 383 * No optics in this port; ask timer routine 384 * to probe for later insertion. 385 */ 386 sc->sfp_probe = TRUE; 387 error = 0; 388 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 389 device_printf(dev, "Unsupported SFP+ module detected!\n"); 390 error = EIO; 391 goto failed; 392 } else if (error) { 393 device_printf(dev, "Unable to initialize the shared code\n"); 394 error = EIO; 395 goto failed; 396 } 397 398 /* Make sure we have a good EEPROM before we read from it */ 399 if (ixgbe_validate_eeprom_checksum(&sc->hw, &csum) < 0) { 400 device_printf(dev, "The EEPROM Checksum Is Not Valid\n"); 401 error = EIO; 402 goto failed; 403 } 404 405 error = ixgbe_init_hw(hw); 406 if (error == IXGBE_ERR_EEPROM_VERSION) { 407 device_printf(dev, "Pre-production device detected\n"); 408 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 409 device_printf(dev, "Unsupported SFP+ Module\n"); 410 error = EIO; 411 goto failed; 412 } else if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 413 device_printf(dev, "No SFP+ Module found\n"); 414 } 415 416 /* Detect and set physical type */ 417 ix_setup_optics(sc); 418 419 /* Setup OS specific network interface */ 420 ix_setup_ifp(sc); 421 422 /* Add sysctl tree */ 423 ix_add_sysctl(sc); 424 425 error = ix_setup_intr(sc); 426 if (error) { 427 ether_ifdetach(&sc->arpcom.ac_if); 428 goto failed; 429 } 430 431 /* Initialize statistics */ 432 ix_update_stats(sc); 433 434 /* 435 * Check PCIE slot type/speed/width 436 */ 437 ix_slot_info(sc); 438 439 /* Set an initial default flow control value */ 440 sc->fc = ixgbe_fc_full; 441 442 /* Let hardware know driver is loaded */ 443 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 444 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 445 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 446 447 return 0; 448 failed: 449 ix_detach(dev); 450 return error; 451 } 452 453 static int 454 ix_detach(device_t dev) 455 { 456 struct ix_softc *sc = device_get_softc(dev); 457 458 if (device_is_attached(dev)) { 459 struct ifnet *ifp = &sc->arpcom.ac_if; 460 uint32_t ctrl_ext; 461 462 ifnet_serialize_all(ifp); 463 464 ix_stop(sc); 465 ix_teardown_intr(sc, sc->intr_cnt); 466 467 ifnet_deserialize_all(ifp); 468 469 callout_terminate(&sc->timer); 470 ether_ifdetach(ifp); 471 472 /* Let hardware know driver is unloading */ 473 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 474 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 475 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 476 } 477 478 ifmedia_removeall(&sc->media); 479 bus_generic_detach(dev); 480 481 if (sc->sysctl_tree != NULL) 482 sysctl_ctx_free(&sc->sysctl_ctx); 483 484 ix_free_intr(sc); 485 486 if (sc->mem_res != NULL) { 487 bus_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid, 488 sc->mem_res); 489 } 490 491 ix_free_rings(sc); 492 493 if (sc->mta != NULL) 494 kfree(sc->mta, M_DEVBUF); 495 if (sc->serializes != NULL) 496 kfree(sc->serializes, M_DEVBUF); 497 498 return 0; 499 } 500 501 static int 502 ix_shutdown(device_t dev) 503 { 504 struct ix_softc *sc = device_get_softc(dev); 505 struct ifnet *ifp = &sc->arpcom.ac_if; 506 507 ifnet_serialize_all(ifp); 508 ix_stop(sc); 509 ifnet_deserialize_all(ifp); 510 511 return 0; 512 } 513 514 static void 515 ix_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 516 { 517 struct ix_softc *sc = ifp->if_softc; 518 struct ix_tx_ring *txr = ifsq_get_priv(ifsq); 519 int idx = -1; 520 uint16_t nsegs; 521 522 KKASSERT(txr->tx_ifsq == ifsq); 523 ASSERT_SERIALIZED(&txr->tx_serialize); 524 525 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq)) 526 return; 527 528 if (!sc->link_active) { 529 ifsq_purge(ifsq); 530 return; 531 } 532 533 while (!ifsq_is_empty(ifsq)) { 534 struct mbuf *m_head; 535 536 if (txr->tx_avail <= IX_MAX_SCATTER + IX_TX_RESERVED) { 537 ifsq_set_oactive(ifsq); 538 txr->tx_watchdog.wd_timer = 5; 539 break; 540 } 541 542 m_head = ifsq_dequeue(ifsq); 543 if (m_head == NULL) 544 break; 545 546 if (ix_encap(txr, &m_head, &nsegs, &idx)) { 547 IFNET_STAT_INC(ifp, oerrors, 1); 548 continue; 549 } 550 551 if (nsegs >= txr->tx_wreg_nsegs) { 552 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->tx_idx), idx); 553 nsegs = 0; 554 idx = -1; 555 } 556 557 ETHER_BPF_MTAP(ifp, m_head); 558 } 559 if (idx >= 0) 560 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->tx_idx), idx); 561 } 562 563 static int 564 ix_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) 565 { 566 struct ix_softc *sc = ifp->if_softc; 567 struct ifreq *ifr = (struct ifreq *) data; 568 int error = 0, mask, reinit; 569 570 ASSERT_IFNET_SERIALIZED_ALL(ifp); 571 572 switch (command) { 573 case SIOCSIFMTU: 574 if (ifr->ifr_mtu > IX_MAX_FRAME_SIZE - ETHER_HDR_LEN) { 575 error = EINVAL; 576 } else { 577 ifp->if_mtu = ifr->ifr_mtu; 578 sc->max_frame_size = 579 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 580 ix_init(sc); 581 } 582 break; 583 584 case SIOCSIFFLAGS: 585 if (ifp->if_flags & IFF_UP) { 586 if (ifp->if_flags & IFF_RUNNING) { 587 if ((ifp->if_flags ^ sc->if_flags) & 588 (IFF_PROMISC | IFF_ALLMULTI)) 589 ix_set_promisc(sc); 590 } else { 591 ix_init(sc); 592 } 593 } else if (ifp->if_flags & IFF_RUNNING) { 594 ix_stop(sc); 595 } 596 sc->if_flags = ifp->if_flags; 597 break; 598 599 case SIOCADDMULTI: 600 case SIOCDELMULTI: 601 if (ifp->if_flags & IFF_RUNNING) { 602 ix_disable_intr(sc); 603 ix_set_multi(sc); 604 ix_enable_intr(sc); 605 } 606 break; 607 608 case SIOCSIFMEDIA: 609 case SIOCGIFMEDIA: 610 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 611 break; 612 613 case SIOCSIFCAP: 614 reinit = 0; 615 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 616 if (mask & IFCAP_RXCSUM) { 617 ifp->if_capenable ^= IFCAP_RXCSUM; 618 reinit = 1; 619 } 620 if (mask & IFCAP_VLAN_HWTAGGING) { 621 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 622 reinit = 1; 623 } 624 if (mask & IFCAP_TXCSUM) { 625 ifp->if_capenable ^= IFCAP_TXCSUM; 626 if (ifp->if_capenable & IFCAP_TXCSUM) 627 ifp->if_hwassist |= CSUM_OFFLOAD; 628 else 629 ifp->if_hwassist &= ~CSUM_OFFLOAD; 630 } 631 if (mask & IFCAP_TSO) { 632 ifp->if_capenable ^= IFCAP_TSO; 633 if (ifp->if_capenable & IFCAP_TSO) 634 ifp->if_hwassist |= CSUM_TSO; 635 else 636 ifp->if_hwassist &= ~CSUM_TSO; 637 } 638 if (mask & IFCAP_RSS) 639 ifp->if_capenable ^= IFCAP_RSS; 640 if (reinit && (ifp->if_flags & IFF_RUNNING)) 641 ix_init(sc); 642 break; 643 644 #if 0 645 case SIOCGI2C: 646 { 647 struct ixgbe_i2c_req i2c; 648 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 649 if (error) 650 break; 651 if ((i2c.dev_addr != 0xA0) || (i2c.dev_addr != 0xA2)){ 652 error = EINVAL; 653 break; 654 } 655 hw->phy.ops.read_i2c_byte(hw, i2c.offset, 656 i2c.dev_addr, i2c.data); 657 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 658 break; 659 } 660 #endif 661 662 default: 663 error = ether_ioctl(ifp, command, data); 664 break; 665 } 666 return error; 667 } 668 669 #define IXGBE_MHADD_MFS_SHIFT 16 670 671 static void 672 ix_init(void *xsc) 673 { 674 struct ix_softc *sc = xsc; 675 struct ifnet *ifp = &sc->arpcom.ac_if; 676 struct ixgbe_hw *hw = &sc->hw; 677 uint32_t rxpb, frame, size, tmp; 678 uint32_t gpie, rxctrl; 679 int i, error; 680 681 ASSERT_IFNET_SERIALIZED_ALL(ifp); 682 683 ix_stop(sc); 684 685 /* Configure # of used RX/TX rings */ 686 ix_set_ring_inuse(sc, FALSE); 687 ifq_set_subq_mask(&ifp->if_snd, sc->tx_ring_inuse - 1); 688 689 /* Get the latest mac address, User can use a LAA */ 690 bcopy(IF_LLADDR(ifp), hw->mac.addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 691 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 692 hw->addr_ctrl.rar_used_count = 1; 693 694 /* Prepare transmit descriptors and buffers */ 695 for (i = 0; i < sc->tx_ring_inuse; ++i) 696 ix_init_tx_ring(&sc->tx_rings[i]); 697 698 ixgbe_init_hw(hw); 699 ix_init_tx_unit(sc); 700 701 /* Setup Multicast table */ 702 ix_set_multi(sc); 703 704 /* Prepare receive descriptors and buffers */ 705 for (i = 0; i < sc->rx_ring_inuse; ++i) { 706 error = ix_init_rx_ring(&sc->rx_rings[i]); 707 if (error) { 708 if_printf(ifp, "Could not initialize RX ring%d\n", i); 709 ix_stop(sc); 710 return; 711 } 712 } 713 714 /* Configure RX settings */ 715 ix_init_rx_unit(sc); 716 717 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 718 719 /* Enable Fan Failure Interrupt */ 720 gpie |= IXGBE_SDP1_GPIEN; 721 722 /* Add for Module detection */ 723 if (hw->mac.type == ixgbe_mac_82599EB) 724 gpie |= IXGBE_SDP2_GPIEN; 725 726 /* Thermal Failure Detection */ 727 if (hw->mac.type == ixgbe_mac_X540) 728 gpie |= IXGBE_SDP0_GPIEN; 729 730 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 731 /* Enable Enhanced MSIX mode */ 732 gpie |= IXGBE_GPIE_MSIX_MODE; 733 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 734 IXGBE_GPIE_OCD; 735 } 736 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 737 738 /* Set MTU size */ 739 if (ifp->if_mtu > ETHERMTU) { 740 uint32_t mhadd; 741 742 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 743 mhadd &= ~IXGBE_MHADD_MFS_MASK; 744 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 745 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 746 } 747 748 /* 749 * Enable TX rings 750 */ 751 for (i = 0; i < sc->tx_ring_inuse; ++i) { 752 uint32_t txdctl; 753 754 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 755 txdctl |= IXGBE_TXDCTL_ENABLE; 756 757 /* 758 * Set WTHRESH to 0, since TX head write-back is used 759 */ 760 txdctl &= ~(0x7f << 16); 761 762 /* 763 * When the internal queue falls below PTHRESH (32), 764 * start prefetching as long as there are at least 765 * HTHRESH (1) buffers ready. The values are taken 766 * from the Intel linux driver 3.8.21. 767 * Prefetching enables tx line rate even with 1 queue. 768 */ 769 txdctl |= (32 << 0) | (1 << 8); 770 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl); 771 } 772 773 /* 774 * Enable RX rings 775 */ 776 for (i = 0; i < sc->rx_ring_inuse; ++i) { 777 uint32_t rxdctl; 778 int k; 779 780 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 781 if (hw->mac.type == ixgbe_mac_82598EB) { 782 /* 783 * PTHRESH = 21 784 * HTHRESH = 4 785 * WTHRESH = 8 786 */ 787 rxdctl &= ~0x3FFFFF; 788 rxdctl |= 0x080420; 789 } 790 rxdctl |= IXGBE_RXDCTL_ENABLE; 791 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl); 792 for (k = 0; k < 10; ++k) { 793 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & 794 IXGBE_RXDCTL_ENABLE) 795 break; 796 else 797 msec_delay(1); 798 } 799 wmb(); 800 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 801 sc->rx_rings[0].rx_ndesc - 1); 802 } 803 804 /* Set up VLAN support and filter */ 805 ix_set_vlan(sc); 806 807 /* Enable Receive engine */ 808 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 809 if (hw->mac.type == ixgbe_mac_82598EB) 810 rxctrl |= IXGBE_RXCTRL_DMBYPS; 811 rxctrl |= IXGBE_RXCTRL_RXEN; 812 ixgbe_enable_rx_dma(hw, rxctrl); 813 814 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 815 #if 0 816 ix_configure_ivars(sc); 817 #endif 818 /* Set up auto-mask */ 819 if (hw->mac.type == ixgbe_mac_82598EB) 820 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 821 else { 822 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 823 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 824 } 825 } else { 826 for (i = 0; i < sc->tx_ring_inuse; ++i) 827 ix_set_ivar(sc, i, sc->tx_rings[i].tx_intr_vec, 1); 828 for (i = 0; i < sc->rx_ring_inuse; ++i) 829 ix_set_ivar(sc, i, sc->rx_rings[i].rx_intr_vec, 0); 830 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 831 ix_set_eitr(sc, 0, sc->intr_data[0].intr_rate); 832 } 833 834 /* 835 * Check on any SFP devices that need to be kick-started 836 */ 837 if (hw->phy.type == ixgbe_phy_none) { 838 error = hw->phy.ops.identify(hw); 839 if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 840 if_printf(ifp, 841 "Unsupported SFP+ module type was detected.\n"); 842 /* XXX stop */ 843 return; 844 } 845 } 846 847 #if 0 848 /* Set moderation on the Link interrupt */ 849 IXGBE_WRITE_REG(hw, IXGBE_EITR(sc->linkvec), IXGBE_LINK_ITR); 850 #endif 851 852 /* Config/Enable Link */ 853 ix_config_link(sc); 854 855 /* 856 * Hardware Packet Buffer & Flow Control setup 857 */ 858 frame = sc->max_frame_size; 859 860 /* Calculate High Water */ 861 if (hw->mac.type == ixgbe_mac_X540) 862 tmp = IXGBE_DV_X540(frame, frame); 863 else 864 tmp = IXGBE_DV(frame, frame); 865 size = IXGBE_BT2KB(tmp); 866 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 867 hw->fc.high_water[0] = rxpb - size; 868 869 /* Now calculate Low Water */ 870 if (hw->mac.type == ixgbe_mac_X540) 871 tmp = IXGBE_LOW_DV_X540(frame); 872 else 873 tmp = IXGBE_LOW_DV(frame); 874 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 875 876 hw->fc.requested_mode = sc->fc; 877 hw->fc.pause_time = IX_FC_PAUSE; 878 hw->fc.send_xon = TRUE; 879 880 /* Initialize the FC settings */ 881 ixgbe_start_hw(hw); 882 883 /* And now turn on interrupts */ 884 ix_enable_intr(sc); 885 886 ifp->if_flags |= IFF_RUNNING; 887 for (i = 0; i < sc->tx_ring_inuse; ++i) { 888 ifsq_clr_oactive(sc->tx_rings[i].tx_ifsq); 889 ifsq_watchdog_start(&sc->tx_rings[i].tx_watchdog); 890 } 891 892 ix_set_timer_cpuid(sc, FALSE); 893 callout_reset_bycpu(&sc->timer, hz, ix_timer, sc, sc->timer_cpuid); 894 } 895 896 #if 0 897 /* 898 ** 899 ** MSIX Interrupt Handlers and Tasklets 900 ** 901 */ 902 903 static __inline void 904 ix_enable_queue(struct ix_softc *sc, uint32_t vector) 905 { 906 struct ixgbe_hw *hw = &sc->hw; 907 uint64_t queue = (uint64_t)(1 << vector); 908 uint32_t mask; 909 910 if (hw->mac.type == ixgbe_mac_82598EB) { 911 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 912 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 913 } else { 914 mask = (queue & 0xFFFFFFFF); 915 if (mask) 916 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 917 mask = (queue >> 32); 918 if (mask) 919 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 920 } 921 } 922 923 static __inline void 924 ix_disable_queue(struct ix_softc *sc, uint32_t vector) 925 { 926 struct ixgbe_hw *hw = &sc->hw; 927 uint64_t queue = (uint64_t)(1 << vector); 928 uint32_t mask; 929 930 if (hw->mac.type == ixgbe_mac_82598EB) { 931 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 932 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 933 } else { 934 mask = (queue & 0xFFFFFFFF); 935 if (mask) 936 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 937 mask = (queue >> 32); 938 if (mask) 939 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 940 } 941 } 942 943 static __inline void 944 ix_rearm_queues(struct ix_softc *sc, uint64_t queues) 945 { 946 uint32_t mask; 947 948 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 949 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 950 IXGBE_WRITE_REG(&sc->hw, IXGBE_EICS, mask); 951 } else { 952 mask = (queues & 0xFFFFFFFF); 953 IXGBE_WRITE_REG(&sc->hw, IXGBE_EICS_EX(0), mask); 954 mask = (queues >> 32); 955 IXGBE_WRITE_REG(&sc->hw, IXGBE_EICS_EX(1), mask); 956 } 957 } 958 959 static void 960 ix_handle_que(void *context, int pending) 961 { 962 struct ix_queue *que = context; 963 struct ix_softc *sc = que->sc; 964 struct ix_tx_ring *txr = que->txr; 965 struct ifnet *ifp = &sc->arpcom.ac_if; 966 967 if (ifp->if_flags & IFF_RUNNING) { 968 more = ix_rxeof(que); 969 ix_txeof(txr); 970 if (!ifq_is_empty(&ifp->if_snd)) 971 ixgbe_start_locked(txr, ifp); 972 } 973 974 /* Reenable this interrupt */ 975 if (que->res != NULL) 976 ixgbe_enable_queue(sc, que->msix); 977 else 978 ix_enable_intr(sc); 979 return; 980 } 981 #endif 982 983 static void 984 ix_intr(void *xsc) 985 { 986 struct ix_softc *sc = xsc; 987 struct ixgbe_hw *hw = &sc->hw; 988 uint32_t eicr; 989 990 ASSERT_SERIALIZED(&sc->main_serialize); 991 992 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 993 if (eicr == 0) { 994 IXGBE_WRITE_REG(hw, IXGBE_EIMS, sc->intr_mask); 995 return; 996 } 997 998 if (eicr & IX_RX0_INTR_MASK) { 999 struct ix_rx_ring *rxr = &sc->rx_rings[0]; 1000 1001 lwkt_serialize_enter(&rxr->rx_serialize); 1002 ix_rxeof(rxr); 1003 lwkt_serialize_exit(&rxr->rx_serialize); 1004 } 1005 if (eicr & IX_RX1_INTR_MASK) { 1006 struct ix_rx_ring *rxr; 1007 1008 KKASSERT(sc->rx_ring_inuse == IX_MIN_RXRING_RSS); 1009 rxr = &sc->rx_rings[1]; 1010 1011 lwkt_serialize_enter(&rxr->rx_serialize); 1012 ix_rxeof(rxr); 1013 lwkt_serialize_exit(&rxr->rx_serialize); 1014 } 1015 1016 if (eicr & IX_TX_INTR_MASK) { 1017 struct ix_tx_ring *txr = &sc->tx_rings[0]; 1018 1019 lwkt_serialize_enter(&txr->tx_serialize); 1020 ix_txeof(txr); 1021 if (!ifsq_is_empty(txr->tx_ifsq)) 1022 ifsq_devstart(txr->tx_ifsq); 1023 lwkt_serialize_exit(&txr->tx_serialize); 1024 } 1025 1026 /* Check for fan failure */ 1027 if (__predict_false((eicr & IXGBE_EICR_GPI_SDP1) && 1028 hw->phy.media_type == ixgbe_media_type_copper)) { 1029 if_printf(&sc->arpcom.ac_if, "CRITICAL: FAN FAILURE!! " 1030 "REPLACE IMMEDIATELY!!\n"); 1031 } 1032 1033 /* Link status change */ 1034 if (__predict_false(eicr & IXGBE_EICR_LSC)) 1035 ix_handle_link(sc); 1036 1037 IXGBE_WRITE_REG(hw, IXGBE_EIMS, sc->intr_mask); 1038 } 1039 1040 #if 0 1041 /********************************************************************* 1042 * 1043 * MSIX Queue Interrupt Service routine 1044 * 1045 **********************************************************************/ 1046 void 1047 ix_msix_que(void *arg) 1048 { 1049 struct ix_queue *que = arg; 1050 struct ix_softc *sc = que->sc; 1051 struct ifnet *ifp = &sc->arpcom.ac_if; 1052 struct ix_tx_ring *txr = que->txr; 1053 struct ix_rx_ring *rxr = que->rxr; 1054 uint32_t newitr = 0; 1055 1056 ixgbe_disable_queue(sc, que->msix); 1057 ++que->irqs; 1058 1059 more = ix_rxeof(que); 1060 1061 ix_txeof(txr); 1062 if (!ifq_is_empty(&ifp->if_snd)) 1063 ixgbe_start_locked(txr, ifp); 1064 1065 /* Do AIM now? */ 1066 1067 if (ixgbe_enable_aim == FALSE) 1068 goto no_calc; 1069 /* 1070 ** Do Adaptive Interrupt Moderation: 1071 ** - Write out last calculated setting 1072 ** - Calculate based on average size over 1073 ** the last interval. 1074 */ 1075 if (que->eitr_setting) 1076 IXGBE_WRITE_REG(&sc->hw, 1077 IXGBE_EITR(que->msix), que->eitr_setting); 1078 1079 que->eitr_setting = 0; 1080 1081 /* Idle, do nothing */ 1082 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1083 goto no_calc; 1084 1085 if ((txr->bytes) && (txr->packets)) 1086 newitr = txr->bytes/txr->packets; 1087 if ((rxr->bytes) && (rxr->packets)) 1088 newitr = max(newitr, 1089 (rxr->bytes / rxr->packets)); 1090 newitr += 24; /* account for hardware frame, crc */ 1091 1092 /* set an upper boundary */ 1093 newitr = min(newitr, 3000); 1094 1095 /* Be nice to the mid range */ 1096 if ((newitr > 300) && (newitr < 1200)) 1097 newitr = (newitr / 3); 1098 else 1099 newitr = (newitr / 2); 1100 1101 if (sc->hw.mac.type == ixgbe_mac_82598EB) 1102 newitr |= newitr << 16; 1103 else 1104 newitr |= IXGBE_EITR_CNT_WDIS; 1105 1106 /* save for next interrupt */ 1107 que->eitr_setting = newitr; 1108 1109 /* Reset state */ 1110 txr->bytes = 0; 1111 txr->packets = 0; 1112 rxr->bytes = 0; 1113 rxr->packets = 0; 1114 1115 no_calc: 1116 #if 0 1117 if (more) 1118 taskqueue_enqueue(que->tq, &que->que_task); 1119 else 1120 #endif 1121 ixgbe_enable_queue(sc, que->msix); 1122 return; 1123 } 1124 1125 1126 static void 1127 ix_msix_link(void *arg) 1128 { 1129 struct ix_softc *sc = arg; 1130 struct ixgbe_hw *hw = &sc->hw; 1131 uint32_t reg_eicr; 1132 1133 ++sc->link_irq; 1134 1135 /* First get the cause */ 1136 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1137 /* Clear interrupt with write */ 1138 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1139 1140 #if 0 1141 /* Link status change */ 1142 if (reg_eicr & IXGBE_EICR_LSC) 1143 taskqueue_enqueue(sc->tq, &sc->link_task); 1144 #endif 1145 1146 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 1147 if (reg_eicr & IXGBE_EICR_ECC) { 1148 device_printf(sc->dev, "\nCRITICAL: ECC ERROR!! " 1149 "Please Reboot!!\n"); 1150 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1151 } else 1152 1153 if (reg_eicr & IXGBE_EICR_GPI_SDP1) { 1154 /* Clear the interrupt */ 1155 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1156 #if 0 1157 taskqueue_enqueue(sc->tq, &sc->msf_task); 1158 #endif 1159 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) { 1160 /* Clear the interrupt */ 1161 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 1162 #if 0 1163 taskqueue_enqueue(sc->tq, &sc->mod_task); 1164 #endif 1165 } 1166 } 1167 1168 /* Check for fan failure */ 1169 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1170 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1171 device_printf(sc->dev, "\nCRITICAL: FAN FAILURE!! " 1172 "REPLACE IMMEDIATELY!!\n"); 1173 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1174 } 1175 1176 /* Check for over temp condition */ 1177 if ((hw->mac.type == ixgbe_mac_X540) && 1178 (reg_eicr & IXGBE_EICR_TS)) { 1179 device_printf(sc->dev, "\nCRITICAL: OVER TEMP!! " 1180 "PHY IS SHUT DOWN!!\n"); 1181 device_printf(sc->dev, "System shutdown required\n"); 1182 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS); 1183 } 1184 1185 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); 1186 return; 1187 } 1188 1189 #endif 1190 1191 static void 1192 ix_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1193 { 1194 struct ix_softc *sc = ifp->if_softc; 1195 1196 ix_update_link_status(sc); 1197 1198 ifmr->ifm_status = IFM_AVALID; 1199 ifmr->ifm_active = IFM_ETHER; 1200 1201 if (!sc->link_active) 1202 return; 1203 1204 ifmr->ifm_status |= IFM_ACTIVE; 1205 1206 switch (sc->link_speed) { 1207 case IXGBE_LINK_SPEED_100_FULL: 1208 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1209 break; 1210 case IXGBE_LINK_SPEED_1GB_FULL: 1211 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 1212 break; 1213 case IXGBE_LINK_SPEED_10GB_FULL: 1214 ifmr->ifm_active |= sc->optics | IFM_FDX; 1215 break; 1216 } 1217 } 1218 1219 static int 1220 ix_media_change(struct ifnet *ifp) 1221 { 1222 struct ix_softc *sc = ifp->if_softc; 1223 struct ifmedia *ifm = &sc->media; 1224 1225 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1226 return EINVAL; 1227 1228 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1229 case IFM_AUTO: 1230 sc->hw.phy.autoneg_advertised = 1231 IXGBE_LINK_SPEED_100_FULL | 1232 IXGBE_LINK_SPEED_1GB_FULL | 1233 IXGBE_LINK_SPEED_10GB_FULL; 1234 break; 1235 default: 1236 if_printf(ifp, "Only auto media type\n"); 1237 return EINVAL; 1238 } 1239 return 0; 1240 } 1241 1242 static __inline int 1243 ix_tso_pullup(struct mbuf **mp) 1244 { 1245 int hoff, iphlen, thoff; 1246 struct mbuf *m; 1247 1248 m = *mp; 1249 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable")); 1250 1251 iphlen = m->m_pkthdr.csum_iphlen; 1252 thoff = m->m_pkthdr.csum_thlen; 1253 hoff = m->m_pkthdr.csum_lhlen; 1254 1255 KASSERT(iphlen > 0, ("invalid ip hlen")); 1256 KASSERT(thoff > 0, ("invalid tcp hlen")); 1257 KASSERT(hoff > 0, ("invalid ether hlen")); 1258 1259 if (__predict_false(m->m_len < hoff + iphlen + thoff)) { 1260 m = m_pullup(m, hoff + iphlen + thoff); 1261 if (m == NULL) { 1262 *mp = NULL; 1263 return ENOBUFS; 1264 } 1265 *mp = m; 1266 } 1267 return 0; 1268 } 1269 1270 static int 1271 ix_encap(struct ix_tx_ring *txr, struct mbuf **m_headp, 1272 uint16_t *segs_used, int *idx) 1273 { 1274 uint32_t olinfo_status = 0, cmd_type_len, cmd_rs = 0; 1275 int i, j, error, nsegs, first, maxsegs; 1276 struct mbuf *m_head = *m_headp; 1277 bus_dma_segment_t segs[IX_MAX_SCATTER]; 1278 bus_dmamap_t map; 1279 struct ix_tx_buf *txbuf; 1280 union ixgbe_adv_tx_desc *txd = NULL; 1281 1282 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1283 error = ix_tso_pullup(m_headp); 1284 if (__predict_false(error)) 1285 return error; 1286 m_head = *m_headp; 1287 } 1288 1289 /* Basic descriptor defines */ 1290 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1291 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1292 1293 if (m_head->m_flags & M_VLANTAG) 1294 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1295 1296 /* 1297 * Important to capture the first descriptor 1298 * used because it will contain the index of 1299 * the one we tell the hardware to report back 1300 */ 1301 first = txr->tx_next_avail; 1302 txbuf = &txr->tx_buf[first]; 1303 map = txbuf->map; 1304 1305 /* 1306 * Map the packet for DMA. 1307 */ 1308 maxsegs = txr->tx_avail - IX_TX_RESERVED; 1309 if (maxsegs > IX_MAX_SCATTER) 1310 maxsegs = IX_MAX_SCATTER; 1311 1312 error = bus_dmamap_load_mbuf_defrag(txr->tx_tag, map, m_headp, 1313 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 1314 if (__predict_false(error)) { 1315 m_freem(*m_headp); 1316 *m_headp = NULL; 1317 return error; 1318 } 1319 bus_dmamap_sync(txr->tx_tag, map, BUS_DMASYNC_PREWRITE); 1320 1321 m_head = *m_headp; 1322 1323 /* 1324 * Set up the appropriate offload context if requested, 1325 * this may consume one TX descriptor. 1326 */ 1327 if (ix_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status)) { 1328 (*segs_used)++; 1329 txr->tx_nsegs++; 1330 } 1331 1332 *segs_used += nsegs; 1333 txr->tx_nsegs += nsegs; 1334 if (txr->tx_nsegs >= txr->tx_intr_nsegs) { 1335 /* 1336 * Report Status (RS) is turned on every intr_nsegs 1337 * descriptors (roughly). 1338 */ 1339 txr->tx_nsegs = 0; 1340 cmd_rs = IXGBE_TXD_CMD_RS; 1341 } 1342 1343 i = txr->tx_next_avail; 1344 for (j = 0; j < nsegs; j++) { 1345 bus_size_t seglen; 1346 bus_addr_t segaddr; 1347 1348 txbuf = &txr->tx_buf[i]; 1349 txd = &txr->tx_base[i]; 1350 seglen = segs[j].ds_len; 1351 segaddr = htole64(segs[j].ds_addr); 1352 1353 txd->read.buffer_addr = segaddr; 1354 txd->read.cmd_type_len = htole32(IXGBE_TXD_CMD_IFCS | 1355 cmd_type_len |seglen); 1356 txd->read.olinfo_status = htole32(olinfo_status); 1357 1358 if (++i == txr->tx_ndesc) 1359 i = 0; 1360 } 1361 txd->read.cmd_type_len |= htole32(IXGBE_TXD_CMD_EOP | cmd_rs); 1362 1363 txr->tx_avail -= nsegs; 1364 txr->tx_next_avail = i; 1365 1366 txbuf->m_head = m_head; 1367 txr->tx_buf[first].map = txbuf->map; 1368 txbuf->map = map; 1369 1370 /* 1371 * Defer TDT updating, until enough descrptors are setup 1372 */ 1373 *idx = i; 1374 1375 return 0; 1376 } 1377 1378 static void 1379 ix_set_promisc(struct ix_softc *sc) 1380 { 1381 struct ifnet *ifp = &sc->arpcom.ac_if; 1382 uint32_t reg_rctl; 1383 int mcnt = 0; 1384 1385 reg_rctl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1386 reg_rctl &= ~IXGBE_FCTRL_UPE; 1387 if (ifp->if_flags & IFF_ALLMULTI) { 1388 mcnt = IX_MAX_MCASTADDR; 1389 } else { 1390 struct ifmultiaddr *ifma; 1391 1392 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1393 if (ifma->ifma_addr->sa_family != AF_LINK) 1394 continue; 1395 if (mcnt == IX_MAX_MCASTADDR) 1396 break; 1397 mcnt++; 1398 } 1399 } 1400 if (mcnt < IX_MAX_MCASTADDR) 1401 reg_rctl &= ~IXGBE_FCTRL_MPE; 1402 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 1403 1404 if (ifp->if_flags & IFF_PROMISC) { 1405 reg_rctl |= IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE; 1406 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 1407 } else if (ifp->if_flags & IFF_ALLMULTI) { 1408 reg_rctl |= IXGBE_FCTRL_MPE; 1409 reg_rctl &= ~IXGBE_FCTRL_UPE; 1410 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 1411 } 1412 } 1413 1414 static void 1415 ix_set_multi(struct ix_softc *sc) 1416 { 1417 struct ifnet *ifp = &sc->arpcom.ac_if; 1418 struct ifmultiaddr *ifma; 1419 uint32_t fctrl; 1420 uint8_t *mta; 1421 int mcnt = 0; 1422 1423 mta = sc->mta; 1424 bzero(mta, IXGBE_ETH_LENGTH_OF_ADDRESS * IX_MAX_MCASTADDR); 1425 1426 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1427 if (ifma->ifma_addr->sa_family != AF_LINK) 1428 continue; 1429 if (mcnt == IX_MAX_MCASTADDR) 1430 break; 1431 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1432 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1433 IXGBE_ETH_LENGTH_OF_ADDRESS); 1434 mcnt++; 1435 } 1436 1437 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1438 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1439 if (ifp->if_flags & IFF_PROMISC) { 1440 fctrl |= IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE; 1441 } else if (mcnt >= IX_MAX_MCASTADDR || (ifp->if_flags & IFF_ALLMULTI)) { 1442 fctrl |= IXGBE_FCTRL_MPE; 1443 fctrl &= ~IXGBE_FCTRL_UPE; 1444 } else { 1445 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1446 } 1447 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1448 1449 if (mcnt < IX_MAX_MCASTADDR) { 1450 ixgbe_update_mc_addr_list(&sc->hw, 1451 mta, mcnt, ix_mc_array_itr, TRUE); 1452 } 1453 } 1454 1455 /* 1456 * This is an iterator function now needed by the multicast 1457 * shared code. It simply feeds the shared code routine the 1458 * addresses in the array of ix_set_multi() one by one. 1459 */ 1460 static uint8_t * 1461 ix_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1462 { 1463 uint8_t *addr = *update_ptr; 1464 uint8_t *newptr; 1465 *vmdq = 0; 1466 1467 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1468 *update_ptr = newptr; 1469 return addr; 1470 } 1471 1472 static void 1473 ix_timer(void *arg) 1474 { 1475 struct ix_softc *sc = arg; 1476 1477 lwkt_serialize_enter(&sc->main_serialize); 1478 1479 if ((sc->arpcom.ac_if.if_flags & IFF_RUNNING) == 0) { 1480 lwkt_serialize_exit(&sc->main_serialize); 1481 return; 1482 } 1483 1484 /* Check for pluggable optics */ 1485 if (sc->sfp_probe) { 1486 if (!ix_sfp_probe(sc)) 1487 goto done; /* Nothing to do */ 1488 } 1489 1490 ix_update_link_status(sc); 1491 ix_update_stats(sc); 1492 1493 done: 1494 callout_reset_bycpu(&sc->timer, hz, ix_timer, sc, sc->timer_cpuid); 1495 lwkt_serialize_exit(&sc->main_serialize); 1496 } 1497 1498 static void 1499 ix_update_link_status(struct ix_softc *sc) 1500 { 1501 struct ifnet *ifp = &sc->arpcom.ac_if; 1502 1503 if (sc->link_up) { 1504 if (sc->link_active == FALSE) { 1505 if (bootverbose) { 1506 if_printf(ifp, "Link is up %d Gbps %s\n", 1507 sc->link_speed == 128 ? 10 : 1, 1508 "Full Duplex"); 1509 } 1510 sc->link_active = TRUE; 1511 1512 /* Update any Flow Control changes */ 1513 ixgbe_fc_enable(&sc->hw); 1514 1515 ifp->if_link_state = LINK_STATE_UP; 1516 if_link_state_change(ifp); 1517 } 1518 } else { /* Link down */ 1519 if (sc->link_active == TRUE) { 1520 if (bootverbose) 1521 if_printf(ifp, "Link is Down\n"); 1522 ifp->if_link_state = LINK_STATE_DOWN; 1523 if_link_state_change(ifp); 1524 1525 sc->link_active = FALSE; 1526 } 1527 } 1528 } 1529 1530 static void 1531 ix_stop(struct ix_softc *sc) 1532 { 1533 struct ixgbe_hw *hw = &sc->hw; 1534 struct ifnet *ifp = &sc->arpcom.ac_if; 1535 int i; 1536 1537 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1538 1539 ix_disable_intr(sc); 1540 callout_stop(&sc->timer); 1541 1542 ifp->if_flags &= ~IFF_RUNNING; 1543 for (i = 0; i < sc->tx_ring_cnt; ++i) { 1544 ifsq_clr_oactive(sc->tx_rings[i].tx_ifsq); 1545 ifsq_watchdog_stop(&sc->tx_rings[i].tx_watchdog); 1546 } 1547 1548 ixgbe_reset_hw(hw); 1549 hw->adapter_stopped = FALSE; 1550 ixgbe_stop_adapter(hw); 1551 if (hw->mac.type == ixgbe_mac_82599EB) 1552 ixgbe_stop_mac_link_on_d3_82599(hw); 1553 /* Turn off the laser - noop with no optics */ 1554 ixgbe_disable_tx_laser(hw); 1555 1556 /* Update the stack */ 1557 sc->link_up = FALSE; 1558 ix_update_link_status(sc); 1559 1560 /* Reprogram the RAR[0] in case user changed it. */ 1561 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1562 1563 for (i = 0; i < sc->tx_ring_cnt; ++i) 1564 ix_free_tx_ring(&sc->tx_rings[i]); 1565 1566 for (i = 0; i < sc->rx_ring_cnt; ++i) 1567 ix_free_rx_ring(&sc->rx_rings[i]); 1568 } 1569 1570 static void 1571 ix_setup_optics(struct ix_softc *sc) 1572 { 1573 struct ixgbe_hw *hw = &sc->hw; 1574 int layer; 1575 1576 layer = ixgbe_get_supported_physical_layer(hw); 1577 1578 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) { 1579 sc->optics = IFM_10G_T; 1580 return; 1581 } 1582 1583 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) { 1584 sc->optics = IFM_1000_T; 1585 return; 1586 } 1587 1588 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 1589 sc->optics = IFM_1000_SX; 1590 return; 1591 } 1592 1593 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR | 1594 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) { 1595 sc->optics = IFM_10G_LR; 1596 return; 1597 } 1598 1599 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1600 sc->optics = IFM_10G_SR; 1601 return; 1602 } 1603 1604 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) { 1605 sc->optics = IFM_10G_TWINAX; 1606 return; 1607 } 1608 1609 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 | 1610 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) { 1611 sc->optics = IFM_10G_CX4; 1612 return; 1613 } 1614 1615 /* If we get here just set the default */ 1616 sc->optics = IFM_ETHER | IFM_AUTO; 1617 } 1618 1619 #if 0 1620 /********************************************************************* 1621 * 1622 * Setup MSIX Interrupt resources and handlers 1623 * 1624 **********************************************************************/ 1625 static int 1626 ix_allocate_msix(struct ix_softc *sc) 1627 { 1628 device_t dev = sc->dev; 1629 struct ix_queue *que = sc->queues; 1630 struct ix_tx_ring *txr = sc->tx_rings; 1631 int error, rid, vector = 0; 1632 1633 for (int i = 0; i < sc->num_queues; i++, vector++, que++, txr++) { 1634 rid = vector + 1; 1635 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 1636 RF_SHAREABLE | RF_ACTIVE); 1637 if (que->res == NULL) { 1638 device_printf(dev,"Unable to allocate" 1639 " bus resource: que interrupt [%d]\n", vector); 1640 return (ENXIO); 1641 } 1642 /* Set the handler function */ 1643 error = bus_setup_intr(dev, que->res, 1644 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1645 ix_msix_que, que, &que->tag); 1646 if (error) { 1647 que->res = NULL; 1648 device_printf(dev, "Failed to register QUE handler"); 1649 return (error); 1650 } 1651 #if __FreeBSD_version >= 800504 1652 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 1653 #endif 1654 que->msix = vector; 1655 sc->que_mask |= (uint64_t)(1 << que->msix); 1656 /* 1657 ** Bind the msix vector, and thus the 1658 ** ring to the corresponding cpu. 1659 */ 1660 if (sc->num_queues > 1) 1661 bus_bind_intr(dev, que->res, i); 1662 1663 TASK_INIT(&que->que_task, 0, ix_handle_que, que); 1664 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT, 1665 taskqueue_thread_enqueue, &que->tq); 1666 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", 1667 device_get_nameunit(sc->dev)); 1668 } 1669 1670 /* and Link */ 1671 rid = vector + 1; 1672 sc->res = bus_alloc_resource_any(dev, 1673 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 1674 if (!sc->res) { 1675 device_printf(dev,"Unable to allocate" 1676 " bus resource: Link interrupt [%d]\n", rid); 1677 return (ENXIO); 1678 } 1679 /* Set the link handler function */ 1680 error = bus_setup_intr(dev, sc->res, 1681 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1682 ix_msix_link, sc, &sc->tag); 1683 if (error) { 1684 sc->res = NULL; 1685 device_printf(dev, "Failed to register LINK handler"); 1686 return (error); 1687 } 1688 #if __FreeBSD_version >= 800504 1689 bus_describe_intr(dev, sc->res, sc->tag, "link"); 1690 #endif 1691 sc->linkvec = vector; 1692 /* Tasklets for Link, SFP and Multispeed Fiber */ 1693 TASK_INIT(&sc->link_task, 0, ix_handle_link, sc); 1694 TASK_INIT(&sc->mod_task, 0, ix_handle_mod, sc); 1695 TASK_INIT(&sc->msf_task, 0, ix_handle_msf, sc); 1696 sc->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT, 1697 taskqueue_thread_enqueue, &sc->tq); 1698 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s linkq", 1699 device_get_nameunit(sc->dev)); 1700 1701 return (0); 1702 } 1703 1704 static int 1705 ix_setup_msix(struct ix_softc *sc) 1706 { 1707 device_t dev = sc->dev; 1708 int rid, want, queues, msgs; 1709 1710 /* Override by tuneable */ 1711 if (ixgbe_enable_msix == 0) 1712 goto msi; 1713 1714 /* First try MSI/X */ 1715 rid = PCIR_BAR(MSIX_82598_BAR); 1716 sc->msix_mem = bus_alloc_resource_any(dev, 1717 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1718 if (!sc->msix_mem) { 1719 rid += 4; /* 82599 maps in higher BAR */ 1720 sc->msix_mem = bus_alloc_resource_any(dev, 1721 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1722 } 1723 if (!sc->msix_mem) { 1724 /* May not be enabled */ 1725 device_printf(sc->dev, 1726 "Unable to map MSIX table \n"); 1727 goto msi; 1728 } 1729 1730 msgs = pci_msix_count(dev); 1731 if (msgs == 0) { /* system has msix disabled */ 1732 bus_release_resource(dev, SYS_RES_MEMORY, 1733 rid, sc->msix_mem); 1734 sc->msix_mem = NULL; 1735 goto msi; 1736 } 1737 1738 /* Figure out a reasonable auto config value */ 1739 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; 1740 1741 if (ixgbe_num_queues != 0) 1742 queues = ixgbe_num_queues; 1743 /* Set max queues to 8 when autoconfiguring */ 1744 else if ((ixgbe_num_queues == 0) && (queues > 8)) 1745 queues = 8; 1746 1747 /* 1748 ** Want one vector (RX/TX pair) per queue 1749 ** plus an additional for Link. 1750 */ 1751 want = queues + 1; 1752 if (msgs >= want) 1753 msgs = want; 1754 else { 1755 device_printf(sc->dev, 1756 "MSIX Configuration Problem, " 1757 "%d vectors but %d queues wanted!\n", 1758 msgs, want); 1759 return (0); /* Will go to Legacy setup */ 1760 } 1761 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 1762 device_printf(sc->dev, 1763 "Using MSIX interrupts with %d vectors\n", msgs); 1764 sc->num_queues = queues; 1765 return (msgs); 1766 } 1767 msi: 1768 msgs = pci_msi_count(dev); 1769 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 1770 device_printf(sc->dev,"Using an MSI interrupt\n"); 1771 else 1772 device_printf(sc->dev,"Using a Legacy interrupt\n"); 1773 return (msgs); 1774 } 1775 #endif 1776 1777 static void 1778 ix_setup_ifp(struct ix_softc *sc) 1779 { 1780 struct ixgbe_hw *hw = &sc->hw; 1781 struct ifnet *ifp = &sc->arpcom.ac_if; 1782 int i; 1783 1784 ifp->if_baudrate = IF_Gbps(10UL); 1785 1786 ifp->if_softc = sc; 1787 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1788 ifp->if_init = ix_init; 1789 ifp->if_ioctl = ix_ioctl; 1790 ifp->if_start = ix_start; 1791 ifp->if_serialize = ix_serialize; 1792 ifp->if_deserialize = ix_deserialize; 1793 ifp->if_tryserialize = ix_tryserialize; 1794 #ifdef INVARIANTS 1795 ifp->if_serialize_assert = ix_serialize_assert; 1796 #endif 1797 1798 ifq_set_maxlen(&ifp->if_snd, sc->tx_rings[0].tx_ndesc - 2); 1799 ifq_set_ready(&ifp->if_snd); 1800 ifq_set_subq_cnt(&ifp->if_snd, sc->tx_ring_cnt); 1801 1802 ifp->if_mapsubq = ifq_mapsubq_mask; 1803 ifq_set_subq_mask(&ifp->if_snd, 0); 1804 1805 ether_ifattach(ifp, hw->mac.addr, NULL); 1806 1807 ifp->if_capabilities = 1808 IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 1809 if (IX_ENABLE_HWRSS(sc)) 1810 ifp->if_capabilities |= IFCAP_RSS; 1811 ifp->if_capenable = ifp->if_capabilities; 1812 ifp->if_hwassist = CSUM_OFFLOAD | CSUM_TSO; 1813 1814 /* 1815 * Tell the upper layer(s) we support long frames. 1816 */ 1817 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 1818 1819 /* Setup TX rings and subqueues */ 1820 for (i = 0; i < sc->tx_ring_cnt; ++i) { 1821 struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i); 1822 struct ix_tx_ring *txr = &sc->tx_rings[i]; 1823 1824 ifsq_set_cpuid(ifsq, txr->tx_intr_cpuid); 1825 ifsq_set_priv(ifsq, txr); 1826 ifsq_set_hw_serialize(ifsq, &txr->tx_serialize); 1827 txr->tx_ifsq = ifsq; 1828 1829 ifsq_watchdog_init(&txr->tx_watchdog, ifsq, ix_watchdog); 1830 } 1831 1832 /* 1833 * Specify the media types supported by this adapter and register 1834 * callbacks to update media and link information 1835 */ 1836 ifmedia_add(&sc->media, IFM_ETHER | sc->optics, 0, NULL); 1837 ifmedia_set(&sc->media, IFM_ETHER | sc->optics); 1838 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 1839 ifmedia_add(&sc->media, 1840 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1841 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 1842 } 1843 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1844 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1845 } 1846 1847 static boolean_t 1848 ix_is_sfp(const struct ixgbe_hw *hw) 1849 { 1850 switch (hw->phy.type) { 1851 case ixgbe_phy_sfp_avago: 1852 case ixgbe_phy_sfp_ftl: 1853 case ixgbe_phy_sfp_intel: 1854 case ixgbe_phy_sfp_unknown: 1855 case ixgbe_phy_sfp_passive_tyco: 1856 case ixgbe_phy_sfp_passive_unknown: 1857 return TRUE; 1858 default: 1859 return FALSE; 1860 } 1861 } 1862 1863 static void 1864 ix_config_link(struct ix_softc *sc) 1865 { 1866 struct ixgbe_hw *hw = &sc->hw; 1867 boolean_t sfp; 1868 1869 sfp = ix_is_sfp(hw); 1870 if (sfp) { 1871 if (hw->phy.multispeed_fiber) { 1872 hw->mac.ops.setup_sfp(hw); 1873 ixgbe_enable_tx_laser(hw); 1874 ix_handle_msf(sc); 1875 } else { 1876 ix_handle_mod(sc); 1877 } 1878 } else { 1879 uint32_t autoneg, err = 0; 1880 1881 if (hw->mac.ops.check_link != NULL) { 1882 err = ixgbe_check_link(hw, &sc->link_speed, 1883 &sc->link_up, FALSE); 1884 if (err) 1885 return; 1886 } 1887 1888 autoneg = hw->phy.autoneg_advertised; 1889 if (!autoneg && hw->mac.ops.get_link_capabilities != NULL) { 1890 bool negotiate; 1891 1892 err = hw->mac.ops.get_link_capabilities(hw, 1893 &autoneg, &negotiate); 1894 if (err) 1895 return; 1896 } 1897 1898 if (hw->mac.ops.setup_link != NULL) { 1899 err = hw->mac.ops.setup_link(hw, 1900 autoneg, sc->link_up); 1901 if (err) 1902 return; 1903 } 1904 } 1905 } 1906 1907 static int 1908 ix_alloc_rings(struct ix_softc *sc) 1909 { 1910 int error, i; 1911 1912 /* 1913 * Create top level busdma tag 1914 */ 1915 error = bus_dma_tag_create(NULL, 1, 0, 1916 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 1917 BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 0, 1918 &sc->parent_tag); 1919 if (error) { 1920 device_printf(sc->dev, "could not create top level DMA tag\n"); 1921 return error; 1922 } 1923 1924 /* 1925 * Allocate TX descriptor rings and buffers 1926 */ 1927 sc->tx_rings = kmalloc_cachealign( 1928 sizeof(struct ix_tx_ring) * sc->tx_ring_cnt, 1929 M_DEVBUF, M_WAITOK | M_ZERO); 1930 for (i = 0; i < sc->tx_ring_cnt; ++i) { 1931 struct ix_tx_ring *txr = &sc->tx_rings[i]; 1932 1933 txr->tx_sc = sc; 1934 txr->tx_idx = i; 1935 lwkt_serialize_init(&txr->tx_serialize); 1936 1937 error = ix_create_tx_ring(txr); 1938 if (error) 1939 return error; 1940 } 1941 1942 /* 1943 * Allocate RX descriptor rings and buffers 1944 */ 1945 sc->rx_rings = kmalloc_cachealign( 1946 sizeof(struct ix_rx_ring) * sc->rx_ring_cnt, 1947 M_DEVBUF, M_WAITOK | M_ZERO); 1948 for (i = 0; i < sc->rx_ring_cnt; ++i) { 1949 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 1950 1951 rxr->rx_sc = sc; 1952 rxr->rx_idx = i; 1953 lwkt_serialize_init(&rxr->rx_serialize); 1954 1955 error = ix_create_rx_ring(rxr); 1956 if (error) 1957 return error; 1958 } 1959 1960 return 0; 1961 } 1962 1963 static int 1964 ix_create_tx_ring(struct ix_tx_ring *txr) 1965 { 1966 int error, i, tsize, ntxd; 1967 1968 /* 1969 * Validate number of transmit descriptors. It must not exceed 1970 * hardware maximum, and must be multiple of IX_DBA_ALIGN. 1971 */ 1972 ntxd = device_getenv_int(txr->tx_sc->dev, "txd", ix_txd); 1973 if (((ntxd * sizeof(union ixgbe_adv_tx_desc)) % IX_DBA_ALIGN) != 0 || 1974 ntxd < IX_MIN_TXD || ntxd > IX_MAX_TXD) { 1975 device_printf(txr->tx_sc->dev, 1976 "Using %d TX descriptors instead of %d!\n", 1977 IX_DEF_TXD, ntxd); 1978 txr->tx_ndesc = IX_DEF_TXD; 1979 } else { 1980 txr->tx_ndesc = ntxd; 1981 } 1982 1983 /* 1984 * Allocate TX head write-back buffer 1985 */ 1986 txr->tx_hdr = bus_dmamem_coherent_any(txr->tx_sc->parent_tag, 1987 __VM_CACHELINE_SIZE, __VM_CACHELINE_SIZE, BUS_DMA_WAITOK, 1988 &txr->tx_hdr_dtag, &txr->tx_hdr_map, &txr->tx_hdr_paddr); 1989 if (txr->tx_hdr == NULL) { 1990 device_printf(txr->tx_sc->dev, 1991 "Unable to allocate TX head write-back buffer\n"); 1992 return ENOMEM; 1993 } 1994 1995 /* 1996 * Allocate TX descriptor ring 1997 */ 1998 tsize = roundup2(txr->tx_ndesc * sizeof(union ixgbe_adv_tx_desc), 1999 IX_DBA_ALIGN); 2000 txr->tx_base = bus_dmamem_coherent_any(txr->tx_sc->parent_tag, 2001 IX_DBA_ALIGN, tsize, BUS_DMA_WAITOK | BUS_DMA_ZERO, 2002 &txr->tx_base_dtag, &txr->tx_base_map, &txr->tx_base_paddr); 2003 if (txr->tx_base == NULL) { 2004 device_printf(txr->tx_sc->dev, 2005 "Unable to allocate TX Descriptor memory\n"); 2006 return ENOMEM; 2007 } 2008 2009 tsize = __VM_CACHELINE_ALIGN(sizeof(struct ix_tx_buf) * txr->tx_ndesc); 2010 txr->tx_buf = kmalloc_cachealign(tsize, M_DEVBUF, M_WAITOK | M_ZERO); 2011 2012 /* 2013 * Create DMA tag for TX buffers 2014 */ 2015 error = bus_dma_tag_create(txr->tx_sc->parent_tag, 2016 1, 0, /* alignment, bounds */ 2017 BUS_SPACE_MAXADDR, /* lowaddr */ 2018 BUS_SPACE_MAXADDR, /* highaddr */ 2019 NULL, NULL, /* filter, filterarg */ 2020 IX_TSO_SIZE, /* maxsize */ 2021 IX_MAX_SCATTER, /* nsegments */ 2022 PAGE_SIZE, /* maxsegsize */ 2023 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | 2024 BUS_DMA_ONEBPAGE, /* flags */ 2025 &txr->tx_tag); 2026 if (error) { 2027 device_printf(txr->tx_sc->dev, 2028 "Unable to allocate TX DMA tag\n"); 2029 kfree(txr->tx_buf, M_DEVBUF); 2030 txr->tx_buf = NULL; 2031 return error; 2032 } 2033 2034 /* 2035 * Create DMA maps for TX buffers 2036 */ 2037 for (i = 0; i < txr->tx_ndesc; ++i) { 2038 struct ix_tx_buf *txbuf = &txr->tx_buf[i]; 2039 2040 error = bus_dmamap_create(txr->tx_tag, 2041 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, &txbuf->map); 2042 if (error) { 2043 device_printf(txr->tx_sc->dev, 2044 "Unable to create TX DMA map\n"); 2045 ix_destroy_tx_ring(txr, i); 2046 return error; 2047 } 2048 } 2049 2050 /* 2051 * Initialize various watermark 2052 */ 2053 txr->tx_wreg_nsegs = IX_DEF_TXWREG_NSEGS; 2054 txr->tx_intr_nsegs = txr->tx_ndesc / 16; 2055 2056 return 0; 2057 } 2058 2059 static void 2060 ix_destroy_tx_ring(struct ix_tx_ring *txr, int ndesc) 2061 { 2062 int i; 2063 2064 if (txr->tx_hdr != NULL) { 2065 bus_dmamap_unload(txr->tx_hdr_dtag, txr->tx_hdr_map); 2066 bus_dmamem_free(txr->tx_hdr_dtag, 2067 __DEVOLATILE(void *, txr->tx_hdr), txr->tx_hdr_map); 2068 bus_dma_tag_destroy(txr->tx_hdr_dtag); 2069 txr->tx_hdr = NULL; 2070 } 2071 2072 if (txr->tx_base != NULL) { 2073 bus_dmamap_unload(txr->tx_base_dtag, txr->tx_base_map); 2074 bus_dmamem_free(txr->tx_base_dtag, txr->tx_base, 2075 txr->tx_base_map); 2076 bus_dma_tag_destroy(txr->tx_base_dtag); 2077 txr->tx_base = NULL; 2078 } 2079 2080 if (txr->tx_buf == NULL) 2081 return; 2082 2083 for (i = 0; i < ndesc; ++i) { 2084 struct ix_tx_buf *txbuf = &txr->tx_buf[i]; 2085 2086 KKASSERT(txbuf->m_head == NULL); 2087 bus_dmamap_destroy(txr->tx_tag, txbuf->map); 2088 } 2089 bus_dma_tag_destroy(txr->tx_tag); 2090 2091 kfree(txr->tx_buf, M_DEVBUF); 2092 txr->tx_buf = NULL; 2093 } 2094 2095 static void 2096 ix_init_tx_ring(struct ix_tx_ring *txr) 2097 { 2098 /* Clear the old ring contents */ 2099 bzero(txr->tx_base, sizeof(union ixgbe_adv_tx_desc) * txr->tx_ndesc); 2100 2101 /* Clear TX head write-back buffer */ 2102 *(txr->tx_hdr) = 0; 2103 2104 /* Reset indices */ 2105 txr->tx_next_avail = 0; 2106 txr->tx_next_clean = 0; 2107 txr->tx_nsegs = 0; 2108 2109 /* Set number of descriptors available */ 2110 txr->tx_avail = txr->tx_ndesc; 2111 } 2112 2113 static void 2114 ix_init_tx_unit(struct ix_softc *sc) 2115 { 2116 struct ixgbe_hw *hw = &sc->hw; 2117 int i; 2118 2119 /* 2120 * Setup the Base and Length of the Tx Descriptor Ring 2121 */ 2122 for (i = 0; i < sc->tx_ring_inuse; ++i) { 2123 struct ix_tx_ring *txr = &sc->tx_rings[i]; 2124 uint64_t tdba = txr->tx_base_paddr; 2125 uint64_t hdr_paddr = txr->tx_hdr_paddr; 2126 uint32_t txctrl; 2127 2128 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), (uint32_t)tdba); 2129 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (uint32_t)(tdba >> 32)); 2130 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 2131 txr->tx_ndesc * sizeof(union ixgbe_adv_tx_desc)); 2132 2133 /* Setup the HW Tx Head and Tail descriptor pointers */ 2134 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 2135 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 2136 2137 /* Disable TX head write-back relax ordering */ 2138 switch (hw->mac.type) { 2139 case ixgbe_mac_82598EB: 2140 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 2141 break; 2142 case ixgbe_mac_82599EB: 2143 case ixgbe_mac_X540: 2144 default: 2145 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 2146 break; 2147 } 2148 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2149 switch (hw->mac.type) { 2150 case ixgbe_mac_82598EB: 2151 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 2152 break; 2153 case ixgbe_mac_82599EB: 2154 case ixgbe_mac_X540: 2155 default: 2156 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 2157 break; 2158 } 2159 2160 /* Enable TX head write-back */ 2161 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(i), 2162 (uint32_t)(hdr_paddr >> 32)); 2163 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(i), 2164 ((uint32_t)hdr_paddr) | IXGBE_TDWBAL_HEAD_WB_ENABLE); 2165 } 2166 2167 if (hw->mac.type != ixgbe_mac_82598EB) { 2168 uint32_t dmatxctl, rttdcs; 2169 2170 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2171 dmatxctl |= IXGBE_DMATXCTL_TE; 2172 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 2173 2174 /* Disable arbiter to set MTQC */ 2175 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2176 rttdcs |= IXGBE_RTTDCS_ARBDIS; 2177 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2178 2179 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2180 2181 /* Reenable aribter */ 2182 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 2183 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2184 } 2185 } 2186 2187 static int 2188 ix_tx_ctx_setup(struct ix_tx_ring *txr, const struct mbuf *mp, 2189 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2190 { 2191 struct ixgbe_adv_tx_context_desc *TXD; 2192 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2193 int ehdrlen, ip_hlen = 0, ctxd; 2194 boolean_t offload = TRUE; 2195 2196 /* First check if TSO is to be used */ 2197 if (mp->m_pkthdr.csum_flags & CSUM_TSO) { 2198 return ix_tso_ctx_setup(txr, mp, 2199 cmd_type_len, olinfo_status); 2200 } 2201 2202 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 2203 offload = FALSE; 2204 2205 /* Indicate the whole packet as payload when not doing TSO */ 2206 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; 2207 2208 /* 2209 * In advanced descriptors the vlan tag must be placed into the 2210 * context descriptor. Hence we need to make one even if not 2211 * doing checksum offloads. 2212 */ 2213 if (mp->m_flags & M_VLANTAG) { 2214 vlan_macip_lens |= htole16(mp->m_pkthdr.ether_vlantag) << 2215 IXGBE_ADVTXD_VLAN_SHIFT; 2216 } else if (!offload) { 2217 /* No TX descriptor is consumed */ 2218 return 0; 2219 } 2220 2221 /* Set the ether header length */ 2222 ehdrlen = mp->m_pkthdr.csum_lhlen; 2223 KASSERT(ehdrlen > 0, ("invalid ether hlen")); 2224 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2225 2226 if (mp->m_pkthdr.csum_flags & CSUM_IP) { 2227 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 2228 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2229 ip_hlen = mp->m_pkthdr.csum_iphlen; 2230 KASSERT(ip_hlen > 0, ("invalid ip hlen")); 2231 } 2232 vlan_macip_lens |= ip_hlen; 2233 2234 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2235 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 2236 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2237 else if (mp->m_pkthdr.csum_flags & CSUM_UDP) 2238 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2239 2240 if (mp->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)) 2241 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2242 2243 /* Now ready a context descriptor */ 2244 ctxd = txr->tx_next_avail; 2245 TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; 2246 2247 /* Now copy bits into descriptor */ 2248 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2249 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2250 TXD->seqnum_seed = htole32(0); 2251 TXD->mss_l4len_idx = htole32(0); 2252 2253 /* We've consumed the first desc, adjust counters */ 2254 if (++ctxd == txr->tx_ndesc) 2255 ctxd = 0; 2256 txr->tx_next_avail = ctxd; 2257 --txr->tx_avail; 2258 2259 /* One TX descriptor is consumed */ 2260 return 1; 2261 } 2262 2263 static int 2264 ix_tso_ctx_setup(struct ix_tx_ring *txr, const struct mbuf *mp, 2265 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2266 { 2267 struct ixgbe_adv_tx_context_desc *TXD; 2268 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2269 uint32_t mss_l4len_idx = 0, paylen; 2270 int ctxd, ehdrlen, ip_hlen, tcp_hlen; 2271 2272 ehdrlen = mp->m_pkthdr.csum_lhlen; 2273 KASSERT(ehdrlen > 0, ("invalid ether hlen")); 2274 2275 ip_hlen = mp->m_pkthdr.csum_iphlen; 2276 KASSERT(ip_hlen > 0, ("invalid ip hlen")); 2277 2278 tcp_hlen = mp->m_pkthdr.csum_thlen; 2279 KASSERT(tcp_hlen > 0, ("invalid tcp hlen")); 2280 2281 ctxd = txr->tx_next_avail; 2282 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2283 2284 if (mp->m_flags & M_VLANTAG) { 2285 vlan_macip_lens |= htole16(mp->m_pkthdr.ether_vlantag) << 2286 IXGBE_ADVTXD_VLAN_SHIFT; 2287 } 2288 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2289 vlan_macip_lens |= ip_hlen; 2290 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2291 2292 /* ADV DTYPE TUCMD */ 2293 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2294 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2295 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2296 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2297 2298 /* MSS L4LEN IDX */ 2299 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2300 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2301 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2302 2303 TXD->seqnum_seed = htole32(0); 2304 2305 if (++ctxd == txr->tx_ndesc) 2306 ctxd = 0; 2307 2308 txr->tx_avail--; 2309 txr->tx_next_avail = ctxd; 2310 2311 *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 2312 2313 /* This is used in the transmit desc in encap */ 2314 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; 2315 2316 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 2317 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2318 *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 2319 2320 /* One TX descriptor is consumed */ 2321 return 1; 2322 } 2323 2324 static void 2325 ix_txeof(struct ix_tx_ring *txr) 2326 { 2327 struct ifnet *ifp = &txr->tx_sc->arpcom.ac_if; 2328 int first, hdr, avail; 2329 2330 if (txr->tx_avail == txr->tx_ndesc) 2331 return; 2332 2333 first = txr->tx_next_clean; 2334 hdr = *(txr->tx_hdr); 2335 2336 if (first == hdr) 2337 return; 2338 2339 avail = txr->tx_avail; 2340 while (first != hdr) { 2341 struct ix_tx_buf *txbuf = &txr->tx_buf[first]; 2342 2343 ++avail; 2344 if (txbuf->m_head) { 2345 bus_dmamap_unload(txr->tx_tag, txbuf->map); 2346 m_freem(txbuf->m_head); 2347 txbuf->m_head = NULL; 2348 IFNET_STAT_INC(ifp, opackets, 1); 2349 } 2350 if (++first == txr->tx_ndesc) 2351 first = 0; 2352 } 2353 txr->tx_next_clean = first; 2354 txr->tx_avail = avail; 2355 2356 if (txr->tx_avail > IX_MAX_SCATTER + IX_TX_RESERVED) { 2357 ifsq_clr_oactive(txr->tx_ifsq); 2358 txr->tx_watchdog.wd_timer = 0; 2359 } 2360 } 2361 2362 static int 2363 ix_create_rx_ring(struct ix_rx_ring *rxr) 2364 { 2365 int i, rsize, error, nrxd; 2366 2367 /* 2368 * Validate number of receive descriptors. It must not exceed 2369 * hardware maximum, and must be multiple of IX_DBA_ALIGN. 2370 */ 2371 nrxd = device_getenv_int(rxr->rx_sc->dev, "rxd", ix_rxd); 2372 if (((nrxd * sizeof(union ixgbe_adv_rx_desc)) % IX_DBA_ALIGN) != 0 || 2373 nrxd < IX_MIN_RXD || nrxd > IX_MAX_RXD) { 2374 device_printf(rxr->rx_sc->dev, 2375 "Using %d RX descriptors instead of %d!\n", 2376 IX_DEF_RXD, nrxd); 2377 rxr->rx_ndesc = IX_DEF_RXD; 2378 } else { 2379 rxr->rx_ndesc = nrxd; 2380 } 2381 2382 /* 2383 * Allocate RX descriptor ring 2384 */ 2385 rsize = roundup2(rxr->rx_ndesc * sizeof(union ixgbe_adv_rx_desc), 2386 IX_DBA_ALIGN); 2387 rxr->rx_base = bus_dmamem_coherent_any(rxr->rx_sc->parent_tag, 2388 IX_DBA_ALIGN, rsize, BUS_DMA_WAITOK | BUS_DMA_ZERO, 2389 &rxr->rx_base_dtag, &rxr->rx_base_map, &rxr->rx_base_paddr); 2390 if (rxr->rx_base == NULL) { 2391 device_printf(rxr->rx_sc->dev, 2392 "Unable to allocate TX Descriptor memory\n"); 2393 return ENOMEM; 2394 } 2395 2396 rsize = __VM_CACHELINE_ALIGN(sizeof(struct ix_rx_buf) * rxr->rx_ndesc); 2397 rxr->rx_buf = kmalloc_cachealign(rsize, M_DEVBUF, M_WAITOK | M_ZERO); 2398 2399 /* 2400 * Create DMA tag for RX buffers 2401 */ 2402 error = bus_dma_tag_create(rxr->rx_sc->parent_tag, 2403 1, 0, /* alignment, bounds */ 2404 BUS_SPACE_MAXADDR, /* lowaddr */ 2405 BUS_SPACE_MAXADDR, /* highaddr */ 2406 NULL, NULL, /* filter, filterarg */ 2407 PAGE_SIZE, /* maxsize */ 2408 1, /* nsegments */ 2409 PAGE_SIZE, /* maxsegsize */ 2410 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, /* flags */ 2411 &rxr->rx_tag); 2412 if (error) { 2413 device_printf(rxr->rx_sc->dev, 2414 "Unable to create RX DMA tag\n"); 2415 kfree(rxr->rx_buf, M_DEVBUF); 2416 rxr->rx_buf = NULL; 2417 return error; 2418 } 2419 2420 /* 2421 * Create spare DMA map for RX buffers 2422 */ 2423 error = bus_dmamap_create(rxr->rx_tag, BUS_DMA_WAITOK, 2424 &rxr->rx_sparemap); 2425 if (error) { 2426 device_printf(rxr->rx_sc->dev, 2427 "Unable to create spare RX DMA map\n"); 2428 bus_dma_tag_destroy(rxr->rx_tag); 2429 kfree(rxr->rx_buf, M_DEVBUF); 2430 rxr->rx_buf = NULL; 2431 return error; 2432 } 2433 2434 /* 2435 * Create DMA maps for RX buffers 2436 */ 2437 for (i = 0; i < rxr->rx_ndesc; ++i) { 2438 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i]; 2439 2440 error = bus_dmamap_create(rxr->rx_tag, 2441 BUS_DMA_WAITOK, &rxbuf->map); 2442 if (error) { 2443 device_printf(rxr->rx_sc->dev, 2444 "Unable to create RX dma map\n"); 2445 ix_destroy_rx_ring(rxr, i); 2446 return error; 2447 } 2448 } 2449 2450 /* 2451 * Initialize various watermark 2452 */ 2453 rxr->rx_wreg_nsegs = IX_DEF_RXWREG_NSEGS; 2454 2455 return 0; 2456 } 2457 2458 static void 2459 ix_destroy_rx_ring(struct ix_rx_ring *rxr, int ndesc) 2460 { 2461 int i; 2462 2463 if (rxr->rx_base != NULL) { 2464 bus_dmamap_unload(rxr->rx_base_dtag, rxr->rx_base_map); 2465 bus_dmamem_free(rxr->rx_base_dtag, rxr->rx_base, 2466 rxr->rx_base_map); 2467 bus_dma_tag_destroy(rxr->rx_base_dtag); 2468 rxr->rx_base = NULL; 2469 } 2470 2471 if (rxr->rx_buf == NULL) 2472 return; 2473 2474 for (i = 0; i < ndesc; ++i) { 2475 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i]; 2476 2477 KKASSERT(rxbuf->m_head == NULL); 2478 bus_dmamap_destroy(rxr->rx_tag, rxbuf->map); 2479 } 2480 bus_dmamap_destroy(rxr->rx_tag, rxr->rx_sparemap); 2481 bus_dma_tag_destroy(rxr->rx_tag); 2482 2483 kfree(rxr->rx_buf, M_DEVBUF); 2484 rxr->rx_buf = NULL; 2485 } 2486 2487 /* 2488 ** Used to detect a descriptor that has 2489 ** been merged by Hardware RSC. 2490 */ 2491 static __inline uint32_t 2492 ix_rsc_count(union ixgbe_adv_rx_desc *rx) 2493 { 2494 return (le32toh(rx->wb.lower.lo_dword.data) & 2495 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; 2496 } 2497 2498 #if 0 2499 /********************************************************************* 2500 * 2501 * Initialize Hardware RSC (LRO) feature on 82599 2502 * for an RX ring, this is toggled by the LRO capability 2503 * even though it is transparent to the stack. 2504 * 2505 * NOTE: since this HW feature only works with IPV4 and 2506 * our testing has shown soft LRO to be as effective 2507 * I have decided to disable this by default. 2508 * 2509 **********************************************************************/ 2510 static void 2511 ix_setup_hw_rsc(struct ix_rx_ring *rxr) 2512 { 2513 struct ix_softc *sc = rxr->rx_sc; 2514 struct ixgbe_hw *hw = &sc->hw; 2515 uint32_t rscctrl, rdrxctl; 2516 2517 #if 0 2518 /* If turning LRO/RSC off we need to disable it */ 2519 if ((sc->arpcom.ac_if.if_capenable & IFCAP_LRO) == 0) { 2520 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 2521 rscctrl &= ~IXGBE_RSCCTL_RSCEN; 2522 return; 2523 } 2524 #endif 2525 2526 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2527 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 2528 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 2529 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 2530 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 2531 2532 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 2533 rscctrl |= IXGBE_RSCCTL_RSCEN; 2534 /* 2535 ** Limit the total number of descriptors that 2536 ** can be combined, so it does not exceed 64K 2537 */ 2538 if (rxr->mbuf_sz == MCLBYTES) 2539 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 2540 else if (rxr->mbuf_sz == MJUMPAGESIZE) 2541 rscctrl |= IXGBE_RSCCTL_MAXDESC_8; 2542 else if (rxr->mbuf_sz == MJUM9BYTES) 2543 rscctrl |= IXGBE_RSCCTL_MAXDESC_4; 2544 else /* Using 16K cluster */ 2545 rscctrl |= IXGBE_RSCCTL_MAXDESC_1; 2546 2547 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); 2548 2549 /* Enable TCP header recognition */ 2550 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 2551 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | 2552 IXGBE_PSRTYPE_TCPHDR)); 2553 2554 /* Disable RSC for ACK packets */ 2555 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 2556 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 2557 2558 rxr->hw_rsc = TRUE; 2559 } 2560 #endif 2561 2562 static int 2563 ix_init_rx_ring(struct ix_rx_ring *rxr) 2564 { 2565 int i; 2566 2567 /* Clear the ring contents */ 2568 bzero(rxr->rx_base, rxr->rx_ndesc * sizeof(union ixgbe_adv_rx_desc)); 2569 2570 /* XXX we need JUMPAGESIZE for RSC too */ 2571 if (rxr->rx_sc->max_frame_size <= MCLBYTES) 2572 rxr->rx_mbuf_sz = MCLBYTES; 2573 else 2574 rxr->rx_mbuf_sz = MJUMPAGESIZE; 2575 2576 /* Now replenish the mbufs */ 2577 for (i = 0; i < rxr->rx_ndesc; ++i) { 2578 int error; 2579 2580 error = ix_newbuf(rxr, i, TRUE); 2581 if (error) 2582 return error; 2583 } 2584 2585 /* Setup our descriptor indices */ 2586 rxr->rx_next_check = 0; 2587 rxr->rx_flags &= ~IX_RXRING_FLAG_DISC; 2588 2589 #if 0 2590 /* 2591 ** Now set up the LRO interface: 2592 */ 2593 if (ixgbe_rsc_enable) 2594 ix_setup_hw_rsc(rxr); 2595 #endif 2596 2597 return 0; 2598 } 2599 2600 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2601 2602 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 2603 2604 static void 2605 ix_init_rx_unit(struct ix_softc *sc) 2606 { 2607 struct ixgbe_hw *hw = &sc->hw; 2608 struct ifnet *ifp = &sc->arpcom.ac_if; 2609 uint32_t bufsz, rxctrl, fctrl, rxcsum, hlreg; 2610 int i; 2611 2612 /* 2613 * Make sure receives are disabled while setting up the descriptor ring 2614 */ 2615 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 2616 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN); 2617 2618 /* Enable broadcasts */ 2619 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2620 fctrl |= IXGBE_FCTRL_BAM; 2621 fctrl |= IXGBE_FCTRL_DPF; 2622 fctrl |= IXGBE_FCTRL_PMCF; 2623 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 2624 2625 /* Set for Jumbo Frames? */ 2626 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2627 if (ifp->if_mtu > ETHERMTU) 2628 hlreg |= IXGBE_HLREG0_JUMBOEN; 2629 else 2630 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 2631 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 2632 2633 KKASSERT(sc->rx_rings[0].rx_mbuf_sz >= MCLBYTES); 2634 bufsz = (sc->rx_rings[0].rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 2635 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2636 2637 for (i = 0; i < sc->rx_ring_inuse; ++i) { 2638 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 2639 uint64_t rdba = rxr->rx_base_paddr; 2640 uint32_t srrctl; 2641 2642 /* Setup the Base and Length of the Rx Descriptor Ring */ 2643 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), (uint32_t)rdba); 2644 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (uint32_t)(rdba >> 32)); 2645 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 2646 rxr->rx_ndesc * sizeof(union ixgbe_adv_rx_desc)); 2647 2648 /* 2649 * Set up the SRRCTL register 2650 */ 2651 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 2652 2653 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 2654 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 2655 srrctl |= bufsz; 2656 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2657 if (sc->rx_ring_inuse > 1) { 2658 /* See the commend near ix_enable_rx_drop() */ 2659 switch (sc->fc) { 2660 case ixgbe_fc_rx_pause: 2661 case ixgbe_fc_tx_pause: 2662 case ixgbe_fc_full: 2663 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 2664 if (i == 0 && bootverbose) { 2665 if_printf(ifp, "flow control %d, " 2666 "disable RX drop\n", sc->fc); 2667 } 2668 break; 2669 2670 case ixgbe_fc_none: 2671 srrctl |= IXGBE_SRRCTL_DROP_EN; 2672 if (i == 0 && bootverbose) { 2673 if_printf(ifp, "flow control %d, " 2674 "enable RX drop\n", sc->fc); 2675 } 2676 break; 2677 2678 default: 2679 break; 2680 } 2681 } 2682 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 2683 2684 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2685 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 2686 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 2687 } 2688 2689 if (sc->hw.mac.type != ixgbe_mac_82598EB) 2690 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 0); 2691 2692 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2693 2694 /* 2695 * Setup RSS 2696 */ 2697 if (IX_ENABLE_HWRSS(sc)) { 2698 uint8_t key[IX_NRSSRK * IX_RSSRK_SIZE]; 2699 int j, r; 2700 2701 /* 2702 * NOTE: 2703 * When we reach here, RSS has already been disabled 2704 * in ix_stop(), so we could safely configure RSS key 2705 * and redirect table. 2706 */ 2707 2708 /* 2709 * Configure RSS key 2710 */ 2711 toeplitz_get_key(key, sizeof(key)); 2712 for (i = 0; i < IX_NRSSRK; ++i) { 2713 uint32_t rssrk; 2714 2715 rssrk = IX_RSSRK_VAL(key, i); 2716 IX_RSS_DPRINTF(sc, 1, "rssrk%d 0x%08x\n", 2717 i, rssrk); 2718 2719 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rssrk); 2720 } 2721 2722 /* 2723 * Configure RSS redirect table in following fashion: 2724 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] 2725 */ 2726 r = 0; 2727 for (j = 0; j < IX_NRETA; ++j) { 2728 uint32_t reta = 0; 2729 2730 for (i = 0; i < IX_RETA_SIZE; ++i) { 2731 uint32_t q; 2732 2733 q = r % sc->rx_ring_inuse; 2734 reta |= q << (8 * i); 2735 ++r; 2736 } 2737 IX_RSS_DPRINTF(sc, 1, "reta 0x%08x\n", reta); 2738 IXGBE_WRITE_REG(hw, IXGBE_RETA(j), reta); 2739 } 2740 2741 /* 2742 * Enable multiple receive queues. 2743 * Enable IPv4 RSS standard hash functions. 2744 */ 2745 IXGBE_WRITE_REG(hw, IXGBE_MRQC, 2746 IXGBE_MRQC_RSSEN | 2747 IXGBE_MRQC_RSS_FIELD_IPV4 | 2748 IXGBE_MRQC_RSS_FIELD_IPV4_TCP); 2749 2750 /* 2751 * NOTE: 2752 * PCSD must be enabled to enable multiple 2753 * receive queues. 2754 */ 2755 rxcsum |= IXGBE_RXCSUM_PCSD; 2756 } 2757 2758 if (ifp->if_capenable & IFCAP_RXCSUM) 2759 rxcsum |= IXGBE_RXCSUM_PCSD; 2760 2761 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2762 } 2763 2764 static __inline void 2765 ix_rx_refresh(struct ix_rx_ring *rxr, int i) 2766 { 2767 if (--i < 0) 2768 i = rxr->rx_ndesc - 1; 2769 IXGBE_WRITE_REG(&rxr->rx_sc->hw, IXGBE_RDT(rxr->rx_idx), i); 2770 } 2771 2772 static __inline void 2773 ix_rxcsum(uint32_t staterr, struct mbuf *mp, uint32_t ptype) 2774 { 2775 if ((ptype & 2776 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_IPV4_EX)) == 0) { 2777 /* Not IPv4 */ 2778 return; 2779 } 2780 2781 if ((staterr & (IXGBE_RXD_STAT_IPCS | IXGBE_RXDADV_ERR_IPE)) == 2782 IXGBE_RXD_STAT_IPCS) 2783 mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID; 2784 2785 if ((ptype & 2786 (IXGBE_RXDADV_PKTTYPE_TCP | IXGBE_RXDADV_PKTTYPE_UDP)) == 0) { 2787 /* 2788 * - Neither TCP nor UDP 2789 * - IPv4 fragment 2790 */ 2791 return; 2792 } 2793 2794 if ((staterr & (IXGBE_RXD_STAT_L4CS | IXGBE_RXDADV_ERR_TCPE)) == 2795 IXGBE_RXD_STAT_L4CS) { 2796 mp->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR | 2797 CSUM_FRAG_NOT_CHECKED; 2798 mp->m_pkthdr.csum_data = htons(0xffff); 2799 } 2800 } 2801 2802 static __inline struct pktinfo * 2803 ix_rssinfo(struct mbuf *m, struct pktinfo *pi, 2804 uint32_t hash, uint32_t hashtype, uint32_t ptype) 2805 { 2806 switch (hashtype) { 2807 case IXGBE_RXDADV_RSSTYPE_IPV4_TCP: 2808 pi->pi_netisr = NETISR_IP; 2809 pi->pi_flags = 0; 2810 pi->pi_l3proto = IPPROTO_TCP; 2811 break; 2812 2813 case IXGBE_RXDADV_RSSTYPE_IPV4: 2814 if ((ptype & IXGBE_RXDADV_PKTTYPE_UDP) == 0) { 2815 /* Not UDP or is fragment */ 2816 return NULL; 2817 } 2818 pi->pi_netisr = NETISR_IP; 2819 pi->pi_flags = 0; 2820 pi->pi_l3proto = IPPROTO_UDP; 2821 break; 2822 2823 default: 2824 return NULL; 2825 } 2826 2827 m->m_flags |= M_HASH; 2828 m->m_pkthdr.hash = toeplitz_hash(hash); 2829 return pi; 2830 } 2831 2832 static __inline void 2833 ix_setup_rxdesc(union ixgbe_adv_rx_desc *rxd, const struct ix_rx_buf *rxbuf) 2834 { 2835 rxd->read.pkt_addr = htole64(rxbuf->paddr); 2836 rxd->wb.upper.status_error = 0; 2837 } 2838 2839 static void 2840 ix_rx_discard(struct ix_rx_ring *rxr, int i, boolean_t eop) 2841 { 2842 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i]; 2843 2844 /* 2845 * XXX discard may not be correct 2846 */ 2847 if (eop) { 2848 IFNET_STAT_INC(&rxr->rx_sc->arpcom.ac_if, ierrors, 1); 2849 rxr->rx_flags &= ~IX_RXRING_FLAG_DISC; 2850 } else { 2851 rxr->rx_flags |= IX_RXRING_FLAG_DISC; 2852 } 2853 if (rxbuf->fmp != NULL) { 2854 m_freem(rxbuf->fmp); 2855 rxbuf->fmp = NULL; 2856 rxbuf->lmp = NULL; 2857 } 2858 ix_setup_rxdesc(&rxr->rx_base[i], rxbuf); 2859 } 2860 2861 static void 2862 ix_rxeof(struct ix_rx_ring *rxr) 2863 { 2864 struct ifnet *ifp = &rxr->rx_sc->arpcom.ac_if; 2865 int i, nsegs = 0; 2866 2867 i = rxr->rx_next_check; 2868 for (;;) { 2869 struct ix_rx_buf *rxbuf, *nbuf = NULL; 2870 union ixgbe_adv_rx_desc *cur; 2871 struct mbuf *sendmp = NULL, *mp; 2872 struct pktinfo *pi = NULL, pi0; 2873 uint32_t rsc = 0, ptype, staterr, hash, hashtype; 2874 uint16_t len; 2875 boolean_t eop; 2876 2877 cur = &rxr->rx_base[i]; 2878 staterr = le32toh(cur->wb.upper.status_error); 2879 2880 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 2881 break; 2882 ++nsegs; 2883 2884 rxbuf = &rxr->rx_buf[i]; 2885 mp = rxbuf->m_head; 2886 2887 len = le16toh(cur->wb.upper.length); 2888 ptype = le32toh(cur->wb.lower.lo_dword.data) & 2889 IXGBE_RXDADV_PKTTYPE_MASK; 2890 hash = le32toh(cur->wb.lower.hi_dword.rss); 2891 hashtype = le32toh(cur->wb.lower.lo_dword.data) & 2892 IXGBE_RXDADV_RSSTYPE_MASK; 2893 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 2894 2895 /* 2896 * Make sure bad packets are discarded 2897 */ 2898 if ((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) || 2899 (rxr->rx_flags & IX_RXRING_FLAG_DISC)) { 2900 ix_rx_discard(rxr, i, eop); 2901 goto next_desc; 2902 } 2903 2904 bus_dmamap_sync(rxr->rx_tag, rxbuf->map, BUS_DMASYNC_POSTREAD); 2905 if (ix_newbuf(rxr, i, FALSE) != 0) { 2906 ix_rx_discard(rxr, i, eop); 2907 goto next_desc; 2908 } 2909 2910 /* 2911 * On 82599 which supports a hardware LRO, packets 2912 * need not be fragmented across sequential descriptors, 2913 * rather the next descriptor is indicated in bits 2914 * of the descriptor. This also means that we might 2915 * proceses more than one packet at a time, something 2916 * that has never been true before, it required 2917 * eliminating global chain pointers in favor of what 2918 * we are doing here. 2919 */ 2920 if (!eop) { 2921 int nextp; 2922 2923 /* 2924 * Figure out the next descriptor 2925 * of this frame. 2926 */ 2927 if (rxr->rx_flags & IX_RXRING_FLAG_LRO) 2928 rsc = ix_rsc_count(cur); 2929 if (rsc) { /* Get hardware index */ 2930 nextp = ((staterr & 2931 IXGBE_RXDADV_NEXTP_MASK) >> 2932 IXGBE_RXDADV_NEXTP_SHIFT); 2933 } else { /* Just sequential */ 2934 nextp = i + 1; 2935 if (nextp == rxr->rx_ndesc) 2936 nextp = 0; 2937 } 2938 nbuf = &rxr->rx_buf[nextp]; 2939 prefetch(nbuf); 2940 } 2941 mp->m_len = len; 2942 2943 /* 2944 * Rather than using the fmp/lmp global pointers 2945 * we now keep the head of a packet chain in the 2946 * buffer struct and pass this along from one 2947 * descriptor to the next, until we get EOP. 2948 */ 2949 if (rxbuf->fmp == NULL) { 2950 mp->m_pkthdr.len = len; 2951 rxbuf->fmp = mp; 2952 rxbuf->lmp = mp; 2953 } else { 2954 rxbuf->fmp->m_pkthdr.len += len; 2955 rxbuf->lmp->m_next = mp; 2956 rxbuf->lmp = mp; 2957 } 2958 2959 if (nbuf != NULL) { 2960 /* 2961 * Not the last fragment of this frame, 2962 * pass this fragment list on 2963 */ 2964 nbuf->fmp = rxbuf->fmp; 2965 nbuf->lmp = rxbuf->lmp; 2966 } else { 2967 /* 2968 * Send this frame 2969 */ 2970 sendmp = rxbuf->fmp; 2971 2972 sendmp->m_pkthdr.rcvif = ifp; 2973 IFNET_STAT_INC(ifp, ipackets, 1); 2974 #ifdef IX_RSS_DEBUG 2975 rxr->rx_pkts++; 2976 #endif 2977 2978 /* Process vlan info */ 2979 if (staterr & IXGBE_RXD_STAT_VP) { 2980 sendmp->m_pkthdr.ether_vlantag = 2981 le16toh(cur->wb.upper.vlan); 2982 sendmp->m_flags |= M_VLANTAG; 2983 } 2984 if (ifp->if_capenable & IFCAP_RXCSUM) 2985 ix_rxcsum(staterr, sendmp, ptype); 2986 if (ifp->if_capenable & IFCAP_RSS) { 2987 pi = ix_rssinfo(sendmp, &pi0, 2988 hash, hashtype, ptype); 2989 } 2990 } 2991 rxbuf->fmp = NULL; 2992 rxbuf->lmp = NULL; 2993 next_desc: 2994 /* Advance our pointers to the next descriptor. */ 2995 if (++i == rxr->rx_ndesc) 2996 i = 0; 2997 2998 if (sendmp != NULL) 2999 ether_input_pkt(ifp, sendmp, pi); 3000 3001 if (nsegs >= rxr->rx_wreg_nsegs) { 3002 ix_rx_refresh(rxr, i); 3003 nsegs = 0; 3004 } 3005 } 3006 rxr->rx_next_check = i; 3007 3008 if (nsegs > 0) 3009 ix_rx_refresh(rxr, i); 3010 } 3011 3012 static void 3013 ix_set_vlan(struct ix_softc *sc) 3014 { 3015 struct ixgbe_hw *hw = &sc->hw; 3016 uint32_t ctrl; 3017 3018 if (hw->mac.type == ixgbe_mac_82598EB) { 3019 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3020 ctrl |= IXGBE_VLNCTRL_VME; 3021 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 3022 } else { 3023 int i; 3024 3025 /* 3026 * On 82599 and later chips the VLAN enable is 3027 * per queue in RXDCTL 3028 */ 3029 for (i = 0; i < sc->rx_ring_inuse; ++i) { 3030 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 3031 ctrl |= IXGBE_RXDCTL_VME; 3032 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); 3033 } 3034 } 3035 } 3036 3037 static void 3038 ix_enable_intr(struct ix_softc *sc) 3039 { 3040 struct ixgbe_hw *hw = &sc->hw; 3041 uint32_t mask, fwsm; 3042 int i; 3043 3044 for (i = 0; i < sc->intr_cnt; ++i) 3045 lwkt_serialize_handler_enable(sc->intr_data[i].intr_serialize); 3046 3047 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3048 3049 /* Enable Fan Failure detection */ 3050 if (hw->device_id == IXGBE_DEV_ID_82598AT) 3051 mask |= IXGBE_EIMS_GPI_SDP1; 3052 3053 switch (sc->hw.mac.type) { 3054 case ixgbe_mac_82599EB: 3055 mask |= IXGBE_EIMS_ECC; 3056 mask |= IXGBE_EIMS_GPI_SDP0; 3057 mask |= IXGBE_EIMS_GPI_SDP1; 3058 mask |= IXGBE_EIMS_GPI_SDP2; 3059 break; 3060 case ixgbe_mac_X540: 3061 mask |= IXGBE_EIMS_ECC; 3062 /* Detect if Thermal Sensor is enabled */ 3063 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 3064 if (fwsm & IXGBE_FWSM_TS_ENABLED) 3065 mask |= IXGBE_EIMS_TS; 3066 /* FALL THROUGH */ 3067 default: 3068 break; 3069 } 3070 sc->intr_mask = mask; 3071 3072 /* With MSI-X we use auto clear */ 3073 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 3074 mask = IXGBE_EIMS_ENABLE_MASK; 3075 /* Don't autoclear Link */ 3076 mask &= ~IXGBE_EIMS_OTHER; 3077 mask &= ~IXGBE_EIMS_LSC; 3078 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 3079 } else { 3080 sc->intr_mask |= IX_TX_INTR_MASK | 3081 IX_RX0_INTR_MASK; 3082 3083 KKASSERT(sc->rx_ring_inuse <= IX_MIN_RXRING_RSS); 3084 if (sc->rx_ring_inuse == IX_MIN_RXRING_RSS) 3085 sc->intr_mask |= IX_RX1_INTR_MASK; 3086 } 3087 3088 #if 0 3089 /* 3090 ** Now enable all queues, this is done separately to 3091 ** allow for handling the extended (beyond 32) MSIX 3092 ** vectors that can be used by 82599 3093 */ 3094 for (int i = 0; i < sc->num_queues; i++, que++) 3095 ixgbe_enable_queue(sc, que->msix); 3096 #else 3097 IXGBE_WRITE_REG(hw, IXGBE_EIMS, sc->intr_mask); 3098 #endif 3099 3100 IXGBE_WRITE_FLUSH(hw); 3101 } 3102 3103 static void 3104 ix_disable_intr(struct ix_softc *sc) 3105 { 3106 int i; 3107 3108 #if 0 3109 if (sc->msix_mem) 3110 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 3111 #endif 3112 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3113 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 3114 } else { 3115 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000); 3116 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0); 3117 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0); 3118 } 3119 IXGBE_WRITE_FLUSH(&sc->hw); 3120 3121 for (i = 0; i < sc->intr_cnt; ++i) 3122 lwkt_serialize_handler_disable(sc->intr_data[i].intr_serialize); 3123 } 3124 3125 uint16_t 3126 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) 3127 { 3128 return pci_read_config(((struct ixgbe_osdep *)hw->back)->dev, 3129 reg, 2); 3130 } 3131 3132 void 3133 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value) 3134 { 3135 pci_write_config(((struct ixgbe_osdep *)hw->back)->dev, 3136 reg, value, 2); 3137 } 3138 3139 static void 3140 ix_slot_info(struct ix_softc *sc) 3141 { 3142 struct ixgbe_hw *hw = &sc->hw; 3143 device_t dev = sc->dev; 3144 struct ixgbe_mac_info *mac = &hw->mac; 3145 uint16_t link; 3146 uint32_t offset; 3147 3148 /* For most devices simply call the shared code routine */ 3149 if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) { 3150 ixgbe_get_bus_info(hw); 3151 goto display; 3152 } 3153 3154 /* 3155 * For the Quad port adapter we need to parse back 3156 * up the PCI tree to find the speed of the expansion 3157 * slot into which this adapter is plugged. A bit more work. 3158 */ 3159 dev = device_get_parent(device_get_parent(dev)); 3160 #ifdef IXGBE_DEBUG 3161 device_printf(dev, "parent pcib = %x,%x,%x\n", 3162 pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev)); 3163 #endif 3164 dev = device_get_parent(device_get_parent(dev)); 3165 #ifdef IXGBE_DEBUG 3166 device_printf(dev, "slot pcib = %x,%x,%x\n", 3167 pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev)); 3168 #endif 3169 /* Now get the PCI Express Capabilities offset */ 3170 offset = pci_get_pciecap_ptr(dev); 3171 /* ...and read the Link Status Register */ 3172 link = pci_read_config(dev, offset + PCIER_LINKSTAT, 2); 3173 switch (link & IXGBE_PCI_LINK_WIDTH) { 3174 case IXGBE_PCI_LINK_WIDTH_1: 3175 hw->bus.width = ixgbe_bus_width_pcie_x1; 3176 break; 3177 case IXGBE_PCI_LINK_WIDTH_2: 3178 hw->bus.width = ixgbe_bus_width_pcie_x2; 3179 break; 3180 case IXGBE_PCI_LINK_WIDTH_4: 3181 hw->bus.width = ixgbe_bus_width_pcie_x4; 3182 break; 3183 case IXGBE_PCI_LINK_WIDTH_8: 3184 hw->bus.width = ixgbe_bus_width_pcie_x8; 3185 break; 3186 default: 3187 hw->bus.width = ixgbe_bus_width_unknown; 3188 break; 3189 } 3190 3191 switch (link & IXGBE_PCI_LINK_SPEED) { 3192 case IXGBE_PCI_LINK_SPEED_2500: 3193 hw->bus.speed = ixgbe_bus_speed_2500; 3194 break; 3195 case IXGBE_PCI_LINK_SPEED_5000: 3196 hw->bus.speed = ixgbe_bus_speed_5000; 3197 break; 3198 case IXGBE_PCI_LINK_SPEED_8000: 3199 hw->bus.speed = ixgbe_bus_speed_8000; 3200 break; 3201 default: 3202 hw->bus.speed = ixgbe_bus_speed_unknown; 3203 break; 3204 } 3205 3206 mac->ops.set_lan_id(hw); 3207 3208 display: 3209 device_printf(dev, "PCI Express Bus: Speed %s %s\n", 3210 hw->bus.speed == ixgbe_bus_speed_8000 ? "8.0GT/s" : 3211 hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0GT/s" : 3212 hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5GT/s" : "Unknown", 3213 hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" : 3214 hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" : 3215 hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" : "Unknown"); 3216 3217 if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP && 3218 hw->bus.width <= ixgbe_bus_width_pcie_x4 && 3219 hw->bus.speed == ixgbe_bus_speed_2500) { 3220 device_printf(dev, "For optimal performance a x8 " 3221 "PCIE, or x4 PCIE Gen2 slot is required.\n"); 3222 } else if (hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP && 3223 hw->bus.width <= ixgbe_bus_width_pcie_x8 && 3224 hw->bus.speed < ixgbe_bus_speed_8000) { 3225 device_printf(dev, "For optimal performance a x8 " 3226 "PCIE Gen3 slot is required.\n"); 3227 } 3228 } 3229 3230 /* 3231 * TODO comment is incorrect 3232 * 3233 * Setup the correct IVAR register for a particular MSIX interrupt 3234 * - entry is the register array entry 3235 * - vector is the MSIX vector for this queue 3236 * - type is RX/TX/MISC 3237 */ 3238 static void 3239 ix_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, 3240 int8_t type) 3241 { 3242 struct ixgbe_hw *hw = &sc->hw; 3243 uint32_t ivar, index; 3244 3245 vector |= IXGBE_IVAR_ALLOC_VAL; 3246 3247 switch (hw->mac.type) { 3248 case ixgbe_mac_82598EB: 3249 if (type == -1) 3250 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3251 else 3252 entry += (type * 64); 3253 index = (entry >> 2) & 0x1F; 3254 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3255 ivar &= ~(0xFF << (8 * (entry & 0x3))); 3256 ivar |= (vector << (8 * (entry & 0x3))); 3257 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 3258 break; 3259 3260 case ixgbe_mac_82599EB: 3261 case ixgbe_mac_X540: 3262 if (type == -1) { /* MISC IVAR */ 3263 index = (entry & 1) * 8; 3264 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3265 ivar &= ~(0xFF << index); 3266 ivar |= (vector << index); 3267 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3268 } else { /* RX/TX IVARS */ 3269 index = (16 * (entry & 1)) + (8 * type); 3270 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 3271 ivar &= ~(0xFF << index); 3272 ivar |= (vector << index); 3273 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 3274 } 3275 3276 default: 3277 break; 3278 } 3279 } 3280 3281 #if 0 3282 static void 3283 ix_configure_ivars(struct ix_softc *sc) 3284 { 3285 struct ix_queue *que = sc->queues; 3286 uint32_t newitr; 3287 3288 if (ixgbe_max_interrupt_rate > 0) 3289 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 3290 else 3291 newitr = 0; 3292 3293 for (int i = 0; i < sc->num_queues; i++, que++) { 3294 /* First the RX queue entry */ 3295 ix_set_ivar(sc, i, que->msix, 0); 3296 /* ... and the TX */ 3297 ix_set_ivar(sc, i, que->msix, 1); 3298 /* Set an Initial EITR value */ 3299 IXGBE_WRITE_REG(&sc->hw, 3300 IXGBE_EITR(que->msix), newitr); 3301 } 3302 3303 /* For the Link interrupt */ 3304 ix_set_ivar(sc, 1, sc->linkvec, -1); 3305 } 3306 #endif 3307 3308 static boolean_t 3309 ix_sfp_probe(struct ix_softc *sc) 3310 { 3311 struct ixgbe_hw *hw = &sc->hw; 3312 3313 if (hw->phy.type == ixgbe_phy_nl && 3314 hw->phy.sfp_type == ixgbe_sfp_type_not_present) { 3315 int32_t ret; 3316 3317 ret = hw->phy.ops.identify_sfp(hw); 3318 if (ret) 3319 return FALSE; 3320 3321 ret = hw->phy.ops.reset(hw); 3322 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3323 if_printf(&sc->arpcom.ac_if, 3324 "Unsupported SFP+ module detected! " 3325 "Reload driver with supported module.\n"); 3326 sc->sfp_probe = FALSE; 3327 return FALSE; 3328 } 3329 if_printf(&sc->arpcom.ac_if, "SFP+ module detected!\n"); 3330 3331 /* We now have supported optics */ 3332 sc->sfp_probe = FALSE; 3333 /* Set the optics type so system reports correctly */ 3334 ix_setup_optics(sc); 3335 3336 return TRUE; 3337 } 3338 return FALSE; 3339 } 3340 3341 static void 3342 ix_handle_link(struct ix_softc *sc) 3343 { 3344 ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0); 3345 ix_update_link_status(sc); 3346 } 3347 3348 /* 3349 * Handling SFP module 3350 */ 3351 static void 3352 ix_handle_mod(struct ix_softc *sc) 3353 { 3354 struct ixgbe_hw *hw = &sc->hw; 3355 uint32_t err; 3356 3357 err = hw->phy.ops.identify_sfp(hw); 3358 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3359 if_printf(&sc->arpcom.ac_if, 3360 "Unsupported SFP+ module type was detected.\n"); 3361 return; 3362 } 3363 err = hw->mac.ops.setup_sfp(hw); 3364 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3365 if_printf(&sc->arpcom.ac_if, 3366 "Setup failure - unsupported SFP+ module type.\n"); 3367 return; 3368 } 3369 ix_handle_msf(sc); 3370 } 3371 3372 /* 3373 * Handling MSF (multispeed fiber) 3374 */ 3375 static void 3376 ix_handle_msf(struct ix_softc *sc) 3377 { 3378 struct ixgbe_hw *hw = &sc->hw; 3379 uint32_t autoneg; 3380 3381 autoneg = hw->phy.autoneg_advertised; 3382 if (!autoneg && hw->mac.ops.get_link_capabilities != NULL) { 3383 bool negotiate; 3384 3385 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 3386 } 3387 if (hw->mac.ops.setup_link != NULL) 3388 hw->mac.ops.setup_link(hw, autoneg, TRUE); 3389 } 3390 3391 static void 3392 ix_update_stats(struct ix_softc *sc) 3393 { 3394 struct ifnet *ifp = &sc->arpcom.ac_if; 3395 struct ixgbe_hw *hw = &sc->hw; 3396 uint32_t missed_rx = 0, bprc, lxon, lxoff, total; 3397 uint64_t total_missed_rx = 0; 3398 int i; 3399 3400 sc->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 3401 sc->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 3402 sc->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC); 3403 sc->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC); 3404 3405 /* 3406 * Note: These are for the 8 possible traffic classes, which 3407 * in current implementation is unused, therefore only 0 should 3408 * read real data. 3409 */ 3410 for (i = 0; i < 8; i++) { 3411 uint32_t mp; 3412 3413 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 3414 /* missed_rx tallies misses for the gprc workaround */ 3415 missed_rx += mp; 3416 /* global total per queue */ 3417 sc->stats.mpc[i] += mp; 3418 3419 /* Running comprehensive total for stats display */ 3420 total_missed_rx += sc->stats.mpc[i]; 3421 3422 if (hw->mac.type == ixgbe_mac_82598EB) { 3423 sc->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3424 sc->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 3425 sc->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 3426 sc->stats.pxonrxc[i] += 3427 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 3428 } else { 3429 sc->stats.pxonrxc[i] += 3430 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i)); 3431 } 3432 sc->stats.pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 3433 sc->stats.pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 3434 sc->stats.pxoffrxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 3435 sc->stats.pxon2offc[i] += 3436 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i)); 3437 } 3438 for (i = 0; i < 16; i++) { 3439 sc->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 3440 sc->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 3441 sc->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 3442 } 3443 sc->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC); 3444 sc->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC); 3445 sc->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 3446 3447 /* Hardware workaround, gprc counts missed packets */ 3448 sc->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 3449 sc->stats.gprc -= missed_rx; 3450 3451 if (hw->mac.type != ixgbe_mac_82598EB) { 3452 sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) + 3453 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 3454 sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) + 3455 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32); 3456 sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) + 3457 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 3458 sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3459 sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3460 } else { 3461 sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3462 sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3463 /* 82598 only has a counter in the high register */ 3464 sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3465 sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3466 sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3467 } 3468 3469 /* 3470 * Workaround: mprc hardware is incorrectly counting 3471 * broadcasts, so for now we subtract those. 3472 */ 3473 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3474 sc->stats.bprc += bprc; 3475 sc->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3476 if (hw->mac.type == ixgbe_mac_82598EB) 3477 sc->stats.mprc -= bprc; 3478 3479 sc->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3480 sc->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3481 sc->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3482 sc->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3483 sc->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3484 sc->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3485 3486 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3487 sc->stats.lxontxc += lxon; 3488 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3489 sc->stats.lxofftxc += lxoff; 3490 total = lxon + lxoff; 3491 3492 sc->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 3493 sc->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3494 sc->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3495 sc->stats.gptc -= total; 3496 sc->stats.mptc -= total; 3497 sc->stats.ptc64 -= total; 3498 sc->stats.gotc -= total * ETHER_MIN_LEN; 3499 3500 sc->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3501 sc->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3502 sc->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3503 sc->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3504 sc->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 3505 sc->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 3506 sc->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 3507 sc->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3508 sc->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT); 3509 sc->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3510 sc->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3511 sc->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3512 sc->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3513 sc->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3514 sc->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3515 sc->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC); 3516 sc->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC); 3517 sc->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST); 3518 /* Only read FCOE on 82599 */ 3519 if (hw->mac.type != ixgbe_mac_82598EB) { 3520 sc->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 3521 sc->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 3522 sc->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 3523 sc->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 3524 sc->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 3525 } 3526 3527 /* Rx Errors */ 3528 IFNET_STAT_SET(ifp, iqdrops, total_missed_rx); 3529 IFNET_STAT_SET(ifp, ierrors, sc->stats.crcerrs + sc->stats.rlec); 3530 } 3531 3532 #if 0 3533 /* 3534 * Add sysctl variables, one per statistic, to the system. 3535 */ 3536 static void 3537 ix_add_hw_stats(struct ix_softc *sc) 3538 { 3539 3540 device_t dev = sc->dev; 3541 3542 struct ix_tx_ring *txr = sc->tx_rings; 3543 struct ix_rx_ring *rxr = sc->rx_rings; 3544 3545 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 3546 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 3547 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 3548 struct ixgbe_hw_stats *stats = &sc->stats; 3549 3550 struct sysctl_oid *stat_node, *queue_node; 3551 struct sysctl_oid_list *stat_list, *queue_list; 3552 3553 #define QUEUE_NAME_LEN 32 3554 char namebuf[QUEUE_NAME_LEN]; 3555 3556 /* MAC stats get the own sub node */ 3557 3558 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 3559 CTLFLAG_RD, NULL, "MAC Statistics"); 3560 stat_list = SYSCTL_CHILDREN(stat_node); 3561 3562 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs", 3563 CTLFLAG_RD, &stats->crcerrs, 3564 "CRC Errors"); 3565 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs", 3566 CTLFLAG_RD, &stats->illerrc, 3567 "Illegal Byte Errors"); 3568 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs", 3569 CTLFLAG_RD, &stats->errbc, 3570 "Byte Errors"); 3571 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards", 3572 CTLFLAG_RD, &stats->mspdc, 3573 "MAC Short Packets Discarded"); 3574 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults", 3575 CTLFLAG_RD, &stats->mlfc, 3576 "MAC Local Faults"); 3577 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults", 3578 CTLFLAG_RD, &stats->mrfc, 3579 "MAC Remote Faults"); 3580 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs", 3581 CTLFLAG_RD, &stats->rlec, 3582 "Receive Length Errors"); 3583 3584 /* Flow Control stats */ 3585 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd", 3586 CTLFLAG_RD, &stats->lxontxc, 3587 "Link XON Transmitted"); 3588 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd", 3589 CTLFLAG_RD, &stats->lxonrxc, 3590 "Link XON Received"); 3591 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd", 3592 CTLFLAG_RD, &stats->lxofftxc, 3593 "Link XOFF Transmitted"); 3594 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", 3595 CTLFLAG_RD, &stats->lxoffrxc, 3596 "Link XOFF Received"); 3597 3598 /* Packet Reception Stats */ 3599 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd", 3600 CTLFLAG_RD, &stats->tor, 3601 "Total Octets Received"); 3602 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd", 3603 CTLFLAG_RD, &stats->gorc, 3604 "Good Octets Received"); 3605 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd", 3606 CTLFLAG_RD, &stats->tpr, 3607 "Total Packets Received"); 3608 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd", 3609 CTLFLAG_RD, &stats->gprc, 3610 "Good Packets Received"); 3611 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd", 3612 CTLFLAG_RD, &stats->mprc, 3613 "Multicast Packets Received"); 3614 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd", 3615 CTLFLAG_RD, &stats->bprc, 3616 "Broadcast Packets Received"); 3617 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 3618 CTLFLAG_RD, &stats->prc64, 3619 "64 byte frames received "); 3620 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 3621 CTLFLAG_RD, &stats->prc127, 3622 "65-127 byte frames received"); 3623 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 3624 CTLFLAG_RD, &stats->prc255, 3625 "128-255 byte frames received"); 3626 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 3627 CTLFLAG_RD, &stats->prc511, 3628 "256-511 byte frames received"); 3629 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 3630 CTLFLAG_RD, &stats->prc1023, 3631 "512-1023 byte frames received"); 3632 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 3633 CTLFLAG_RD, &stats->prc1522, 3634 "1023-1522 byte frames received"); 3635 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized", 3636 CTLFLAG_RD, &stats->ruc, 3637 "Receive Undersized"); 3638 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 3639 CTLFLAG_RD, &stats->rfc, 3640 "Fragmented Packets Received "); 3641 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized", 3642 CTLFLAG_RD, &stats->roc, 3643 "Oversized Packets Received"); 3644 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd", 3645 CTLFLAG_RD, &stats->rjc, 3646 "Received Jabber"); 3647 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd", 3648 CTLFLAG_RD, &stats->mngprc, 3649 "Management Packets Received"); 3650 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd", 3651 CTLFLAG_RD, &stats->mngptc, 3652 "Management Packets Dropped"); 3653 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs", 3654 CTLFLAG_RD, &stats->xec, 3655 "Checksum Errors"); 3656 3657 /* Packet Transmission Stats */ 3658 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 3659 CTLFLAG_RD, &stats->gotc, 3660 "Good Octets Transmitted"); 3661 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 3662 CTLFLAG_RD, &stats->tpt, 3663 "Total Packets Transmitted"); 3664 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 3665 CTLFLAG_RD, &stats->gptc, 3666 "Good Packets Transmitted"); 3667 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 3668 CTLFLAG_RD, &stats->bptc, 3669 "Broadcast Packets Transmitted"); 3670 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 3671 CTLFLAG_RD, &stats->mptc, 3672 "Multicast Packets Transmitted"); 3673 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd", 3674 CTLFLAG_RD, &stats->mngptc, 3675 "Management Packets Transmitted"); 3676 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 3677 CTLFLAG_RD, &stats->ptc64, 3678 "64 byte frames transmitted "); 3679 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 3680 CTLFLAG_RD, &stats->ptc127, 3681 "65-127 byte frames transmitted"); 3682 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 3683 CTLFLAG_RD, &stats->ptc255, 3684 "128-255 byte frames transmitted"); 3685 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 3686 CTLFLAG_RD, &stats->ptc511, 3687 "256-511 byte frames transmitted"); 3688 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 3689 CTLFLAG_RD, &stats->ptc1023, 3690 "512-1023 byte frames transmitted"); 3691 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 3692 CTLFLAG_RD, &stats->ptc1522, 3693 "1024-1522 byte frames transmitted"); 3694 } 3695 #endif 3696 3697 /* 3698 * Enable the hardware to drop packets when the buffer is full. 3699 * This is useful when multiple RX rings are used, so that no 3700 * single RX ring being full stalls the entire RX engine. We 3701 * only enable this when multiple RX rings are used and when 3702 * flow control is disabled. 3703 */ 3704 static void 3705 ix_enable_rx_drop(struct ix_softc *sc) 3706 { 3707 struct ixgbe_hw *hw = &sc->hw; 3708 int i; 3709 3710 if (bootverbose) { 3711 if_printf(&sc->arpcom.ac_if, 3712 "flow control %d, enable RX drop\n", sc->fc); 3713 } 3714 3715 for (i = 0; i < sc->rx_ring_inuse; ++i) { 3716 uint32_t srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 3717 3718 srrctl |= IXGBE_SRRCTL_DROP_EN; 3719 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 3720 } 3721 } 3722 3723 static void 3724 ix_disable_rx_drop(struct ix_softc *sc) 3725 { 3726 struct ixgbe_hw *hw = &sc->hw; 3727 int i; 3728 3729 if (bootverbose) { 3730 if_printf(&sc->arpcom.ac_if, 3731 "flow control %d, disable RX drop\n", sc->fc); 3732 } 3733 3734 for (i = 0; i < sc->rx_ring_inuse; ++i) { 3735 uint32_t srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 3736 3737 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 3738 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 3739 } 3740 } 3741 3742 static int 3743 ix_sysctl_flowctrl(SYSCTL_HANDLER_ARGS) 3744 { 3745 struct ix_softc *sc = (struct ix_softc *)arg1; 3746 struct ifnet *ifp = &sc->arpcom.ac_if; 3747 int error, fc; 3748 3749 fc = sc->fc; 3750 error = sysctl_handle_int(oidp, &fc, 0, req); 3751 if (error || req->newptr == NULL) 3752 return error; 3753 3754 switch (fc) { 3755 case ixgbe_fc_rx_pause: 3756 case ixgbe_fc_tx_pause: 3757 case ixgbe_fc_full: 3758 case ixgbe_fc_none: 3759 break; 3760 default: 3761 return EINVAL; 3762 } 3763 3764 ifnet_serialize_all(ifp); 3765 3766 /* Don't bother if it's not changed */ 3767 if (sc->fc == fc) 3768 goto done; 3769 sc->fc = fc; 3770 3771 /* Don't do anything, if the interface is not up yet */ 3772 if ((ifp->if_flags & IFF_RUNNING) == 0) 3773 goto done; 3774 3775 if (sc->rx_ring_inuse > 1) { 3776 switch (sc->fc) { 3777 case ixgbe_fc_rx_pause: 3778 case ixgbe_fc_tx_pause: 3779 case ixgbe_fc_full: 3780 ix_disable_rx_drop(sc); 3781 break; 3782 3783 case ixgbe_fc_none: 3784 ix_enable_rx_drop(sc); 3785 break; 3786 3787 default: 3788 panic("leading fc check mismatch"); 3789 } 3790 } 3791 3792 sc->hw.fc.requested_mode = sc->fc; 3793 /* Don't autoneg if forcing a value */ 3794 sc->hw.fc.disable_fc_autoneg = TRUE; 3795 ixgbe_fc_enable(&sc->hw); 3796 3797 done: 3798 ifnet_deserialize_all(ifp); 3799 return error; 3800 } 3801 3802 #ifdef foo 3803 /* XXX not working properly w/ 82599 connected w/ DAC */ 3804 /* XXX only work after the interface is up */ 3805 static int 3806 ix_sysctl_advspeed(SYSCTL_HANDLER_ARGS) 3807 { 3808 struct ix_softc *sc = (struct ix_softc *)arg1; 3809 struct ifnet *ifp = &sc->arpcom.ac_if; 3810 struct ixgbe_hw *hw = &sc->hw; 3811 ixgbe_link_speed speed; 3812 int error, advspeed; 3813 3814 advspeed = sc->advspeed; 3815 error = sysctl_handle_int(oidp, &advspeed, 0, req); 3816 if (error || req->newptr == NULL) 3817 return error; 3818 3819 if (!(hw->phy.media_type == ixgbe_media_type_copper || 3820 hw->phy.multispeed_fiber)) 3821 return EOPNOTSUPP; 3822 if (hw->mac.ops.setup_link == NULL) 3823 return EOPNOTSUPP; 3824 3825 switch (advspeed) { 3826 case 0: /* auto */ 3827 speed = IXGBE_LINK_SPEED_UNKNOWN; 3828 break; 3829 3830 case 1: /* 1Gb */ 3831 speed = IXGBE_LINK_SPEED_1GB_FULL; 3832 break; 3833 3834 case 2: /* 100Mb */ 3835 speed = IXGBE_LINK_SPEED_100_FULL; 3836 break; 3837 3838 case 3: /* 1Gb/10Gb */ 3839 speed = IXGBE_LINK_SPEED_1GB_FULL | 3840 IXGBE_LINK_SPEED_10GB_FULL; 3841 break; 3842 3843 default: 3844 return EINVAL; 3845 } 3846 3847 ifnet_serialize_all(ifp); 3848 3849 if (sc->advspeed == advspeed) /* no change */ 3850 goto done; 3851 3852 if ((speed & IXGBE_LINK_SPEED_100_FULL) && 3853 hw->mac.type != ixgbe_mac_X540) { 3854 error = EOPNOTSUPP; 3855 goto done; 3856 } 3857 3858 sc->advspeed = advspeed; 3859 3860 if ((ifp->if_flags & IFF_RUNNING) == 0) 3861 goto done; 3862 3863 if (speed == IXGBE_LINK_SPEED_UNKNOWN) { 3864 ix_config_link(sc); 3865 } else { 3866 hw->mac.autotry_restart = TRUE; 3867 hw->mac.ops.setup_link(hw, speed, sc->link_up); 3868 } 3869 3870 done: 3871 ifnet_deserialize_all(ifp); 3872 return error; 3873 } 3874 #endif 3875 3876 static void 3877 ix_setup_serialize(struct ix_softc *sc) 3878 { 3879 int i = 0, j; 3880 3881 /* Main + RX + TX */ 3882 sc->nserialize = 1 + sc->rx_ring_cnt + sc->tx_ring_cnt; 3883 sc->serializes = 3884 kmalloc(sc->nserialize * sizeof(struct lwkt_serialize *), 3885 M_DEVBUF, M_WAITOK | M_ZERO); 3886 3887 /* 3888 * Setup serializes 3889 * 3890 * NOTE: Order is critical 3891 */ 3892 3893 KKASSERT(i < sc->nserialize); 3894 sc->serializes[i++] = &sc->main_serialize; 3895 3896 for (j = 0; j < sc->rx_ring_cnt; ++j) { 3897 KKASSERT(i < sc->nserialize); 3898 sc->serializes[i++] = &sc->rx_rings[j].rx_serialize; 3899 } 3900 3901 for (j = 0; j < sc->tx_ring_cnt; ++j) { 3902 KKASSERT(i < sc->nserialize); 3903 sc->serializes[i++] = &sc->tx_rings[j].tx_serialize; 3904 } 3905 3906 KKASSERT(i == sc->nserialize); 3907 } 3908 3909 static int 3910 ix_alloc_intr(struct ix_softc *sc) 3911 { 3912 struct ix_intr_data *intr; 3913 u_int intr_flags; 3914 int i; 3915 3916 if (sc->intr_data != NULL) 3917 kfree(sc->intr_data, M_DEVBUF); 3918 3919 sc->intr_cnt = 1; 3920 sc->intr_data = kmalloc(sizeof(struct ix_intr_data), M_DEVBUF, 3921 M_WAITOK | M_ZERO); 3922 intr = &sc->intr_data[0]; 3923 3924 /* 3925 * Allocate MSI/legacy interrupt resource 3926 */ 3927 sc->intr_type = pci_alloc_1intr(sc->dev, ix_msi_enable, 3928 &intr->intr_rid, &intr_flags); 3929 3930 intr->intr_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 3931 &intr->intr_rid, intr_flags); 3932 if (intr->intr_res == NULL) { 3933 device_printf(sc->dev, "Unable to allocate bus resource: " 3934 "interrupt\n"); 3935 return ENXIO; 3936 } 3937 3938 intr->intr_serialize = &sc->main_serialize; 3939 intr->intr_cpuid = rman_get_cpuid(intr->intr_res); 3940 intr->intr_func = ix_intr; 3941 intr->intr_funcarg = sc; 3942 intr->intr_rate = IX_INTR_RATE; 3943 intr->intr_use = IX_INTR_USE_RXTX; 3944 3945 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3946 sc->tx_rings[i].tx_intr_cpuid = intr->intr_cpuid; 3947 sc->tx_rings[i].tx_intr_vec = IX_TX_INTR_VEC; 3948 } 3949 3950 for (i = 0; i < sc->rx_ring_cnt; ++i) 3951 sc->rx_rings[i].rx_intr_vec = IX_RX0_INTR_VEC; 3952 3953 ix_set_ring_inuse(sc, FALSE); 3954 3955 KKASSERT(sc->rx_ring_inuse <= IX_MIN_RXRING_RSS); 3956 if (sc->rx_ring_inuse == IX_MIN_RXRING_RSS) 3957 sc->rx_rings[1].rx_intr_vec = IX_RX1_INTR_VEC; 3958 3959 return 0; 3960 } 3961 3962 static void 3963 ix_free_intr(struct ix_softc *sc) 3964 { 3965 if (sc->intr_data == NULL) 3966 return; 3967 3968 if (sc->intr_type != PCI_INTR_TYPE_MSIX) { 3969 struct ix_intr_data *intr = &sc->intr_data[0]; 3970 3971 KKASSERT(sc->intr_cnt == 1); 3972 if (intr->intr_res != NULL) { 3973 bus_release_resource(sc->dev, SYS_RES_IRQ, 3974 intr->intr_rid, intr->intr_res); 3975 } 3976 if (sc->intr_type == PCI_INTR_TYPE_MSI) 3977 pci_release_msi(sc->dev); 3978 } else { 3979 /* TODO */ 3980 } 3981 kfree(sc->intr_data, M_DEVBUF); 3982 } 3983 3984 static void 3985 ix_set_ring_inuse(struct ix_softc *sc, boolean_t polling) 3986 { 3987 sc->rx_ring_inuse = ix_get_rxring_inuse(sc, polling); 3988 sc->tx_ring_inuse = ix_get_txring_inuse(sc, polling); 3989 if (bootverbose) { 3990 if_printf(&sc->arpcom.ac_if, 3991 "RX rings %d/%d, TX rings %d/%d\n", 3992 sc->rx_ring_inuse, sc->rx_ring_cnt, 3993 sc->tx_ring_inuse, sc->tx_ring_cnt); 3994 } 3995 } 3996 3997 static int 3998 ix_get_rxring_inuse(const struct ix_softc *sc, boolean_t polling) 3999 { 4000 if (!IX_ENABLE_HWRSS(sc)) 4001 return 1; 4002 4003 if (polling) 4004 return sc->rx_ring_cnt; 4005 else if (sc->intr_type != PCI_INTR_TYPE_MSIX) 4006 return IX_MIN_RXRING_RSS; 4007 else 4008 return 1; /* TODO */ 4009 } 4010 4011 static int 4012 ix_get_txring_inuse(const struct ix_softc *sc, boolean_t polling) 4013 { 4014 if (!IX_ENABLE_HWTSS(sc)) 4015 return 1; 4016 4017 if (polling) 4018 return sc->tx_ring_cnt; 4019 else if (sc->intr_type != PCI_INTR_TYPE_MSIX) 4020 return 1; 4021 else 4022 return 1; /* TODO */ 4023 } 4024 4025 static int 4026 ix_setup_intr(struct ix_softc *sc) 4027 { 4028 int i; 4029 4030 for (i = 0; i < sc->intr_cnt; ++i) { 4031 struct ix_intr_data *intr = &sc->intr_data[i]; 4032 int error; 4033 4034 error = bus_setup_intr_descr(sc->dev, intr->intr_res, 4035 INTR_MPSAFE, intr->intr_func, intr->intr_funcarg, 4036 &intr->intr_hand, intr->intr_serialize, intr->intr_desc); 4037 if (error) { 4038 device_printf(sc->dev, "can't setup %dth intr\n", i); 4039 ix_teardown_intr(sc, i); 4040 return error; 4041 } 4042 } 4043 return 0; 4044 } 4045 4046 static void 4047 ix_teardown_intr(struct ix_softc *sc, int intr_cnt) 4048 { 4049 int i; 4050 4051 if (sc->intr_data == NULL) 4052 return; 4053 4054 for (i = 0; i < intr_cnt; ++i) { 4055 struct ix_intr_data *intr = &sc->intr_data[i]; 4056 4057 bus_teardown_intr(sc->dev, intr->intr_res, intr->intr_hand); 4058 } 4059 } 4060 4061 static void 4062 ix_serialize(struct ifnet *ifp, enum ifnet_serialize slz) 4063 { 4064 struct ix_softc *sc = ifp->if_softc; 4065 4066 ifnet_serialize_array_enter(sc->serializes, sc->nserialize, slz); 4067 } 4068 4069 static void 4070 ix_deserialize(struct ifnet *ifp, enum ifnet_serialize slz) 4071 { 4072 struct ix_softc *sc = ifp->if_softc; 4073 4074 ifnet_serialize_array_exit(sc->serializes, sc->nserialize, slz); 4075 } 4076 4077 static int 4078 ix_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz) 4079 { 4080 struct ix_softc *sc = ifp->if_softc; 4081 4082 return ifnet_serialize_array_try(sc->serializes, sc->nserialize, slz); 4083 } 4084 4085 #ifdef INVARIANTS 4086 4087 static void 4088 ix_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz, 4089 boolean_t serialized) 4090 { 4091 struct ix_softc *sc = ifp->if_softc; 4092 4093 ifnet_serialize_array_assert(sc->serializes, sc->nserialize, slz, 4094 serialized); 4095 } 4096 4097 #endif /* INVARIANTS */ 4098 4099 static void 4100 ix_free_rings(struct ix_softc *sc) 4101 { 4102 int i; 4103 4104 if (sc->tx_rings != NULL) { 4105 for (i = 0; i < sc->tx_ring_cnt; ++i) { 4106 struct ix_tx_ring *txr = &sc->tx_rings[i]; 4107 4108 ix_destroy_tx_ring(txr, txr->tx_ndesc); 4109 } 4110 kfree(sc->tx_rings, M_DEVBUF); 4111 } 4112 4113 if (sc->rx_rings != NULL) { 4114 for (i =0; i < sc->rx_ring_cnt; ++i) { 4115 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 4116 4117 ix_destroy_rx_ring(rxr, rxr->rx_ndesc); 4118 } 4119 kfree(sc->rx_rings, M_DEVBUF); 4120 } 4121 4122 if (sc->parent_tag != NULL) 4123 bus_dma_tag_destroy(sc->parent_tag); 4124 } 4125 4126 static void 4127 ix_watchdog(struct ifaltq_subque *ifsq) 4128 { 4129 struct ix_tx_ring *txr = ifsq_get_priv(ifsq); 4130 struct ifnet *ifp = ifsq_get_ifp(ifsq); 4131 struct ix_softc *sc = ifp->if_softc; 4132 int i; 4133 4134 KKASSERT(txr->tx_ifsq == ifsq); 4135 ASSERT_IFNET_SERIALIZED_ALL(ifp); 4136 4137 /* 4138 * If the interface has been paused then don't do the watchdog check 4139 */ 4140 if (IXGBE_READ_REG(&sc->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) { 4141 txr->tx_watchdog.wd_timer = 5; 4142 return; 4143 } 4144 4145 if_printf(ifp, "Watchdog timeout -- resetting\n"); 4146 if_printf(ifp, "Queue(%d) tdh = %d, hw tdt = %d\n", txr->tx_idx, 4147 IXGBE_READ_REG(&sc->hw, IXGBE_TDH(txr->tx_idx)), 4148 IXGBE_READ_REG(&sc->hw, IXGBE_TDT(txr->tx_idx))); 4149 if_printf(ifp, "TX(%d) desc avail = %d, next TX to Clean = %d\n", 4150 txr->tx_idx, txr->tx_avail, txr->tx_next_clean); 4151 4152 ix_init(sc); 4153 for (i = 0; i < sc->tx_ring_inuse; ++i) 4154 ifsq_devstart_sched(sc->tx_rings[i].tx_ifsq); 4155 } 4156 4157 static void 4158 ix_free_tx_ring(struct ix_tx_ring *txr) 4159 { 4160 int i; 4161 4162 for (i = 0; i < txr->tx_ndesc; ++i) { 4163 struct ix_tx_buf *txbuf = &txr->tx_buf[i]; 4164 4165 if (txbuf->m_head != NULL) { 4166 bus_dmamap_unload(txr->tx_tag, txbuf->map); 4167 m_freem(txbuf->m_head); 4168 txbuf->m_head = NULL; 4169 } 4170 } 4171 } 4172 4173 static void 4174 ix_free_rx_ring(struct ix_rx_ring *rxr) 4175 { 4176 int i; 4177 4178 for (i = 0; i < rxr->rx_ndesc; ++i) { 4179 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i]; 4180 4181 if (rxbuf->fmp != NULL) { 4182 m_freem(rxbuf->fmp); 4183 rxbuf->fmp = NULL; 4184 rxbuf->lmp = NULL; 4185 } else { 4186 KKASSERT(rxbuf->lmp == NULL); 4187 } 4188 if (rxbuf->m_head != NULL) { 4189 bus_dmamap_unload(rxr->rx_tag, rxbuf->map); 4190 m_freem(rxbuf->m_head); 4191 rxbuf->m_head = NULL; 4192 } 4193 } 4194 } 4195 4196 static int 4197 ix_newbuf(struct ix_rx_ring *rxr, int i, boolean_t wait) 4198 { 4199 struct mbuf *m; 4200 bus_dma_segment_t seg; 4201 bus_dmamap_t map; 4202 struct ix_rx_buf *rxbuf; 4203 int flags, error, nseg; 4204 4205 flags = MB_DONTWAIT; 4206 if (__predict_false(wait)) 4207 flags = MB_WAIT; 4208 4209 m = m_getjcl(flags, MT_DATA, M_PKTHDR, rxr->rx_mbuf_sz); 4210 if (m == NULL) { 4211 if (wait) { 4212 if_printf(&rxr->rx_sc->arpcom.ac_if, 4213 "Unable to allocate RX mbuf\n"); 4214 } 4215 return ENOBUFS; 4216 } 4217 m->m_len = m->m_pkthdr.len = rxr->rx_mbuf_sz; 4218 4219 error = bus_dmamap_load_mbuf_segment(rxr->rx_tag, 4220 rxr->rx_sparemap, m, &seg, 1, &nseg, BUS_DMA_NOWAIT); 4221 if (error) { 4222 m_freem(m); 4223 if (wait) { 4224 if_printf(&rxr->rx_sc->arpcom.ac_if, 4225 "Unable to load RX mbuf\n"); 4226 } 4227 return error; 4228 } 4229 4230 rxbuf = &rxr->rx_buf[i]; 4231 if (rxbuf->m_head != NULL) 4232 bus_dmamap_unload(rxr->rx_tag, rxbuf->map); 4233 4234 map = rxbuf->map; 4235 rxbuf->map = rxr->rx_sparemap; 4236 rxr->rx_sparemap = map; 4237 4238 rxbuf->m_head = m; 4239 rxbuf->paddr = seg.ds_addr; 4240 4241 ix_setup_rxdesc(&rxr->rx_base[i], rxbuf); 4242 return 0; 4243 } 4244 4245 static void 4246 ix_add_sysctl(struct ix_softc *sc) 4247 { 4248 #ifdef IX_RSS_DEBUG 4249 char node[32]; 4250 #endif 4251 int i, add; 4252 4253 sysctl_ctx_init(&sc->sysctl_ctx); 4254 sc->sysctl_tree = SYSCTL_ADD_NODE(&sc->sysctl_ctx, 4255 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO, 4256 device_get_nameunit(sc->dev), CTLFLAG_RD, 0, ""); 4257 if (sc->sysctl_tree == NULL) { 4258 device_printf(sc->dev, "can't add sysctl node\n"); 4259 return; 4260 } 4261 4262 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 4263 OID_AUTO, "rxr", CTLFLAG_RD, &sc->rx_ring_cnt, 0, "# of RX rings"); 4264 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 4265 OID_AUTO, "rxr_inuse", CTLFLAG_RD, &sc->rx_ring_inuse, 0, 4266 "# of RX rings used"); 4267 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 4268 OID_AUTO, "txr", CTLFLAG_RD, &sc->tx_ring_cnt, 0, "# of TX rings"); 4269 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 4270 OID_AUTO, "txr_inuse", CTLFLAG_RD, &sc->tx_ring_inuse, 0, 4271 "# of TX rings used"); 4272 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 4273 OID_AUTO, "rxd", CTLTYPE_INT | CTLFLAG_RD, 4274 sc, 0, ix_sysctl_rxd, "I", 4275 "# of RX descs"); 4276 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 4277 OID_AUTO, "txd", CTLTYPE_INT | CTLFLAG_RD, 4278 sc, 0, ix_sysctl_txd, "I", 4279 "# of TX descs"); 4280 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 4281 OID_AUTO, "tx_wreg_nsegs", CTLTYPE_INT | CTLFLAG_RW, 4282 sc, 0, ix_sysctl_tx_wreg_nsegs, "I", 4283 "# of segments sent before write to hardware register"); 4284 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 4285 OID_AUTO, "rx_wreg_nsegs", CTLTYPE_INT | CTLFLAG_RW, 4286 sc, 0, ix_sysctl_rx_wreg_nsegs, "I", 4287 "# of received segments sent before write to hardware register"); 4288 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 4289 OID_AUTO, "tx_intr_nsegs", CTLTYPE_INT | CTLFLAG_RW, 4290 sc, 0, ix_sysctl_tx_intr_nsegs, "I", 4291 "# of segments per TX interrupt"); 4292 4293 add = 0; 4294 for (i = 0; i < sc->intr_cnt; ++i) { 4295 if (sc->intr_data[i].intr_use == IX_INTR_USE_RXTX) { 4296 add = 1; 4297 break; 4298 } 4299 } 4300 if (add) { 4301 SYSCTL_ADD_PROC(&sc->sysctl_ctx, 4302 SYSCTL_CHILDREN(sc->sysctl_tree), 4303 OID_AUTO, "intr_rate", CTLTYPE_INT | CTLFLAG_RW, 4304 sc, 0, ix_sysctl_intr_rate, "I", 4305 "interrupt rate"); 4306 } 4307 4308 #ifdef IX_RSS_DEBUG 4309 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 4310 OID_AUTO, "rss_debug", CTLFLAG_RW, &sc->rss_debug, 0, 4311 "RSS debug level"); 4312 for (i = 0; i < sc->rx_ring_cnt; ++i) { 4313 ksnprintf(node, sizeof(node), "rx%d_pkt", i); 4314 SYSCTL_ADD_ULONG(&sc->sysctl_ctx, 4315 SYSCTL_CHILDREN(sc->sysctl_tree), OID_AUTO, node, 4316 CTLFLAG_RW, &sc->rx_rings[i].rx_pkts, "RXed packets"); 4317 } 4318 #endif 4319 4320 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 4321 OID_AUTO, "flowctrl", CTLTYPE_INT | CTLFLAG_RW, 4322 sc, 0, ix_sysctl_flowctrl, "I", 4323 "flow control, 0 - off, 1 - rx pause, 2 - tx pause, 3 - full"); 4324 4325 #ifdef foo 4326 /* 4327 * Allow a kind of speed control by forcing the autoneg 4328 * advertised speed list to only a certain value, this 4329 * supports 1G on 82599 devices, and 100Mb on X540. 4330 */ 4331 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 4332 OID_AUTO, "advspeed", CTLTYPE_INT | CTLFLAG_RW, 4333 sc, 0, ix_sysctl_advspeed, "I", 4334 "advertised link speed, " 4335 "0 - auto, 1 - 1Gb, 2 - 100Mb, 3 - 1Gb/10Gb"); 4336 #endif 4337 4338 #if 0 4339 ix_add_hw_stats(sc); 4340 #endif 4341 4342 } 4343 4344 static int 4345 ix_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS) 4346 { 4347 struct ix_softc *sc = (void *)arg1; 4348 struct ifnet *ifp = &sc->arpcom.ac_if; 4349 int error, nsegs, i; 4350 4351 nsegs = sc->tx_rings[0].tx_wreg_nsegs; 4352 error = sysctl_handle_int(oidp, &nsegs, 0, req); 4353 if (error || req->newptr == NULL) 4354 return error; 4355 if (nsegs < 0) 4356 return EINVAL; 4357 4358 ifnet_serialize_all(ifp); 4359 for (i = 0; i < sc->tx_ring_cnt; ++i) 4360 sc->tx_rings[i].tx_wreg_nsegs = nsegs; 4361 ifnet_deserialize_all(ifp); 4362 4363 return 0; 4364 } 4365 4366 static int 4367 ix_sysctl_rx_wreg_nsegs(SYSCTL_HANDLER_ARGS) 4368 { 4369 struct ix_softc *sc = (void *)arg1; 4370 struct ifnet *ifp = &sc->arpcom.ac_if; 4371 int error, nsegs, i; 4372 4373 nsegs = sc->rx_rings[0].rx_wreg_nsegs; 4374 error = sysctl_handle_int(oidp, &nsegs, 0, req); 4375 if (error || req->newptr == NULL) 4376 return error; 4377 if (nsegs < 0) 4378 return EINVAL; 4379 4380 ifnet_serialize_all(ifp); 4381 for (i = 0; i < sc->rx_ring_cnt; ++i) 4382 sc->rx_rings[i].rx_wreg_nsegs =nsegs; 4383 ifnet_deserialize_all(ifp); 4384 4385 return 0; 4386 } 4387 4388 static int 4389 ix_sysctl_txd(SYSCTL_HANDLER_ARGS) 4390 { 4391 struct ix_softc *sc = (void *)arg1; 4392 int txd; 4393 4394 txd = sc->tx_rings[0].tx_ndesc; 4395 return sysctl_handle_int(oidp, &txd, 0, req); 4396 } 4397 4398 static int 4399 ix_sysctl_rxd(SYSCTL_HANDLER_ARGS) 4400 { 4401 struct ix_softc *sc = (void *)arg1; 4402 int rxd; 4403 4404 rxd = sc->rx_rings[0].rx_ndesc; 4405 return sysctl_handle_int(oidp, &rxd, 0, req); 4406 } 4407 4408 static int 4409 ix_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS) 4410 { 4411 struct ix_softc *sc = (void *)arg1; 4412 struct ifnet *ifp = &sc->arpcom.ac_if; 4413 struct ix_tx_ring *txr = &sc->tx_rings[0]; 4414 int error, nsegs; 4415 4416 nsegs = txr->tx_intr_nsegs; 4417 error = sysctl_handle_int(oidp, &nsegs, 0, req); 4418 if (error || req->newptr == NULL) 4419 return error; 4420 if (nsegs < 0) 4421 return EINVAL; 4422 4423 ifnet_serialize_all(ifp); 4424 4425 if (nsegs >= txr->tx_ndesc - IX_MAX_SCATTER - IX_TX_RESERVED) { 4426 error = EINVAL; 4427 } else { 4428 int i; 4429 4430 error = 0; 4431 for (i = 0; i < sc->tx_ring_cnt; ++i) 4432 sc->tx_rings[i].tx_intr_nsegs = nsegs; 4433 } 4434 4435 ifnet_deserialize_all(ifp); 4436 4437 return error; 4438 } 4439 4440 static void 4441 ix_set_eitr(struct ix_softc *sc, int idx, int rate) 4442 { 4443 uint32_t eitr, eitr_intvl; 4444 4445 eitr = IXGBE_READ_REG(&sc->hw, IXGBE_EITR(idx)); 4446 eitr_intvl = 1000000000 / 256 / rate; 4447 4448 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 4449 eitr &= ~IX_EITR_INTVL_MASK_82598; 4450 if (eitr_intvl == 0) 4451 eitr_intvl = 1; 4452 else if (eitr_intvl > IX_EITR_INTVL_MASK_82598) 4453 eitr_intvl = IX_EITR_INTVL_MASK_82598; 4454 } else { 4455 eitr &= ~IX_EITR_INTVL_MASK; 4456 4457 eitr_intvl &= ~IX_EITR_INTVL_RSVD_MASK; 4458 if (eitr_intvl == 0) 4459 eitr_intvl = IX_EITR_INTVL_MIN; 4460 else if (eitr_intvl > IX_EITR_INTVL_MAX) 4461 eitr_intvl = IX_EITR_INTVL_MAX; 4462 } 4463 eitr |= eitr_intvl; 4464 4465 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(idx), eitr); 4466 } 4467 4468 static int 4469 ix_sysctl_intr_rate(SYSCTL_HANDLER_ARGS) 4470 { 4471 struct ix_softc *sc = (void *)arg1; 4472 struct ifnet *ifp = &sc->arpcom.ac_if; 4473 int error, rate, i; 4474 4475 rate = 0; 4476 for (i = 0; i < sc->intr_cnt; ++i) { 4477 if (sc->intr_data[i].intr_use == IX_INTR_USE_RXTX) { 4478 rate = sc->intr_data[i].intr_rate; 4479 break; 4480 } 4481 } 4482 4483 error = sysctl_handle_int(oidp, &rate, 0, req); 4484 if (error || req->newptr == NULL) 4485 return error; 4486 if (rate <= 0) 4487 return EINVAL; 4488 4489 ifnet_serialize_all(ifp); 4490 4491 for (i = 0; i < sc->intr_cnt; ++i) { 4492 if (sc->intr_data[i].intr_use == IX_INTR_USE_RXTX) { 4493 sc->intr_data[i].intr_rate = rate; 4494 if (ifp->if_flags & IFF_RUNNING) 4495 ix_set_eitr(sc, i, rate); 4496 } 4497 } 4498 4499 ifnet_deserialize_all(ifp); 4500 4501 return error; 4502 } 4503 4504 static void 4505 ix_set_timer_cpuid(struct ix_softc *sc, boolean_t polling) 4506 { 4507 if (polling || sc->intr_type == PCI_INTR_TYPE_MSIX) 4508 sc->timer_cpuid = 0; /* XXX fixed */ 4509 else 4510 sc->timer_cpuid = rman_get_cpuid(sc->intr_data[0].intr_res); 4511 } 4512