1 /* 2 * Copyright (c) 2001-2017, Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the Intel Corporation nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include "opt_ifpoll.h" 33 #include "opt_ix.h" 34 35 #include <sys/param.h> 36 #include <sys/bus.h> 37 #include <sys/endian.h> 38 #include <sys/interrupt.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/mbuf.h> 42 #include <sys/proc.h> 43 #include <sys/rman.h> 44 #include <sys/serialize.h> 45 #include <sys/serialize2.h> 46 #include <sys/socket.h> 47 #include <sys/sockio.h> 48 #include <sys/sysctl.h> 49 #include <sys/systm.h> 50 #include <sys/taskqueue.h> 51 52 #include <net/bpf.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_arp.h> 56 #include <net/if_dl.h> 57 #include <net/if_media.h> 58 #include <net/ifq_var.h> 59 #include <net/if_ringmap.h> 60 #include <net/toeplitz.h> 61 #include <net/toeplitz2.h> 62 #include <net/vlan/if_vlan_var.h> 63 #include <net/vlan/if_vlan_ether.h> 64 #include <net/if_poll.h> 65 66 #include <netinet/in_systm.h> 67 #include <netinet/in.h> 68 #include <netinet/ip.h> 69 70 #include <bus/pci/pcivar.h> 71 #include <bus/pci/pcireg.h> 72 73 #include <dev/netif/ix/ixgbe_common.h> 74 #include <dev/netif/ix/ixgbe_api.h> 75 #include <dev/netif/ix/if_ix.h> 76 77 #define IX_IFM_DEFAULT (IFM_ETHER | IFM_AUTO) 78 79 #ifdef IX_RSS_DEBUG 80 #define IX_RSS_DPRINTF(sc, lvl, fmt, ...) \ 81 do { \ 82 if (sc->rss_debug >= lvl) \ 83 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \ 84 } while (0) 85 #else /* !IX_RSS_DEBUG */ 86 #define IX_RSS_DPRINTF(sc, lvl, fmt, ...) ((void)0) 87 #endif /* IX_RSS_DEBUG */ 88 89 #define IX_NAME "Intel(R) PRO/10GbE " 90 #define IX_DEVICE(id) \ 91 { IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_##id, IX_NAME #id } 92 #define IX_DEVICE_NULL { 0, 0, NULL } 93 94 static struct ix_device { 95 uint16_t vid; 96 uint16_t did; 97 const char *desc; 98 } ix_devices[] = { 99 IX_DEVICE(82598AF_DUAL_PORT), 100 IX_DEVICE(82598AF_SINGLE_PORT), 101 IX_DEVICE(82598EB_CX4), 102 IX_DEVICE(82598AT), 103 IX_DEVICE(82598AT2), 104 IX_DEVICE(82598), 105 IX_DEVICE(82598_DA_DUAL_PORT), 106 IX_DEVICE(82598_CX4_DUAL_PORT), 107 IX_DEVICE(82598EB_XF_LR), 108 IX_DEVICE(82598_SR_DUAL_PORT_EM), 109 IX_DEVICE(82598EB_SFP_LOM), 110 IX_DEVICE(82599_KX4), 111 IX_DEVICE(82599_KX4_MEZZ), 112 IX_DEVICE(82599_SFP), 113 IX_DEVICE(82599_XAUI_LOM), 114 IX_DEVICE(82599_CX4), 115 IX_DEVICE(82599_T3_LOM), 116 IX_DEVICE(82599_COMBO_BACKPLANE), 117 IX_DEVICE(82599_BACKPLANE_FCOE), 118 IX_DEVICE(82599_SFP_SF2), 119 IX_DEVICE(82599_SFP_FCOE), 120 IX_DEVICE(82599EN_SFP), 121 IX_DEVICE(82599_SFP_SF_QP), 122 IX_DEVICE(82599_QSFP_SF_QP), 123 IX_DEVICE(X540T), 124 IX_DEVICE(X540T1), 125 IX_DEVICE(X550T), 126 IX_DEVICE(X550T1), 127 IX_DEVICE(X550EM_X_KR), 128 IX_DEVICE(X550EM_X_KX4), 129 IX_DEVICE(X550EM_X_10G_T), 130 IX_DEVICE(X550EM_X_1G_T), 131 IX_DEVICE(X550EM_X_SFP), 132 IX_DEVICE(X550EM_A_KR), 133 IX_DEVICE(X550EM_A_KR_L), 134 IX_DEVICE(X550EM_A_SFP), 135 IX_DEVICE(X550EM_A_SFP_N), 136 IX_DEVICE(X550EM_A_SGMII), 137 IX_DEVICE(X550EM_A_SGMII_L), 138 IX_DEVICE(X550EM_A_10G_T), 139 IX_DEVICE(X550EM_A_1G_T), 140 IX_DEVICE(X550EM_A_1G_T_L), 141 #if 0 142 IX_DEVICE(X540_BYPASS), 143 IX_DEVICE(82599_BYPASS), 144 #endif 145 146 /* required last entry */ 147 IX_DEVICE_NULL 148 }; 149 150 static int ix_probe(device_t); 151 static int ix_attach(device_t); 152 static int ix_detach(device_t); 153 static int ix_shutdown(device_t); 154 155 static void ix_serialize(struct ifnet *, enum ifnet_serialize); 156 static void ix_deserialize(struct ifnet *, enum ifnet_serialize); 157 static int ix_tryserialize(struct ifnet *, enum ifnet_serialize); 158 #ifdef INVARIANTS 159 static void ix_serialize_assert(struct ifnet *, enum ifnet_serialize, 160 boolean_t); 161 #endif 162 static void ix_start(struct ifnet *, struct ifaltq_subque *); 163 static void ix_watchdog(struct ifaltq_subque *); 164 static int ix_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 165 static void ix_init(void *); 166 static void ix_stop(struct ix_softc *); 167 static void ix_media_status(struct ifnet *, struct ifmediareq *); 168 static int ix_media_change(struct ifnet *); 169 static void ix_timer(void *); 170 static void ix_fw_timer(void *); 171 #ifdef IFPOLL_ENABLE 172 static void ix_npoll(struct ifnet *, struct ifpoll_info *); 173 static void ix_npoll_rx(struct ifnet *, void *, int); 174 static void ix_npoll_rx_direct(struct ifnet *, void *, int); 175 static void ix_npoll_tx(struct ifnet *, void *, int); 176 static void ix_npoll_status(struct ifnet *); 177 #endif 178 179 static void ix_add_sysctl(struct ix_softc *); 180 static void ix_add_intr_rate_sysctl(struct ix_softc *, int, 181 const char *, int (*)(SYSCTL_HANDLER_ARGS), const char *); 182 static int ix_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS); 183 static int ix_sysctl_tx_nmbuf(SYSCTL_HANDLER_ARGS); 184 static int ix_sysctl_rx_wreg_nsegs(SYSCTL_HANDLER_ARGS); 185 static int ix_sysctl_txd(SYSCTL_HANDLER_ARGS); 186 static int ix_sysctl_rxd(SYSCTL_HANDLER_ARGS); 187 static int ix_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS); 188 static int ix_sysctl_intr_rate(SYSCTL_HANDLER_ARGS, int); 189 static int ix_sysctl_rxtx_intr_rate(SYSCTL_HANDLER_ARGS); 190 static int ix_sysctl_rx_intr_rate(SYSCTL_HANDLER_ARGS); 191 static int ix_sysctl_tx_intr_rate(SYSCTL_HANDLER_ARGS); 192 static int ix_sysctl_sts_intr_rate(SYSCTL_HANDLER_ARGS); 193 #if 0 194 static void ix_add_hw_stats(struct ix_softc *); 195 #endif 196 197 static void ix_watchdog_reset(struct ix_softc *); 198 static void ix_watchdog_task(void *, int); 199 static void ix_sync_netisr(struct ix_softc *, int); 200 static void ix_slot_info(struct ix_softc *); 201 static int ix_alloc_rings(struct ix_softc *); 202 static void ix_free_rings(struct ix_softc *); 203 static void ix_setup_ifp(struct ix_softc *); 204 static void ix_setup_serialize(struct ix_softc *); 205 static void ix_setup_caps(struct ix_softc *); 206 static void ix_set_ring_inuse(struct ix_softc *, boolean_t); 207 static int ix_get_timer_cpuid(const struct ix_softc *, boolean_t); 208 static void ix_update_stats(struct ix_softc *); 209 static void ix_detect_fanfail(struct ix_softc *, uint32_t, boolean_t); 210 211 static void ix_set_promisc(struct ix_softc *); 212 static void ix_set_multi(struct ix_softc *); 213 static void ix_set_vlan(struct ix_softc *); 214 static uint8_t *ix_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 215 static enum ixgbe_fc_mode ix_ifmedia2fc(int); 216 static const char *ix_ifmedia2str(int); 217 static const char *ix_fc2str(enum ixgbe_fc_mode); 218 219 static void ix_get_txring_cnt(const struct ix_softc *, int *, int *); 220 static int ix_get_txring_inuse(const struct ix_softc *, boolean_t); 221 static void ix_init_tx_ring(struct ix_tx_ring *); 222 static void ix_free_tx_ring(struct ix_tx_ring *); 223 static int ix_create_tx_ring(struct ix_tx_ring *); 224 static void ix_destroy_tx_ring(struct ix_tx_ring *, int); 225 static void ix_init_tx_unit(struct ix_softc *); 226 static int ix_encap(struct ix_tx_ring *, struct mbuf **, 227 uint16_t *, int *); 228 static int ix_tx_ctx_setup(struct ix_tx_ring *, 229 const struct mbuf *, uint32_t *, uint32_t *); 230 static int ix_tso_ctx_setup(struct ix_tx_ring *, 231 const struct mbuf *, uint32_t *, uint32_t *); 232 static void ix_txeof(struct ix_tx_ring *, int); 233 static void ix_txgc(struct ix_tx_ring *); 234 static void ix_txgc_timer(void *); 235 236 static void ix_get_rxring_cnt(const struct ix_softc *, int *, int *); 237 static int ix_get_rxring_inuse(const struct ix_softc *, boolean_t); 238 static int ix_init_rx_ring(struct ix_rx_ring *); 239 static void ix_free_rx_ring(struct ix_rx_ring *); 240 static int ix_create_rx_ring(struct ix_rx_ring *); 241 static void ix_destroy_rx_ring(struct ix_rx_ring *, int); 242 static void ix_init_rx_unit(struct ix_softc *, boolean_t); 243 #if 0 244 static void ix_setup_hw_rsc(struct ix_rx_ring *); 245 #endif 246 static int ix_newbuf(struct ix_rx_ring *, int, boolean_t); 247 static void ix_rxeof(struct ix_rx_ring *, int); 248 static void ix_rx_discard(struct ix_rx_ring *, int, boolean_t); 249 static void ix_enable_rx_drop(struct ix_softc *); 250 static void ix_disable_rx_drop(struct ix_softc *); 251 252 static void ix_config_gpie(struct ix_softc *); 253 static void ix_alloc_msix(struct ix_softc *); 254 static void ix_free_msix(struct ix_softc *, boolean_t); 255 static void ix_setup_msix_eims(const struct ix_softc *, int, 256 uint32_t *, uint32_t *); 257 static int ix_alloc_intr(struct ix_softc *); 258 static void ix_free_intr(struct ix_softc *); 259 static int ix_setup_intr(struct ix_softc *); 260 static void ix_teardown_intr(struct ix_softc *, int); 261 static void ix_enable_intr(struct ix_softc *); 262 static void ix_disable_intr(struct ix_softc *); 263 static void ix_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t); 264 static void ix_set_eitr(struct ix_softc *, int, int); 265 static void ix_intr_status(struct ix_softc *, uint32_t); 266 static void ix_intr_82598(void *); 267 static void ix_intr(void *); 268 static void ix_msix_rxtx(void *); 269 static void ix_msix_rx(void *); 270 static void ix_msix_tx(void *); 271 static void ix_msix_status(void *); 272 273 static void ix_config_link(struct ix_softc *); 274 static boolean_t ix_sfp_probe(struct ix_softc *); 275 static boolean_t ix_is_sfp(struct ixgbe_hw *); 276 static void ix_update_link_status(struct ix_softc *); 277 static void ix_handle_link(struct ix_softc *); 278 static void ix_handle_mod(struct ix_softc *); 279 static void ix_handle_msf(struct ix_softc *); 280 static void ix_handle_phy(struct ix_softc *); 281 static int ix_powerdown(struct ix_softc *); 282 static void ix_config_flowctrl(struct ix_softc *); 283 static void ix_config_dmac(struct ix_softc *); 284 static void ix_init_media(struct ix_softc *); 285 286 static void ix_serialize_skipmain(struct ix_softc *); 287 static void ix_deserialize_skipmain(struct ix_softc *); 288 289 static device_method_t ix_methods[] = { 290 /* Device interface */ 291 DEVMETHOD(device_probe, ix_probe), 292 DEVMETHOD(device_attach, ix_attach), 293 DEVMETHOD(device_detach, ix_detach), 294 DEVMETHOD(device_shutdown, ix_shutdown), 295 DEVMETHOD_END 296 }; 297 298 static driver_t ix_driver = { 299 "ix", 300 ix_methods, 301 sizeof(struct ix_softc) 302 }; 303 304 static devclass_t ix_devclass; 305 306 DECLARE_DUMMY_MODULE(if_ix); 307 DRIVER_MODULE(if_ix, pci, ix_driver, ix_devclass, NULL, NULL); 308 309 static int ix_msi_enable = 1; 310 static int ix_msix_enable = 1; 311 static int ix_rxr = 0; 312 static int ix_txr = 0; 313 static int ix_txd = IX_PERF_TXD; 314 static int ix_rxd = IX_PERF_RXD; 315 static int ix_unsupported_sfp = 0; 316 static int ix_direct_input = 1; 317 318 static char ix_flowctrl[IFM_ETH_FC_STRLEN] = IFM_ETH_FC_NONE; 319 320 TUNABLE_INT("hw.ix.msi.enable", &ix_msi_enable); 321 TUNABLE_INT("hw.ix.msix.enable", &ix_msix_enable); 322 TUNABLE_INT("hw.ix.rxr", &ix_rxr); 323 TUNABLE_INT("hw.ix.txr", &ix_txr); 324 TUNABLE_INT("hw.ix.txd", &ix_txd); 325 TUNABLE_INT("hw.ix.rxd", &ix_rxd); 326 TUNABLE_INT("hw.ix.unsupported_sfp", &ix_unsupported_sfp); 327 TUNABLE_STR("hw.ix.flow_ctrl", ix_flowctrl, sizeof(ix_flowctrl)); 328 TUNABLE_INT("hw.ix.direct_input", &ix_direct_input); 329 330 /* 331 * Smart speed setting, default to on. This only works 332 * as a compile option right now as its during attach, 333 * set this to 'ixgbe_smart_speed_off' to disable. 334 */ 335 static const enum ixgbe_smart_speed ix_smart_speed = 336 ixgbe_smart_speed_on; 337 338 static __inline void 339 ix_try_txgc(struct ix_tx_ring *txr, int8_t dec) 340 { 341 342 if (txr->tx_running > 0) { 343 txr->tx_running -= dec; 344 if (txr->tx_running <= 0 && txr->tx_nmbuf && 345 txr->tx_avail < txr->tx_ndesc && 346 txr->tx_avail + txr->tx_intr_nsegs > txr->tx_ndesc) 347 ix_txgc(txr); 348 } 349 } 350 351 static void 352 ix_txgc_timer(void *xtxr) 353 { 354 struct ix_tx_ring *txr = xtxr; 355 struct ifnet *ifp = &txr->tx_sc->arpcom.ac_if; 356 357 if ((ifp->if_flags & (IFF_RUNNING | IFF_UP | IFF_NPOLLING)) != 358 (IFF_RUNNING | IFF_UP)) 359 return; 360 361 if (!lwkt_serialize_try(&txr->tx_serialize)) 362 goto done; 363 364 if ((ifp->if_flags & (IFF_RUNNING | IFF_UP | IFF_NPOLLING)) != 365 (IFF_RUNNING | IFF_UP)) { 366 lwkt_serialize_exit(&txr->tx_serialize); 367 return; 368 } 369 ix_try_txgc(txr, IX_TX_RUNNING_DEC); 370 371 lwkt_serialize_exit(&txr->tx_serialize); 372 done: 373 callout_reset(&txr->tx_gc_timer, 1, ix_txgc_timer, txr); 374 } 375 376 static __inline void 377 ix_tx_intr(struct ix_tx_ring *txr, int hdr) 378 { 379 380 ix_txeof(txr, hdr); 381 if (!ifsq_is_empty(txr->tx_ifsq)) 382 ifsq_devstart(txr->tx_ifsq); 383 } 384 385 static __inline void 386 ix_free_txbuf(struct ix_tx_ring *txr, struct ix_tx_buf *txbuf) 387 { 388 389 KKASSERT(txbuf->m_head != NULL); 390 KKASSERT(txr->tx_nmbuf > 0); 391 txr->tx_nmbuf--; 392 393 bus_dmamap_unload(txr->tx_tag, txbuf->map); 394 m_freem(txbuf->m_head); 395 txbuf->m_head = NULL; 396 } 397 398 static int 399 ix_probe(device_t dev) 400 { 401 const struct ix_device *d; 402 uint16_t vid, did; 403 404 vid = pci_get_vendor(dev); 405 did = pci_get_device(dev); 406 407 for (d = ix_devices; d->desc != NULL; ++d) { 408 if (vid == d->vid && did == d->did) { 409 device_set_desc(dev, d->desc); 410 return 0; 411 } 412 } 413 return ENXIO; 414 } 415 416 static void 417 ix_get_rxring_cnt(const struct ix_softc *sc, int *ring_cnt, int *ring_cntmax) 418 { 419 420 switch (sc->hw.mac.type) { 421 case ixgbe_mac_X550: 422 case ixgbe_mac_X550EM_x: 423 case ixgbe_mac_X550EM_a: 424 *ring_cntmax = IX_MAX_RXRING_X550; 425 break; 426 427 default: 428 *ring_cntmax = IX_MAX_RXRING; 429 break; 430 } 431 *ring_cnt = device_getenv_int(sc->dev, "rxr", ix_rxr); 432 } 433 434 static void 435 ix_get_txring_cnt(const struct ix_softc *sc, int *ring_cnt, int *ring_cntmax) 436 { 437 438 switch (sc->hw.mac.type) { 439 case ixgbe_mac_82598EB: 440 *ring_cntmax = IX_MAX_TXRING_82598; 441 break; 442 443 case ixgbe_mac_82599EB: 444 *ring_cntmax = IX_MAX_TXRING_82599; 445 break; 446 447 case ixgbe_mac_X540: 448 *ring_cntmax = IX_MAX_TXRING_X540; 449 break; 450 451 case ixgbe_mac_X550: 452 case ixgbe_mac_X550EM_x: 453 case ixgbe_mac_X550EM_a: 454 *ring_cntmax = IX_MAX_TXRING_X550; 455 break; 456 457 default: 458 *ring_cntmax = IX_MAX_TXRING; 459 break; 460 } 461 *ring_cnt = device_getenv_int(sc->dev, "txr", ix_txr); 462 } 463 464 static int 465 ix_attach(device_t dev) 466 { 467 struct ix_softc *sc = device_get_softc(dev); 468 struct ixgbe_hw *hw; 469 int error, ring_cnt, ring_cntmax; 470 uint32_t ctrl_ext; 471 char flowctrl[IFM_ETH_FC_STRLEN]; 472 473 sc->dev = dev; 474 hw = &sc->hw; 475 hw->back = sc; 476 477 if_initname(&sc->arpcom.ac_if, device_get_name(dev), 478 device_get_unit(dev)); 479 ifmedia_init(&sc->media, IFM_IMASK | IFM_ETH_FCMASK, 480 ix_media_change, ix_media_status); 481 482 /* Save frame size */ 483 sc->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN; 484 485 sc->direct_input = ix_direct_input; 486 TASK_INIT(&sc->wdog_task, 0, ix_watchdog_task, sc); 487 488 callout_init_mp(&sc->fw_timer); 489 callout_init_mp(&sc->timer); 490 lwkt_serialize_init(&sc->main_serialize); 491 492 /* 493 * Save off the information about this board 494 */ 495 hw->vendor_id = pci_get_vendor(dev); 496 hw->device_id = pci_get_device(dev); 497 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); 498 hw->subsystem_vendor_id = pci_read_config(dev, PCIR_SUBVEND_0, 2); 499 hw->subsystem_device_id = pci_read_config(dev, PCIR_SUBDEV_0, 2); 500 501 /* Enable bus mastering */ 502 pci_enable_busmaster(dev); 503 504 /* 505 * Allocate IO memory 506 */ 507 sc->mem_rid = PCIR_BAR(0); 508 sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 509 &sc->mem_rid, RF_ACTIVE); 510 if (sc->mem_res == NULL) { 511 device_printf(dev, "Unable to allocate bus resource: memory\n"); 512 error = ENXIO; 513 goto failed; 514 } 515 516 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->mem_res); 517 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->mem_res); 518 519 sc->hw.hw_addr = (uint8_t *)&sc->osdep.mem_bus_space_handle; 520 521 /* Let hardware know driver is loaded */ 522 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 523 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 524 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 525 526 /* 527 * Initialize the shared code 528 */ 529 if (ixgbe_init_shared_code(hw)) { 530 device_printf(dev, "Unable to initialize the shared code\n"); 531 error = ENXIO; 532 goto failed; 533 } 534 535 if (hw->mbx.ops.init_params) 536 hw->mbx.ops.init_params(hw); 537 538 hw->allow_unsupported_sfp = ix_unsupported_sfp; 539 540 /* Pick up the 82599 settings */ 541 if (hw->mac.type != ixgbe_mac_82598EB) 542 hw->phy.smart_speed = ix_smart_speed; 543 544 /* Setup hardware capabilities */ 545 ix_setup_caps(sc); 546 547 /* Allocate multicast array memory. */ 548 sc->mta = kmalloc(sizeof(*sc->mta) * IX_MAX_MCASTADDR, 549 M_DEVBUF, M_WAITOK); 550 551 /* Save initial wake up filter configuration; WOL is disabled. */ 552 sc->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC); 553 554 /* Verify adapter fan is still functional (if applicable) */ 555 if (sc->caps & IX_CAP_DETECT_FANFAIL) 556 ix_detect_fanfail(sc, IXGBE_READ_REG(hw, IXGBE_ESDP), FALSE); 557 558 /* Ensure SW/FW semaphore is free */ 559 ixgbe_init_swfw_semaphore(hw); 560 561 #ifdef notyet 562 /* Enable EEE power saving */ 563 if (sc->caps & IX_CAP_EEE) 564 hw->mac.ops.setup_eee(hw, true); 565 #endif 566 567 /* 568 * Configure total supported RX/TX ring count 569 */ 570 ix_get_rxring_cnt(sc, &ring_cnt, &ring_cntmax); 571 sc->rx_rmap = if_ringmap_alloc(dev, ring_cnt, ring_cntmax); 572 ix_get_txring_cnt(sc, &ring_cnt, &ring_cntmax); 573 sc->tx_rmap = if_ringmap_alloc(dev, ring_cnt, ring_cntmax); 574 if_ringmap_match(dev, sc->rx_rmap, sc->tx_rmap); 575 576 sc->rx_ring_cnt = if_ringmap_count(sc->rx_rmap); 577 sc->rx_ring_inuse = sc->rx_ring_cnt; 578 sc->tx_ring_cnt = if_ringmap_count(sc->tx_rmap); 579 sc->tx_ring_inuse = sc->tx_ring_cnt; 580 581 /* Allocate TX/RX rings */ 582 error = ix_alloc_rings(sc); 583 if (error) 584 goto failed; 585 586 /* Allocate interrupt */ 587 error = ix_alloc_intr(sc); 588 if (error) 589 goto failed; 590 591 /* Setup serializes */ 592 ix_setup_serialize(sc); 593 594 hw->phy.reset_if_overtemp = TRUE; 595 error = ixgbe_reset_hw(hw); 596 hw->phy.reset_if_overtemp = FALSE; 597 if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 598 /* 599 * No optics in this port; ask timer routine 600 * to probe for later insertion. 601 */ 602 sc->sfp_probe = TRUE; 603 error = 0; 604 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 605 device_printf(dev, "Unsupported SFP+ module detected!\n"); 606 error = EIO; 607 goto failed; 608 } else if (error) { 609 device_printf(dev, "Hardware initialization failed\n"); 610 error = EIO; 611 goto failed; 612 } 613 614 /* Make sure we have a good EEPROM before we read from it */ 615 if (ixgbe_validate_eeprom_checksum(&sc->hw, NULL) < 0) { 616 device_printf(dev, "The EEPROM Checksum Is Not Valid\n"); 617 error = EIO; 618 goto failed; 619 } 620 621 error = ixgbe_start_hw(hw); 622 if (error == IXGBE_ERR_EEPROM_VERSION) { 623 device_printf(dev, "Pre-production device detected\n"); 624 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 625 device_printf(dev, "Unsupported SFP+ Module\n"); 626 error = EIO; 627 goto failed; 628 } else if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 629 device_printf(dev, "No SFP+ Module found\n"); 630 } 631 632 /* Enable the optics for 82599 SFP+ fiber */ 633 ixgbe_enable_tx_laser(hw); 634 635 /* Enable power to the phy. */ 636 ixgbe_set_phy_power(hw, TRUE); 637 638 sc->ifm_media = IX_IFM_DEFAULT; 639 /* Get default flow control settings */ 640 device_getenv_string(dev, "flow_ctrl", flowctrl, sizeof(flowctrl), 641 ix_flowctrl); 642 sc->ifm_media |= ifmedia_str2ethfc(flowctrl); 643 sc->advspeed = IXGBE_LINK_SPEED_UNKNOWN; 644 645 /* Setup OS specific network interface */ 646 ix_setup_ifp(sc); 647 648 /* Add sysctl tree */ 649 ix_add_sysctl(sc); 650 651 error = ix_setup_intr(sc); 652 if (error) { 653 ether_ifdetach(&sc->arpcom.ac_if); 654 goto failed; 655 } 656 657 /* Initialize statistics */ 658 ix_update_stats(sc); 659 660 /* Check PCIE slot type/speed/width */ 661 ix_slot_info(sc); 662 663 if (sc->caps & IX_CAP_FW_RECOVERY) { 664 device_printf(dev, "start fw timer\n"); 665 callout_reset_bycpu(&sc->fw_timer, hz, 666 ix_fw_timer, sc, ix_get_timer_cpuid(sc, FALSE)); 667 } 668 669 return 0; 670 failed: 671 ix_detach(dev); 672 return error; 673 } 674 675 static int 676 ix_detach(device_t dev) 677 { 678 struct ix_softc *sc = device_get_softc(dev); 679 680 if (device_is_attached(dev)) { 681 struct ifnet *ifp = &sc->arpcom.ac_if; 682 683 ix_sync_netisr(sc, IFF_UP); 684 taskqueue_drain(taskqueue_thread[0], &sc->wdog_task); 685 686 ifnet_serialize_all(ifp); 687 688 ix_powerdown(sc); 689 ix_teardown_intr(sc, sc->intr_cnt); 690 691 ifnet_deserialize_all(ifp); 692 693 callout_terminate(&sc->timer); 694 ether_ifdetach(ifp); 695 } 696 callout_terminate(&sc->fw_timer); 697 698 if (sc->mem_res != NULL) { 699 uint32_t ctrl_ext; 700 701 /* Let hardware know driver is unloading */ 702 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 703 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 704 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 705 } 706 707 ifmedia_removeall(&sc->media); 708 bus_generic_detach(dev); 709 710 ix_free_intr(sc); 711 712 if (sc->msix_mem_res != NULL) { 713 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_mem_rid, 714 sc->msix_mem_res); 715 } 716 if (sc->mem_res != NULL) { 717 bus_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid, 718 sc->mem_res); 719 } 720 721 ix_free_rings(sc); 722 723 if (sc->mta != NULL) 724 kfree(sc->mta, M_DEVBUF); 725 if (sc->serializes != NULL) 726 kfree(sc->serializes, M_DEVBUF); 727 728 if (sc->rx_rmap != NULL) 729 if_ringmap_free(sc->rx_rmap); 730 if (sc->rx_rmap_intr != NULL) 731 if_ringmap_free(sc->rx_rmap_intr); 732 if (sc->tx_rmap != NULL) 733 if_ringmap_free(sc->tx_rmap); 734 if (sc->tx_rmap_intr != NULL) 735 if_ringmap_free(sc->tx_rmap_intr); 736 737 return 0; 738 } 739 740 static int 741 ix_shutdown(device_t dev) 742 { 743 struct ix_softc *sc = device_get_softc(dev); 744 struct ifnet *ifp = &sc->arpcom.ac_if; 745 746 ix_sync_netisr(sc, IFF_UP); 747 taskqueue_drain(taskqueue_thread[0], &sc->wdog_task); 748 749 ifnet_serialize_all(ifp); 750 ix_powerdown(sc); 751 ifnet_deserialize_all(ifp); 752 753 return 0; 754 } 755 756 static void 757 ix_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 758 { 759 struct ix_softc *sc = ifp->if_softc; 760 struct ix_tx_ring *txr = ifsq_get_priv(ifsq); 761 int idx = -1; 762 uint16_t nsegs; 763 764 KKASSERT(txr->tx_ifsq == ifsq); 765 ASSERT_SERIALIZED(&txr->tx_serialize); 766 767 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq)) 768 return; 769 770 if (!sc->link_active || (txr->tx_flags & IX_TXFLAG_ENABLED) == 0) { 771 ifsq_purge(ifsq); 772 return; 773 } 774 775 while (!ifsq_is_empty(ifsq)) { 776 struct mbuf *m_head; 777 778 if (txr->tx_avail <= IX_MAX_SCATTER + IX_TX_RESERVED) { 779 ifsq_set_oactive(ifsq); 780 ifsq_watchdog_set_count(&txr->tx_watchdog, 5); 781 break; 782 } 783 784 m_head = ifsq_dequeue(ifsq); 785 if (m_head == NULL) 786 break; 787 788 if (ix_encap(txr, &m_head, &nsegs, &idx)) { 789 IFNET_STAT_INC(ifp, oerrors, 1); 790 continue; 791 } 792 793 /* 794 * TX interrupt are aggressively aggregated, so increasing 795 * opackets at TX interrupt time will make the opackets 796 * statistics vastly inaccurate; we do the opackets increment 797 * now. 798 */ 799 IFNET_STAT_INC(ifp, opackets, 1); 800 801 if (nsegs >= txr->tx_wreg_nsegs) { 802 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->tx_idx), idx); 803 nsegs = 0; 804 idx = -1; 805 } 806 807 ETHER_BPF_MTAP(ifp, m_head); 808 } 809 if (idx >= 0) 810 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->tx_idx), idx); 811 txr->tx_running = IX_TX_RUNNING; 812 } 813 814 static int 815 ix_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) 816 { 817 struct ix_softc *sc = ifp->if_softc; 818 struct ifreq *ifr = (struct ifreq *) data; 819 int error = 0, mask, reinit; 820 821 ASSERT_IFNET_SERIALIZED_ALL(ifp); 822 823 switch (command) { 824 case SIOCSIFMTU: 825 if (ifr->ifr_mtu > IX_MAX_MTU) { 826 error = EINVAL; 827 } else { 828 ifp->if_mtu = ifr->ifr_mtu; 829 sc->max_frame_size = ifp->if_mtu + IX_MTU_HDR; 830 ix_init(sc); 831 } 832 break; 833 834 case SIOCSIFFLAGS: 835 if (ifp->if_flags & IFF_UP) { 836 if (ifp->if_flags & IFF_RUNNING) { 837 if ((ifp->if_flags ^ sc->if_flags) & 838 (IFF_PROMISC | IFF_ALLMULTI)) 839 ix_set_promisc(sc); 840 } else { 841 ix_init(sc); 842 } 843 } else if (ifp->if_flags & IFF_RUNNING) { 844 ix_stop(sc); 845 } 846 sc->if_flags = ifp->if_flags; 847 break; 848 849 case SIOCADDMULTI: 850 case SIOCDELMULTI: 851 if (ifp->if_flags & IFF_RUNNING) { 852 ix_disable_intr(sc); 853 ix_set_multi(sc); 854 #ifdef IFPOLL_ENABLE 855 if ((ifp->if_flags & IFF_NPOLLING) == 0) 856 #endif 857 ix_enable_intr(sc); 858 } 859 break; 860 861 case SIOCSIFMEDIA: 862 case SIOCGIFMEDIA: 863 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 864 break; 865 866 case SIOCSIFCAP: 867 reinit = 0; 868 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 869 if (mask & IFCAP_RXCSUM) { 870 ifp->if_capenable ^= IFCAP_RXCSUM; 871 reinit = 1; 872 } 873 if (mask & IFCAP_VLAN_HWTAGGING) { 874 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 875 reinit = 1; 876 } 877 if (mask & IFCAP_TXCSUM) { 878 ifp->if_capenable ^= IFCAP_TXCSUM; 879 if (ifp->if_capenable & IFCAP_TXCSUM) 880 ifp->if_hwassist |= CSUM_OFFLOAD; 881 else 882 ifp->if_hwassist &= ~CSUM_OFFLOAD; 883 } 884 if (mask & IFCAP_TSO) { 885 ifp->if_capenable ^= IFCAP_TSO; 886 if (ifp->if_capenable & IFCAP_TSO) 887 ifp->if_hwassist |= CSUM_TSO; 888 else 889 ifp->if_hwassist &= ~CSUM_TSO; 890 } 891 if (mask & IFCAP_RSS) 892 ifp->if_capenable ^= IFCAP_RSS; 893 if (reinit && (ifp->if_flags & IFF_RUNNING)) 894 ix_init(sc); 895 break; 896 897 #if 0 898 case SIOCGI2C: 899 { 900 struct ixgbe_i2c_req i2c; 901 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 902 if (error) 903 break; 904 if ((i2c.dev_addr != 0xA0) || (i2c.dev_addr != 0xA2)){ 905 error = EINVAL; 906 break; 907 } 908 hw->phy.ops.read_i2c_byte(hw, i2c.offset, 909 i2c.dev_addr, i2c.data); 910 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 911 break; 912 } 913 #endif 914 915 default: 916 error = ether_ioctl(ifp, command, data); 917 break; 918 } 919 return error; 920 } 921 922 #define IXGBE_MHADD_MFS_SHIFT 16 923 924 static void 925 ix_init(void *xsc) 926 { 927 struct ix_softc *sc = xsc; 928 struct ifnet *ifp = &sc->arpcom.ac_if; 929 struct ixgbe_hw *hw = &sc->hw; 930 uint32_t rxctrl; 931 int i, error; 932 boolean_t polling; 933 934 ASSERT_IFNET_SERIALIZED_ALL(ifp); 935 936 ix_stop(sc); 937 938 if (sc->flags & IX_FLAG_FW_RECOVERY) 939 return; 940 941 polling = FALSE; 942 #ifdef IFPOLL_ENABLE 943 if (ifp->if_flags & IFF_NPOLLING) 944 polling = TRUE; 945 #endif 946 947 /* Configure # of used RX/TX rings */ 948 ix_set_ring_inuse(sc, polling); 949 ifq_set_subq_divisor(&ifp->if_snd, sc->tx_ring_inuse); 950 951 /* Get the latest mac address, User can use a LAA */ 952 bcopy(IF_LLADDR(ifp), hw->mac.addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 953 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 954 hw->addr_ctrl.rar_used_count = 1; 955 956 /* Prepare transmit descriptors and buffers */ 957 for (i = 0; i < sc->tx_ring_inuse; ++i) 958 ix_init_tx_ring(&sc->tx_rings[i]); 959 960 ixgbe_init_hw(hw); 961 ix_init_tx_unit(sc); 962 963 /* Setup Multicast table */ 964 ix_set_multi(sc); 965 966 /* Prepare receive descriptors and buffers */ 967 for (i = 0; i < sc->rx_ring_inuse; ++i) { 968 error = ix_init_rx_ring(&sc->rx_rings[i]); 969 if (error) { 970 if_printf(ifp, "Could not initialize RX ring%d\n", i); 971 ix_stop(sc); 972 return; 973 } 974 } 975 976 /* Configure RX settings */ 977 ix_init_rx_unit(sc, polling); 978 979 /* Enable SDP & MSI-X interrupts based on adapter */ 980 ix_config_gpie(sc); 981 982 /* Set MTU size */ 983 if (ifp->if_mtu > ETHERMTU) { 984 uint32_t mhadd; 985 986 /* aka IXGBE_MAXFRS on 82599 and newer */ 987 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 988 mhadd &= ~IXGBE_MHADD_MFS_MASK; 989 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 990 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 991 } 992 993 /* 994 * Enable TX rings 995 */ 996 for (i = 0; i < sc->tx_ring_inuse; ++i) { 997 uint32_t txdctl; 998 999 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 1000 txdctl |= IXGBE_TXDCTL_ENABLE; 1001 1002 /* 1003 * Set WTHRESH to 0, since TX head write-back is used 1004 */ 1005 txdctl &= ~(0x7f << 16); 1006 1007 /* 1008 * When the internal queue falls below PTHRESH (32), 1009 * start prefetching as long as there are at least 1010 * HTHRESH (1) buffers ready. The values are taken 1011 * from the Intel linux driver 3.8.21. 1012 * Prefetching enables tx line rate even with 1 queue. 1013 */ 1014 txdctl |= (32 << 0) | (1 << 8); 1015 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl); 1016 } 1017 1018 /* 1019 * Enable RX rings 1020 */ 1021 for (i = 0; i < sc->rx_ring_inuse; ++i) { 1022 uint32_t rxdctl; 1023 int k; 1024 1025 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 1026 if (hw->mac.type == ixgbe_mac_82598EB) { 1027 /* 1028 * PTHRESH = 21 1029 * HTHRESH = 4 1030 * WTHRESH = 8 1031 */ 1032 rxdctl &= ~0x3FFFFF; 1033 rxdctl |= 0x080420; 1034 } 1035 rxdctl |= IXGBE_RXDCTL_ENABLE; 1036 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl); 1037 for (k = 0; k < 10; ++k) { 1038 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & 1039 IXGBE_RXDCTL_ENABLE) 1040 break; 1041 else 1042 msec_delay(1); 1043 } 1044 wmb(); 1045 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 1046 sc->rx_rings[0].rx_ndesc - 1); 1047 } 1048 1049 /* Enable Receive engine */ 1050 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 1051 if (hw->mac.type == ixgbe_mac_82598EB) 1052 rxctrl |= IXGBE_RXCTRL_DMBYPS; 1053 rxctrl |= IXGBE_RXCTRL_RXEN; 1054 ixgbe_enable_rx_dma(hw, rxctrl); 1055 1056 for (i = 0; i < sc->tx_ring_inuse; ++i) { 1057 const struct ix_tx_ring *txr = &sc->tx_rings[i]; 1058 1059 if (txr->tx_intr_vec >= 0) { 1060 ix_set_ivar(sc, i, txr->tx_intr_vec, 1); 1061 } else if (!polling) { 1062 /* 1063 * Unconfigured TX interrupt vector could only 1064 * happen for MSI-X. 1065 */ 1066 KASSERT(sc->intr_type == PCI_INTR_TYPE_MSIX, 1067 ("TX intr vector is not set")); 1068 if (bootverbose) 1069 if_printf(ifp, "IVAR skips TX ring %d\n", i); 1070 } 1071 } 1072 for (i = 0; i < sc->rx_ring_inuse; ++i) { 1073 const struct ix_rx_ring *rxr = &sc->rx_rings[i]; 1074 1075 if (polling && rxr->rx_intr_vec < 0) 1076 continue; 1077 1078 KKASSERT(rxr->rx_intr_vec >= 0); 1079 ix_set_ivar(sc, i, rxr->rx_intr_vec, 0); 1080 if (rxr->rx_txr != NULL) { 1081 /* 1082 * Piggyback the TX ring interrupt onto the RX 1083 * ring interrupt vector. 1084 */ 1085 KASSERT(rxr->rx_txr->tx_intr_vec < 0, 1086 ("piggybacked TX ring configured intr vector")); 1087 ix_set_ivar(sc, rxr->rx_txr->tx_idx, 1088 rxr->rx_intr_vec, 1); 1089 if (bootverbose) { 1090 if_printf(ifp, "IVAR RX ring %d piggybacks " 1091 "TX ring %u\n", i, rxr->rx_txr->tx_idx); 1092 } 1093 } 1094 } 1095 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 1096 /* Set up status MSI-X vector; it is using fixed entry 1 */ 1097 ix_set_ivar(sc, 1, sc->sts_msix_vec, -1); 1098 1099 /* Set up auto-mask for TX and RX rings */ 1100 if (hw->mac.type == ixgbe_mac_82598EB) { 1101 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EIMS_RTX_QUEUE); 1102 } else { 1103 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 1104 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 1105 } 1106 } else { 1107 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EIMS_RTX_QUEUE); 1108 } 1109 for (i = 0; i < sc->intr_cnt; ++i) 1110 ix_set_eitr(sc, i, sc->intr_data[i].intr_rate); 1111 1112 /* 1113 * Check on any SFP devices that need to be kick-started 1114 */ 1115 if (hw->phy.type == ixgbe_phy_none) { 1116 error = hw->phy.ops.identify(hw); 1117 if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1118 if_printf(ifp, 1119 "Unsupported SFP+ module type was detected.\n"); 1120 /* XXX stop */ 1121 return; 1122 } 1123 } 1124 1125 /* Config/Enable Link */ 1126 ix_config_link(sc); 1127 1128 /* Hardware Packet Buffer & Flow Control setup */ 1129 ix_config_flowctrl(sc); 1130 1131 /* Initialize the FC settings */ 1132 ixgbe_start_hw(hw); 1133 1134 /* Set up VLAN support and filter */ 1135 ix_set_vlan(sc); 1136 1137 /* Setup DMA Coalescing */ 1138 ix_config_dmac(sc); 1139 1140 /* 1141 * Only enable interrupts if we are not polling, make sure 1142 * they are off otherwise. 1143 */ 1144 if (polling) 1145 ix_disable_intr(sc); 1146 else 1147 ix_enable_intr(sc); 1148 1149 ifp->if_flags |= IFF_RUNNING; 1150 for (i = 0; i < sc->tx_ring_inuse; ++i) { 1151 struct ix_tx_ring *txr = &sc->tx_rings[i]; 1152 1153 ifsq_clr_oactive(txr->tx_ifsq); 1154 ifsq_watchdog_start(&txr->tx_watchdog); 1155 1156 if (!polling) { 1157 callout_reset_bycpu(&txr->tx_gc_timer, 1, 1158 ix_txgc_timer, txr, txr->tx_intr_cpuid); 1159 } 1160 } 1161 1162 sc->timer_cpuid = ix_get_timer_cpuid(sc, polling); 1163 callout_reset_bycpu(&sc->timer, hz, ix_timer, sc, sc->timer_cpuid); 1164 } 1165 1166 static void 1167 ix_intr(void *xsc) 1168 { 1169 struct ix_softc *sc = xsc; 1170 struct ixgbe_hw *hw = &sc->hw; 1171 uint32_t eicr; 1172 1173 ASSERT_SERIALIZED(&sc->main_serialize); 1174 1175 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1176 if (eicr == 0) { 1177 IXGBE_WRITE_REG(hw, IXGBE_EIMS, sc->intr_mask); 1178 return; 1179 } 1180 1181 if (eicr & IX_RX0_INTR_MASK) { 1182 struct ix_rx_ring *rxr = &sc->rx_rings[0]; 1183 1184 lwkt_serialize_enter(&rxr->rx_serialize); 1185 ix_rxeof(rxr, -1); 1186 lwkt_serialize_exit(&rxr->rx_serialize); 1187 } 1188 if (eicr & IX_RX1_INTR_MASK) { 1189 struct ix_rx_ring *rxr; 1190 1191 KKASSERT(sc->rx_ring_inuse == IX_MIN_RXRING_RSS); 1192 rxr = &sc->rx_rings[1]; 1193 1194 lwkt_serialize_enter(&rxr->rx_serialize); 1195 ix_rxeof(rxr, -1); 1196 lwkt_serialize_exit(&rxr->rx_serialize); 1197 } 1198 1199 if (eicr & IX_TX_INTR_MASK) { 1200 struct ix_tx_ring *txr = &sc->tx_rings[0]; 1201 1202 lwkt_serialize_enter(&txr->tx_serialize); 1203 ix_tx_intr(txr, *(txr->tx_hdr)); 1204 lwkt_serialize_exit(&txr->tx_serialize); 1205 } 1206 1207 if (__predict_false(eicr & IX_EICR_STATUS)) 1208 ix_intr_status(sc, eicr); 1209 1210 IXGBE_WRITE_REG(hw, IXGBE_EIMS, sc->intr_mask); 1211 } 1212 1213 static void 1214 ix_intr_82598(void *xsc) 1215 { 1216 struct ix_softc *sc = xsc; 1217 1218 ASSERT_SERIALIZED(&sc->main_serialize); 1219 1220 /* Software workaround for 82598 errata #26 */ 1221 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); 1222 1223 ix_intr(sc); 1224 } 1225 1226 static void 1227 ix_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1228 { 1229 struct ix_softc *sc = ifp->if_softc; 1230 struct ifmedia *ifm = &sc->media; 1231 int layer; 1232 boolean_t link_active; 1233 1234 if (sc->flags & IX_FLAG_FW_RECOVERY) { 1235 link_active = FALSE; 1236 } else { 1237 ix_update_link_status(sc); 1238 link_active = sc->link_active; 1239 } 1240 1241 ifmr->ifm_status = IFM_AVALID; 1242 ifmr->ifm_active = IFM_ETHER; 1243 1244 if (!link_active) { 1245 if (IFM_SUBTYPE(ifm->ifm_media) != IFM_AUTO) 1246 ifmr->ifm_active |= ifm->ifm_media; 1247 else 1248 ifmr->ifm_active |= IFM_NONE; 1249 return; 1250 } 1251 ifmr->ifm_status |= IFM_ACTIVE; 1252 1253 layer = sc->phy_layer; 1254 1255 if ((layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) || 1256 (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) || 1257 (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) || 1258 (layer & IXGBE_PHYSICAL_LAYER_10BASE_T)) { 1259 switch (sc->link_speed) { 1260 case IXGBE_LINK_SPEED_10GB_FULL: 1261 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1262 break; 1263 case IXGBE_LINK_SPEED_1GB_FULL: 1264 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1265 break; 1266 case IXGBE_LINK_SPEED_100_FULL: 1267 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1268 break; 1269 case IXGBE_LINK_SPEED_10_FULL: 1270 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1271 break; 1272 } 1273 } else if ((layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) || 1274 (layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)) { 1275 switch (sc->link_speed) { 1276 case IXGBE_LINK_SPEED_10GB_FULL: 1277 ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX; 1278 break; 1279 } 1280 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1281 switch (sc->link_speed) { 1282 case IXGBE_LINK_SPEED_10GB_FULL: 1283 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 1284 break; 1285 case IXGBE_LINK_SPEED_1GB_FULL: 1286 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 1287 break; 1288 } 1289 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM) { 1290 switch (sc->link_speed) { 1291 case IXGBE_LINK_SPEED_10GB_FULL: 1292 ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX; 1293 break; 1294 case IXGBE_LINK_SPEED_1GB_FULL: 1295 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 1296 break; 1297 } 1298 } else if ((layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) || 1299 (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)) { 1300 switch (sc->link_speed) { 1301 case IXGBE_LINK_SPEED_10GB_FULL: 1302 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 1303 break; 1304 case IXGBE_LINK_SPEED_1GB_FULL: 1305 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 1306 break; 1307 } 1308 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1309 switch (sc->link_speed) { 1310 case IXGBE_LINK_SPEED_10GB_FULL: 1311 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 1312 break; 1313 } 1314 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1315 /* 1316 * XXX: These need to use the proper media types once 1317 * they're added. 1318 */ 1319 switch (sc->link_speed) { 1320 case IXGBE_LINK_SPEED_10GB_FULL: 1321 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 1322 break; 1323 case IXGBE_LINK_SPEED_2_5GB_FULL: 1324 ifmr->ifm_active |= IFM_2500_SX | IFM_FDX; 1325 break; 1326 case IXGBE_LINK_SPEED_1GB_FULL: 1327 ifmr->ifm_active |= IFM_1000_CX | IFM_FDX; 1328 break; 1329 } 1330 } else if ((layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) || 1331 (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) || 1332 (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)) { 1333 /* 1334 * XXX: These need to use the proper media types once 1335 * they're added. 1336 */ 1337 switch (sc->link_speed) { 1338 case IXGBE_LINK_SPEED_10GB_FULL: 1339 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 1340 break; 1341 case IXGBE_LINK_SPEED_2_5GB_FULL: 1342 ifmr->ifm_active |= IFM_2500_SX | IFM_FDX; 1343 break; 1344 case IXGBE_LINK_SPEED_1GB_FULL: 1345 ifmr->ifm_active |= IFM_1000_CX | IFM_FDX; 1346 break; 1347 } 1348 } 1349 1350 /* If nothing is recognized... */ 1351 if (IFM_SUBTYPE(ifmr->ifm_active) == 0) 1352 ifmr->ifm_active |= IFM_NONE; 1353 1354 if (sc->ifm_media & IFM_ETH_FORCEPAUSE) 1355 ifmr->ifm_active |= (sc->ifm_media & IFM_ETH_FCMASK); 1356 1357 switch (sc->hw.fc.current_mode) { 1358 case ixgbe_fc_full: 1359 ifmr->ifm_active |= IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE; 1360 break; 1361 case ixgbe_fc_rx_pause: 1362 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 1363 break; 1364 case ixgbe_fc_tx_pause: 1365 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 1366 break; 1367 default: 1368 break; 1369 } 1370 } 1371 1372 static int 1373 ix_media_change(struct ifnet *ifp) 1374 { 1375 struct ix_softc *sc = ifp->if_softc; 1376 struct ifmedia *ifm = &sc->media; 1377 struct ixgbe_hw *hw = &sc->hw; 1378 1379 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1380 return (EINVAL); 1381 1382 if (hw->phy.media_type == ixgbe_media_type_backplane || 1383 hw->mac.ops.setup_link == NULL) { 1384 if ((ifm->ifm_media ^ sc->ifm_media) & IFM_ETH_FCMASK) { 1385 /* Only flow control setting changes are allowed */ 1386 return (EOPNOTSUPP); 1387 } 1388 } 1389 1390 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1391 case IFM_AUTO: 1392 sc->advspeed = IXGBE_LINK_SPEED_UNKNOWN; 1393 break; 1394 1395 case IFM_10G_T: 1396 case IFM_10G_LRM: 1397 case IFM_10G_SR: /* XXX also KR */ 1398 case IFM_10G_LR: 1399 case IFM_10G_CX4: /* XXX also KX4 */ 1400 case IFM_10G_TWINAX: 1401 sc->advspeed = IXGBE_LINK_SPEED_10GB_FULL; 1402 break; 1403 1404 case IFM_1000_T: 1405 case IFM_1000_LX: 1406 case IFM_1000_SX: 1407 case IFM_1000_CX: /* XXX is KX */ 1408 sc->advspeed = IXGBE_LINK_SPEED_1GB_FULL; 1409 break; 1410 1411 case IFM_100_TX: 1412 sc->advspeed = IXGBE_LINK_SPEED_100_FULL; 1413 break; 1414 1415 default: 1416 if (bootverbose) { 1417 if_printf(ifp, "Invalid media type %d!\n", 1418 ifm->ifm_media); 1419 } 1420 return EINVAL; 1421 } 1422 sc->ifm_media = ifm->ifm_media; 1423 1424 #if 0 1425 if (hw->mac.ops.setup_link != NULL) { 1426 hw->mac.autotry_restart = TRUE; 1427 hw->mac.ops.setup_link(hw, sc->advspeed, TRUE); 1428 } 1429 #else 1430 if (ifp->if_flags & IFF_RUNNING) 1431 ix_init(sc); 1432 #endif 1433 return 0; 1434 } 1435 1436 static __inline int 1437 ix_tso_pullup(struct mbuf **mp) 1438 { 1439 int hoff, iphlen, thoff; 1440 struct mbuf *m; 1441 1442 m = *mp; 1443 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable")); 1444 1445 iphlen = m->m_pkthdr.csum_iphlen; 1446 thoff = m->m_pkthdr.csum_thlen; 1447 hoff = m->m_pkthdr.csum_lhlen; 1448 1449 KASSERT(iphlen > 0, ("invalid ip hlen")); 1450 KASSERT(thoff > 0, ("invalid tcp hlen")); 1451 KASSERT(hoff > 0, ("invalid ether hlen")); 1452 1453 if (__predict_false(m->m_len < hoff + iphlen + thoff)) { 1454 m = m_pullup(m, hoff + iphlen + thoff); 1455 if (m == NULL) { 1456 *mp = NULL; 1457 return ENOBUFS; 1458 } 1459 *mp = m; 1460 } 1461 return 0; 1462 } 1463 1464 static int 1465 ix_encap(struct ix_tx_ring *txr, struct mbuf **m_headp, 1466 uint16_t *segs_used, int *idx) 1467 { 1468 uint32_t olinfo_status = 0, cmd_type_len, cmd_rs = 0; 1469 int i, j, error, nsegs, first, maxsegs; 1470 struct mbuf *m_head = *m_headp; 1471 bus_dma_segment_t segs[IX_MAX_SCATTER]; 1472 bus_dmamap_t map; 1473 struct ix_tx_buf *txbuf; 1474 union ixgbe_adv_tx_desc *txd = NULL; 1475 1476 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1477 error = ix_tso_pullup(m_headp); 1478 if (__predict_false(error)) 1479 return error; 1480 m_head = *m_headp; 1481 } 1482 1483 /* Basic descriptor defines */ 1484 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1485 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1486 1487 if (m_head->m_flags & M_VLANTAG) 1488 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1489 1490 /* 1491 * Important to capture the first descriptor 1492 * used because it will contain the index of 1493 * the one we tell the hardware to report back 1494 */ 1495 first = txr->tx_next_avail; 1496 txbuf = &txr->tx_buf[first]; 1497 map = txbuf->map; 1498 1499 /* 1500 * Map the packet for DMA. 1501 */ 1502 maxsegs = txr->tx_avail - IX_TX_RESERVED; 1503 if (maxsegs > IX_MAX_SCATTER) 1504 maxsegs = IX_MAX_SCATTER; 1505 1506 error = bus_dmamap_load_mbuf_defrag(txr->tx_tag, map, m_headp, 1507 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 1508 if (__predict_false(error)) { 1509 m_freem(*m_headp); 1510 *m_headp = NULL; 1511 return error; 1512 } 1513 bus_dmamap_sync(txr->tx_tag, map, BUS_DMASYNC_PREWRITE); 1514 1515 m_head = *m_headp; 1516 1517 /* 1518 * Set up the appropriate offload context if requested, 1519 * this may consume one TX descriptor. 1520 */ 1521 if (ix_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status)) { 1522 (*segs_used)++; 1523 txr->tx_nsegs++; 1524 } 1525 1526 *segs_used += nsegs; 1527 txr->tx_nsegs += nsegs; 1528 if (txr->tx_nsegs >= txr->tx_intr_nsegs) { 1529 /* 1530 * Report Status (RS) is turned on every intr_nsegs 1531 * descriptors (roughly). 1532 */ 1533 txr->tx_nsegs = 0; 1534 cmd_rs = IXGBE_TXD_CMD_RS; 1535 } 1536 1537 i = txr->tx_next_avail; 1538 for (j = 0; j < nsegs; j++) { 1539 bus_size_t seglen; 1540 bus_addr_t segaddr; 1541 1542 txbuf = &txr->tx_buf[i]; 1543 txd = &txr->tx_base[i]; 1544 seglen = segs[j].ds_len; 1545 segaddr = htole64(segs[j].ds_addr); 1546 1547 txd->read.buffer_addr = segaddr; 1548 txd->read.cmd_type_len = htole32(IXGBE_TXD_CMD_IFCS | 1549 cmd_type_len |seglen); 1550 txd->read.olinfo_status = htole32(olinfo_status); 1551 1552 if (++i == txr->tx_ndesc) 1553 i = 0; 1554 } 1555 txd->read.cmd_type_len |= htole32(IXGBE_TXD_CMD_EOP | cmd_rs); 1556 1557 txr->tx_avail -= nsegs; 1558 txr->tx_next_avail = i; 1559 txr->tx_nmbuf++; 1560 1561 txbuf->m_head = m_head; 1562 txr->tx_buf[first].map = txbuf->map; 1563 txbuf->map = map; 1564 1565 /* 1566 * Defer TDT updating, until enough descrptors are setup 1567 */ 1568 *idx = i; 1569 1570 return 0; 1571 } 1572 1573 static void 1574 ix_set_promisc(struct ix_softc *sc) 1575 { 1576 struct ifnet *ifp = &sc->arpcom.ac_if; 1577 uint32_t reg_rctl; 1578 int mcnt = 0; 1579 1580 reg_rctl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1581 reg_rctl &= ~IXGBE_FCTRL_UPE; 1582 if (ifp->if_flags & IFF_ALLMULTI) { 1583 mcnt = IX_MAX_MCASTADDR; 1584 } else { 1585 struct ifmultiaddr *ifma; 1586 1587 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1588 if (ifma->ifma_addr->sa_family != AF_LINK) 1589 continue; 1590 if (mcnt == IX_MAX_MCASTADDR) 1591 break; 1592 mcnt++; 1593 } 1594 } 1595 if (mcnt < IX_MAX_MCASTADDR) 1596 reg_rctl &= ~IXGBE_FCTRL_MPE; 1597 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 1598 1599 if (ifp->if_flags & IFF_PROMISC) { 1600 reg_rctl |= IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE; 1601 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 1602 } else if (ifp->if_flags & IFF_ALLMULTI) { 1603 reg_rctl |= IXGBE_FCTRL_MPE; 1604 reg_rctl &= ~IXGBE_FCTRL_UPE; 1605 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 1606 } 1607 } 1608 1609 static void 1610 ix_set_multi(struct ix_softc *sc) 1611 { 1612 struct ifnet *ifp = &sc->arpcom.ac_if; 1613 struct ifmultiaddr *ifma; 1614 uint32_t fctrl; 1615 struct ix_mc_addr *mta; 1616 int mcnt = 0; 1617 1618 mta = sc->mta; 1619 bzero(mta, sizeof(*mta) * IX_MAX_MCASTADDR); 1620 1621 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1622 if (ifma->ifma_addr->sa_family != AF_LINK) 1623 continue; 1624 if (mcnt == IX_MAX_MCASTADDR) 1625 break; 1626 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1627 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 1628 mcnt++; 1629 } 1630 1631 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1632 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1633 if (ifp->if_flags & IFF_PROMISC) { 1634 fctrl |= IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE; 1635 } else if (mcnt >= IX_MAX_MCASTADDR || (ifp->if_flags & IFF_ALLMULTI)) { 1636 fctrl |= IXGBE_FCTRL_MPE; 1637 fctrl &= ~IXGBE_FCTRL_UPE; 1638 } else { 1639 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1640 } 1641 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1642 1643 if (mcnt < IX_MAX_MCASTADDR) { 1644 ixgbe_update_mc_addr_list(&sc->hw, 1645 (uint8_t *)mta, mcnt, ix_mc_array_itr, TRUE); 1646 } 1647 } 1648 1649 /* 1650 * This is an iterator function now needed by the multicast 1651 * shared code. It simply feeds the shared code routine the 1652 * addresses in the array of ix_set_multi() one by one. 1653 */ 1654 static uint8_t * 1655 ix_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1656 { 1657 struct ix_mc_addr *mta = (struct ix_mc_addr *)*update_ptr; 1658 1659 *vmdq = mta->vmdq; 1660 *update_ptr = (uint8_t *)(mta + 1); 1661 1662 return (mta->addr); 1663 } 1664 1665 static void 1666 ix_timer(void *arg) 1667 { 1668 struct ix_softc *sc = arg; 1669 1670 lwkt_serialize_enter(&sc->main_serialize); 1671 1672 if ((sc->arpcom.ac_if.if_flags & IFF_RUNNING) == 0) { 1673 lwkt_serialize_exit(&sc->main_serialize); 1674 return; 1675 } 1676 1677 /* Check for pluggable optics */ 1678 if (sc->sfp_probe) { 1679 if (!ix_sfp_probe(sc)) 1680 goto done; /* Nothing to do */ 1681 } 1682 1683 ix_update_link_status(sc); 1684 ix_update_stats(sc); 1685 1686 done: 1687 callout_reset_bycpu(&sc->timer, hz, ix_timer, sc, sc->timer_cpuid); 1688 lwkt_serialize_exit(&sc->main_serialize); 1689 } 1690 1691 static void 1692 ix_update_link_status(struct ix_softc *sc) 1693 { 1694 struct ifnet *ifp = &sc->arpcom.ac_if; 1695 1696 if (sc->link_up) { 1697 if (sc->link_active == FALSE) { 1698 if (bootverbose) { 1699 if_printf(ifp, "Link is up %d Gbps %s\n", 1700 sc->link_speed == 128 ? 10 : 1, 1701 "Full Duplex"); 1702 } 1703 1704 /* 1705 * Update any Flow Control changes 1706 */ 1707 ixgbe_fc_enable(&sc->hw); 1708 /* MUST after ixgbe_fc_enable() */ 1709 if (sc->rx_ring_inuse > 1) { 1710 switch (sc->hw.fc.current_mode) { 1711 case ixgbe_fc_rx_pause: 1712 case ixgbe_fc_tx_pause: 1713 case ixgbe_fc_full: 1714 ix_disable_rx_drop(sc); 1715 break; 1716 1717 case ixgbe_fc_none: 1718 ix_enable_rx_drop(sc); 1719 break; 1720 1721 default: 1722 break; 1723 } 1724 } 1725 1726 /* Update DMA coalescing config */ 1727 ix_config_dmac(sc); 1728 1729 sc->link_active = TRUE; 1730 1731 ifp->if_link_state = LINK_STATE_UP; 1732 if_link_state_change(ifp); 1733 } 1734 } else { /* Link down */ 1735 if (sc->link_active == TRUE) { 1736 if (bootverbose) 1737 if_printf(ifp, "Link is Down\n"); 1738 ifp->if_link_state = LINK_STATE_DOWN; 1739 if_link_state_change(ifp); 1740 1741 sc->link_active = FALSE; 1742 } 1743 } 1744 } 1745 1746 static void 1747 ix_stop(struct ix_softc *sc) 1748 { 1749 struct ixgbe_hw *hw = &sc->hw; 1750 struct ifnet *ifp = &sc->arpcom.ac_if; 1751 int i; 1752 1753 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1754 1755 ix_disable_intr(sc); 1756 callout_stop(&sc->timer); 1757 1758 ifp->if_flags &= ~IFF_RUNNING; 1759 for (i = 0; i < sc->tx_ring_cnt; ++i) { 1760 struct ix_tx_ring *txr = &sc->tx_rings[i]; 1761 1762 ifsq_clr_oactive(txr->tx_ifsq); 1763 ifsq_watchdog_stop(&txr->tx_watchdog); 1764 txr->tx_flags &= ~IX_TXFLAG_ENABLED; 1765 1766 txr->tx_running = 0; 1767 callout_stop(&txr->tx_gc_timer); 1768 } 1769 1770 ixgbe_reset_hw(hw); 1771 hw->adapter_stopped = FALSE; 1772 ixgbe_stop_adapter(hw); 1773 if (hw->mac.type == ixgbe_mac_82599EB) 1774 ixgbe_stop_mac_link_on_d3_82599(hw); 1775 /* Turn off the laser - noop with no optics */ 1776 ixgbe_disable_tx_laser(hw); 1777 1778 /* Update the stack */ 1779 sc->link_up = FALSE; 1780 ix_update_link_status(sc); 1781 1782 /* Reprogram the RAR[0] in case user changed it. */ 1783 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1784 1785 for (i = 0; i < sc->tx_ring_cnt; ++i) 1786 ix_free_tx_ring(&sc->tx_rings[i]); 1787 1788 for (i = 0; i < sc->rx_ring_cnt; ++i) 1789 ix_free_rx_ring(&sc->rx_rings[i]); 1790 } 1791 1792 static void 1793 ix_setup_ifp(struct ix_softc *sc) 1794 { 1795 struct ixgbe_hw *hw = &sc->hw; 1796 struct ifnet *ifp = &sc->arpcom.ac_if; 1797 int i; 1798 1799 ifp->if_baudrate = IF_Gbps(10UL); 1800 1801 ifp->if_softc = sc; 1802 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1803 ifp->if_init = ix_init; 1804 ifp->if_ioctl = ix_ioctl; 1805 ifp->if_start = ix_start; 1806 ifp->if_serialize = ix_serialize; 1807 ifp->if_deserialize = ix_deserialize; 1808 ifp->if_tryserialize = ix_tryserialize; 1809 #ifdef INVARIANTS 1810 ifp->if_serialize_assert = ix_serialize_assert; 1811 #endif 1812 #ifdef IFPOLL_ENABLE 1813 ifp->if_npoll = ix_npoll; 1814 #endif 1815 1816 /* Increase TSO burst length */ 1817 ifp->if_tsolen = (8 * ETHERMTU); 1818 1819 ifp->if_nmbclusters = sc->rx_ring_cnt * sc->rx_rings[0].rx_ndesc; 1820 ifp->if_nmbjclusters = ifp->if_nmbclusters; 1821 1822 ifq_set_maxlen(&ifp->if_snd, sc->tx_rings[0].tx_ndesc - 2); 1823 ifq_set_ready(&ifp->if_snd); 1824 ifq_set_subq_cnt(&ifp->if_snd, sc->tx_ring_cnt); 1825 1826 ifp->if_mapsubq = ifq_mapsubq_modulo; 1827 ifq_set_subq_divisor(&ifp->if_snd, 1); 1828 1829 ether_ifattach(ifp, hw->mac.addr, NULL); 1830 1831 ifp->if_capabilities = 1832 IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 1833 if (IX_ENABLE_HWRSS(sc)) 1834 ifp->if_capabilities |= IFCAP_RSS; 1835 ifp->if_capenable = ifp->if_capabilities; 1836 ifp->if_hwassist = CSUM_OFFLOAD | CSUM_TSO; 1837 1838 /* 1839 * Tell the upper layer(s) we support long frames. 1840 */ 1841 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 1842 1843 /* Setup TX rings and subqueues */ 1844 for (i = 0; i < sc->tx_ring_cnt; ++i) { 1845 struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i); 1846 struct ix_tx_ring *txr = &sc->tx_rings[i]; 1847 1848 ifsq_set_cpuid(ifsq, txr->tx_intr_cpuid); 1849 ifsq_set_priv(ifsq, txr); 1850 ifsq_set_hw_serialize(ifsq, &txr->tx_serialize); 1851 txr->tx_ifsq = ifsq; 1852 1853 ifsq_watchdog_init(&txr->tx_watchdog, ifsq, ix_watchdog, 0); 1854 } 1855 1856 /* Specify the media types supported by this adapter */ 1857 sc->phy_layer = ixgbe_get_supported_physical_layer(hw); 1858 ix_init_media(sc); 1859 } 1860 1861 static boolean_t 1862 ix_is_sfp(struct ixgbe_hw *hw) 1863 { 1864 switch (hw->mac.type) { 1865 case ixgbe_mac_82598EB: 1866 if (hw->phy.type == ixgbe_phy_nl) 1867 return TRUE; 1868 return FALSE; 1869 1870 case ixgbe_mac_82599EB: 1871 switch (hw->mac.ops.get_media_type(hw)) { 1872 case ixgbe_media_type_fiber: 1873 case ixgbe_media_type_fiber_qsfp: 1874 return TRUE; 1875 default: 1876 return FALSE; 1877 } 1878 1879 case ixgbe_mac_X550EM_x: 1880 case ixgbe_mac_X550EM_a: 1881 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) 1882 return TRUE; 1883 return FALSE; 1884 1885 default: 1886 return FALSE; 1887 } 1888 } 1889 1890 static void 1891 ix_config_link(struct ix_softc *sc) 1892 { 1893 struct ixgbe_hw *hw = &sc->hw; 1894 boolean_t sfp; 1895 1896 sfp = ix_is_sfp(hw); 1897 if (sfp) { 1898 if (hw->phy.multispeed_fiber) 1899 ixgbe_enable_tx_laser(hw); 1900 ix_handle_mod(sc); 1901 } else { 1902 uint32_t autoneg, err = 0; 1903 1904 if (hw->mac.ops.check_link != NULL) { 1905 err = ixgbe_check_link(hw, &sc->link_speed, 1906 &sc->link_up, FALSE); 1907 if (err) 1908 return; 1909 } 1910 1911 if (sc->advspeed != IXGBE_LINK_SPEED_UNKNOWN) 1912 autoneg = sc->advspeed; 1913 else 1914 autoneg = hw->phy.autoneg_advertised; 1915 if (!autoneg && hw->mac.ops.get_link_capabilities != NULL) { 1916 bool negotiate; 1917 1918 err = hw->mac.ops.get_link_capabilities(hw, 1919 &autoneg, &negotiate); 1920 if (err) 1921 return; 1922 } 1923 1924 if (hw->mac.ops.setup_link != NULL) { 1925 err = hw->mac.ops.setup_link(hw, 1926 autoneg, sc->link_up); 1927 if (err) 1928 return; 1929 } 1930 } 1931 } 1932 1933 static int 1934 ix_alloc_rings(struct ix_softc *sc) 1935 { 1936 int error, i; 1937 1938 /* 1939 * Create top level busdma tag 1940 */ 1941 error = bus_dma_tag_create(NULL, 1, 0, 1942 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 1943 BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 0, 1944 &sc->parent_tag); 1945 if (error) { 1946 device_printf(sc->dev, "could not create top level DMA tag\n"); 1947 return error; 1948 } 1949 1950 /* 1951 * Allocate TX descriptor rings and buffers 1952 */ 1953 sc->tx_rings = kmalloc(sizeof(struct ix_tx_ring) * sc->tx_ring_cnt, 1954 M_DEVBUF, 1955 M_WAITOK | M_ZERO | M_CACHEALIGN); 1956 for (i = 0; i < sc->tx_ring_cnt; ++i) { 1957 struct ix_tx_ring *txr = &sc->tx_rings[i]; 1958 1959 txr->tx_sc = sc; 1960 txr->tx_idx = i; 1961 txr->tx_intr_vec = -1; 1962 txr->tx_intr_cpuid = -1; 1963 lwkt_serialize_init(&txr->tx_serialize); 1964 callout_init_mp(&txr->tx_gc_timer); 1965 1966 error = ix_create_tx_ring(txr); 1967 if (error) 1968 return error; 1969 } 1970 1971 /* 1972 * Allocate RX descriptor rings and buffers 1973 */ 1974 sc->rx_rings = kmalloc(sizeof(struct ix_rx_ring) * sc->rx_ring_cnt, 1975 M_DEVBUF, 1976 M_WAITOK | M_ZERO | M_CACHEALIGN); 1977 for (i = 0; i < sc->rx_ring_cnt; ++i) { 1978 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 1979 1980 rxr->rx_sc = sc; 1981 rxr->rx_idx = i; 1982 rxr->rx_intr_vec = -1; 1983 lwkt_serialize_init(&rxr->rx_serialize); 1984 1985 error = ix_create_rx_ring(rxr); 1986 if (error) 1987 return error; 1988 } 1989 1990 return 0; 1991 } 1992 1993 static int 1994 ix_create_tx_ring(struct ix_tx_ring *txr) 1995 { 1996 int error, i, tsize, ntxd; 1997 1998 /* 1999 * Validate number of transmit descriptors. It must not exceed 2000 * hardware maximum, and must be multiple of IX_DBA_ALIGN. 2001 */ 2002 ntxd = device_getenv_int(txr->tx_sc->dev, "txd", ix_txd); 2003 if (((ntxd * sizeof(union ixgbe_adv_tx_desc)) % IX_DBA_ALIGN) != 0 || 2004 ntxd < IX_MIN_TXD || ntxd > IX_MAX_TXD) { 2005 device_printf(txr->tx_sc->dev, 2006 "Using %d TX descriptors instead of %d!\n", 2007 IX_DEF_TXD, ntxd); 2008 txr->tx_ndesc = IX_DEF_TXD; 2009 } else { 2010 txr->tx_ndesc = ntxd; 2011 } 2012 2013 /* 2014 * Allocate TX head write-back buffer 2015 */ 2016 txr->tx_hdr = bus_dmamem_coherent_any(txr->tx_sc->parent_tag, 2017 __VM_CACHELINE_SIZE, __VM_CACHELINE_SIZE, BUS_DMA_WAITOK, 2018 &txr->tx_hdr_dtag, &txr->tx_hdr_map, &txr->tx_hdr_paddr); 2019 if (txr->tx_hdr == NULL) { 2020 device_printf(txr->tx_sc->dev, 2021 "Unable to allocate TX head write-back buffer\n"); 2022 return ENOMEM; 2023 } 2024 2025 /* 2026 * Allocate TX descriptor ring 2027 */ 2028 tsize = roundup2(txr->tx_ndesc * sizeof(union ixgbe_adv_tx_desc), 2029 IX_DBA_ALIGN); 2030 txr->tx_base = bus_dmamem_coherent_any(txr->tx_sc->parent_tag, 2031 IX_DBA_ALIGN, tsize, BUS_DMA_WAITOK | BUS_DMA_ZERO, 2032 &txr->tx_base_dtag, &txr->tx_base_map, &txr->tx_base_paddr); 2033 if (txr->tx_base == NULL) { 2034 device_printf(txr->tx_sc->dev, 2035 "Unable to allocate TX Descriptor memory\n"); 2036 return ENOMEM; 2037 } 2038 2039 tsize = __VM_CACHELINE_ALIGN(sizeof(struct ix_tx_buf) * txr->tx_ndesc); 2040 txr->tx_buf = kmalloc(tsize, M_DEVBUF, 2041 M_WAITOK | M_ZERO | M_CACHEALIGN); 2042 2043 /* 2044 * Create DMA tag for TX buffers 2045 */ 2046 error = bus_dma_tag_create(txr->tx_sc->parent_tag, 2047 1, 0, /* alignment, bounds */ 2048 BUS_SPACE_MAXADDR, /* lowaddr */ 2049 BUS_SPACE_MAXADDR, /* highaddr */ 2050 IX_TSO_SIZE, /* maxsize */ 2051 IX_MAX_SCATTER, /* nsegments */ 2052 PAGE_SIZE, /* maxsegsize */ 2053 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | 2054 BUS_DMA_ONEBPAGE, /* flags */ 2055 &txr->tx_tag); 2056 if (error) { 2057 device_printf(txr->tx_sc->dev, 2058 "Unable to allocate TX DMA tag\n"); 2059 kfree(txr->tx_buf, M_DEVBUF); 2060 txr->tx_buf = NULL; 2061 return error; 2062 } 2063 2064 /* 2065 * Create DMA maps for TX buffers 2066 */ 2067 for (i = 0; i < txr->tx_ndesc; ++i) { 2068 struct ix_tx_buf *txbuf = &txr->tx_buf[i]; 2069 2070 error = bus_dmamap_create(txr->tx_tag, 2071 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, &txbuf->map); 2072 if (error) { 2073 device_printf(txr->tx_sc->dev, 2074 "Unable to create TX DMA map\n"); 2075 ix_destroy_tx_ring(txr, i); 2076 return error; 2077 } 2078 } 2079 2080 /* 2081 * Initialize various watermark 2082 */ 2083 txr->tx_wreg_nsegs = IX_DEF_TXWREG_NSEGS; 2084 txr->tx_intr_nsegs = txr->tx_ndesc / 16; 2085 2086 return 0; 2087 } 2088 2089 static void 2090 ix_destroy_tx_ring(struct ix_tx_ring *txr, int ndesc) 2091 { 2092 int i; 2093 2094 if (txr->tx_hdr != NULL) { 2095 bus_dmamap_unload(txr->tx_hdr_dtag, txr->tx_hdr_map); 2096 bus_dmamem_free(txr->tx_hdr_dtag, 2097 __DEVOLATILE(void *, txr->tx_hdr), txr->tx_hdr_map); 2098 bus_dma_tag_destroy(txr->tx_hdr_dtag); 2099 txr->tx_hdr = NULL; 2100 } 2101 2102 if (txr->tx_base != NULL) { 2103 bus_dmamap_unload(txr->tx_base_dtag, txr->tx_base_map); 2104 bus_dmamem_free(txr->tx_base_dtag, txr->tx_base, 2105 txr->tx_base_map); 2106 bus_dma_tag_destroy(txr->tx_base_dtag); 2107 txr->tx_base = NULL; 2108 } 2109 2110 if (txr->tx_buf == NULL) 2111 return; 2112 2113 for (i = 0; i < ndesc; ++i) { 2114 struct ix_tx_buf *txbuf = &txr->tx_buf[i]; 2115 2116 KKASSERT(txbuf->m_head == NULL); 2117 bus_dmamap_destroy(txr->tx_tag, txbuf->map); 2118 } 2119 bus_dma_tag_destroy(txr->tx_tag); 2120 2121 kfree(txr->tx_buf, M_DEVBUF); 2122 txr->tx_buf = NULL; 2123 } 2124 2125 static void 2126 ix_init_tx_ring(struct ix_tx_ring *txr) 2127 { 2128 /* Clear the old ring contents */ 2129 bzero(txr->tx_base, sizeof(union ixgbe_adv_tx_desc) * txr->tx_ndesc); 2130 2131 /* Clear TX head write-back buffer */ 2132 *(txr->tx_hdr) = 0; 2133 2134 /* Reset indices */ 2135 txr->tx_next_avail = 0; 2136 txr->tx_next_clean = 0; 2137 txr->tx_nsegs = 0; 2138 txr->tx_nmbuf = 0; 2139 txr->tx_running = 0; 2140 2141 /* Set number of descriptors available */ 2142 txr->tx_avail = txr->tx_ndesc; 2143 2144 /* Enable this TX ring */ 2145 txr->tx_flags |= IX_TXFLAG_ENABLED; 2146 } 2147 2148 static void 2149 ix_init_tx_unit(struct ix_softc *sc) 2150 { 2151 struct ixgbe_hw *hw = &sc->hw; 2152 int i; 2153 2154 /* 2155 * Setup the Base and Length of the Tx Descriptor Ring 2156 */ 2157 for (i = 0; i < sc->tx_ring_inuse; ++i) { 2158 struct ix_tx_ring *txr = &sc->tx_rings[i]; 2159 uint64_t tdba = txr->tx_base_paddr; 2160 uint64_t hdr_paddr = txr->tx_hdr_paddr; 2161 uint32_t txctrl; 2162 2163 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), (uint32_t)tdba); 2164 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (uint32_t)(tdba >> 32)); 2165 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 2166 txr->tx_ndesc * sizeof(union ixgbe_adv_tx_desc)); 2167 2168 /* Setup the HW Tx Head and Tail descriptor pointers */ 2169 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 2170 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 2171 2172 /* Disable TX head write-back relax ordering */ 2173 switch (hw->mac.type) { 2174 case ixgbe_mac_82598EB: 2175 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 2176 break; 2177 default: 2178 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 2179 break; 2180 } 2181 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2182 switch (hw->mac.type) { 2183 case ixgbe_mac_82598EB: 2184 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 2185 break; 2186 default: 2187 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 2188 break; 2189 } 2190 2191 /* Enable TX head write-back */ 2192 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(i), 2193 (uint32_t)(hdr_paddr >> 32)); 2194 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(i), 2195 ((uint32_t)hdr_paddr) | IXGBE_TDWBAL_HEAD_WB_ENABLE); 2196 } 2197 2198 if (hw->mac.type != ixgbe_mac_82598EB) { 2199 uint32_t dmatxctl, rttdcs; 2200 2201 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2202 dmatxctl |= IXGBE_DMATXCTL_TE; 2203 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 2204 2205 /* Disable arbiter to set MTQC */ 2206 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2207 rttdcs |= IXGBE_RTTDCS_ARBDIS; 2208 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2209 2210 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2211 2212 /* Reenable aribter */ 2213 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 2214 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2215 } 2216 } 2217 2218 static int 2219 ix_tx_ctx_setup(struct ix_tx_ring *txr, const struct mbuf *mp, 2220 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2221 { 2222 struct ixgbe_adv_tx_context_desc *TXD; 2223 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2224 int ehdrlen, ip_hlen = 0, ctxd; 2225 boolean_t offload = TRUE; 2226 2227 /* First check if TSO is to be used */ 2228 if (mp->m_pkthdr.csum_flags & CSUM_TSO) { 2229 return ix_tso_ctx_setup(txr, mp, 2230 cmd_type_len, olinfo_status); 2231 } 2232 2233 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 2234 offload = FALSE; 2235 2236 /* Indicate the whole packet as payload when not doing TSO */ 2237 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; 2238 2239 /* 2240 * In advanced descriptors the vlan tag must be placed into the 2241 * context descriptor. Hence we need to make one even if not 2242 * doing checksum offloads. 2243 */ 2244 if (mp->m_flags & M_VLANTAG) { 2245 vlan_macip_lens |= htole16(mp->m_pkthdr.ether_vlantag) << 2246 IXGBE_ADVTXD_VLAN_SHIFT; 2247 } else if (!offload) { 2248 /* No TX descriptor is consumed */ 2249 return 0; 2250 } 2251 2252 /* Set the ether header length */ 2253 ehdrlen = mp->m_pkthdr.csum_lhlen; 2254 KASSERT(ehdrlen > 0, ("invalid ether hlen")); 2255 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2256 2257 if (mp->m_pkthdr.csum_flags & CSUM_IP) { 2258 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 2259 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2260 ip_hlen = mp->m_pkthdr.csum_iphlen; 2261 KASSERT(ip_hlen > 0, ("invalid ip hlen")); 2262 } 2263 vlan_macip_lens |= ip_hlen; 2264 2265 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2266 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 2267 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2268 else if (mp->m_pkthdr.csum_flags & CSUM_UDP) 2269 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2270 2271 if (mp->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)) 2272 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2273 2274 /* Now ready a context descriptor */ 2275 ctxd = txr->tx_next_avail; 2276 TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; 2277 2278 /* Now copy bits into descriptor */ 2279 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2280 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2281 TXD->seqnum_seed = htole32(0); 2282 TXD->mss_l4len_idx = htole32(0); 2283 2284 /* We've consumed the first desc, adjust counters */ 2285 if (++ctxd == txr->tx_ndesc) 2286 ctxd = 0; 2287 txr->tx_next_avail = ctxd; 2288 --txr->tx_avail; 2289 2290 /* One TX descriptor is consumed */ 2291 return 1; 2292 } 2293 2294 static int 2295 ix_tso_ctx_setup(struct ix_tx_ring *txr, const struct mbuf *mp, 2296 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2297 { 2298 struct ixgbe_adv_tx_context_desc *TXD; 2299 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2300 uint32_t mss_l4len_idx = 0, paylen; 2301 int ctxd, ehdrlen, ip_hlen, tcp_hlen; 2302 2303 ehdrlen = mp->m_pkthdr.csum_lhlen; 2304 KASSERT(ehdrlen > 0, ("invalid ether hlen")); 2305 2306 ip_hlen = mp->m_pkthdr.csum_iphlen; 2307 KASSERT(ip_hlen > 0, ("invalid ip hlen")); 2308 2309 tcp_hlen = mp->m_pkthdr.csum_thlen; 2310 KASSERT(tcp_hlen > 0, ("invalid tcp hlen")); 2311 2312 ctxd = txr->tx_next_avail; 2313 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2314 2315 if (mp->m_flags & M_VLANTAG) { 2316 vlan_macip_lens |= htole16(mp->m_pkthdr.ether_vlantag) << 2317 IXGBE_ADVTXD_VLAN_SHIFT; 2318 } 2319 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2320 vlan_macip_lens |= ip_hlen; 2321 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2322 2323 /* ADV DTYPE TUCMD */ 2324 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2325 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2326 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2327 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2328 2329 /* MSS L4LEN IDX */ 2330 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2331 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2332 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2333 2334 TXD->seqnum_seed = htole32(0); 2335 2336 if (++ctxd == txr->tx_ndesc) 2337 ctxd = 0; 2338 2339 txr->tx_avail--; 2340 txr->tx_next_avail = ctxd; 2341 2342 *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 2343 2344 /* This is used in the transmit desc in encap */ 2345 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; 2346 2347 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 2348 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2349 *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 2350 2351 /* One TX descriptor is consumed */ 2352 return 1; 2353 } 2354 2355 static void 2356 ix_txeof(struct ix_tx_ring *txr, int hdr) 2357 { 2358 int first, avail; 2359 2360 if (txr->tx_avail == txr->tx_ndesc) 2361 return; 2362 2363 first = txr->tx_next_clean; 2364 if (first == hdr) 2365 return; 2366 2367 avail = txr->tx_avail; 2368 while (first != hdr) { 2369 struct ix_tx_buf *txbuf = &txr->tx_buf[first]; 2370 2371 KKASSERT(avail < txr->tx_ndesc); 2372 ++avail; 2373 2374 if (txbuf->m_head != NULL) 2375 ix_free_txbuf(txr, txbuf); 2376 if (++first == txr->tx_ndesc) 2377 first = 0; 2378 } 2379 txr->tx_next_clean = first; 2380 txr->tx_avail = avail; 2381 2382 if (txr->tx_avail > IX_MAX_SCATTER + IX_TX_RESERVED) { 2383 ifsq_clr_oactive(txr->tx_ifsq); 2384 ifsq_watchdog_set_count(&txr->tx_watchdog, 0); 2385 } 2386 txr->tx_running = IX_TX_RUNNING; 2387 } 2388 2389 static void 2390 ix_txgc(struct ix_tx_ring *txr) 2391 { 2392 int first, hdr; 2393 #ifdef INVARIANTS 2394 int avail; 2395 #endif 2396 2397 if (txr->tx_avail == txr->tx_ndesc) 2398 return; 2399 2400 hdr = IXGBE_READ_REG(&txr->tx_sc->hw, IXGBE_TDH(txr->tx_idx)); 2401 first = txr->tx_next_clean; 2402 if (first == hdr) 2403 goto done; 2404 txr->tx_gc++; 2405 2406 #ifdef INVARIANTS 2407 avail = txr->tx_avail; 2408 #endif 2409 while (first != hdr) { 2410 struct ix_tx_buf *txbuf = &txr->tx_buf[first]; 2411 2412 #ifdef INVARIANTS 2413 KKASSERT(avail < txr->tx_ndesc); 2414 ++avail; 2415 #endif 2416 if (txbuf->m_head != NULL) 2417 ix_free_txbuf(txr, txbuf); 2418 if (++first == txr->tx_ndesc) 2419 first = 0; 2420 } 2421 done: 2422 if (txr->tx_nmbuf) 2423 txr->tx_running = IX_TX_RUNNING; 2424 } 2425 2426 static int 2427 ix_create_rx_ring(struct ix_rx_ring *rxr) 2428 { 2429 int i, rsize, error, nrxd; 2430 2431 /* 2432 * Validate number of receive descriptors. It must not exceed 2433 * hardware maximum, and must be multiple of IX_DBA_ALIGN. 2434 */ 2435 nrxd = device_getenv_int(rxr->rx_sc->dev, "rxd", ix_rxd); 2436 if (((nrxd * sizeof(union ixgbe_adv_rx_desc)) % IX_DBA_ALIGN) != 0 || 2437 nrxd < IX_MIN_RXD || nrxd > IX_MAX_RXD) { 2438 device_printf(rxr->rx_sc->dev, 2439 "Using %d RX descriptors instead of %d!\n", 2440 IX_DEF_RXD, nrxd); 2441 rxr->rx_ndesc = IX_DEF_RXD; 2442 } else { 2443 rxr->rx_ndesc = nrxd; 2444 } 2445 2446 /* 2447 * Allocate RX descriptor ring 2448 */ 2449 rsize = roundup2(rxr->rx_ndesc * sizeof(union ixgbe_adv_rx_desc), 2450 IX_DBA_ALIGN); 2451 rxr->rx_base = bus_dmamem_coherent_any(rxr->rx_sc->parent_tag, 2452 IX_DBA_ALIGN, rsize, BUS_DMA_WAITOK | BUS_DMA_ZERO, 2453 &rxr->rx_base_dtag, &rxr->rx_base_map, &rxr->rx_base_paddr); 2454 if (rxr->rx_base == NULL) { 2455 device_printf(rxr->rx_sc->dev, 2456 "Unable to allocate TX Descriptor memory\n"); 2457 return ENOMEM; 2458 } 2459 2460 rsize = __VM_CACHELINE_ALIGN(sizeof(struct ix_rx_buf) * rxr->rx_ndesc); 2461 rxr->rx_buf = kmalloc(rsize, M_DEVBUF, 2462 M_WAITOK | M_ZERO | M_CACHEALIGN); 2463 2464 /* 2465 * Create DMA tag for RX buffers 2466 */ 2467 error = bus_dma_tag_create(rxr->rx_sc->parent_tag, 2468 1, 0, /* alignment, bounds */ 2469 BUS_SPACE_MAXADDR, /* lowaddr */ 2470 BUS_SPACE_MAXADDR, /* highaddr */ 2471 PAGE_SIZE, /* maxsize */ 2472 1, /* nsegments */ 2473 PAGE_SIZE, /* maxsegsize */ 2474 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, /* flags */ 2475 &rxr->rx_tag); 2476 if (error) { 2477 device_printf(rxr->rx_sc->dev, 2478 "Unable to create RX DMA tag\n"); 2479 kfree(rxr->rx_buf, M_DEVBUF); 2480 rxr->rx_buf = NULL; 2481 return error; 2482 } 2483 2484 /* 2485 * Create spare DMA map for RX buffers 2486 */ 2487 error = bus_dmamap_create(rxr->rx_tag, BUS_DMA_WAITOK, 2488 &rxr->rx_sparemap); 2489 if (error) { 2490 device_printf(rxr->rx_sc->dev, 2491 "Unable to create spare RX DMA map\n"); 2492 bus_dma_tag_destroy(rxr->rx_tag); 2493 kfree(rxr->rx_buf, M_DEVBUF); 2494 rxr->rx_buf = NULL; 2495 return error; 2496 } 2497 2498 /* 2499 * Create DMA maps for RX buffers 2500 */ 2501 for (i = 0; i < rxr->rx_ndesc; ++i) { 2502 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i]; 2503 2504 error = bus_dmamap_create(rxr->rx_tag, 2505 BUS_DMA_WAITOK, &rxbuf->map); 2506 if (error) { 2507 device_printf(rxr->rx_sc->dev, 2508 "Unable to create RX dma map\n"); 2509 ix_destroy_rx_ring(rxr, i); 2510 return error; 2511 } 2512 } 2513 2514 /* 2515 * Initialize various watermark 2516 */ 2517 rxr->rx_wreg_nsegs = IX_DEF_RXWREG_NSEGS; 2518 2519 return 0; 2520 } 2521 2522 static void 2523 ix_destroy_rx_ring(struct ix_rx_ring *rxr, int ndesc) 2524 { 2525 int i; 2526 2527 if (rxr->rx_base != NULL) { 2528 bus_dmamap_unload(rxr->rx_base_dtag, rxr->rx_base_map); 2529 bus_dmamem_free(rxr->rx_base_dtag, rxr->rx_base, 2530 rxr->rx_base_map); 2531 bus_dma_tag_destroy(rxr->rx_base_dtag); 2532 rxr->rx_base = NULL; 2533 } 2534 2535 if (rxr->rx_buf == NULL) 2536 return; 2537 2538 for (i = 0; i < ndesc; ++i) { 2539 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i]; 2540 2541 KKASSERT(rxbuf->m_head == NULL); 2542 bus_dmamap_destroy(rxr->rx_tag, rxbuf->map); 2543 } 2544 bus_dmamap_destroy(rxr->rx_tag, rxr->rx_sparemap); 2545 bus_dma_tag_destroy(rxr->rx_tag); 2546 2547 kfree(rxr->rx_buf, M_DEVBUF); 2548 rxr->rx_buf = NULL; 2549 } 2550 2551 /* 2552 ** Used to detect a descriptor that has 2553 ** been merged by Hardware RSC. 2554 */ 2555 static __inline uint32_t 2556 ix_rsc_count(union ixgbe_adv_rx_desc *rx) 2557 { 2558 return (le32toh(rx->wb.lower.lo_dword.data) & 2559 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; 2560 } 2561 2562 #if 0 2563 /********************************************************************* 2564 * 2565 * Initialize Hardware RSC (LRO) feature on 82599 2566 * for an RX ring, this is toggled by the LRO capability 2567 * even though it is transparent to the stack. 2568 * 2569 * NOTE: since this HW feature only works with IPV4 and 2570 * our testing has shown soft LRO to be as effective 2571 * I have decided to disable this by default. 2572 * 2573 **********************************************************************/ 2574 static void 2575 ix_setup_hw_rsc(struct ix_rx_ring *rxr) 2576 { 2577 struct ix_softc *sc = rxr->rx_sc; 2578 struct ixgbe_hw *hw = &sc->hw; 2579 uint32_t rscctrl, rdrxctl; 2580 2581 #if 0 2582 /* If turning LRO/RSC off we need to disable it */ 2583 if ((sc->arpcom.ac_if.if_capenable & IFCAP_LRO) == 0) { 2584 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 2585 rscctrl &= ~IXGBE_RSCCTL_RSCEN; 2586 return; 2587 } 2588 #endif 2589 2590 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2591 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 2592 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 2593 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 2594 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 2595 2596 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 2597 rscctrl |= IXGBE_RSCCTL_RSCEN; 2598 /* 2599 ** Limit the total number of descriptors that 2600 ** can be combined, so it does not exceed 64K 2601 */ 2602 if (rxr->mbuf_sz == MCLBYTES) 2603 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 2604 else if (rxr->mbuf_sz == MJUMPAGESIZE) 2605 rscctrl |= IXGBE_RSCCTL_MAXDESC_8; 2606 else if (rxr->mbuf_sz == MJUM9BYTES) 2607 rscctrl |= IXGBE_RSCCTL_MAXDESC_4; 2608 else /* Using 16K cluster */ 2609 rscctrl |= IXGBE_RSCCTL_MAXDESC_1; 2610 2611 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); 2612 2613 /* Enable TCP header recognition */ 2614 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 2615 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | 2616 IXGBE_PSRTYPE_TCPHDR)); 2617 2618 /* Disable RSC for ACK packets */ 2619 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 2620 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 2621 2622 rxr->hw_rsc = TRUE; 2623 } 2624 #endif 2625 2626 static int 2627 ix_init_rx_ring(struct ix_rx_ring *rxr) 2628 { 2629 int i; 2630 2631 /* Clear the ring contents */ 2632 bzero(rxr->rx_base, rxr->rx_ndesc * sizeof(union ixgbe_adv_rx_desc)); 2633 2634 /* XXX we need JUMPAGESIZE for RSC too */ 2635 if (rxr->rx_sc->max_frame_size <= MCLBYTES) 2636 rxr->rx_mbuf_sz = MCLBYTES; 2637 else 2638 rxr->rx_mbuf_sz = MJUMPAGESIZE; 2639 2640 /* Now replenish the mbufs */ 2641 for (i = 0; i < rxr->rx_ndesc; ++i) { 2642 int error; 2643 2644 error = ix_newbuf(rxr, i, TRUE); 2645 if (error) 2646 return error; 2647 } 2648 2649 /* Setup our descriptor indices */ 2650 rxr->rx_next_check = 0; 2651 rxr->rx_flags &= ~IX_RXRING_FLAG_DISC; 2652 2653 #if 0 2654 /* 2655 ** Now set up the LRO interface: 2656 */ 2657 if (ixgbe_rsc_enable) 2658 ix_setup_hw_rsc(rxr); 2659 #endif 2660 2661 return 0; 2662 } 2663 2664 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2665 2666 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 2667 2668 static void 2669 ix_init_rx_unit(struct ix_softc *sc, boolean_t polling) 2670 { 2671 struct ixgbe_hw *hw = &sc->hw; 2672 struct ifnet *ifp = &sc->arpcom.ac_if; 2673 uint32_t bufsz, fctrl, rxcsum, hlreg; 2674 int i; 2675 2676 /* 2677 * Make sure receives are disabled while setting up the descriptor ring 2678 */ 2679 ixgbe_disable_rx(hw); 2680 2681 /* Enable broadcasts */ 2682 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2683 fctrl |= IXGBE_FCTRL_BAM; 2684 if (hw->mac.type == ixgbe_mac_82598EB) { 2685 fctrl |= IXGBE_FCTRL_DPF; 2686 fctrl |= IXGBE_FCTRL_PMCF; 2687 } 2688 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 2689 2690 /* Set for Jumbo Frames? */ 2691 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2692 if (ifp->if_mtu > ETHERMTU) 2693 hlreg |= IXGBE_HLREG0_JUMBOEN; 2694 else 2695 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 2696 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 2697 2698 KKASSERT(sc->rx_rings[0].rx_mbuf_sz >= MCLBYTES); 2699 bufsz = (sc->rx_rings[0].rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 2700 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2701 2702 for (i = 0; i < sc->rx_ring_inuse; ++i) { 2703 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 2704 uint64_t rdba = rxr->rx_base_paddr; 2705 uint32_t srrctl; 2706 2707 /* Setup the Base and Length of the Rx Descriptor Ring */ 2708 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), (uint32_t)rdba); 2709 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (uint32_t)(rdba >> 32)); 2710 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 2711 rxr->rx_ndesc * sizeof(union ixgbe_adv_rx_desc)); 2712 2713 /* 2714 * Set up the SRRCTL register 2715 */ 2716 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 2717 2718 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 2719 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 2720 srrctl |= bufsz; 2721 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2722 if (sc->rx_ring_inuse > 1) { 2723 /* See the commend near ix_enable_rx_drop() */ 2724 if (sc->ifm_media & 2725 (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) { 2726 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 2727 if (i == 0 && bootverbose) { 2728 if_printf(ifp, "flow control %s, " 2729 "disable RX drop\n", 2730 ix_ifmedia2str(sc->ifm_media)); 2731 } 2732 } else { 2733 srrctl |= IXGBE_SRRCTL_DROP_EN; 2734 if (i == 0 && bootverbose) { 2735 if_printf(ifp, "flow control %s, " 2736 "enable RX drop\n", 2737 ix_ifmedia2str(sc->ifm_media)); 2738 } 2739 } 2740 } 2741 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 2742 2743 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2744 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 2745 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 2746 } 2747 2748 if (sc->hw.mac.type != ixgbe_mac_82598EB) 2749 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 0); 2750 2751 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2752 2753 /* 2754 * Setup RSS 2755 */ 2756 if (sc->rx_ring_inuse > 1) { 2757 uint8_t key[IX_NRSSRK * IX_RSSRK_SIZE]; 2758 const struct if_ringmap *rm; 2759 int j, r, nreta, table_nent; 2760 2761 /* 2762 * NOTE: 2763 * When we reach here, RSS has already been disabled 2764 * in ix_stop(), so we could safely configure RSS key 2765 * and redirect table. 2766 */ 2767 2768 /* 2769 * Configure RSS key 2770 */ 2771 toeplitz_get_key(key, sizeof(key)); 2772 for (i = 0; i < IX_NRSSRK; ++i) { 2773 uint32_t rssrk; 2774 2775 rssrk = IX_RSSRK_VAL(key, i); 2776 IX_RSS_DPRINTF(sc, 1, "rssrk%d 0x%08x\n", 2777 i, rssrk); 2778 2779 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rssrk); 2780 } 2781 2782 /* 2783 * Configure RSS redirect table. 2784 */ 2785 2786 /* Table size will differ based on MAC */ 2787 switch (hw->mac.type) { 2788 case ixgbe_mac_X550: 2789 case ixgbe_mac_X550EM_x: 2790 case ixgbe_mac_X550EM_a: 2791 nreta = IX_NRETA_X550; 2792 break; 2793 default: 2794 nreta = IX_NRETA; 2795 break; 2796 } 2797 2798 table_nent = nreta * IX_RETA_SIZE; 2799 KASSERT(table_nent <= IX_RDRTABLE_SIZE, 2800 ("invalid RETA count %d", nreta)); 2801 if (polling) 2802 rm = sc->rx_rmap; 2803 else 2804 rm = sc->rx_rmap_intr; 2805 if_ringmap_rdrtable(rm, sc->rdr_table, table_nent); 2806 2807 r = 0; 2808 for (j = 0; j < nreta; ++j) { 2809 uint32_t reta = 0; 2810 2811 for (i = 0; i < IX_RETA_SIZE; ++i) { 2812 uint32_t q; 2813 2814 q = sc->rdr_table[r]; 2815 KASSERT(q < sc->rx_ring_inuse, 2816 ("invalid RX ring index %d", q)); 2817 reta |= q << (8 * i); 2818 ++r; 2819 } 2820 IX_RSS_DPRINTF(sc, 1, "reta 0x%08x\n", reta); 2821 if (j < IX_NRETA) { 2822 IXGBE_WRITE_REG(hw, IXGBE_RETA(j), reta); 2823 } else { 2824 IXGBE_WRITE_REG(hw, IXGBE_ERETA(j - IX_NRETA), 2825 reta); 2826 } 2827 } 2828 2829 /* 2830 * Enable multiple receive queues. 2831 * Enable IPv4 RSS standard hash functions. 2832 */ 2833 IXGBE_WRITE_REG(hw, IXGBE_MRQC, 2834 IXGBE_MRQC_RSSEN | 2835 IXGBE_MRQC_RSS_FIELD_IPV4 | 2836 IXGBE_MRQC_RSS_FIELD_IPV4_TCP); 2837 2838 /* 2839 * NOTE: 2840 * PCSD must be enabled to enable multiple 2841 * receive queues. 2842 */ 2843 rxcsum |= IXGBE_RXCSUM_PCSD; 2844 } 2845 2846 if (ifp->if_capenable & IFCAP_RXCSUM) 2847 rxcsum |= IXGBE_RXCSUM_PCSD; 2848 2849 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2850 } 2851 2852 static __inline void 2853 ix_rx_refresh(struct ix_rx_ring *rxr, int i) 2854 { 2855 if (--i < 0) 2856 i = rxr->rx_ndesc - 1; 2857 IXGBE_WRITE_REG(&rxr->rx_sc->hw, IXGBE_RDT(rxr->rx_idx), i); 2858 } 2859 2860 static __inline void 2861 ix_rxcsum(uint32_t staterr, struct mbuf *mp, uint32_t ptype) 2862 { 2863 if ((ptype & 2864 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_IPV4_EX)) == 0) { 2865 /* Not IPv4 */ 2866 return; 2867 } 2868 2869 if ((staterr & (IXGBE_RXD_STAT_IPCS | IXGBE_RXDADV_ERR_IPE)) == 2870 IXGBE_RXD_STAT_IPCS) 2871 mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID; 2872 2873 if ((ptype & 2874 (IXGBE_RXDADV_PKTTYPE_TCP | IXGBE_RXDADV_PKTTYPE_UDP)) == 0) { 2875 /* 2876 * - Neither TCP nor UDP 2877 * - IPv4 fragment 2878 */ 2879 return; 2880 } 2881 2882 if ((staterr & (IXGBE_RXD_STAT_L4CS | IXGBE_RXDADV_ERR_TCPE)) == 2883 IXGBE_RXD_STAT_L4CS) { 2884 mp->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR | 2885 CSUM_FRAG_NOT_CHECKED; 2886 mp->m_pkthdr.csum_data = htons(0xffff); 2887 } 2888 } 2889 2890 static __inline struct pktinfo * 2891 ix_rssinfo(struct mbuf *m, struct pktinfo *pi, 2892 uint32_t hash, uint32_t hashtype, uint32_t ptype) 2893 { 2894 switch (hashtype) { 2895 case IXGBE_RXDADV_RSSTYPE_IPV4_TCP: 2896 pi->pi_netisr = NETISR_IP; 2897 pi->pi_flags = 0; 2898 pi->pi_l3proto = IPPROTO_TCP; 2899 break; 2900 2901 case IXGBE_RXDADV_RSSTYPE_IPV4: 2902 if ((ptype & IXGBE_RXDADV_PKTTYPE_UDP) == 0) { 2903 /* Not UDP or is fragment */ 2904 return NULL; 2905 } 2906 pi->pi_netisr = NETISR_IP; 2907 pi->pi_flags = 0; 2908 pi->pi_l3proto = IPPROTO_UDP; 2909 break; 2910 2911 default: 2912 return NULL; 2913 } 2914 2915 m_sethash(m, toeplitz_hash(hash)); 2916 return pi; 2917 } 2918 2919 static __inline void 2920 ix_setup_rxdesc(union ixgbe_adv_rx_desc *rxd, const struct ix_rx_buf *rxbuf) 2921 { 2922 rxd->read.pkt_addr = htole64(rxbuf->paddr); 2923 rxd->wb.upper.status_error = 0; 2924 } 2925 2926 static void 2927 ix_rx_discard(struct ix_rx_ring *rxr, int i, boolean_t eop) 2928 { 2929 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i]; 2930 2931 /* 2932 * XXX discard may not be correct 2933 */ 2934 if (eop) { 2935 IFNET_STAT_INC(&rxr->rx_sc->arpcom.ac_if, ierrors, 1); 2936 rxr->rx_flags &= ~IX_RXRING_FLAG_DISC; 2937 } else { 2938 rxr->rx_flags |= IX_RXRING_FLAG_DISC; 2939 } 2940 if (rxbuf->fmp != NULL) { 2941 m_freem(rxbuf->fmp); 2942 rxbuf->fmp = NULL; 2943 rxbuf->lmp = NULL; 2944 } 2945 ix_setup_rxdesc(&rxr->rx_base[i], rxbuf); 2946 } 2947 2948 static void 2949 ix_rxeof(struct ix_rx_ring *rxr, int count) 2950 { 2951 struct ifnet *ifp = &rxr->rx_sc->arpcom.ac_if; 2952 int i, nsegs = 0, cpuid = mycpuid; 2953 2954 i = rxr->rx_next_check; 2955 while (count != 0) { 2956 struct ix_rx_buf *rxbuf, *nbuf = NULL; 2957 union ixgbe_adv_rx_desc *cur; 2958 struct mbuf *sendmp = NULL, *mp; 2959 struct pktinfo *pi = NULL, pi0; 2960 uint32_t rsc = 0, ptype, staterr, hash, hashtype; 2961 uint16_t len; 2962 boolean_t eop; 2963 2964 cur = &rxr->rx_base[i]; 2965 staterr = le32toh(cur->wb.upper.status_error); 2966 2967 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 2968 break; 2969 ++nsegs; 2970 2971 rxbuf = &rxr->rx_buf[i]; 2972 mp = rxbuf->m_head; 2973 2974 len = le16toh(cur->wb.upper.length); 2975 ptype = le32toh(cur->wb.lower.lo_dword.data) & 2976 IXGBE_RXDADV_PKTTYPE_MASK; 2977 hash = le32toh(cur->wb.lower.hi_dword.rss); 2978 hashtype = le32toh(cur->wb.lower.lo_dword.data) & 2979 IXGBE_RXDADV_RSSTYPE_MASK; 2980 2981 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 2982 if (eop) 2983 --count; 2984 2985 /* 2986 * Make sure bad packets are discarded 2987 */ 2988 if ((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) || 2989 (rxr->rx_flags & IX_RXRING_FLAG_DISC)) { 2990 ix_rx_discard(rxr, i, eop); 2991 goto next_desc; 2992 } 2993 2994 bus_dmamap_sync(rxr->rx_tag, rxbuf->map, BUS_DMASYNC_POSTREAD); 2995 if (ix_newbuf(rxr, i, FALSE) != 0) { 2996 ix_rx_discard(rxr, i, eop); 2997 goto next_desc; 2998 } 2999 3000 /* 3001 * On 82599 which supports a hardware LRO, packets 3002 * need not be fragmented across sequential descriptors, 3003 * rather the next descriptor is indicated in bits 3004 * of the descriptor. This also means that we might 3005 * proceses more than one packet at a time, something 3006 * that has never been true before, it required 3007 * eliminating global chain pointers in favor of what 3008 * we are doing here. 3009 */ 3010 if (!eop) { 3011 int nextp; 3012 3013 /* 3014 * Figure out the next descriptor 3015 * of this frame. 3016 */ 3017 if (rxr->rx_flags & IX_RXRING_FLAG_LRO) 3018 rsc = ix_rsc_count(cur); 3019 if (rsc) { /* Get hardware index */ 3020 nextp = ((staterr & 3021 IXGBE_RXDADV_NEXTP_MASK) >> 3022 IXGBE_RXDADV_NEXTP_SHIFT); 3023 } else { /* Just sequential */ 3024 nextp = i + 1; 3025 if (nextp == rxr->rx_ndesc) 3026 nextp = 0; 3027 } 3028 nbuf = &rxr->rx_buf[nextp]; 3029 prefetch(nbuf); 3030 } 3031 mp->m_len = len; 3032 3033 /* 3034 * Rather than using the fmp/lmp global pointers 3035 * we now keep the head of a packet chain in the 3036 * buffer struct and pass this along from one 3037 * descriptor to the next, until we get EOP. 3038 */ 3039 if (rxbuf->fmp == NULL) { 3040 mp->m_pkthdr.len = len; 3041 rxbuf->fmp = mp; 3042 rxbuf->lmp = mp; 3043 } else { 3044 rxbuf->fmp->m_pkthdr.len += len; 3045 rxbuf->lmp->m_next = mp; 3046 rxbuf->lmp = mp; 3047 } 3048 3049 if (nbuf != NULL) { 3050 /* 3051 * Not the last fragment of this frame, 3052 * pass this fragment list on 3053 */ 3054 nbuf->fmp = rxbuf->fmp; 3055 nbuf->lmp = rxbuf->lmp; 3056 } else { 3057 /* 3058 * Send this frame 3059 */ 3060 sendmp = rxbuf->fmp; 3061 3062 sendmp->m_pkthdr.rcvif = ifp; 3063 IFNET_STAT_INC(ifp, ipackets, 1); 3064 #ifdef IX_RSS_DEBUG 3065 rxr->rx_pkts++; 3066 #endif 3067 3068 /* Process vlan info */ 3069 if (staterr & IXGBE_RXD_STAT_VP) { 3070 sendmp->m_pkthdr.ether_vlantag = 3071 le16toh(cur->wb.upper.vlan); 3072 sendmp->m_flags |= M_VLANTAG; 3073 } 3074 if (ifp->if_capenable & IFCAP_RXCSUM) 3075 ix_rxcsum(staterr, sendmp, ptype); 3076 if (ifp->if_capenable & IFCAP_RSS) { 3077 pi = ix_rssinfo(sendmp, &pi0, 3078 hash, hashtype, ptype); 3079 } 3080 } 3081 rxbuf->fmp = NULL; 3082 rxbuf->lmp = NULL; 3083 next_desc: 3084 /* Advance our pointers to the next descriptor. */ 3085 if (++i == rxr->rx_ndesc) 3086 i = 0; 3087 3088 if (sendmp != NULL) 3089 ifp->if_input(ifp, sendmp, pi, cpuid); 3090 3091 if (nsegs >= rxr->rx_wreg_nsegs) { 3092 ix_rx_refresh(rxr, i); 3093 nsegs = 0; 3094 } 3095 } 3096 rxr->rx_next_check = i; 3097 3098 if (nsegs > 0) 3099 ix_rx_refresh(rxr, i); 3100 } 3101 3102 static void 3103 ix_set_vlan(struct ix_softc *sc) 3104 { 3105 struct ixgbe_hw *hw = &sc->hw; 3106 uint32_t ctrl; 3107 3108 if ((sc->arpcom.ac_if.if_capenable & IFCAP_VLAN_HWTAGGING) == 0) 3109 return; 3110 3111 if (hw->mac.type == ixgbe_mac_82598EB) { 3112 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3113 ctrl |= IXGBE_VLNCTRL_VME; 3114 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 3115 } else { 3116 int i; 3117 3118 /* 3119 * On 82599 and later chips the VLAN enable is 3120 * per queue in RXDCTL 3121 */ 3122 for (i = 0; i < sc->rx_ring_inuse; ++i) { 3123 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 3124 ctrl |= IXGBE_RXDCTL_VME; 3125 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); 3126 } 3127 } 3128 } 3129 3130 static void 3131 ix_enable_intr(struct ix_softc *sc) 3132 { 3133 struct ixgbe_hw *hw = &sc->hw; 3134 uint32_t fwsm; 3135 int i; 3136 3137 for (i = 0; i < sc->intr_cnt; ++i) 3138 lwkt_serialize_handler_enable(sc->intr_data[i].intr_serialize); 3139 3140 sc->intr_mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3141 3142 switch (hw->mac.type) { 3143 case ixgbe_mac_82599EB: 3144 sc->intr_mask |= IXGBE_EIMS_ECC; 3145 /* Temperature sensor on some adapters */ 3146 sc->intr_mask |= IXGBE_EIMS_GPI_SDP0; 3147 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 3148 sc->intr_mask |= IXGBE_EIMS_GPI_SDP1; 3149 sc->intr_mask |= IXGBE_EIMS_GPI_SDP2; 3150 break; 3151 3152 case ixgbe_mac_X540: 3153 sc->intr_mask |= IXGBE_EIMS_ECC; 3154 /* Detect if Thermal Sensor is enabled */ 3155 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 3156 if (fwsm & IXGBE_FWSM_TS_ENABLED) 3157 sc->intr_mask |= IXGBE_EIMS_TS; 3158 break; 3159 3160 case ixgbe_mac_X550: 3161 sc->intr_mask |= IXGBE_EIMS_ECC; 3162 /* MAC thermal sensor is automatically enabled */ 3163 sc->intr_mask |= IXGBE_EIMS_TS; 3164 break; 3165 3166 case ixgbe_mac_X550EM_a: 3167 case ixgbe_mac_X550EM_x: 3168 sc->intr_mask |= IXGBE_EIMS_ECC; 3169 /* Some devices use SDP0 for important information */ 3170 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 3171 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP || 3172 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N || 3173 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 3174 sc->intr_mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw); 3175 if (hw->phy.type == ixgbe_phy_x550em_ext_t) 3176 sc->intr_mask |= IXGBE_EICR_GPI_SDP0_X540; 3177 break; 3178 3179 default: 3180 break; 3181 } 3182 3183 /* Enable Fan Failure detection */ 3184 if (sc->caps & IX_CAP_DETECT_FANFAIL) 3185 sc->intr_mask |= IXGBE_EIMS_GPI_SDP1; 3186 3187 /* With MSI-X we use auto clear for RX and TX rings */ 3188 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 3189 /* 3190 * There are no EIAC1/EIAC2 for newer chips; the related 3191 * bits for TX and RX rings > 16 are always auto clear. 3192 * 3193 * XXX which bits? There are _no_ documented EICR1 and 3194 * EICR2 at all; only EICR. 3195 */ 3196 IXGBE_WRITE_REG(hw, IXGBE_EIAC, IXGBE_EIMS_RTX_QUEUE); 3197 } else { 3198 sc->intr_mask |= IX_TX_INTR_MASK | IX_RX0_INTR_MASK; 3199 3200 KKASSERT(sc->rx_ring_inuse <= IX_MIN_RXRING_RSS); 3201 if (sc->rx_ring_inuse == IX_MIN_RXRING_RSS) 3202 sc->intr_mask |= IX_RX1_INTR_MASK; 3203 } 3204 3205 IXGBE_WRITE_REG(hw, IXGBE_EIMS, sc->intr_mask); 3206 3207 /* 3208 * Enable RX and TX rings for MSI-X 3209 */ 3210 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 3211 for (i = 0; i < sc->tx_ring_inuse; ++i) { 3212 const struct ix_tx_ring *txr = &sc->tx_rings[i]; 3213 3214 if (txr->tx_intr_vec >= 0) { 3215 IXGBE_WRITE_REG(hw, txr->tx_eims, 3216 txr->tx_eims_val); 3217 } 3218 } 3219 for (i = 0; i < sc->rx_ring_inuse; ++i) { 3220 const struct ix_rx_ring *rxr = &sc->rx_rings[i]; 3221 3222 KKASSERT(rxr->rx_intr_vec >= 0); 3223 IXGBE_WRITE_REG(hw, rxr->rx_eims, rxr->rx_eims_val); 3224 } 3225 } 3226 3227 IXGBE_WRITE_FLUSH(hw); 3228 } 3229 3230 static void 3231 ix_disable_intr(struct ix_softc *sc) 3232 { 3233 int i; 3234 3235 if (sc->intr_type == PCI_INTR_TYPE_MSIX) 3236 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 3237 3238 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3239 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 3240 } else { 3241 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000); 3242 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0); 3243 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0); 3244 } 3245 IXGBE_WRITE_FLUSH(&sc->hw); 3246 3247 for (i = 0; i < sc->intr_cnt; ++i) 3248 lwkt_serialize_handler_disable(sc->intr_data[i].intr_serialize); 3249 } 3250 3251 static void 3252 ix_slot_info(struct ix_softc *sc) 3253 { 3254 device_t dev = sc->dev; 3255 struct ixgbe_hw *hw = &sc->hw; 3256 uint32_t offset; 3257 uint16_t link; 3258 boolean_t bus_info_valid = TRUE; 3259 3260 /* Some devices are behind an internal bridge */ 3261 switch (hw->device_id) { 3262 case IXGBE_DEV_ID_82599_SFP_SF_QP: 3263 case IXGBE_DEV_ID_82599_QSFP_SF_QP: 3264 goto get_parent_info; 3265 default: 3266 break; 3267 } 3268 3269 ixgbe_get_bus_info(hw); 3270 3271 /* 3272 * Some devices don't use PCI-E, but there is no need 3273 * to display "Unknown" for bus speed and width. 3274 */ 3275 switch (hw->mac.type) { 3276 case ixgbe_mac_X550EM_x: 3277 case ixgbe_mac_X550EM_a: 3278 return; 3279 default: 3280 goto display; 3281 } 3282 3283 get_parent_info: 3284 /* 3285 * For the Quad port adapter we need to parse back up 3286 * the PCI tree to find the speed of the expansion slot 3287 * into which this adapter is plugged. A bit more work. 3288 */ 3289 dev = device_get_parent(device_get_parent(dev)); 3290 #ifdef IXGBE_DEBUG 3291 device_printf(dev, "parent pcib = %x,%x,%x\n", pci_get_bus(dev), 3292 pci_get_slot(dev), pci_get_function(dev)); 3293 #endif 3294 dev = device_get_parent(device_get_parent(dev)); 3295 #ifdef IXGBE_DEBUG 3296 device_printf(dev, "slot pcib = %x,%x,%x\n", pci_get_bus(dev), 3297 pci_get_slot(dev), pci_get_function(dev)); 3298 #endif 3299 /* Now get the PCI Express Capabilities offset */ 3300 offset = pci_get_pciecap_ptr(dev); 3301 if (offset == 0) { 3302 /* 3303 * Hmm...can't get PCI-Express capabilities. 3304 * Falling back to default method. 3305 */ 3306 bus_info_valid = FALSE; 3307 ixgbe_get_bus_info(hw); 3308 goto display; 3309 } 3310 /* ...and read the Link Status Register */ 3311 link = pci_read_config(dev, offset + PCIER_LINKSTAT, 2); 3312 ixgbe_set_pci_config_data_generic(hw, link); 3313 3314 display: 3315 device_printf(dev, "PCI Express Bus: Speed %s %s\n", 3316 hw->bus.speed == ixgbe_bus_speed_8000 ? "8.0GT/s" : 3317 hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0GT/s" : 3318 hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5GT/s" : "Unknown", 3319 hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" : 3320 hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" : 3321 hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" : "Unknown"); 3322 3323 if (bus_info_valid) { 3324 if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP && 3325 hw->bus.width <= ixgbe_bus_width_pcie_x4 && 3326 hw->bus.speed == ixgbe_bus_speed_2500) { 3327 device_printf(dev, "PCI-Express bandwidth available " 3328 "for this card is not sufficient for optimal " 3329 "performance.\n"); 3330 device_printf(dev, "For optimal performance a " 3331 "x8 PCIE, or x4 PCIE Gen2 slot is required.\n"); 3332 } 3333 if (hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP && 3334 hw->bus.width <= ixgbe_bus_width_pcie_x8 && 3335 hw->bus.speed < ixgbe_bus_speed_8000) { 3336 device_printf(dev, "PCI-Express bandwidth available " 3337 "for this card is not sufficient for optimal " 3338 "performance.\n"); 3339 device_printf(dev, "For optimal performance a " 3340 "x8 PCIE Gen3 slot is required.\n"); 3341 } 3342 } else { 3343 device_printf(dev, "Unable to determine slot speed/width. " 3344 "The speed/width reported are that of the internal " 3345 "switch.\n"); 3346 } 3347 } 3348 3349 /* 3350 * TODO comment is incorrect 3351 * 3352 * Setup the correct IVAR register for a particular MSIX interrupt 3353 * - entry is the register array entry 3354 * - vector is the MSIX vector for this queue 3355 * - type is RX/TX/MISC 3356 */ 3357 static void 3358 ix_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, 3359 int8_t type) 3360 { 3361 struct ixgbe_hw *hw = &sc->hw; 3362 uint32_t ivar, index; 3363 3364 vector |= IXGBE_IVAR_ALLOC_VAL; 3365 3366 switch (hw->mac.type) { 3367 case ixgbe_mac_82598EB: 3368 if (type == -1) 3369 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3370 else 3371 entry += (type * 64); 3372 index = (entry >> 2) & 0x1F; 3373 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3374 ivar &= ~(0xFF << (8 * (entry & 0x3))); 3375 ivar |= (vector << (8 * (entry & 0x3))); 3376 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 3377 break; 3378 3379 case ixgbe_mac_82599EB: 3380 case ixgbe_mac_X540: 3381 case ixgbe_mac_X550: 3382 case ixgbe_mac_X550EM_a: 3383 case ixgbe_mac_X550EM_x: 3384 if (type == -1) { /* MISC IVAR */ 3385 index = (entry & 1) * 8; 3386 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3387 ivar &= ~(0xFF << index); 3388 ivar |= (vector << index); 3389 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3390 } else { /* RX/TX IVARS */ 3391 index = (16 * (entry & 1)) + (8 * type); 3392 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 3393 ivar &= ~(0xFF << index); 3394 ivar |= (vector << index); 3395 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 3396 } 3397 /* FALL THROUGH */ 3398 default: 3399 break; 3400 } 3401 } 3402 3403 static boolean_t 3404 ix_sfp_probe(struct ix_softc *sc) 3405 { 3406 struct ixgbe_hw *hw = &sc->hw; 3407 3408 if (hw->phy.type == ixgbe_phy_nl && 3409 hw->phy.sfp_type == ixgbe_sfp_type_not_present) { 3410 int32_t ret; 3411 3412 ret = hw->phy.ops.identify_sfp(hw); 3413 if (ret) 3414 return FALSE; 3415 3416 ret = hw->phy.ops.reset(hw); 3417 sc->sfp_probe = FALSE; 3418 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3419 if_printf(&sc->arpcom.ac_if, 3420 "Unsupported SFP+ module detected! " 3421 "Reload driver with supported module.\n"); 3422 return FALSE; 3423 } 3424 if_printf(&sc->arpcom.ac_if, "SFP+ module detected!\n"); 3425 3426 /* We now have supported optics */ 3427 return TRUE; 3428 } 3429 return FALSE; 3430 } 3431 3432 static void 3433 ix_handle_link(struct ix_softc *sc) 3434 { 3435 ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0); 3436 ix_update_link_status(sc); 3437 } 3438 3439 /* 3440 * Handling SFP module 3441 */ 3442 static void 3443 ix_handle_mod(struct ix_softc *sc) 3444 { 3445 struct ixgbe_hw *hw = &sc->hw; 3446 uint32_t err; 3447 3448 if (sc->hw.need_crosstalk_fix) { 3449 uint32_t cage_full = 0; 3450 3451 switch (hw->mac.type) { 3452 case ixgbe_mac_82599EB: 3453 cage_full = IXGBE_READ_REG(hw, IXGBE_ESDP) & 3454 IXGBE_ESDP_SDP2; 3455 break; 3456 3457 case ixgbe_mac_X550EM_x: 3458 case ixgbe_mac_X550EM_a: 3459 cage_full = IXGBE_READ_REG(hw, IXGBE_ESDP) & 3460 IXGBE_ESDP_SDP0; 3461 break; 3462 3463 default: 3464 break; 3465 } 3466 3467 if (!cage_full) 3468 return; 3469 } 3470 3471 err = hw->phy.ops.identify_sfp(hw); 3472 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3473 if_printf(&sc->arpcom.ac_if, 3474 "Unsupported SFP+ module type was detected.\n"); 3475 return; 3476 } 3477 3478 if (hw->mac.type == ixgbe_mac_82598EB) 3479 err = hw->phy.ops.reset(hw); 3480 else 3481 err = hw->mac.ops.setup_sfp(hw); 3482 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3483 if_printf(&sc->arpcom.ac_if, 3484 "Setup failure - unsupported SFP+ module type.\n"); 3485 return; 3486 } 3487 ix_handle_msf(sc); 3488 } 3489 3490 /* 3491 * Handling MSF (multispeed fiber) 3492 */ 3493 static void 3494 ix_handle_msf(struct ix_softc *sc) 3495 { 3496 struct ixgbe_hw *hw = &sc->hw; 3497 uint32_t autoneg; 3498 3499 sc->phy_layer = ixgbe_get_supported_physical_layer(hw); 3500 ix_init_media(sc); 3501 3502 if (sc->advspeed != IXGBE_LINK_SPEED_UNKNOWN) 3503 autoneg = sc->advspeed; 3504 else 3505 autoneg = hw->phy.autoneg_advertised; 3506 if (!autoneg && hw->mac.ops.get_link_capabilities != NULL) { 3507 bool negotiate; 3508 3509 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 3510 } 3511 if (hw->mac.ops.setup_link != NULL) 3512 hw->mac.ops.setup_link(hw, autoneg, TRUE); 3513 } 3514 3515 static void 3516 ix_handle_phy(struct ix_softc *sc) 3517 { 3518 struct ixgbe_hw *hw = &sc->hw; 3519 int error; 3520 3521 error = hw->phy.ops.handle_lasi(hw); 3522 if (error == IXGBE_ERR_OVERTEMP) { 3523 if_printf(&sc->arpcom.ac_if, 3524 "CRITICAL: EXTERNAL PHY OVER TEMP!! " 3525 "PHY will downshift to lower power state!\n"); 3526 } else if (error) { 3527 if_printf(&sc->arpcom.ac_if, 3528 "Error handling LASI interrupt: %d\n", error); 3529 } 3530 } 3531 3532 static void 3533 ix_update_stats(struct ix_softc *sc) 3534 { 3535 struct ifnet *ifp = &sc->arpcom.ac_if; 3536 struct ixgbe_hw *hw = &sc->hw; 3537 struct ixgbe_hw_stats *stats = &sc->stats; 3538 uint32_t missed_rx = 0, bprc, lxon, lxoff, total; 3539 uint64_t total_missed_rx = 0; 3540 int i; 3541 3542 stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 3543 stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 3544 stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC); 3545 stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC); 3546 stats->mpc[0] += IXGBE_READ_REG(hw, IXGBE_MPC(0)); 3547 3548 for (i = 0; i < 16; i++) { 3549 stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 3550 stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 3551 stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 3552 } 3553 stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC); 3554 stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC); 3555 stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 3556 3557 /* Hardware workaround, gprc counts missed packets */ 3558 stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 3559 stats->gprc -= missed_rx; 3560 3561 if (hw->mac.type != ixgbe_mac_82598EB) { 3562 stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) + 3563 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 3564 stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) + 3565 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32); 3566 stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL) + 3567 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 3568 stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3569 stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3570 } else { 3571 stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3572 stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3573 /* 82598 only has a counter in the high register */ 3574 stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3575 stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3576 stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3577 } 3578 3579 /* 3580 * Workaround: mprc hardware is incorrectly counting 3581 * broadcasts, so for now we subtract those. 3582 */ 3583 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3584 stats->bprc += bprc; 3585 stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3586 if (hw->mac.type == ixgbe_mac_82598EB) 3587 stats->mprc -= bprc; 3588 3589 stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3590 stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3591 stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3592 stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3593 stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3594 stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3595 3596 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3597 stats->lxontxc += lxon; 3598 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3599 stats->lxofftxc += lxoff; 3600 total = lxon + lxoff; 3601 3602 stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 3603 stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3604 stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3605 stats->gptc -= total; 3606 stats->mptc -= total; 3607 stats->ptc64 -= total; 3608 stats->gotc -= total * ETHER_MIN_LEN; 3609 3610 stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3611 stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3612 stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3613 stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3614 stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 3615 stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 3616 stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 3617 stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3618 stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT); 3619 stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3620 stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3621 stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3622 stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3623 stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3624 stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3625 stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC); 3626 stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC); 3627 stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST); 3628 /* Only read FCOE on 82599 */ 3629 if (hw->mac.type != ixgbe_mac_82598EB) { 3630 stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 3631 stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 3632 stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 3633 stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 3634 stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 3635 } 3636 3637 /* Rx Errors */ 3638 IFNET_STAT_SET(ifp, iqdrops, total_missed_rx); 3639 IFNET_STAT_SET(ifp, ierrors, sc->stats.crcerrs + sc->stats.rlec); 3640 } 3641 3642 #if 0 3643 /* 3644 * Add sysctl variables, one per statistic, to the system. 3645 */ 3646 static void 3647 ix_add_hw_stats(struct ix_softc *sc) 3648 { 3649 3650 device_t dev = sc->dev; 3651 3652 struct ix_tx_ring *txr = sc->tx_rings; 3653 struct ix_rx_ring *rxr = sc->rx_rings; 3654 3655 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 3656 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 3657 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 3658 struct ixgbe_hw_stats *stats = &sc->stats; 3659 3660 struct sysctl_oid *stat_node, *queue_node; 3661 struct sysctl_oid_list *stat_list, *queue_list; 3662 3663 #define QUEUE_NAME_LEN 32 3664 char namebuf[QUEUE_NAME_LEN]; 3665 3666 /* MAC stats get the own sub node */ 3667 3668 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 3669 CTLFLAG_RD, NULL, "MAC Statistics"); 3670 stat_list = SYSCTL_CHILDREN(stat_node); 3671 3672 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs", 3673 CTLFLAG_RD, &stats->crcerrs, 3674 "CRC Errors"); 3675 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs", 3676 CTLFLAG_RD, &stats->illerrc, 3677 "Illegal Byte Errors"); 3678 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs", 3679 CTLFLAG_RD, &stats->errbc, 3680 "Byte Errors"); 3681 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards", 3682 CTLFLAG_RD, &stats->mspdc, 3683 "MAC Short Packets Discarded"); 3684 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults", 3685 CTLFLAG_RD, &stats->mlfc, 3686 "MAC Local Faults"); 3687 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults", 3688 CTLFLAG_RD, &stats->mrfc, 3689 "MAC Remote Faults"); 3690 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs", 3691 CTLFLAG_RD, &stats->rlec, 3692 "Receive Length Errors"); 3693 3694 /* Flow Control stats */ 3695 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd", 3696 CTLFLAG_RD, &stats->lxontxc, 3697 "Link XON Transmitted"); 3698 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd", 3699 CTLFLAG_RD, &stats->lxonrxc, 3700 "Link XON Received"); 3701 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd", 3702 CTLFLAG_RD, &stats->lxofftxc, 3703 "Link XOFF Transmitted"); 3704 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", 3705 CTLFLAG_RD, &stats->lxoffrxc, 3706 "Link XOFF Received"); 3707 3708 /* Packet Reception Stats */ 3709 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd", 3710 CTLFLAG_RD, &stats->tor, 3711 "Total Octets Received"); 3712 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd", 3713 CTLFLAG_RD, &stats->gorc, 3714 "Good Octets Received"); 3715 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd", 3716 CTLFLAG_RD, &stats->tpr, 3717 "Total Packets Received"); 3718 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd", 3719 CTLFLAG_RD, &stats->gprc, 3720 "Good Packets Received"); 3721 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd", 3722 CTLFLAG_RD, &stats->mprc, 3723 "Multicast Packets Received"); 3724 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd", 3725 CTLFLAG_RD, &stats->bprc, 3726 "Broadcast Packets Received"); 3727 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 3728 CTLFLAG_RD, &stats->prc64, 3729 "64 byte frames received "); 3730 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 3731 CTLFLAG_RD, &stats->prc127, 3732 "65-127 byte frames received"); 3733 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 3734 CTLFLAG_RD, &stats->prc255, 3735 "128-255 byte frames received"); 3736 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 3737 CTLFLAG_RD, &stats->prc511, 3738 "256-511 byte frames received"); 3739 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 3740 CTLFLAG_RD, &stats->prc1023, 3741 "512-1023 byte frames received"); 3742 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 3743 CTLFLAG_RD, &stats->prc1522, 3744 "1023-1522 byte frames received"); 3745 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized", 3746 CTLFLAG_RD, &stats->ruc, 3747 "Receive Undersized"); 3748 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 3749 CTLFLAG_RD, &stats->rfc, 3750 "Fragmented Packets Received "); 3751 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized", 3752 CTLFLAG_RD, &stats->roc, 3753 "Oversized Packets Received"); 3754 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd", 3755 CTLFLAG_RD, &stats->rjc, 3756 "Received Jabber"); 3757 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd", 3758 CTLFLAG_RD, &stats->mngprc, 3759 "Management Packets Received"); 3760 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd", 3761 CTLFLAG_RD, &stats->mngptc, 3762 "Management Packets Dropped"); 3763 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs", 3764 CTLFLAG_RD, &stats->xec, 3765 "Checksum Errors"); 3766 3767 /* Packet Transmission Stats */ 3768 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 3769 CTLFLAG_RD, &stats->gotc, 3770 "Good Octets Transmitted"); 3771 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 3772 CTLFLAG_RD, &stats->tpt, 3773 "Total Packets Transmitted"); 3774 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 3775 CTLFLAG_RD, &stats->gptc, 3776 "Good Packets Transmitted"); 3777 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 3778 CTLFLAG_RD, &stats->bptc, 3779 "Broadcast Packets Transmitted"); 3780 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 3781 CTLFLAG_RD, &stats->mptc, 3782 "Multicast Packets Transmitted"); 3783 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd", 3784 CTLFLAG_RD, &stats->mngptc, 3785 "Management Packets Transmitted"); 3786 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 3787 CTLFLAG_RD, &stats->ptc64, 3788 "64 byte frames transmitted "); 3789 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 3790 CTLFLAG_RD, &stats->ptc127, 3791 "65-127 byte frames transmitted"); 3792 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 3793 CTLFLAG_RD, &stats->ptc255, 3794 "128-255 byte frames transmitted"); 3795 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 3796 CTLFLAG_RD, &stats->ptc511, 3797 "256-511 byte frames transmitted"); 3798 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 3799 CTLFLAG_RD, &stats->ptc1023, 3800 "512-1023 byte frames transmitted"); 3801 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 3802 CTLFLAG_RD, &stats->ptc1522, 3803 "1024-1522 byte frames transmitted"); 3804 } 3805 #endif 3806 3807 /* 3808 * Enable the hardware to drop packets when the buffer is full. 3809 * This is useful when multiple RX rings are used, so that no 3810 * single RX ring being full stalls the entire RX engine. We 3811 * only enable this when multiple RX rings are used and when 3812 * flow control is disabled. 3813 */ 3814 static void 3815 ix_enable_rx_drop(struct ix_softc *sc) 3816 { 3817 struct ixgbe_hw *hw = &sc->hw; 3818 int i; 3819 3820 if (bootverbose) { 3821 if_printf(&sc->arpcom.ac_if, 3822 "flow control %s, enable RX drop\n", 3823 ix_fc2str(sc->hw.fc.current_mode)); 3824 } 3825 3826 for (i = 0; i < sc->rx_ring_inuse; ++i) { 3827 uint32_t srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 3828 3829 srrctl |= IXGBE_SRRCTL_DROP_EN; 3830 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 3831 } 3832 } 3833 3834 static void 3835 ix_disable_rx_drop(struct ix_softc *sc) 3836 { 3837 struct ixgbe_hw *hw = &sc->hw; 3838 int i; 3839 3840 if (bootverbose) { 3841 if_printf(&sc->arpcom.ac_if, 3842 "flow control %s, disable RX drop\n", 3843 ix_fc2str(sc->hw.fc.current_mode)); 3844 } 3845 3846 for (i = 0; i < sc->rx_ring_inuse; ++i) { 3847 uint32_t srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 3848 3849 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 3850 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 3851 } 3852 } 3853 3854 static void 3855 ix_setup_serialize(struct ix_softc *sc) 3856 { 3857 int i = 0, j; 3858 3859 /* Main + RX + TX */ 3860 sc->nserialize = 1 + sc->rx_ring_cnt + sc->tx_ring_cnt; 3861 sc->serializes = 3862 kmalloc(sc->nserialize * sizeof(struct lwkt_serialize *), 3863 M_DEVBUF, M_WAITOK | M_ZERO); 3864 3865 /* 3866 * Setup serializes 3867 * 3868 * NOTE: Order is critical 3869 */ 3870 3871 KKASSERT(i < sc->nserialize); 3872 sc->serializes[i++] = &sc->main_serialize; 3873 3874 for (j = 0; j < sc->rx_ring_cnt; ++j) { 3875 KKASSERT(i < sc->nserialize); 3876 sc->serializes[i++] = &sc->rx_rings[j].rx_serialize; 3877 } 3878 3879 for (j = 0; j < sc->tx_ring_cnt; ++j) { 3880 KKASSERT(i < sc->nserialize); 3881 sc->serializes[i++] = &sc->tx_rings[j].tx_serialize; 3882 } 3883 3884 KKASSERT(i == sc->nserialize); 3885 } 3886 3887 static int 3888 ix_alloc_intr(struct ix_softc *sc) 3889 { 3890 struct ix_intr_data *intr; 3891 struct ix_tx_ring *txr; 3892 u_int intr_flags; 3893 int i; 3894 3895 ix_alloc_msix(sc); 3896 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 3897 ix_set_ring_inuse(sc, FALSE); 3898 goto done; 3899 } 3900 3901 /* 3902 * Reset some settings changed by ix_alloc_msix(). 3903 */ 3904 if (sc->rx_rmap_intr != NULL) { 3905 if_ringmap_free(sc->rx_rmap_intr); 3906 sc->rx_rmap_intr = NULL; 3907 } 3908 if (sc->tx_rmap_intr != NULL) { 3909 if_ringmap_free(sc->tx_rmap_intr); 3910 sc->tx_rmap_intr = NULL; 3911 } 3912 if (sc->intr_data != NULL) { 3913 kfree(sc->intr_data, M_DEVBUF); 3914 sc->intr_data = NULL; 3915 } 3916 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3917 txr = &sc->tx_rings[i]; 3918 txr->tx_intr_vec = -1; 3919 txr->tx_intr_cpuid = -1; 3920 } 3921 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3922 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 3923 3924 rxr->rx_intr_vec = -1; 3925 rxr->rx_txr = NULL; 3926 } 3927 3928 sc->intr_cnt = 1; 3929 sc->intr_data = kmalloc(sizeof(struct ix_intr_data), M_DEVBUF, 3930 M_WAITOK | M_ZERO); 3931 intr = &sc->intr_data[0]; 3932 3933 /* 3934 * Allocate MSI/legacy interrupt resource 3935 */ 3936 if (sc->caps & IX_CAP_LEGACY_INTR) { 3937 sc->intr_type = pci_alloc_1intr(sc->dev, ix_msi_enable, 3938 &intr->intr_rid, &intr_flags); 3939 } else { 3940 int cpu; 3941 3942 /* 3943 * Only MSI is supported. 3944 */ 3945 cpu = device_getenv_int(sc->dev, "msi.cpu", -1); 3946 if (cpu >= ncpus) 3947 cpu = ncpus - 1; 3948 3949 if (pci_alloc_msi(sc->dev, &intr->intr_rid, 1, cpu) == 0) { 3950 sc->intr_type = PCI_INTR_TYPE_MSI; 3951 intr_flags = RF_ACTIVE; 3952 } else { 3953 sc->intr_type = PCI_INTR_TYPE_LEGACY; 3954 device_printf(sc->dev, "Unable to allocate MSI\n"); 3955 return ENXIO; 3956 } 3957 } 3958 3959 intr->intr_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 3960 &intr->intr_rid, intr_flags); 3961 if (intr->intr_res == NULL) { 3962 device_printf(sc->dev, "Unable to allocate bus resource: " 3963 "interrupt\n"); 3964 return ENXIO; 3965 } 3966 3967 intr->intr_serialize = &sc->main_serialize; 3968 intr->intr_cpuid = rman_get_cpuid(intr->intr_res); 3969 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3970 intr->intr_func = ix_intr_82598; 3971 else 3972 intr->intr_func = ix_intr; 3973 intr->intr_funcarg = sc; 3974 intr->intr_rate = IX_INTR_RATE; 3975 intr->intr_use = IX_INTR_USE_RXTX; 3976 3977 sc->tx_rings[0].tx_intr_vec = IX_TX_INTR_VEC; 3978 sc->tx_rings[0].tx_intr_cpuid = intr->intr_cpuid; 3979 3980 sc->rx_rings[0].rx_intr_vec = IX_RX0_INTR_VEC; 3981 3982 ix_set_ring_inuse(sc, FALSE); 3983 3984 KKASSERT(sc->rx_ring_inuse <= IX_MIN_RXRING_RSS); 3985 if (sc->rx_ring_inuse == IX_MIN_RXRING_RSS) { 3986 sc->rx_rings[1].rx_intr_vec = IX_RX1_INTR_VEC; 3987 3988 /* 3989 * Allocate RX ring map for RSS setup. 3990 */ 3991 sc->rx_rmap_intr = if_ringmap_alloc(sc->dev, 3992 IX_MIN_RXRING_RSS, IX_MIN_RXRING_RSS); 3993 KASSERT(if_ringmap_count(sc->rx_rmap_intr) == 3994 sc->rx_ring_inuse, ("RX ring inuse mismatch")); 3995 } 3996 done: 3997 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3998 txr = &sc->tx_rings[i]; 3999 if (txr->tx_intr_cpuid < 0) 4000 txr->tx_intr_cpuid = 0; 4001 } 4002 return 0; 4003 } 4004 4005 static void 4006 ix_free_intr(struct ix_softc *sc) 4007 { 4008 if (sc->intr_data == NULL) 4009 return; 4010 4011 if (sc->intr_type != PCI_INTR_TYPE_MSIX) { 4012 struct ix_intr_data *intr = &sc->intr_data[0]; 4013 4014 KKASSERT(sc->intr_cnt == 1); 4015 if (intr->intr_res != NULL) { 4016 bus_release_resource(sc->dev, SYS_RES_IRQ, 4017 intr->intr_rid, intr->intr_res); 4018 } 4019 if (sc->intr_type == PCI_INTR_TYPE_MSI) 4020 pci_release_msi(sc->dev); 4021 4022 kfree(sc->intr_data, M_DEVBUF); 4023 } else { 4024 ix_free_msix(sc, TRUE); 4025 } 4026 } 4027 4028 static void 4029 ix_set_ring_inuse(struct ix_softc *sc, boolean_t polling) 4030 { 4031 sc->rx_ring_inuse = ix_get_rxring_inuse(sc, polling); 4032 sc->tx_ring_inuse = ix_get_txring_inuse(sc, polling); 4033 if (bootverbose) { 4034 if_printf(&sc->arpcom.ac_if, 4035 "RX rings %d/%d, TX rings %d/%d\n", 4036 sc->rx_ring_inuse, sc->rx_ring_cnt, 4037 sc->tx_ring_inuse, sc->tx_ring_cnt); 4038 } 4039 } 4040 4041 static int 4042 ix_get_rxring_inuse(const struct ix_softc *sc, boolean_t polling) 4043 { 4044 if (!IX_ENABLE_HWRSS(sc)) 4045 return 1; 4046 4047 if (polling) 4048 return sc->rx_ring_cnt; 4049 else if (sc->intr_type != PCI_INTR_TYPE_MSIX) 4050 return IX_MIN_RXRING_RSS; 4051 else 4052 return sc->rx_ring_msix; 4053 } 4054 4055 static int 4056 ix_get_txring_inuse(const struct ix_softc *sc, boolean_t polling) 4057 { 4058 if (!IX_ENABLE_HWTSS(sc)) 4059 return 1; 4060 4061 if (polling) 4062 return sc->tx_ring_cnt; 4063 else if (sc->intr_type != PCI_INTR_TYPE_MSIX) 4064 return 1; 4065 else 4066 return sc->tx_ring_msix; 4067 } 4068 4069 static int 4070 ix_setup_intr(struct ix_softc *sc) 4071 { 4072 int i; 4073 4074 for (i = 0; i < sc->intr_cnt; ++i) { 4075 struct ix_intr_data *intr = &sc->intr_data[i]; 4076 int error; 4077 4078 error = bus_setup_intr_descr(sc->dev, intr->intr_res, 4079 INTR_MPSAFE, intr->intr_func, intr->intr_funcarg, 4080 &intr->intr_hand, intr->intr_serialize, intr->intr_desc); 4081 if (error) { 4082 device_printf(sc->dev, "can't setup %dth intr\n", i); 4083 ix_teardown_intr(sc, i); 4084 return error; 4085 } 4086 } 4087 return 0; 4088 } 4089 4090 static void 4091 ix_teardown_intr(struct ix_softc *sc, int intr_cnt) 4092 { 4093 int i; 4094 4095 if (sc->intr_data == NULL) 4096 return; 4097 4098 for (i = 0; i < intr_cnt; ++i) { 4099 struct ix_intr_data *intr = &sc->intr_data[i]; 4100 4101 bus_teardown_intr(sc->dev, intr->intr_res, intr->intr_hand); 4102 } 4103 } 4104 4105 static void 4106 ix_serialize(struct ifnet *ifp, enum ifnet_serialize slz) 4107 { 4108 struct ix_softc *sc = ifp->if_softc; 4109 4110 ifnet_serialize_array_enter(sc->serializes, sc->nserialize, slz); 4111 } 4112 4113 static void 4114 ix_deserialize(struct ifnet *ifp, enum ifnet_serialize slz) 4115 { 4116 struct ix_softc *sc = ifp->if_softc; 4117 4118 ifnet_serialize_array_exit(sc->serializes, sc->nserialize, slz); 4119 } 4120 4121 static int 4122 ix_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz) 4123 { 4124 struct ix_softc *sc = ifp->if_softc; 4125 4126 return ifnet_serialize_array_try(sc->serializes, sc->nserialize, slz); 4127 } 4128 4129 static void 4130 ix_serialize_skipmain(struct ix_softc *sc) 4131 { 4132 lwkt_serialize_array_enter(sc->serializes, sc->nserialize, 1); 4133 } 4134 4135 static void 4136 ix_deserialize_skipmain(struct ix_softc *sc) 4137 { 4138 lwkt_serialize_array_exit(sc->serializes, sc->nserialize, 1); 4139 } 4140 4141 #ifdef INVARIANTS 4142 4143 static void 4144 ix_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz, 4145 boolean_t serialized) 4146 { 4147 struct ix_softc *sc = ifp->if_softc; 4148 4149 ifnet_serialize_array_assert(sc->serializes, sc->nserialize, slz, 4150 serialized); 4151 } 4152 4153 #endif /* INVARIANTS */ 4154 4155 static void 4156 ix_free_rings(struct ix_softc *sc) 4157 { 4158 int i; 4159 4160 if (sc->tx_rings != NULL) { 4161 for (i = 0; i < sc->tx_ring_cnt; ++i) { 4162 struct ix_tx_ring *txr = &sc->tx_rings[i]; 4163 4164 ix_destroy_tx_ring(txr, txr->tx_ndesc); 4165 } 4166 kfree(sc->tx_rings, M_DEVBUF); 4167 } 4168 4169 if (sc->rx_rings != NULL) { 4170 for (i =0; i < sc->rx_ring_cnt; ++i) { 4171 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 4172 4173 ix_destroy_rx_ring(rxr, rxr->rx_ndesc); 4174 } 4175 kfree(sc->rx_rings, M_DEVBUF); 4176 } 4177 4178 if (sc->parent_tag != NULL) 4179 bus_dma_tag_destroy(sc->parent_tag); 4180 } 4181 4182 static void 4183 ix_watchdog_reset(struct ix_softc *sc) 4184 { 4185 int i; 4186 4187 ASSERT_IFNET_SERIALIZED_ALL(&sc->arpcom.ac_if); 4188 ix_init(sc); 4189 for (i = 0; i < sc->tx_ring_inuse; ++i) 4190 ifsq_devstart_sched(sc->tx_rings[i].tx_ifsq); 4191 } 4192 4193 static void 4194 ix_sync_netisr(struct ix_softc *sc, int flags) 4195 { 4196 struct ifnet *ifp = &sc->arpcom.ac_if; 4197 4198 ifnet_serialize_all(ifp); 4199 if (ifp->if_flags & IFF_RUNNING) { 4200 ifp->if_flags &= ~(IFF_RUNNING | flags); 4201 } else { 4202 ifnet_deserialize_all(ifp); 4203 return; 4204 } 4205 ifnet_deserialize_all(ifp); 4206 4207 /* Make sure that polling stopped. */ 4208 netmsg_service_sync(); 4209 } 4210 4211 static void 4212 ix_watchdog_task(void *xsc, int pending __unused) 4213 { 4214 struct ix_softc *sc = xsc; 4215 struct ifnet *ifp = &sc->arpcom.ac_if; 4216 4217 ix_sync_netisr(sc, 0); 4218 4219 ifnet_serialize_all(ifp); 4220 if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) == IFF_UP) 4221 ix_watchdog_reset(sc); 4222 ifnet_deserialize_all(ifp); 4223 } 4224 4225 static void 4226 ix_watchdog(struct ifaltq_subque *ifsq) 4227 { 4228 struct ix_tx_ring *txr = ifsq_get_priv(ifsq); 4229 struct ifnet *ifp = ifsq_get_ifp(ifsq); 4230 struct ix_softc *sc = ifp->if_softc; 4231 4232 KKASSERT(txr->tx_ifsq == ifsq); 4233 ASSERT_IFNET_SERIALIZED_ALL(ifp); 4234 4235 /* 4236 * If the interface has been paused then don't do the watchdog check 4237 */ 4238 if (IXGBE_READ_REG(&sc->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) { 4239 ifsq_watchdog_set_count(&txr->tx_watchdog, 5); 4240 return; 4241 } 4242 4243 if_printf(ifp, "Watchdog timeout -- resetting\n"); 4244 if_printf(ifp, "Queue(%d) tdh = %d, hw tdt = %d\n", txr->tx_idx, 4245 IXGBE_READ_REG(&sc->hw, IXGBE_TDH(txr->tx_idx)), 4246 IXGBE_READ_REG(&sc->hw, IXGBE_TDT(txr->tx_idx))); 4247 if_printf(ifp, "TX(%d) desc avail = %d, next TX to Clean = %d\n", 4248 txr->tx_idx, txr->tx_avail, txr->tx_next_clean); 4249 4250 if ((ifp->if_flags & (IFF_IDIRECT | IFF_NPOLLING | IFF_RUNNING)) == 4251 (IFF_IDIRECT | IFF_NPOLLING | IFF_RUNNING)) 4252 taskqueue_enqueue(taskqueue_thread[0], &sc->wdog_task); 4253 else 4254 ix_watchdog_reset(sc); 4255 } 4256 4257 static void 4258 ix_free_tx_ring(struct ix_tx_ring *txr) 4259 { 4260 int i; 4261 4262 for (i = 0; i < txr->tx_ndesc; ++i) { 4263 struct ix_tx_buf *txbuf = &txr->tx_buf[i]; 4264 4265 if (txbuf->m_head != NULL) 4266 ix_free_txbuf(txr, txbuf); 4267 } 4268 } 4269 4270 static void 4271 ix_free_rx_ring(struct ix_rx_ring *rxr) 4272 { 4273 int i; 4274 4275 for (i = 0; i < rxr->rx_ndesc; ++i) { 4276 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i]; 4277 4278 if (rxbuf->fmp != NULL) { 4279 m_freem(rxbuf->fmp); 4280 rxbuf->fmp = NULL; 4281 rxbuf->lmp = NULL; 4282 } else { 4283 KKASSERT(rxbuf->lmp == NULL); 4284 } 4285 if (rxbuf->m_head != NULL) { 4286 bus_dmamap_unload(rxr->rx_tag, rxbuf->map); 4287 m_freem(rxbuf->m_head); 4288 rxbuf->m_head = NULL; 4289 } 4290 } 4291 } 4292 4293 static int 4294 ix_newbuf(struct ix_rx_ring *rxr, int i, boolean_t wait) 4295 { 4296 struct mbuf *m; 4297 bus_dma_segment_t seg; 4298 bus_dmamap_t map; 4299 struct ix_rx_buf *rxbuf; 4300 int flags, error, nseg; 4301 4302 flags = M_NOWAIT; 4303 if (__predict_false(wait)) 4304 flags = M_WAITOK; 4305 4306 m = m_getjcl(flags, MT_DATA, M_PKTHDR, rxr->rx_mbuf_sz); 4307 if (m == NULL) { 4308 if (wait) { 4309 if_printf(&rxr->rx_sc->arpcom.ac_if, 4310 "Unable to allocate RX mbuf\n"); 4311 } 4312 return ENOBUFS; 4313 } 4314 m->m_len = m->m_pkthdr.len = rxr->rx_mbuf_sz; 4315 4316 error = bus_dmamap_load_mbuf_segment(rxr->rx_tag, 4317 rxr->rx_sparemap, m, &seg, 1, &nseg, BUS_DMA_NOWAIT); 4318 if (error) { 4319 m_freem(m); 4320 if (wait) { 4321 if_printf(&rxr->rx_sc->arpcom.ac_if, 4322 "Unable to load RX mbuf\n"); 4323 } 4324 return error; 4325 } 4326 4327 rxbuf = &rxr->rx_buf[i]; 4328 if (rxbuf->m_head != NULL) 4329 bus_dmamap_unload(rxr->rx_tag, rxbuf->map); 4330 4331 map = rxbuf->map; 4332 rxbuf->map = rxr->rx_sparemap; 4333 rxr->rx_sparemap = map; 4334 4335 rxbuf->m_head = m; 4336 rxbuf->paddr = seg.ds_addr; 4337 4338 ix_setup_rxdesc(&rxr->rx_base[i], rxbuf); 4339 return 0; 4340 } 4341 4342 static void 4343 ix_add_sysctl(struct ix_softc *sc) 4344 { 4345 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 4346 struct sysctl_oid *tree = device_get_sysctl_tree(sc->dev); 4347 char node[32]; 4348 int i; 4349 4350 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4351 OID_AUTO, "rxr", CTLFLAG_RD, &sc->rx_ring_cnt, 0, "# of RX rings"); 4352 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4353 OID_AUTO, "rxr_inuse", CTLFLAG_RD, &sc->rx_ring_inuse, 0, 4354 "# of RX rings used"); 4355 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4356 OID_AUTO, "txr", CTLFLAG_RD, &sc->tx_ring_cnt, 0, "# of TX rings"); 4357 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4358 OID_AUTO, "txr_inuse", CTLFLAG_RD, &sc->tx_ring_inuse, 0, 4359 "# of TX rings used"); 4360 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4361 OID_AUTO, "rxd", CTLTYPE_INT | CTLFLAG_RD, 4362 sc, 0, ix_sysctl_rxd, "I", 4363 "# of RX descs"); 4364 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4365 OID_AUTO, "txd", CTLTYPE_INT | CTLFLAG_RD, 4366 sc, 0, ix_sysctl_txd, "I", 4367 "# of TX descs"); 4368 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4369 OID_AUTO, "tx_wreg_nsegs", CTLTYPE_INT | CTLFLAG_RW, 4370 sc, 0, ix_sysctl_tx_wreg_nsegs, "I", 4371 "# of segments sent before write to hardware register"); 4372 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4373 OID_AUTO, "rx_wreg_nsegs", CTLTYPE_INT | CTLFLAG_RW, 4374 sc, 0, ix_sysctl_rx_wreg_nsegs, "I", 4375 "# of received segments sent before write to hardware register"); 4376 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4377 OID_AUTO, "tx_intr_nsegs", CTLTYPE_INT | CTLFLAG_RW, 4378 sc, 0, ix_sysctl_tx_intr_nsegs, "I", 4379 "# of segments per TX interrupt"); 4380 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4381 OID_AUTO, "direct_input", CTLFLAG_RW, &sc->direct_input, 0, 4382 "Enable direct input"); 4383 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 4384 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4385 OID_AUTO, "tx_msix_cpumap", CTLTYPE_OPAQUE | CTLFLAG_RD, 4386 sc->tx_rmap_intr, 0, if_ringmap_cpumap_sysctl, "I", 4387 "TX MSI-X CPU map"); 4388 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4389 OID_AUTO, "rx_msix_cpumap", CTLTYPE_OPAQUE | CTLFLAG_RD, 4390 sc->rx_rmap_intr, 0, if_ringmap_cpumap_sysctl, "I", 4391 "RX MSI-X CPU map"); 4392 } 4393 #ifdef IFPOLL_ENABLE 4394 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4395 OID_AUTO, "tx_poll_cpumap", CTLTYPE_OPAQUE | CTLFLAG_RD, 4396 sc->tx_rmap, 0, if_ringmap_cpumap_sysctl, "I", 4397 "TX polling CPU map"); 4398 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4399 OID_AUTO, "rx_poll_cpumap", CTLTYPE_OPAQUE | CTLFLAG_RD, 4400 sc->rx_rmap, 0, if_ringmap_cpumap_sysctl, "I", 4401 "RX polling CPU map"); 4402 #endif 4403 4404 #define IX_ADD_INTR_RATE_SYSCTL(sc, use, name) \ 4405 do { \ 4406 ix_add_intr_rate_sysctl(sc, IX_INTR_USE_##use, #name, \ 4407 ix_sysctl_##name, #use " interrupt rate"); \ 4408 } while (0) 4409 4410 IX_ADD_INTR_RATE_SYSCTL(sc, RXTX, rxtx_intr_rate); 4411 IX_ADD_INTR_RATE_SYSCTL(sc, RX, rx_intr_rate); 4412 IX_ADD_INTR_RATE_SYSCTL(sc, TX, tx_intr_rate); 4413 IX_ADD_INTR_RATE_SYSCTL(sc, STATUS, sts_intr_rate); 4414 4415 #undef IX_ADD_INTR_RATE_SYSCTL 4416 4417 #ifdef IX_RSS_DEBUG 4418 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4419 OID_AUTO, "rss_debug", CTLFLAG_RW, &sc->rss_debug, 0, 4420 "RSS debug level"); 4421 for (i = 0; i < sc->rx_ring_cnt; ++i) { 4422 ksnprintf(node, sizeof(node), "rx%d_pkt", i); 4423 SYSCTL_ADD_ULONG(ctx, 4424 SYSCTL_CHILDREN(tree), OID_AUTO, node, 4425 CTLFLAG_RW, &sc->rx_rings[i].rx_pkts, "RXed packets"); 4426 } 4427 #endif 4428 for (i = 0; i < sc->tx_ring_cnt; ++i) { 4429 struct ix_tx_ring *txr = &sc->tx_rings[i]; 4430 4431 ksnprintf(node, sizeof(node), "tx%d_nmbuf", i); 4432 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, node, 4433 CTLTYPE_INT | CTLFLAG_RD, txr, 0, ix_sysctl_tx_nmbuf, "I", 4434 "# of pending TX mbufs"); 4435 4436 ksnprintf(node, sizeof(node), "tx%d_gc", i); 4437 SYSCTL_ADD_ULONG(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, node, 4438 CTLFLAG_RW, &txr->tx_gc, "# of TX desc GC"); 4439 } 4440 4441 #if 0 4442 ix_add_hw_stats(sc); 4443 #endif 4444 4445 } 4446 4447 static int 4448 ix_sysctl_tx_nmbuf(SYSCTL_HANDLER_ARGS) 4449 { 4450 struct ix_tx_ring *txr = (void *)arg1; 4451 int nmbuf; 4452 4453 nmbuf = txr->tx_nmbuf; 4454 return (sysctl_handle_int(oidp, &nmbuf, 0, req)); 4455 } 4456 4457 static int 4458 ix_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS) 4459 { 4460 struct ix_softc *sc = (void *)arg1; 4461 struct ifnet *ifp = &sc->arpcom.ac_if; 4462 int error, nsegs, i; 4463 4464 nsegs = sc->tx_rings[0].tx_wreg_nsegs; 4465 error = sysctl_handle_int(oidp, &nsegs, 0, req); 4466 if (error || req->newptr == NULL) 4467 return error; 4468 if (nsegs < 0) 4469 return EINVAL; 4470 4471 ifnet_serialize_all(ifp); 4472 for (i = 0; i < sc->tx_ring_cnt; ++i) 4473 sc->tx_rings[i].tx_wreg_nsegs = nsegs; 4474 ifnet_deserialize_all(ifp); 4475 4476 return 0; 4477 } 4478 4479 static int 4480 ix_sysctl_rx_wreg_nsegs(SYSCTL_HANDLER_ARGS) 4481 { 4482 struct ix_softc *sc = (void *)arg1; 4483 struct ifnet *ifp = &sc->arpcom.ac_if; 4484 int error, nsegs, i; 4485 4486 nsegs = sc->rx_rings[0].rx_wreg_nsegs; 4487 error = sysctl_handle_int(oidp, &nsegs, 0, req); 4488 if (error || req->newptr == NULL) 4489 return error; 4490 if (nsegs < 0) 4491 return EINVAL; 4492 4493 ifnet_serialize_all(ifp); 4494 for (i = 0; i < sc->rx_ring_cnt; ++i) 4495 sc->rx_rings[i].rx_wreg_nsegs =nsegs; 4496 ifnet_deserialize_all(ifp); 4497 4498 return 0; 4499 } 4500 4501 static int 4502 ix_sysctl_txd(SYSCTL_HANDLER_ARGS) 4503 { 4504 struct ix_softc *sc = (void *)arg1; 4505 int txd; 4506 4507 txd = sc->tx_rings[0].tx_ndesc; 4508 return sysctl_handle_int(oidp, &txd, 0, req); 4509 } 4510 4511 static int 4512 ix_sysctl_rxd(SYSCTL_HANDLER_ARGS) 4513 { 4514 struct ix_softc *sc = (void *)arg1; 4515 int rxd; 4516 4517 rxd = sc->rx_rings[0].rx_ndesc; 4518 return sysctl_handle_int(oidp, &rxd, 0, req); 4519 } 4520 4521 static int 4522 ix_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS) 4523 { 4524 struct ix_softc *sc = (void *)arg1; 4525 struct ifnet *ifp = &sc->arpcom.ac_if; 4526 struct ix_tx_ring *txr = &sc->tx_rings[0]; 4527 int error, nsegs; 4528 4529 nsegs = txr->tx_intr_nsegs; 4530 error = sysctl_handle_int(oidp, &nsegs, 0, req); 4531 if (error || req->newptr == NULL) 4532 return error; 4533 if (nsegs < 0) 4534 return EINVAL; 4535 4536 ifnet_serialize_all(ifp); 4537 4538 if (nsegs >= txr->tx_ndesc - IX_MAX_SCATTER - IX_TX_RESERVED) { 4539 error = EINVAL; 4540 } else { 4541 int i; 4542 4543 error = 0; 4544 for (i = 0; i < sc->tx_ring_cnt; ++i) 4545 sc->tx_rings[i].tx_intr_nsegs = nsegs; 4546 } 4547 4548 ifnet_deserialize_all(ifp); 4549 4550 return error; 4551 } 4552 4553 static void 4554 ix_set_eitr(struct ix_softc *sc, int idx, int rate) 4555 { 4556 uint32_t eitr, eitr_intvl; 4557 4558 eitr = IXGBE_READ_REG(&sc->hw, IXGBE_EITR(idx)); 4559 eitr_intvl = 1000000000 / 256 / rate; 4560 4561 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 4562 eitr &= ~IX_EITR_INTVL_MASK_82598; 4563 if (eitr_intvl == 0) 4564 eitr_intvl = 1; 4565 else if (eitr_intvl > IX_EITR_INTVL_MASK_82598) 4566 eitr_intvl = IX_EITR_INTVL_MASK_82598; 4567 } else { 4568 eitr &= ~IX_EITR_INTVL_MASK; 4569 4570 eitr_intvl &= ~IX_EITR_INTVL_RSVD_MASK; 4571 if (eitr_intvl == 0) 4572 eitr_intvl = IX_EITR_INTVL_MIN; 4573 else if (eitr_intvl > IX_EITR_INTVL_MAX) 4574 eitr_intvl = IX_EITR_INTVL_MAX; 4575 } 4576 eitr |= eitr_intvl; 4577 4578 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(idx), eitr); 4579 } 4580 4581 static int 4582 ix_sysctl_rxtx_intr_rate(SYSCTL_HANDLER_ARGS) 4583 { 4584 return ix_sysctl_intr_rate(oidp, arg1, arg2, req, IX_INTR_USE_RXTX); 4585 } 4586 4587 static int 4588 ix_sysctl_rx_intr_rate(SYSCTL_HANDLER_ARGS) 4589 { 4590 return ix_sysctl_intr_rate(oidp, arg1, arg2, req, IX_INTR_USE_RX); 4591 } 4592 4593 static int 4594 ix_sysctl_tx_intr_rate(SYSCTL_HANDLER_ARGS) 4595 { 4596 return ix_sysctl_intr_rate(oidp, arg1, arg2, req, IX_INTR_USE_TX); 4597 } 4598 4599 static int 4600 ix_sysctl_sts_intr_rate(SYSCTL_HANDLER_ARGS) 4601 { 4602 return ix_sysctl_intr_rate(oidp, arg1, arg2, req, IX_INTR_USE_STATUS); 4603 } 4604 4605 static int 4606 ix_sysctl_intr_rate(SYSCTL_HANDLER_ARGS, int use) 4607 { 4608 struct ix_softc *sc = (void *)arg1; 4609 struct ifnet *ifp = &sc->arpcom.ac_if; 4610 int error, rate, i; 4611 4612 rate = 0; 4613 for (i = 0; i < sc->intr_cnt; ++i) { 4614 if (sc->intr_data[i].intr_use == use) { 4615 rate = sc->intr_data[i].intr_rate; 4616 break; 4617 } 4618 } 4619 4620 error = sysctl_handle_int(oidp, &rate, 0, req); 4621 if (error || req->newptr == NULL) 4622 return error; 4623 if (rate <= 0) 4624 return EINVAL; 4625 4626 ifnet_serialize_all(ifp); 4627 4628 for (i = 0; i < sc->intr_cnt; ++i) { 4629 if (sc->intr_data[i].intr_use == use) { 4630 sc->intr_data[i].intr_rate = rate; 4631 if (ifp->if_flags & IFF_RUNNING) 4632 ix_set_eitr(sc, i, rate); 4633 } 4634 } 4635 4636 ifnet_deserialize_all(ifp); 4637 4638 return error; 4639 } 4640 4641 static void 4642 ix_add_intr_rate_sysctl(struct ix_softc *sc, int use, 4643 const char *name, int (*handler)(SYSCTL_HANDLER_ARGS), const char *desc) 4644 { 4645 int i; 4646 4647 for (i = 0; i < sc->intr_cnt; ++i) { 4648 if (sc->intr_data[i].intr_use == use) { 4649 SYSCTL_ADD_PROC(device_get_sysctl_ctx(sc->dev), 4650 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), 4651 OID_AUTO, name, CTLTYPE_INT | CTLFLAG_RW, 4652 sc, 0, handler, "I", desc); 4653 break; 4654 } 4655 } 4656 } 4657 4658 static int 4659 ix_get_timer_cpuid(const struct ix_softc *sc, boolean_t polling) 4660 { 4661 if (polling || sc->intr_type == PCI_INTR_TYPE_MSIX) 4662 return 0; /* XXX fixed */ 4663 else 4664 return rman_get_cpuid(sc->intr_data[0].intr_res); 4665 } 4666 4667 static void 4668 ix_alloc_msix(struct ix_softc *sc) 4669 { 4670 int msix_enable, msix_cnt, msix_ring, alloc_cnt; 4671 struct ix_intr_data *intr; 4672 int i, x, error; 4673 int ring_cnt, ring_cntmax; 4674 boolean_t setup = FALSE; 4675 4676 msix_enable = ix_msix_enable; 4677 /* 4678 * Don't enable MSI-X on 82598 by default, see: 4679 * 82598 specification update errata #38 4680 */ 4681 if (sc->hw.mac.type == ixgbe_mac_82598EB) 4682 msix_enable = 0; 4683 msix_enable = device_getenv_int(sc->dev, "msix.enable", msix_enable); 4684 if (!msix_enable) 4685 return; 4686 4687 msix_cnt = pci_msix_count(sc->dev); 4688 #ifdef IX_MSIX_DEBUG 4689 msix_cnt = device_getenv_int(sc->dev, "msix.count", msix_cnt); 4690 #endif 4691 if (msix_cnt <= 1) { 4692 /* One MSI-X model does not make sense. */ 4693 return; 4694 } 4695 4696 /* 4697 * Make sure that we don't break interrupt related registers 4698 * (EIMS, etc) limitation. 4699 */ 4700 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 4701 if (msix_cnt > IX_MAX_MSIX_82598) 4702 msix_cnt = IX_MAX_MSIX_82598; 4703 } else { 4704 if (msix_cnt > IX_MAX_MSIX) 4705 msix_cnt = IX_MAX_MSIX; 4706 } 4707 if (bootverbose) 4708 device_printf(sc->dev, "MSI-X count %d\n", msix_cnt); 4709 msix_ring = msix_cnt - 1; /* -1 for status */ 4710 4711 /* 4712 * Configure # of RX/TX rings usable by MSI-X. 4713 */ 4714 ix_get_rxring_cnt(sc, &ring_cnt, &ring_cntmax); 4715 if (ring_cntmax > msix_ring) 4716 ring_cntmax = msix_ring; 4717 sc->rx_rmap_intr = if_ringmap_alloc(sc->dev, ring_cnt, ring_cntmax); 4718 4719 ix_get_txring_cnt(sc, &ring_cnt, &ring_cntmax); 4720 if (ring_cntmax > msix_ring) 4721 ring_cntmax = msix_ring; 4722 sc->tx_rmap_intr = if_ringmap_alloc(sc->dev, ring_cnt, ring_cntmax); 4723 4724 if_ringmap_match(sc->dev, sc->rx_rmap_intr, sc->tx_rmap_intr); 4725 sc->rx_ring_msix = if_ringmap_count(sc->rx_rmap_intr); 4726 KASSERT(sc->rx_ring_msix <= sc->rx_ring_cnt, 4727 ("total RX ring count %d, MSI-X RX ring count %d", 4728 sc->rx_ring_cnt, sc->rx_ring_msix)); 4729 sc->tx_ring_msix = if_ringmap_count(sc->tx_rmap_intr); 4730 KASSERT(sc->tx_ring_msix <= sc->tx_ring_cnt, 4731 ("total TX ring count %d, MSI-X TX ring count %d", 4732 sc->tx_ring_cnt, sc->tx_ring_msix)); 4733 4734 /* 4735 * Aggregate TX/RX MSI-X 4736 */ 4737 ring_cntmax = sc->rx_ring_msix; 4738 if (ring_cntmax < sc->tx_ring_msix) 4739 ring_cntmax = sc->tx_ring_msix; 4740 KASSERT(ring_cntmax <= msix_ring, 4741 ("invalid ring count max %d, MSI-X count for rings %d", 4742 ring_cntmax, msix_ring)); 4743 4744 alloc_cnt = ring_cntmax + 1; /* +1 for status */ 4745 if (bootverbose) { 4746 device_printf(sc->dev, "MSI-X alloc %d, " 4747 "RX ring %d, TX ring %d\n", alloc_cnt, 4748 sc->rx_ring_msix, sc->tx_ring_msix); 4749 } 4750 4751 sc->msix_mem_rid = PCIR_BAR(IX_MSIX_BAR_82598); 4752 sc->msix_mem_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 4753 &sc->msix_mem_rid, RF_ACTIVE); 4754 if (sc->msix_mem_res == NULL) { 4755 sc->msix_mem_rid = PCIR_BAR(IX_MSIX_BAR_82599); 4756 sc->msix_mem_res = bus_alloc_resource_any(sc->dev, 4757 SYS_RES_MEMORY, &sc->msix_mem_rid, RF_ACTIVE); 4758 if (sc->msix_mem_res == NULL) { 4759 device_printf(sc->dev, "Unable to map MSI-X table\n"); 4760 return; 4761 } 4762 } 4763 4764 sc->intr_cnt = alloc_cnt; 4765 sc->intr_data = kmalloc(sizeof(struct ix_intr_data) * sc->intr_cnt, 4766 M_DEVBUF, M_WAITOK | M_ZERO); 4767 for (x = 0; x < sc->intr_cnt; ++x) { 4768 intr = &sc->intr_data[x]; 4769 intr->intr_rid = -1; 4770 intr->intr_rate = IX_INTR_RATE; 4771 } 4772 4773 x = 0; 4774 for (i = 0; i < sc->rx_ring_msix; ++i) { 4775 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 4776 struct ix_tx_ring *txr = NULL; 4777 int cpuid, j; 4778 4779 KKASSERT(x < sc->intr_cnt); 4780 rxr->rx_intr_vec = x; 4781 ix_setup_msix_eims(sc, x, 4782 &rxr->rx_eims, &rxr->rx_eims_val); 4783 4784 cpuid = if_ringmap_cpumap(sc->rx_rmap_intr, i); 4785 4786 /* 4787 * Try finding TX ring to piggyback. 4788 */ 4789 for (j = 0; j < sc->tx_ring_msix; ++j) { 4790 if (cpuid == 4791 if_ringmap_cpumap(sc->tx_rmap_intr, j)) { 4792 txr = &sc->tx_rings[j]; 4793 KKASSERT(txr->tx_intr_cpuid < 0); 4794 break; 4795 } 4796 } 4797 rxr->rx_txr = txr; 4798 4799 intr = &sc->intr_data[x++]; 4800 intr->intr_serialize = &rxr->rx_serialize; 4801 if (txr != NULL) { 4802 ksnprintf(intr->intr_desc0, 4803 sizeof(intr->intr_desc0), "%s rx%dtx%d", 4804 device_get_nameunit(sc->dev), i, txr->tx_idx); 4805 intr->intr_use = IX_INTR_USE_RXTX; 4806 intr->intr_func = ix_msix_rxtx; 4807 } else { 4808 ksnprintf(intr->intr_desc0, 4809 sizeof(intr->intr_desc0), "%s rx%d", 4810 device_get_nameunit(sc->dev), i); 4811 intr->intr_rate = IX_MSIX_RX_RATE; 4812 intr->intr_use = IX_INTR_USE_RX; 4813 intr->intr_func = ix_msix_rx; 4814 } 4815 intr->intr_funcarg = rxr; 4816 intr->intr_cpuid = cpuid; 4817 KKASSERT(intr->intr_cpuid < netisr_ncpus); 4818 intr->intr_desc = intr->intr_desc0; 4819 4820 if (txr != NULL) { 4821 txr->tx_intr_cpuid = intr->intr_cpuid; 4822 /* NOTE: Leave TX ring's intr_vec negative. */ 4823 } 4824 } 4825 4826 for (i = 0; i < sc->tx_ring_msix; ++i) { 4827 struct ix_tx_ring *txr = &sc->tx_rings[i]; 4828 4829 if (txr->tx_intr_cpuid >= 0) { 4830 /* Piggybacked by RX ring. */ 4831 continue; 4832 } 4833 4834 KKASSERT(x < sc->intr_cnt); 4835 txr->tx_intr_vec = x; 4836 ix_setup_msix_eims(sc, x, &txr->tx_eims, &txr->tx_eims_val); 4837 4838 intr = &sc->intr_data[x++]; 4839 intr->intr_serialize = &txr->tx_serialize; 4840 intr->intr_rate = IX_MSIX_TX_RATE; 4841 intr->intr_use = IX_INTR_USE_TX; 4842 intr->intr_func = ix_msix_tx; 4843 intr->intr_funcarg = txr; 4844 intr->intr_cpuid = if_ringmap_cpumap(sc->tx_rmap_intr, i); 4845 KKASSERT(intr->intr_cpuid < netisr_ncpus); 4846 ksnprintf(intr->intr_desc0, sizeof(intr->intr_desc0), "%s tx%d", 4847 device_get_nameunit(sc->dev), i); 4848 intr->intr_desc = intr->intr_desc0; 4849 4850 txr->tx_intr_cpuid = intr->intr_cpuid; 4851 } 4852 4853 /* 4854 * Status MSI-X 4855 */ 4856 KKASSERT(x < sc->intr_cnt); 4857 sc->sts_msix_vec = x; 4858 4859 intr = &sc->intr_data[x++]; 4860 4861 intr->intr_serialize = &sc->main_serialize; 4862 intr->intr_func = ix_msix_status; 4863 intr->intr_funcarg = sc; 4864 intr->intr_cpuid = 0; 4865 intr->intr_use = IX_INTR_USE_STATUS; 4866 4867 ksnprintf(intr->intr_desc0, sizeof(intr->intr_desc0), "%s sts", 4868 device_get_nameunit(sc->dev)); 4869 intr->intr_desc = intr->intr_desc0; 4870 4871 KKASSERT(x == sc->intr_cnt); 4872 4873 error = pci_setup_msix(sc->dev); 4874 if (error) { 4875 device_printf(sc->dev, "Setup MSI-X failed\n"); 4876 goto back; 4877 } 4878 setup = TRUE; 4879 4880 for (i = 0; i < sc->intr_cnt; ++i) { 4881 intr = &sc->intr_data[i]; 4882 4883 error = pci_alloc_msix_vector(sc->dev, i, &intr->intr_rid, 4884 intr->intr_cpuid); 4885 if (error) { 4886 device_printf(sc->dev, 4887 "Unable to allocate MSI-X %d on cpu%d\n", i, 4888 intr->intr_cpuid); 4889 goto back; 4890 } 4891 4892 intr->intr_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 4893 &intr->intr_rid, RF_ACTIVE); 4894 if (intr->intr_res == NULL) { 4895 device_printf(sc->dev, 4896 "Unable to allocate MSI-X %d resource\n", i); 4897 error = ENOMEM; 4898 goto back; 4899 } 4900 } 4901 4902 pci_enable_msix(sc->dev); 4903 sc->intr_type = PCI_INTR_TYPE_MSIX; 4904 back: 4905 if (error) 4906 ix_free_msix(sc, setup); 4907 } 4908 4909 static void 4910 ix_free_msix(struct ix_softc *sc, boolean_t setup) 4911 { 4912 int i; 4913 4914 KKASSERT(sc->intr_cnt > 1); 4915 4916 for (i = 0; i < sc->intr_cnt; ++i) { 4917 struct ix_intr_data *intr = &sc->intr_data[i]; 4918 4919 if (intr->intr_res != NULL) { 4920 bus_release_resource(sc->dev, SYS_RES_IRQ, 4921 intr->intr_rid, intr->intr_res); 4922 } 4923 if (intr->intr_rid >= 0) 4924 pci_release_msix_vector(sc->dev, intr->intr_rid); 4925 } 4926 if (setup) 4927 pci_teardown_msix(sc->dev); 4928 4929 sc->intr_cnt = 0; 4930 kfree(sc->intr_data, M_DEVBUF); 4931 sc->intr_data = NULL; 4932 } 4933 4934 static void 4935 ix_msix_rx(void *xrxr) 4936 { 4937 struct ix_rx_ring *rxr = xrxr; 4938 4939 ASSERT_SERIALIZED(&rxr->rx_serialize); 4940 4941 ix_rxeof(rxr, -1); 4942 IXGBE_WRITE_REG(&rxr->rx_sc->hw, rxr->rx_eims, rxr->rx_eims_val); 4943 } 4944 4945 static void 4946 ix_msix_tx(void *xtxr) 4947 { 4948 struct ix_tx_ring *txr = xtxr; 4949 4950 ASSERT_SERIALIZED(&txr->tx_serialize); 4951 4952 ix_tx_intr(txr, *(txr->tx_hdr)); 4953 IXGBE_WRITE_REG(&txr->tx_sc->hw, txr->tx_eims, txr->tx_eims_val); 4954 } 4955 4956 static void 4957 ix_msix_rxtx(void *xrxr) 4958 { 4959 struct ix_rx_ring *rxr = xrxr; 4960 struct ix_tx_ring *txr; 4961 int hdr; 4962 4963 ASSERT_SERIALIZED(&rxr->rx_serialize); 4964 4965 ix_rxeof(rxr, -1); 4966 4967 /* 4968 * NOTE: 4969 * Since tx_next_clean is only changed by ix_txeof(), 4970 * which is called only in interrupt handler, the 4971 * check w/o holding tx serializer is MPSAFE. 4972 */ 4973 txr = rxr->rx_txr; 4974 hdr = *(txr->tx_hdr); 4975 if (hdr != txr->tx_next_clean) { 4976 lwkt_serialize_enter(&txr->tx_serialize); 4977 ix_tx_intr(txr, hdr); 4978 lwkt_serialize_exit(&txr->tx_serialize); 4979 } 4980 4981 IXGBE_WRITE_REG(&rxr->rx_sc->hw, rxr->rx_eims, rxr->rx_eims_val); 4982 } 4983 4984 static void 4985 ix_intr_status(struct ix_softc *sc, uint32_t eicr) 4986 { 4987 struct ixgbe_hw *hw = &sc->hw; 4988 4989 /* Link status change */ 4990 if (eicr & IXGBE_EICR_LSC) 4991 ix_handle_link(sc); 4992 4993 if (hw->mac.type != ixgbe_mac_82598EB) { 4994 if (eicr & IXGBE_EICR_ECC) 4995 if_printf(&sc->arpcom.ac_if, "ECC ERROR!! REBOOT!!\n"); 4996 4997 /* Check for over temp condition */ 4998 if (sc->caps & IX_CAP_TEMP_SENSOR) { 4999 int32_t retval; 5000 5001 switch (sc->hw.mac.type) { 5002 case ixgbe_mac_X550EM_a: 5003 if ((eicr & IXGBE_EICR_GPI_SDP0_X550EM_a) == 0) 5004 break; 5005 retval = hw->phy.ops.check_overtemp(hw); 5006 if (retval != IXGBE_ERR_OVERTEMP) 5007 break; 5008 5009 /* Disable more temp sensor interrupts. */ 5010 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 5011 IXGBE_EICR_GPI_SDP0_X550EM_a); 5012 if_printf(&sc->arpcom.ac_if, "CRITICAL: " 5013 "OVER TEMP!! PHY IS SHUT DOWN!! " 5014 "SHUTDOWN!!\n"); 5015 break; 5016 5017 default: 5018 if ((eicr & IXGBE_EICR_TS) == 0) 5019 break; 5020 retval = hw->phy.ops.check_overtemp(hw); 5021 if (retval != IXGBE_ERR_OVERTEMP) 5022 break; 5023 5024 /* Disable more temp sensor interrupts. */ 5025 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_TS); 5026 if_printf(&sc->arpcom.ac_if, "CRITICAL: " 5027 "OVER TEMP!! PHY IS SHUT DOWN!! " 5028 "SHUTDOWN!!\n"); 5029 break; 5030 } 5031 } 5032 } 5033 5034 if (ix_is_sfp(hw)) { 5035 uint32_t eicr_mask; 5036 5037 /* Pluggable optics-related interrupt */ 5038 if (hw->mac.type >= ixgbe_mac_X540) 5039 eicr_mask = IXGBE_EICR_GPI_SDP0_X540; 5040 else 5041 eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 5042 5043 if (eicr & eicr_mask) 5044 ix_handle_mod(sc); 5045 5046 if (hw->mac.type == ixgbe_mac_82599EB && 5047 (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) 5048 ix_handle_msf(sc); 5049 } 5050 5051 /* Check for fan failure */ 5052 if (sc->caps & IX_CAP_DETECT_FANFAIL) 5053 ix_detect_fanfail(sc, eicr, TRUE); 5054 5055 /* External PHY interrupt */ 5056 if (hw->phy.type == ixgbe_phy_x550em_ext_t && 5057 (eicr & IXGBE_EICR_GPI_SDP0_X540)) 5058 ix_handle_phy(sc); 5059 } 5060 5061 static void 5062 ix_msix_status(void *xsc) 5063 { 5064 struct ix_softc *sc = xsc; 5065 uint32_t eicr; 5066 5067 ASSERT_SERIALIZED(&sc->main_serialize); 5068 5069 eicr = IXGBE_READ_REG(&sc->hw, IXGBE_EICR); 5070 ix_intr_status(sc, eicr); 5071 5072 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, sc->intr_mask); 5073 } 5074 5075 static void 5076 ix_setup_msix_eims(const struct ix_softc *sc, int x, 5077 uint32_t *eims, uint32_t *eims_val) 5078 { 5079 if (x < 32) { 5080 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 5081 KASSERT(x < IX_MAX_MSIX_82598, 5082 ("%s: invalid vector %d for 82598", 5083 device_get_nameunit(sc->dev), x)); 5084 *eims = IXGBE_EIMS; 5085 } else { 5086 *eims = IXGBE_EIMS_EX(0); 5087 } 5088 *eims_val = 1 << x; 5089 } else { 5090 KASSERT(x < IX_MAX_MSIX, ("%s: invalid vector %d", 5091 device_get_nameunit(sc->dev), x)); 5092 KASSERT(sc->hw.mac.type != ixgbe_mac_82598EB, 5093 ("%s: invalid vector %d for 82598", 5094 device_get_nameunit(sc->dev), x)); 5095 *eims = IXGBE_EIMS_EX(1); 5096 *eims_val = 1 << (x - 32); 5097 } 5098 } 5099 5100 #ifdef IFPOLL_ENABLE 5101 5102 static void 5103 ix_npoll_status(struct ifnet *ifp) 5104 { 5105 struct ix_softc *sc = ifp->if_softc; 5106 uint32_t eicr; 5107 5108 ASSERT_SERIALIZED(&sc->main_serialize); 5109 5110 eicr = IXGBE_READ_REG(&sc->hw, IXGBE_EICR); 5111 ix_intr_status(sc, eicr); 5112 } 5113 5114 static void 5115 ix_npoll_tx(struct ifnet *ifp, void *arg, int cycle __unused) 5116 { 5117 struct ix_tx_ring *txr = arg; 5118 5119 ASSERT_SERIALIZED(&txr->tx_serialize); 5120 5121 ix_tx_intr(txr, *(txr->tx_hdr)); 5122 ix_try_txgc(txr, 1); 5123 } 5124 5125 static void 5126 ix_npoll_rx(struct ifnet *ifp __unused, void *arg, int cycle) 5127 { 5128 struct ix_rx_ring *rxr = arg; 5129 5130 ASSERT_SERIALIZED(&rxr->rx_serialize); 5131 ix_rxeof(rxr, cycle); 5132 } 5133 5134 static void 5135 ix_npoll_rx_direct(struct ifnet *ifp __unused, void *arg, int cycle) 5136 { 5137 struct ix_rx_ring *rxr = arg; 5138 5139 ASSERT_NOT_SERIALIZED(&rxr->rx_serialize); 5140 ix_rxeof(rxr, cycle); 5141 } 5142 5143 static void 5144 ix_npoll(struct ifnet *ifp, struct ifpoll_info *info) 5145 { 5146 struct ix_softc *sc = ifp->if_softc; 5147 int i, txr_cnt, rxr_cnt, idirect; 5148 5149 ASSERT_IFNET_SERIALIZED_ALL(ifp); 5150 5151 idirect = sc->direct_input; 5152 cpu_ccfence(); 5153 5154 if (info) { 5155 int cpu; 5156 5157 info->ifpi_status.status_func = ix_npoll_status; 5158 info->ifpi_status.serializer = &sc->main_serialize; 5159 5160 txr_cnt = ix_get_txring_inuse(sc, TRUE); 5161 for (i = 0; i < txr_cnt; ++i) { 5162 struct ix_tx_ring *txr = &sc->tx_rings[i]; 5163 5164 cpu = if_ringmap_cpumap(sc->tx_rmap, i); 5165 KKASSERT(cpu < netisr_ncpus); 5166 info->ifpi_tx[cpu].poll_func = ix_npoll_tx; 5167 info->ifpi_tx[cpu].arg = txr; 5168 info->ifpi_tx[cpu].serializer = &txr->tx_serialize; 5169 ifsq_set_cpuid(txr->tx_ifsq, cpu); 5170 } 5171 5172 rxr_cnt = ix_get_rxring_inuse(sc, TRUE); 5173 for (i = 0; i < rxr_cnt; ++i) { 5174 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 5175 5176 cpu = if_ringmap_cpumap(sc->rx_rmap, i); 5177 KKASSERT(cpu < netisr_ncpus); 5178 info->ifpi_rx[cpu].arg = rxr; 5179 if (idirect) { 5180 info->ifpi_rx[cpu].poll_func = 5181 ix_npoll_rx_direct; 5182 info->ifpi_rx[cpu].serializer = NULL; 5183 } else { 5184 info->ifpi_rx[cpu].poll_func = ix_npoll_rx; 5185 info->ifpi_rx[cpu].serializer = 5186 &rxr->rx_serialize; 5187 } 5188 } 5189 if (idirect) 5190 ifp->if_flags |= IFF_IDIRECT; 5191 } else { 5192 ifp->if_flags &= ~IFF_IDIRECT; 5193 for (i = 0; i < sc->tx_ring_cnt; ++i) { 5194 struct ix_tx_ring *txr = &sc->tx_rings[i]; 5195 5196 ifsq_set_cpuid(txr->tx_ifsq, txr->tx_intr_cpuid); 5197 } 5198 } 5199 if (ifp->if_flags & IFF_RUNNING) 5200 ix_init(sc); 5201 } 5202 5203 #endif /* IFPOLL_ENABLE */ 5204 5205 static enum ixgbe_fc_mode 5206 ix_ifmedia2fc(int ifm) 5207 { 5208 int fc_opt = ifm & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE); 5209 5210 switch (fc_opt) { 5211 case (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE): 5212 return ixgbe_fc_full; 5213 5214 case IFM_ETH_RXPAUSE: 5215 return ixgbe_fc_rx_pause; 5216 5217 case IFM_ETH_TXPAUSE: 5218 return ixgbe_fc_tx_pause; 5219 5220 default: 5221 return ixgbe_fc_none; 5222 } 5223 } 5224 5225 static const char * 5226 ix_ifmedia2str(int ifm) 5227 { 5228 int fc_opt = ifm & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE); 5229 5230 switch (fc_opt) { 5231 case (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE): 5232 return IFM_ETH_FC_FULL; 5233 5234 case IFM_ETH_RXPAUSE: 5235 return IFM_ETH_FC_RXPAUSE; 5236 5237 case IFM_ETH_TXPAUSE: 5238 return IFM_ETH_FC_TXPAUSE; 5239 5240 default: 5241 return IFM_ETH_FC_NONE; 5242 } 5243 } 5244 5245 static const char * 5246 ix_fc2str(enum ixgbe_fc_mode fc) 5247 { 5248 switch (fc) { 5249 case ixgbe_fc_full: 5250 return IFM_ETH_FC_FULL; 5251 5252 case ixgbe_fc_rx_pause: 5253 return IFM_ETH_FC_RXPAUSE; 5254 5255 case ixgbe_fc_tx_pause: 5256 return IFM_ETH_FC_TXPAUSE; 5257 5258 default: 5259 return IFM_ETH_FC_NONE; 5260 } 5261 } 5262 5263 static int 5264 ix_powerdown(struct ix_softc *sc) 5265 { 5266 struct ixgbe_hw *hw = &sc->hw; 5267 int error = 0; 5268 5269 /* Limit power management flow to X550EM baseT */ 5270 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 5271 hw->phy.ops.enter_lplu) { 5272 /* Turn off support for APM wakeup. (Using ACPI instead) */ 5273 IXGBE_WRITE_REG(hw, IXGBE_GRC, 5274 IXGBE_READ_REG(hw, IXGBE_GRC) & ~(uint32_t)2); 5275 5276 /* 5277 * Clear Wake Up Status register to prevent any previous wakeup 5278 * events from waking us up immediately after we suspend. 5279 */ 5280 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 5281 5282 /* 5283 * Program the Wakeup Filter Control register with user filter 5284 * settings 5285 */ 5286 IXGBE_WRITE_REG(hw, IXGBE_WUFC, sc->wufc); 5287 5288 /* Enable wakeups and power management in Wakeup Control */ 5289 IXGBE_WRITE_REG(hw, IXGBE_WUC, 5290 IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN); 5291 5292 /* X550EM baseT adapters need a special LPLU flow */ 5293 hw->phy.reset_disable = true; 5294 ix_stop(sc); 5295 error = hw->phy.ops.enter_lplu(hw); 5296 if (error) { 5297 if_printf(&sc->arpcom.ac_if, 5298 "Error entering LPLU: %d\n", error); 5299 } 5300 hw->phy.reset_disable = false; 5301 } else { 5302 /* Just stop for other adapters */ 5303 ix_stop(sc); 5304 } 5305 return error; 5306 } 5307 5308 static void 5309 ix_config_flowctrl(struct ix_softc *sc) 5310 { 5311 struct ixgbe_hw *hw = &sc->hw; 5312 uint32_t rxpb, frame, size, tmp; 5313 5314 frame = sc->max_frame_size; 5315 5316 /* Calculate High Water */ 5317 switch (hw->mac.type) { 5318 case ixgbe_mac_X540: 5319 case ixgbe_mac_X550: 5320 case ixgbe_mac_X550EM_a: 5321 case ixgbe_mac_X550EM_x: 5322 tmp = IXGBE_DV_X540(frame, frame); 5323 break; 5324 default: 5325 tmp = IXGBE_DV(frame, frame); 5326 break; 5327 } 5328 size = IXGBE_BT2KB(tmp); 5329 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 5330 hw->fc.high_water[0] = rxpb - size; 5331 5332 /* Now calculate Low Water */ 5333 switch (hw->mac.type) { 5334 case ixgbe_mac_X540: 5335 case ixgbe_mac_X550: 5336 case ixgbe_mac_X550EM_a: 5337 case ixgbe_mac_X550EM_x: 5338 tmp = IXGBE_LOW_DV_X540(frame); 5339 break; 5340 default: 5341 tmp = IXGBE_LOW_DV(frame); 5342 break; 5343 } 5344 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 5345 5346 hw->fc.requested_mode = ix_ifmedia2fc(sc->ifm_media); 5347 if (sc->ifm_media & IFM_ETH_FORCEPAUSE) 5348 hw->fc.disable_fc_autoneg = TRUE; 5349 else 5350 hw->fc.disable_fc_autoneg = FALSE; 5351 hw->fc.pause_time = IX_FC_PAUSE; 5352 hw->fc.send_xon = TRUE; 5353 } 5354 5355 static void 5356 ix_config_dmac(struct ix_softc *sc) 5357 { 5358 struct ixgbe_hw *hw = &sc->hw; 5359 struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config; 5360 5361 if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config) 5362 return; 5363 5364 if ((dcfg->watchdog_timer ^ sc->dmac) || 5365 (dcfg->link_speed ^ sc->link_speed)) { 5366 dcfg->watchdog_timer = sc->dmac; 5367 dcfg->fcoe_en = false; 5368 dcfg->link_speed = sc->link_speed; 5369 dcfg->num_tcs = 1; 5370 5371 if (bootverbose) { 5372 if_printf(&sc->arpcom.ac_if, "dmac settings: " 5373 "watchdog %d, link speed %d\n", 5374 dcfg->watchdog_timer, dcfg->link_speed); 5375 } 5376 5377 hw->mac.ops.dmac_config(hw); 5378 } 5379 } 5380 5381 static void 5382 ix_init_media(struct ix_softc *sc) 5383 { 5384 struct ixgbe_hw *hw = &sc->hw; 5385 uint32_t layer; 5386 5387 ifmedia_removeall(&sc->media); 5388 5389 layer = sc->phy_layer; 5390 5391 /* 5392 * Media types with matching DragonFlyBSD media defines 5393 */ 5394 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) { 5395 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX, 5396 0, NULL); 5397 } 5398 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) { 5399 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 5400 0, NULL); 5401 } 5402 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) { 5403 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 5404 0, NULL); 5405 /* No half-duplex support */ 5406 } 5407 if (layer & IXGBE_PHYSICAL_LAYER_10BASE_T) { 5408 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 5409 0, NULL); 5410 /* No half-duplex support */ 5411 } 5412 5413 if ((layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) || 5414 (layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)) { 5415 ifmedia_add_nodup(&sc->media, 5416 IFM_ETHER | IFM_10G_TWINAX | IFM_FDX, 0, NULL); 5417 } 5418 5419 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 5420 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_LR | IFM_FDX, 5421 0, NULL); 5422 if (hw->phy.multispeed_fiber) { 5423 ifmedia_add_nodup(&sc->media, 5424 IFM_ETHER | IFM_1000_LX | IFM_FDX, 0, NULL); 5425 } 5426 } 5427 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM) { 5428 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_LRM | IFM_FDX, 5429 0, NULL); 5430 if (hw->phy.multispeed_fiber) { 5431 ifmedia_add_nodup(&sc->media, 5432 IFM_ETHER | IFM_1000_LX | IFM_FDX, 0, NULL); 5433 } 5434 } 5435 5436 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 5437 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_SR | IFM_FDX, 5438 0, NULL); 5439 if (hw->phy.multispeed_fiber) { 5440 ifmedia_add_nodup(&sc->media, 5441 IFM_ETHER | IFM_1000_SX | IFM_FDX, 0, NULL); 5442 } 5443 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 5444 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 5445 0, NULL); 5446 } 5447 5448 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 5449 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_CX4 | IFM_FDX, 5450 0, NULL); 5451 } 5452 5453 /* 5454 * XXX Other (no matching DragonFlyBSD media type): 5455 * To workaround this, we'll assign these completely 5456 * inappropriate media types. 5457 */ 5458 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 5459 if_printf(&sc->arpcom.ac_if, "Media supported: 10GbaseKR\n"); 5460 if_printf(&sc->arpcom.ac_if, "10GbaseKR mapped to 10GbaseSR\n"); 5461 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_SR | IFM_FDX, 5462 0, NULL); 5463 } 5464 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) { 5465 if_printf(&sc->arpcom.ac_if, "Media supported: 10GbaseKX4\n"); 5466 if_printf(&sc->arpcom.ac_if, 5467 "10GbaseKX4 mapped to 10GbaseCX4\n"); 5468 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_CX4 | IFM_FDX, 5469 0, NULL); 5470 } 5471 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 5472 if_printf(&sc->arpcom.ac_if, "Media supported: 1000baseKX\n"); 5473 if_printf(&sc->arpcom.ac_if, 5474 "1000baseKX mapped to 1000baseCX\n"); 5475 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_1000_CX | IFM_FDX, 5476 0, NULL); 5477 } 5478 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) { 5479 if_printf(&sc->arpcom.ac_if, "Media supported: 2500baseKX\n"); 5480 if_printf(&sc->arpcom.ac_if, 5481 "2500baseKX mapped to 2500baseSX\n"); 5482 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_2500_SX | IFM_FDX, 5483 0, NULL); 5484 } 5485 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX) { 5486 if_printf(&sc->arpcom.ac_if, 5487 "Media supported: 1000baseBX, ignored\n"); 5488 } 5489 5490 /* XXX we probably don't need this */ 5491 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 5492 ifmedia_add_nodup(&sc->media, 5493 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 5494 } 5495 5496 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 5497 5498 if (ifmedia_tryset(&sc->media, sc->ifm_media)) { 5499 int flowctrl = (sc->ifm_media & IFM_ETH_FCMASK); 5500 5501 sc->advspeed = IXGBE_LINK_SPEED_UNKNOWN; 5502 sc->ifm_media = IX_IFM_DEFAULT | flowctrl; 5503 ifmedia_set(&sc->media, sc->ifm_media); 5504 } 5505 } 5506 5507 static void 5508 ix_setup_caps(struct ix_softc *sc) 5509 { 5510 5511 sc->caps |= IX_CAP_LEGACY_INTR; 5512 5513 switch (sc->hw.mac.type) { 5514 case ixgbe_mac_82598EB: 5515 if (sc->hw.device_id == IXGBE_DEV_ID_82598AT) 5516 sc->caps |= IX_CAP_DETECT_FANFAIL; 5517 break; 5518 5519 case ixgbe_mac_X550: 5520 sc->caps |= IX_CAP_TEMP_SENSOR | IX_CAP_FW_RECOVERY; 5521 break; 5522 5523 case ixgbe_mac_X550EM_x: 5524 if (sc->hw.device_id == IXGBE_DEV_ID_X550EM_X_KR) 5525 sc->caps |= IX_CAP_EEE; 5526 sc->caps |= IX_CAP_FW_RECOVERY; 5527 break; 5528 5529 case ixgbe_mac_X550EM_a: 5530 sc->caps &= ~IX_CAP_LEGACY_INTR; 5531 if (sc->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T || 5532 sc->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L) 5533 sc->caps |= IX_CAP_TEMP_SENSOR | IX_CAP_EEE; 5534 sc->caps |= IX_CAP_FW_RECOVERY; 5535 break; 5536 5537 case ixgbe_mac_82599EB: 5538 if (sc->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP) 5539 sc->caps &= ~IX_CAP_LEGACY_INTR; 5540 break; 5541 5542 default: 5543 break; 5544 } 5545 } 5546 5547 static void 5548 ix_detect_fanfail(struct ix_softc *sc, uint32_t reg, boolean_t intr) 5549 { 5550 uint32_t mask; 5551 5552 mask = intr ? IXGBE_EICR_GPI_SDP1_BY_MAC(&sc->hw) : IXGBE_ESDP_SDP1; 5553 if (reg & mask) { 5554 if_printf(&sc->arpcom.ac_if, 5555 "CRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n"); 5556 } 5557 } 5558 5559 static void 5560 ix_config_gpie(struct ix_softc *sc) 5561 { 5562 struct ixgbe_hw *hw = &sc->hw; 5563 uint32_t gpie; 5564 5565 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 5566 5567 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 5568 /* Enable Enhanced MSI-X mode */ 5569 gpie |= IXGBE_GPIE_MSIX_MODE | 5570 IXGBE_GPIE_EIAME | 5571 IXGBE_GPIE_PBA_SUPPORT | 5572 IXGBE_GPIE_OCD; 5573 } 5574 5575 /* Fan Failure Interrupt */ 5576 if (sc->caps & IX_CAP_DETECT_FANFAIL) 5577 gpie |= IXGBE_SDP1_GPIEN; 5578 5579 /* Thermal Sensor Interrupt */ 5580 if (sc->caps & IX_CAP_TEMP_SENSOR) 5581 gpie |= IXGBE_SDP0_GPIEN_X540; 5582 5583 /* Link detection */ 5584 switch (hw->mac.type) { 5585 case ixgbe_mac_82599EB: 5586 gpie |= IXGBE_SDP1_GPIEN | IXGBE_SDP2_GPIEN; 5587 break; 5588 5589 case ixgbe_mac_X550EM_x: 5590 case ixgbe_mac_X550EM_a: 5591 gpie |= IXGBE_SDP0_GPIEN_X540; 5592 break; 5593 5594 default: 5595 break; 5596 } 5597 5598 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 5599 } 5600 5601 static void 5602 ix_fw_timer(void *xsc) 5603 { 5604 struct ix_softc *sc = xsc; 5605 struct ifnet *ifp = &sc->arpcom.ac_if; 5606 5607 lwkt_serialize_enter(&sc->main_serialize); 5608 5609 if (ixgbe_fw_recovery_mode(&sc->hw)) { 5610 if ((sc->flags & IX_FLAG_FW_RECOVERY) == 0) { 5611 sc->flags |= IX_FLAG_FW_RECOVERY; 5612 if (ifp->if_flags & IFF_RUNNING) { 5613 if_printf(ifp, 5614 "fw recovery mode entered, stop\n"); 5615 ix_serialize_skipmain(sc); 5616 ix_stop(sc); 5617 ix_deserialize_skipmain(sc); 5618 } else { 5619 if_printf(ifp, "fw recovery mode entered\n"); 5620 } 5621 } 5622 } else { 5623 if (sc->flags & IX_FLAG_FW_RECOVERY) { 5624 sc->flags &= ~IX_FLAG_FW_RECOVERY; 5625 if (ifp->if_flags & IFF_UP) { 5626 if_printf(ifp, 5627 "fw recovery mode exited, reinit\n"); 5628 ix_serialize_skipmain(sc); 5629 ix_init(sc); 5630 ix_deserialize_skipmain(sc); 5631 } else { 5632 if_printf(ifp, "fw recovery mode exited\n"); 5633 } 5634 } 5635 } 5636 5637 callout_reset_bycpu(&sc->fw_timer, hz, ix_fw_timer, sc, 5638 ix_get_timer_cpuid(sc, 5639 (ifp->if_flags & IFF_NPOLLING) ? TRUE : FALSE)); 5640 5641 lwkt_serialize_exit(&sc->main_serialize); 5642 } 5643