1 /* 2 * Copyright (c) 2001-2014, Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the Intel Corporation nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include "opt_ifpoll.h" 33 #include "opt_ix.h" 34 35 #include <sys/param.h> 36 #include <sys/bus.h> 37 #include <sys/endian.h> 38 #include <sys/interrupt.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/mbuf.h> 42 #include <sys/proc.h> 43 #include <sys/rman.h> 44 #include <sys/serialize.h> 45 #include <sys/serialize2.h> 46 #include <sys/socket.h> 47 #include <sys/sockio.h> 48 #include <sys/sysctl.h> 49 #include <sys/systm.h> 50 51 #include <net/bpf.h> 52 #include <net/ethernet.h> 53 #include <net/if.h> 54 #include <net/if_arp.h> 55 #include <net/if_dl.h> 56 #include <net/if_media.h> 57 #include <net/ifq_var.h> 58 #include <net/toeplitz.h> 59 #include <net/toeplitz2.h> 60 #include <net/vlan/if_vlan_var.h> 61 #include <net/vlan/if_vlan_ether.h> 62 #include <net/if_poll.h> 63 64 #include <netinet/in_systm.h> 65 #include <netinet/in.h> 66 #include <netinet/ip.h> 67 68 #include <bus/pci/pcivar.h> 69 #include <bus/pci/pcireg.h> 70 71 #include <dev/netif/ix/ixgbe_api.h> 72 #include <dev/netif/ix/if_ix.h> 73 74 #define IX_IFM_DEFAULT (IFM_ETHER | IFM_AUTO) 75 76 #ifdef IX_RSS_DEBUG 77 #define IX_RSS_DPRINTF(sc, lvl, fmt, ...) \ 78 do { \ 79 if (sc->rss_debug >= lvl) \ 80 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \ 81 } while (0) 82 #else /* !IX_RSS_DEBUG */ 83 #define IX_RSS_DPRINTF(sc, lvl, fmt, ...) ((void)0) 84 #endif /* IX_RSS_DEBUG */ 85 86 #define IX_NAME "Intel(R) PRO/10GbE " 87 #define IX_DEVICE(id) \ 88 { IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_##id, IX_NAME #id } 89 #define IX_DEVICE_NULL { 0, 0, NULL } 90 91 static struct ix_device { 92 uint16_t vid; 93 uint16_t did; 94 const char *desc; 95 } ix_devices[] = { 96 IX_DEVICE(82598AF_DUAL_PORT), 97 IX_DEVICE(82598AF_SINGLE_PORT), 98 IX_DEVICE(82598EB_CX4), 99 IX_DEVICE(82598AT), 100 IX_DEVICE(82598AT2), 101 IX_DEVICE(82598), 102 IX_DEVICE(82598_DA_DUAL_PORT), 103 IX_DEVICE(82598_CX4_DUAL_PORT), 104 IX_DEVICE(82598EB_XF_LR), 105 IX_DEVICE(82598_SR_DUAL_PORT_EM), 106 IX_DEVICE(82598EB_SFP_LOM), 107 IX_DEVICE(82599_KX4), 108 IX_DEVICE(82599_KX4_MEZZ), 109 IX_DEVICE(82599_SFP), 110 IX_DEVICE(82599_XAUI_LOM), 111 IX_DEVICE(82599_CX4), 112 IX_DEVICE(82599_T3_LOM), 113 IX_DEVICE(82599_COMBO_BACKPLANE), 114 IX_DEVICE(82599_BACKPLANE_FCOE), 115 IX_DEVICE(82599_SFP_SF2), 116 IX_DEVICE(82599_SFP_FCOE), 117 IX_DEVICE(82599EN_SFP), 118 IX_DEVICE(82599_SFP_SF_QP), 119 IX_DEVICE(82599_QSFP_SF_QP), 120 IX_DEVICE(X540T), 121 IX_DEVICE(X540T1), 122 IX_DEVICE(X550T), 123 IX_DEVICE(X550EM_X_KR), 124 IX_DEVICE(X550EM_X_KX4), 125 IX_DEVICE(X550EM_X_10G_T), 126 127 /* required last entry */ 128 IX_DEVICE_NULL 129 }; 130 131 static int ix_probe(device_t); 132 static int ix_attach(device_t); 133 static int ix_detach(device_t); 134 static int ix_shutdown(device_t); 135 136 static void ix_serialize(struct ifnet *, enum ifnet_serialize); 137 static void ix_deserialize(struct ifnet *, enum ifnet_serialize); 138 static int ix_tryserialize(struct ifnet *, enum ifnet_serialize); 139 #ifdef INVARIANTS 140 static void ix_serialize_assert(struct ifnet *, enum ifnet_serialize, 141 boolean_t); 142 #endif 143 static void ix_start(struct ifnet *, struct ifaltq_subque *); 144 static void ix_watchdog(struct ifaltq_subque *); 145 static int ix_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 146 static void ix_init(void *); 147 static void ix_stop(struct ix_softc *); 148 static void ix_media_status(struct ifnet *, struct ifmediareq *); 149 static int ix_media_change(struct ifnet *); 150 static void ix_timer(void *); 151 #ifdef IFPOLL_ENABLE 152 static void ix_npoll(struct ifnet *, struct ifpoll_info *); 153 static void ix_npoll_rx(struct ifnet *, void *, int); 154 static void ix_npoll_tx(struct ifnet *, void *, int); 155 static void ix_npoll_status(struct ifnet *); 156 #endif 157 158 static void ix_add_sysctl(struct ix_softc *); 159 static void ix_add_intr_rate_sysctl(struct ix_softc *, int, 160 const char *, int (*)(SYSCTL_HANDLER_ARGS), const char *); 161 static int ix_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS); 162 static int ix_sysctl_rx_wreg_nsegs(SYSCTL_HANDLER_ARGS); 163 static int ix_sysctl_txd(SYSCTL_HANDLER_ARGS); 164 static int ix_sysctl_rxd(SYSCTL_HANDLER_ARGS); 165 static int ix_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS); 166 static int ix_sysctl_intr_rate(SYSCTL_HANDLER_ARGS, int); 167 static int ix_sysctl_rxtx_intr_rate(SYSCTL_HANDLER_ARGS); 168 static int ix_sysctl_rx_intr_rate(SYSCTL_HANDLER_ARGS); 169 static int ix_sysctl_tx_intr_rate(SYSCTL_HANDLER_ARGS); 170 static int ix_sysctl_sts_intr_rate(SYSCTL_HANDLER_ARGS); 171 #if 0 172 static void ix_add_hw_stats(struct ix_softc *); 173 #endif 174 #ifdef IFPOLL_ENABLE 175 static int ix_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS); 176 static int ix_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS); 177 #endif 178 179 static void ix_slot_info(struct ix_softc *); 180 static int ix_alloc_rings(struct ix_softc *); 181 static void ix_free_rings(struct ix_softc *); 182 static void ix_setup_ifp(struct ix_softc *); 183 static void ix_setup_serialize(struct ix_softc *); 184 static void ix_set_ring_inuse(struct ix_softc *, boolean_t); 185 static void ix_set_timer_cpuid(struct ix_softc *, boolean_t); 186 static void ix_update_stats(struct ix_softc *); 187 188 static void ix_set_promisc(struct ix_softc *); 189 static void ix_set_multi(struct ix_softc *); 190 static void ix_set_vlan(struct ix_softc *); 191 static uint8_t *ix_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 192 static enum ixgbe_fc_mode ix_ifmedia2fc(int); 193 static const char *ix_ifmedia2str(int); 194 static const char *ix_fc2str(enum ixgbe_fc_mode); 195 196 static int ix_get_txring_inuse(const struct ix_softc *, boolean_t); 197 static void ix_init_tx_ring(struct ix_tx_ring *); 198 static void ix_free_tx_ring(struct ix_tx_ring *); 199 static int ix_create_tx_ring(struct ix_tx_ring *); 200 static void ix_destroy_tx_ring(struct ix_tx_ring *, int); 201 static void ix_init_tx_unit(struct ix_softc *); 202 static int ix_encap(struct ix_tx_ring *, struct mbuf **, 203 uint16_t *, int *); 204 static int ix_tx_ctx_setup(struct ix_tx_ring *, 205 const struct mbuf *, uint32_t *, uint32_t *); 206 static int ix_tso_ctx_setup(struct ix_tx_ring *, 207 const struct mbuf *, uint32_t *, uint32_t *); 208 static void ix_txeof(struct ix_tx_ring *, int); 209 210 static int ix_get_rxring_inuse(const struct ix_softc *, boolean_t); 211 static int ix_init_rx_ring(struct ix_rx_ring *); 212 static void ix_free_rx_ring(struct ix_rx_ring *); 213 static int ix_create_rx_ring(struct ix_rx_ring *); 214 static void ix_destroy_rx_ring(struct ix_rx_ring *, int); 215 static void ix_init_rx_unit(struct ix_softc *); 216 #if 0 217 static void ix_setup_hw_rsc(struct ix_rx_ring *); 218 #endif 219 static int ix_newbuf(struct ix_rx_ring *, int, boolean_t); 220 static void ix_rxeof(struct ix_rx_ring *, int); 221 static void ix_rx_discard(struct ix_rx_ring *, int, boolean_t); 222 static void ix_enable_rx_drop(struct ix_softc *); 223 static void ix_disable_rx_drop(struct ix_softc *); 224 225 static void ix_alloc_msix(struct ix_softc *); 226 static void ix_free_msix(struct ix_softc *, boolean_t); 227 static void ix_conf_rx_msix(struct ix_softc *, int, int *, int); 228 static void ix_conf_tx_msix(struct ix_softc *, int, int *, int); 229 static void ix_setup_msix_eims(const struct ix_softc *, int, 230 uint32_t *, uint32_t *); 231 static int ix_alloc_intr(struct ix_softc *); 232 static void ix_free_intr(struct ix_softc *); 233 static int ix_setup_intr(struct ix_softc *); 234 static void ix_teardown_intr(struct ix_softc *, int); 235 static void ix_enable_intr(struct ix_softc *); 236 static void ix_disable_intr(struct ix_softc *); 237 static void ix_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t); 238 static void ix_set_eitr(struct ix_softc *, int, int); 239 static void ix_intr_status(struct ix_softc *, uint32_t); 240 static void ix_intr(void *); 241 static void ix_msix_rxtx(void *); 242 static void ix_msix_rx(void *); 243 static void ix_msix_tx(void *); 244 static void ix_msix_status(void *); 245 246 static void ix_config_link(struct ix_softc *); 247 static boolean_t ix_sfp_probe(struct ix_softc *); 248 static boolean_t ix_is_sfp(const struct ixgbe_hw *); 249 static void ix_update_link_status(struct ix_softc *); 250 static void ix_handle_link(struct ix_softc *); 251 static void ix_handle_mod(struct ix_softc *); 252 static void ix_handle_msf(struct ix_softc *); 253 static void ix_handle_phy(struct ix_softc *); 254 static int ix_powerdown(struct ix_softc *); 255 static void ix_config_flowctrl(struct ix_softc *); 256 static void ix_config_dmac(struct ix_softc *); 257 static void ix_init_media(struct ix_softc *); 258 259 /* XXX Missing shared code prototype */ 260 extern void ixgbe_stop_mac_link_on_d3_82599(struct ixgbe_hw *); 261 262 static device_method_t ix_methods[] = { 263 /* Device interface */ 264 DEVMETHOD(device_probe, ix_probe), 265 DEVMETHOD(device_attach, ix_attach), 266 DEVMETHOD(device_detach, ix_detach), 267 DEVMETHOD(device_shutdown, ix_shutdown), 268 DEVMETHOD_END 269 }; 270 271 static driver_t ix_driver = { 272 "ix", 273 ix_methods, 274 sizeof(struct ix_softc) 275 }; 276 277 static devclass_t ix_devclass; 278 279 DECLARE_DUMMY_MODULE(if_ix); 280 DRIVER_MODULE(if_ix, pci, ix_driver, ix_devclass, NULL, NULL); 281 282 static int ix_msi_enable = 1; 283 static int ix_msix_enable = 1; 284 static int ix_msix_agg_rxtx = 1; 285 static int ix_rxr = 0; 286 static int ix_txr = 0; 287 static int ix_txd = IX_PERF_TXD; 288 static int ix_rxd = IX_PERF_RXD; 289 static int ix_unsupported_sfp = 0; 290 291 static char ix_flowctrl[IFM_ETH_FC_STRLEN] = IFM_ETH_FC_FULL; 292 293 TUNABLE_INT("hw.ix.msi.enable", &ix_msi_enable); 294 TUNABLE_INT("hw.ix.msix.enable", &ix_msix_enable); 295 TUNABLE_INT("hw.ix.msix.agg_rxtx", &ix_msix_agg_rxtx); 296 TUNABLE_INT("hw.ix.rxr", &ix_rxr); 297 TUNABLE_INT("hw.ix.txr", &ix_txr); 298 TUNABLE_INT("hw.ix.txd", &ix_txd); 299 TUNABLE_INT("hw.ix.rxd", &ix_rxd); 300 TUNABLE_INT("hw.ix.unsupported_sfp", &ix_unsupported_sfp); 301 TUNABLE_STR("hw.ix.flow_ctrl", ix_flowctrl, sizeof(ix_flowctrl)); 302 303 /* 304 * Smart speed setting, default to on. This only works 305 * as a compile option right now as its during attach, 306 * set this to 'ixgbe_smart_speed_off' to disable. 307 */ 308 static const enum ixgbe_smart_speed ix_smart_speed = 309 ixgbe_smart_speed_on; 310 311 static int 312 ix_probe(device_t dev) 313 { 314 const struct ix_device *d; 315 uint16_t vid, did; 316 317 vid = pci_get_vendor(dev); 318 did = pci_get_device(dev); 319 320 for (d = ix_devices; d->desc != NULL; ++d) { 321 if (vid == d->vid && did == d->did) { 322 device_set_desc(dev, d->desc); 323 return 0; 324 } 325 } 326 return ENXIO; 327 } 328 329 static int 330 ix_attach(device_t dev) 331 { 332 struct ix_softc *sc = device_get_softc(dev); 333 struct ixgbe_hw *hw; 334 int error, ring_cnt_max; 335 uint16_t csum; 336 uint32_t ctrl_ext; 337 #ifdef IFPOLL_ENABLE 338 int offset, offset_def; 339 #endif 340 char flowctrl[IFM_ETH_FC_STRLEN]; 341 342 sc->dev = sc->osdep.dev = dev; 343 hw = &sc->hw; 344 345 if_initname(&sc->arpcom.ac_if, device_get_name(dev), 346 device_get_unit(dev)); 347 ifmedia_init(&sc->media, IFM_IMASK | IFM_ETH_FCMASK, 348 ix_media_change, ix_media_status); 349 350 /* Save frame size */ 351 sc->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN; 352 353 callout_init_mp(&sc->timer); 354 lwkt_serialize_init(&sc->main_serialize); 355 356 /* 357 * Save off the information about this board 358 */ 359 hw->vendor_id = pci_get_vendor(dev); 360 hw->device_id = pci_get_device(dev); 361 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); 362 hw->subsystem_vendor_id = pci_read_config(dev, PCIR_SUBVEND_0, 2); 363 hw->subsystem_device_id = pci_read_config(dev, PCIR_SUBDEV_0, 2); 364 365 ixgbe_set_mac_type(hw); 366 367 /* Pick up the 82599 */ 368 if (hw->mac.type != ixgbe_mac_82598EB) 369 hw->phy.smart_speed = ix_smart_speed; 370 371 /* Enable bus mastering */ 372 pci_enable_busmaster(dev); 373 374 /* 375 * Allocate IO memory 376 */ 377 sc->mem_rid = PCIR_BAR(0); 378 sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 379 &sc->mem_rid, RF_ACTIVE); 380 if (sc->mem_res == NULL) { 381 device_printf(dev, "Unable to allocate bus resource: memory\n"); 382 error = ENXIO; 383 goto failed; 384 } 385 386 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->mem_res); 387 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->mem_res); 388 389 sc->hw.hw_addr = (uint8_t *)&sc->osdep.mem_bus_space_handle; 390 sc->hw.back = &sc->osdep; 391 392 /* 393 * Configure total supported RX/TX ring count 394 */ 395 sc->rx_ring_cnt = device_getenv_int(dev, "rxr", ix_rxr); 396 sc->rx_ring_cnt = if_ring_count2(sc->rx_ring_cnt, IX_MAX_RXRING); 397 sc->rx_ring_inuse = sc->rx_ring_cnt; 398 399 switch (hw->mac.type) { 400 case ixgbe_mac_82598EB: 401 ring_cnt_max = IX_MAX_TXRING_82598; 402 break; 403 404 case ixgbe_mac_82599EB: 405 ring_cnt_max = IX_MAX_TXRING_82599; 406 break; 407 408 case ixgbe_mac_X540: 409 ring_cnt_max = IX_MAX_TXRING_X540; 410 break; 411 412 default: 413 ring_cnt_max = 1; 414 break; 415 } 416 sc->tx_ring_cnt = device_getenv_int(dev, "txr", ix_txr); 417 sc->tx_ring_cnt = if_ring_count2(sc->tx_ring_cnt, ring_cnt_max); 418 sc->tx_ring_inuse = sc->tx_ring_cnt; 419 420 /* Allocate TX/RX rings */ 421 error = ix_alloc_rings(sc); 422 if (error) 423 goto failed; 424 425 #ifdef IFPOLL_ENABLE 426 /* 427 * NPOLLING RX CPU offset 428 */ 429 if (sc->rx_ring_cnt == ncpus2) { 430 offset = 0; 431 } else { 432 offset_def = (sc->rx_ring_cnt * device_get_unit(dev)) % ncpus2; 433 offset = device_getenv_int(dev, "npoll.rxoff", offset_def); 434 if (offset >= ncpus2 || 435 offset % sc->rx_ring_cnt != 0) { 436 device_printf(dev, "invalid npoll.rxoff %d, use %d\n", 437 offset, offset_def); 438 offset = offset_def; 439 } 440 } 441 sc->rx_npoll_off = offset; 442 443 /* 444 * NPOLLING TX CPU offset 445 */ 446 if (sc->tx_ring_cnt == ncpus2) { 447 offset = 0; 448 } else { 449 offset_def = (sc->tx_ring_cnt * device_get_unit(dev)) % ncpus2; 450 offset = device_getenv_int(dev, "npoll.txoff", offset_def); 451 if (offset >= ncpus2 || 452 offset % sc->tx_ring_cnt != 0) { 453 device_printf(dev, "invalid npoll.txoff %d, use %d\n", 454 offset, offset_def); 455 offset = offset_def; 456 } 457 } 458 sc->tx_npoll_off = offset; 459 #endif 460 461 /* Allocate interrupt */ 462 error = ix_alloc_intr(sc); 463 if (error) 464 goto failed; 465 466 /* Setup serializes */ 467 ix_setup_serialize(sc); 468 469 /* Allocate multicast array memory. */ 470 sc->mta = kmalloc(IXGBE_ETH_LENGTH_OF_ADDRESS * IX_MAX_MCASTADDR, 471 M_DEVBUF, M_WAITOK); 472 473 /* Initialize the shared code */ 474 hw->allow_unsupported_sfp = ix_unsupported_sfp; 475 error = ixgbe_init_shared_code(hw); 476 if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 477 /* 478 * No optics in this port; ask timer routine 479 * to probe for later insertion. 480 */ 481 sc->sfp_probe = TRUE; 482 error = 0; 483 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 484 device_printf(dev, "Unsupported SFP+ module detected!\n"); 485 error = EIO; 486 goto failed; 487 } else if (error) { 488 device_printf(dev, "Unable to initialize the shared code\n"); 489 error = EIO; 490 goto failed; 491 } 492 493 /* Make sure we have a good EEPROM before we read from it */ 494 if (ixgbe_validate_eeprom_checksum(&sc->hw, &csum) < 0) { 495 device_printf(dev, "The EEPROM Checksum Is Not Valid\n"); 496 error = EIO; 497 goto failed; 498 } 499 500 error = ixgbe_init_hw(hw); 501 if (error == IXGBE_ERR_EEPROM_VERSION) { 502 device_printf(dev, "Pre-production device detected\n"); 503 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 504 device_printf(dev, "Unsupported SFP+ Module\n"); 505 error = EIO; 506 goto failed; 507 } else if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 508 device_printf(dev, "No SFP+ Module found\n"); 509 } 510 511 sc->ifm_media = IX_IFM_DEFAULT; 512 /* Get default flow control settings */ 513 device_getenv_string(dev, "flow_ctrl", flowctrl, sizeof(flowctrl), 514 ix_flowctrl); 515 sc->ifm_media |= ifmedia_str2ethfc(flowctrl); 516 sc->advspeed = IXGBE_LINK_SPEED_UNKNOWN; 517 518 /* Setup OS specific network interface */ 519 ix_setup_ifp(sc); 520 521 /* Add sysctl tree */ 522 ix_add_sysctl(sc); 523 524 error = ix_setup_intr(sc); 525 if (error) { 526 ether_ifdetach(&sc->arpcom.ac_if); 527 goto failed; 528 } 529 530 /* Initialize statistics */ 531 ix_update_stats(sc); 532 533 /* Check PCIE slot type/speed/width */ 534 ix_slot_info(sc); 535 536 /* Save initial wake up filter configuration */ 537 sc->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC); 538 539 /* Let hardware know driver is loaded */ 540 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 541 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 542 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 543 544 return 0; 545 failed: 546 ix_detach(dev); 547 return error; 548 } 549 550 static int 551 ix_detach(device_t dev) 552 { 553 struct ix_softc *sc = device_get_softc(dev); 554 555 if (device_is_attached(dev)) { 556 struct ifnet *ifp = &sc->arpcom.ac_if; 557 uint32_t ctrl_ext; 558 559 ifnet_serialize_all(ifp); 560 561 ix_powerdown(sc); 562 ix_teardown_intr(sc, sc->intr_cnt); 563 564 ifnet_deserialize_all(ifp); 565 566 callout_terminate(&sc->timer); 567 ether_ifdetach(ifp); 568 569 /* Let hardware know driver is unloading */ 570 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 571 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 572 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 573 } 574 575 ifmedia_removeall(&sc->media); 576 bus_generic_detach(dev); 577 578 ix_free_intr(sc); 579 580 if (sc->msix_mem_res != NULL) { 581 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_mem_rid, 582 sc->msix_mem_res); 583 } 584 if (sc->mem_res != NULL) { 585 bus_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid, 586 sc->mem_res); 587 } 588 589 ix_free_rings(sc); 590 591 if (sc->mta != NULL) 592 kfree(sc->mta, M_DEVBUF); 593 if (sc->serializes != NULL) 594 kfree(sc->serializes, M_DEVBUF); 595 596 return 0; 597 } 598 599 static int 600 ix_shutdown(device_t dev) 601 { 602 struct ix_softc *sc = device_get_softc(dev); 603 struct ifnet *ifp = &sc->arpcom.ac_if; 604 605 ifnet_serialize_all(ifp); 606 ix_powerdown(sc); 607 ifnet_deserialize_all(ifp); 608 609 return 0; 610 } 611 612 static void 613 ix_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 614 { 615 struct ix_softc *sc = ifp->if_softc; 616 struct ix_tx_ring *txr = ifsq_get_priv(ifsq); 617 int idx = -1; 618 uint16_t nsegs; 619 620 KKASSERT(txr->tx_ifsq == ifsq); 621 ASSERT_SERIALIZED(&txr->tx_serialize); 622 623 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq)) 624 return; 625 626 if (!sc->link_active || (txr->tx_flags & IX_TXFLAG_ENABLED) == 0) { 627 ifsq_purge(ifsq); 628 return; 629 } 630 631 while (!ifsq_is_empty(ifsq)) { 632 struct mbuf *m_head; 633 634 if (txr->tx_avail <= IX_MAX_SCATTER + IX_TX_RESERVED) { 635 ifsq_set_oactive(ifsq); 636 txr->tx_watchdog.wd_timer = 5; 637 break; 638 } 639 640 m_head = ifsq_dequeue(ifsq); 641 if (m_head == NULL) 642 break; 643 644 if (ix_encap(txr, &m_head, &nsegs, &idx)) { 645 IFNET_STAT_INC(ifp, oerrors, 1); 646 continue; 647 } 648 649 /* 650 * TX interrupt are aggressively aggregated, so increasing 651 * opackets at TX interrupt time will make the opackets 652 * statistics vastly inaccurate; we do the opackets increment 653 * now. 654 */ 655 IFNET_STAT_INC(ifp, opackets, 1); 656 657 if (nsegs >= txr->tx_wreg_nsegs) { 658 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->tx_idx), idx); 659 nsegs = 0; 660 idx = -1; 661 } 662 663 ETHER_BPF_MTAP(ifp, m_head); 664 } 665 if (idx >= 0) 666 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->tx_idx), idx); 667 } 668 669 static int 670 ix_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) 671 { 672 struct ix_softc *sc = ifp->if_softc; 673 struct ifreq *ifr = (struct ifreq *) data; 674 int error = 0, mask, reinit; 675 676 ASSERT_IFNET_SERIALIZED_ALL(ifp); 677 678 switch (command) { 679 case SIOCSIFMTU: 680 if (ifr->ifr_mtu > IX_MAX_MTU) { 681 error = EINVAL; 682 } else { 683 ifp->if_mtu = ifr->ifr_mtu; 684 sc->max_frame_size = ifp->if_mtu + IX_MTU_HDR; 685 ix_init(sc); 686 } 687 break; 688 689 case SIOCSIFFLAGS: 690 if (ifp->if_flags & IFF_UP) { 691 if (ifp->if_flags & IFF_RUNNING) { 692 if ((ifp->if_flags ^ sc->if_flags) & 693 (IFF_PROMISC | IFF_ALLMULTI)) 694 ix_set_promisc(sc); 695 } else { 696 ix_init(sc); 697 } 698 } else if (ifp->if_flags & IFF_RUNNING) { 699 ix_stop(sc); 700 } 701 sc->if_flags = ifp->if_flags; 702 break; 703 704 case SIOCADDMULTI: 705 case SIOCDELMULTI: 706 if (ifp->if_flags & IFF_RUNNING) { 707 ix_disable_intr(sc); 708 ix_set_multi(sc); 709 #ifdef IFPOLL_ENABLE 710 if ((ifp->if_flags & IFF_NPOLLING) == 0) 711 #endif 712 ix_enable_intr(sc); 713 } 714 break; 715 716 case SIOCSIFMEDIA: 717 case SIOCGIFMEDIA: 718 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 719 break; 720 721 case SIOCSIFCAP: 722 reinit = 0; 723 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 724 if (mask & IFCAP_RXCSUM) { 725 ifp->if_capenable ^= IFCAP_RXCSUM; 726 reinit = 1; 727 } 728 if (mask & IFCAP_VLAN_HWTAGGING) { 729 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 730 reinit = 1; 731 } 732 if (mask & IFCAP_TXCSUM) { 733 ifp->if_capenable ^= IFCAP_TXCSUM; 734 if (ifp->if_capenable & IFCAP_TXCSUM) 735 ifp->if_hwassist |= CSUM_OFFLOAD; 736 else 737 ifp->if_hwassist &= ~CSUM_OFFLOAD; 738 } 739 if (mask & IFCAP_TSO) { 740 ifp->if_capenable ^= IFCAP_TSO; 741 if (ifp->if_capenable & IFCAP_TSO) 742 ifp->if_hwassist |= CSUM_TSO; 743 else 744 ifp->if_hwassist &= ~CSUM_TSO; 745 } 746 if (mask & IFCAP_RSS) 747 ifp->if_capenable ^= IFCAP_RSS; 748 if (reinit && (ifp->if_flags & IFF_RUNNING)) 749 ix_init(sc); 750 break; 751 752 #if 0 753 case SIOCGI2C: 754 { 755 struct ixgbe_i2c_req i2c; 756 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 757 if (error) 758 break; 759 if ((i2c.dev_addr != 0xA0) || (i2c.dev_addr != 0xA2)){ 760 error = EINVAL; 761 break; 762 } 763 hw->phy.ops.read_i2c_byte(hw, i2c.offset, 764 i2c.dev_addr, i2c.data); 765 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 766 break; 767 } 768 #endif 769 770 default: 771 error = ether_ioctl(ifp, command, data); 772 break; 773 } 774 return error; 775 } 776 777 #define IXGBE_MHADD_MFS_SHIFT 16 778 779 static void 780 ix_init(void *xsc) 781 { 782 struct ix_softc *sc = xsc; 783 struct ifnet *ifp = &sc->arpcom.ac_if; 784 struct ixgbe_hw *hw = &sc->hw; 785 uint32_t gpie, rxctrl; 786 int i, error; 787 boolean_t polling; 788 789 ASSERT_IFNET_SERIALIZED_ALL(ifp); 790 791 ix_stop(sc); 792 793 polling = FALSE; 794 #ifdef IFPOLL_ENABLE 795 if (ifp->if_flags & IFF_NPOLLING) 796 polling = TRUE; 797 #endif 798 799 /* Configure # of used RX/TX rings */ 800 ix_set_ring_inuse(sc, polling); 801 ifq_set_subq_mask(&ifp->if_snd, sc->tx_ring_inuse - 1); 802 803 /* Get the latest mac address, User can use a LAA */ 804 bcopy(IF_LLADDR(ifp), hw->mac.addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 805 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 806 hw->addr_ctrl.rar_used_count = 1; 807 808 /* Prepare transmit descriptors and buffers */ 809 for (i = 0; i < sc->tx_ring_inuse; ++i) 810 ix_init_tx_ring(&sc->tx_rings[i]); 811 812 ixgbe_init_hw(hw); 813 ix_init_tx_unit(sc); 814 815 /* Setup Multicast table */ 816 ix_set_multi(sc); 817 818 /* Prepare receive descriptors and buffers */ 819 for (i = 0; i < sc->rx_ring_inuse; ++i) { 820 error = ix_init_rx_ring(&sc->rx_rings[i]); 821 if (error) { 822 if_printf(ifp, "Could not initialize RX ring%d\n", i); 823 ix_stop(sc); 824 return; 825 } 826 } 827 828 /* Configure RX settings */ 829 ix_init_rx_unit(sc); 830 831 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 832 833 /* Enable Fan Failure Interrupt */ 834 gpie |= IXGBE_SDP1_GPIEN_BY_MAC(hw); 835 836 /* Add for Module detection */ 837 if (hw->mac.type == ixgbe_mac_82599EB) 838 gpie |= IXGBE_SDP2_GPIEN; 839 840 /* 841 * Thermal Failure Detection (X540) 842 * Link Detection (X552) 843 */ 844 if (hw->mac.type == ixgbe_mac_X540 || 845 hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 846 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 847 gpie |= IXGBE_SDP0_GPIEN_X540; 848 849 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 850 /* Enable Enhanced MSIX mode */ 851 gpie |= IXGBE_GPIE_MSIX_MODE; 852 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 853 IXGBE_GPIE_OCD; 854 } 855 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 856 857 /* Set MTU size */ 858 if (ifp->if_mtu > ETHERMTU) { 859 uint32_t mhadd; 860 861 /* aka IXGBE_MAXFRS on 82599 and newer */ 862 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 863 mhadd &= ~IXGBE_MHADD_MFS_MASK; 864 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 865 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 866 } 867 868 /* 869 * Enable TX rings 870 */ 871 for (i = 0; i < sc->tx_ring_inuse; ++i) { 872 uint32_t txdctl; 873 874 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 875 txdctl |= IXGBE_TXDCTL_ENABLE; 876 877 /* 878 * Set WTHRESH to 0, since TX head write-back is used 879 */ 880 txdctl &= ~(0x7f << 16); 881 882 /* 883 * When the internal queue falls below PTHRESH (32), 884 * start prefetching as long as there are at least 885 * HTHRESH (1) buffers ready. The values are taken 886 * from the Intel linux driver 3.8.21. 887 * Prefetching enables tx line rate even with 1 queue. 888 */ 889 txdctl |= (32 << 0) | (1 << 8); 890 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl); 891 } 892 893 /* 894 * Enable RX rings 895 */ 896 for (i = 0; i < sc->rx_ring_inuse; ++i) { 897 uint32_t rxdctl; 898 int k; 899 900 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 901 if (hw->mac.type == ixgbe_mac_82598EB) { 902 /* 903 * PTHRESH = 21 904 * HTHRESH = 4 905 * WTHRESH = 8 906 */ 907 rxdctl &= ~0x3FFFFF; 908 rxdctl |= 0x080420; 909 } 910 rxdctl |= IXGBE_RXDCTL_ENABLE; 911 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl); 912 for (k = 0; k < 10; ++k) { 913 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & 914 IXGBE_RXDCTL_ENABLE) 915 break; 916 else 917 msec_delay(1); 918 } 919 wmb(); 920 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 921 sc->rx_rings[0].rx_ndesc - 1); 922 } 923 924 /* Enable Receive engine */ 925 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 926 if (hw->mac.type == ixgbe_mac_82598EB) 927 rxctrl |= IXGBE_RXCTRL_DMBYPS; 928 rxctrl |= IXGBE_RXCTRL_RXEN; 929 ixgbe_enable_rx_dma(hw, rxctrl); 930 931 for (i = 0; i < sc->tx_ring_inuse; ++i) { 932 const struct ix_tx_ring *txr = &sc->tx_rings[i]; 933 934 if (txr->tx_intr_vec >= 0) { 935 ix_set_ivar(sc, i, txr->tx_intr_vec, 1); 936 } else { 937 /* 938 * Unconfigured TX interrupt vector could only 939 * happen for MSI-X. 940 */ 941 KASSERT(sc->intr_type == PCI_INTR_TYPE_MSIX, 942 ("TX intr vector is not set")); 943 KASSERT(i < sc->rx_ring_inuse, 944 ("invalid TX ring %d, no piggyback RX ring", i)); 945 KASSERT(sc->rx_rings[i].rx_txr == txr, 946 ("RX ring %d piggybacked TX ring mismatch", i)); 947 if (bootverbose) 948 if_printf(ifp, "IVAR skips TX ring %d\n", i); 949 } 950 } 951 for (i = 0; i < sc->rx_ring_inuse; ++i) { 952 const struct ix_rx_ring *rxr = &sc->rx_rings[i]; 953 954 KKASSERT(rxr->rx_intr_vec >= 0); 955 ix_set_ivar(sc, i, rxr->rx_intr_vec, 0); 956 if (rxr->rx_txr != NULL) { 957 /* 958 * Piggyback the TX ring interrupt onto the RX 959 * ring interrupt vector. 960 */ 961 KASSERT(rxr->rx_txr->tx_intr_vec < 0, 962 ("piggybacked TX ring configured intr vector")); 963 KASSERT(rxr->rx_txr->tx_idx == i, 964 ("RX ring %d piggybacked TX ring %u", 965 i, rxr->rx_txr->tx_idx)); 966 ix_set_ivar(sc, i, rxr->rx_intr_vec, 1); 967 if (bootverbose) { 968 if_printf(ifp, "IVAR RX ring %d piggybacks " 969 "TX ring %u\n", i, rxr->rx_txr->tx_idx); 970 } 971 } 972 } 973 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 974 /* Set up status MSI-X vector; it is using fixed entry 1 */ 975 ix_set_ivar(sc, 1, sc->sts_msix_vec, -1); 976 977 /* Set up auto-mask for TX and RX rings */ 978 if (hw->mac.type == ixgbe_mac_82598EB) { 979 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EIMS_RTX_QUEUE); 980 } else { 981 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 982 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 983 } 984 } else { 985 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EIMS_RTX_QUEUE); 986 } 987 for (i = 0; i < sc->intr_cnt; ++i) 988 ix_set_eitr(sc, i, sc->intr_data[i].intr_rate); 989 990 /* 991 * Check on any SFP devices that need to be kick-started 992 */ 993 if (hw->phy.type == ixgbe_phy_none) { 994 error = hw->phy.ops.identify(hw); 995 if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 996 if_printf(ifp, 997 "Unsupported SFP+ module type was detected.\n"); 998 /* XXX stop */ 999 return; 1000 } 1001 } 1002 1003 /* Config/Enable Link */ 1004 ix_config_link(sc); 1005 1006 /* Hardware Packet Buffer & Flow Control setup */ 1007 ix_config_flowctrl(sc); 1008 1009 /* Initialize the FC settings */ 1010 ixgbe_start_hw(hw); 1011 1012 /* Set up VLAN support and filter */ 1013 ix_set_vlan(sc); 1014 1015 /* Setup DMA Coalescing */ 1016 ix_config_dmac(sc); 1017 1018 /* 1019 * Only enable interrupts if we are not polling, make sure 1020 * they are off otherwise. 1021 */ 1022 if (polling) 1023 ix_disable_intr(sc); 1024 else 1025 ix_enable_intr(sc); 1026 1027 ifp->if_flags |= IFF_RUNNING; 1028 for (i = 0; i < sc->tx_ring_inuse; ++i) { 1029 ifsq_clr_oactive(sc->tx_rings[i].tx_ifsq); 1030 ifsq_watchdog_start(&sc->tx_rings[i].tx_watchdog); 1031 } 1032 1033 ix_set_timer_cpuid(sc, polling); 1034 callout_reset_bycpu(&sc->timer, hz, ix_timer, sc, sc->timer_cpuid); 1035 } 1036 1037 static void 1038 ix_intr(void *xsc) 1039 { 1040 struct ix_softc *sc = xsc; 1041 struct ixgbe_hw *hw = &sc->hw; 1042 uint32_t eicr; 1043 1044 ASSERT_SERIALIZED(&sc->main_serialize); 1045 1046 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1047 if (eicr == 0) { 1048 IXGBE_WRITE_REG(hw, IXGBE_EIMS, sc->intr_mask); 1049 return; 1050 } 1051 1052 if (eicr & IX_RX0_INTR_MASK) { 1053 struct ix_rx_ring *rxr = &sc->rx_rings[0]; 1054 1055 lwkt_serialize_enter(&rxr->rx_serialize); 1056 ix_rxeof(rxr, -1); 1057 lwkt_serialize_exit(&rxr->rx_serialize); 1058 } 1059 if (eicr & IX_RX1_INTR_MASK) { 1060 struct ix_rx_ring *rxr; 1061 1062 KKASSERT(sc->rx_ring_inuse == IX_MIN_RXRING_RSS); 1063 rxr = &sc->rx_rings[1]; 1064 1065 lwkt_serialize_enter(&rxr->rx_serialize); 1066 ix_rxeof(rxr, -1); 1067 lwkt_serialize_exit(&rxr->rx_serialize); 1068 } 1069 1070 if (eicr & IX_TX_INTR_MASK) { 1071 struct ix_tx_ring *txr = &sc->tx_rings[0]; 1072 1073 lwkt_serialize_enter(&txr->tx_serialize); 1074 ix_txeof(txr, *(txr->tx_hdr)); 1075 if (!ifsq_is_empty(txr->tx_ifsq)) 1076 ifsq_devstart(txr->tx_ifsq); 1077 lwkt_serialize_exit(&txr->tx_serialize); 1078 } 1079 1080 if (__predict_false(eicr & IX_EICR_STATUS)) 1081 ix_intr_status(sc, eicr); 1082 1083 IXGBE_WRITE_REG(hw, IXGBE_EIMS, sc->intr_mask); 1084 } 1085 1086 static void 1087 ix_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1088 { 1089 struct ix_softc *sc = ifp->if_softc; 1090 struct ifmedia *ifm = &sc->media; 1091 int layer; 1092 1093 ix_update_link_status(sc); 1094 1095 ifmr->ifm_status = IFM_AVALID; 1096 ifmr->ifm_active = IFM_ETHER; 1097 1098 if (!sc->link_active) { 1099 if (IFM_SUBTYPE(ifm->ifm_media) != IFM_AUTO) 1100 ifmr->ifm_active |= ifm->ifm_media; 1101 else 1102 ifmr->ifm_active |= IFM_NONE; 1103 return; 1104 } 1105 ifmr->ifm_status |= IFM_ACTIVE; 1106 1107 layer = ixgbe_get_supported_physical_layer(&sc->hw); 1108 1109 if ((layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) || 1110 (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) || 1111 (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX)) { 1112 switch (sc->link_speed) { 1113 case IXGBE_LINK_SPEED_10GB_FULL: 1114 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1115 break; 1116 case IXGBE_LINK_SPEED_1GB_FULL: 1117 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1118 break; 1119 case IXGBE_LINK_SPEED_100_FULL: 1120 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1121 break; 1122 } 1123 } else if ((layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) || 1124 (layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)) { 1125 switch (sc->link_speed) { 1126 case IXGBE_LINK_SPEED_10GB_FULL: 1127 ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX; 1128 break; 1129 } 1130 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1131 switch (sc->link_speed) { 1132 case IXGBE_LINK_SPEED_10GB_FULL: 1133 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 1134 break; 1135 case IXGBE_LINK_SPEED_1GB_FULL: 1136 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 1137 break; 1138 } 1139 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM) { 1140 switch (sc->link_speed) { 1141 case IXGBE_LINK_SPEED_10GB_FULL: 1142 ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX; 1143 break; 1144 case IXGBE_LINK_SPEED_1GB_FULL: 1145 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 1146 break; 1147 } 1148 } else if ((layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) || 1149 (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)) { 1150 switch (sc->link_speed) { 1151 case IXGBE_LINK_SPEED_10GB_FULL: 1152 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 1153 break; 1154 case IXGBE_LINK_SPEED_1GB_FULL: 1155 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 1156 break; 1157 } 1158 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1159 switch (sc->link_speed) { 1160 case IXGBE_LINK_SPEED_10GB_FULL: 1161 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 1162 break; 1163 } 1164 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1165 /* 1166 * XXX: These need to use the proper media types once 1167 * they're added. 1168 */ 1169 switch (sc->link_speed) { 1170 case IXGBE_LINK_SPEED_10GB_FULL: 1171 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 1172 break; 1173 case IXGBE_LINK_SPEED_2_5GB_FULL: 1174 ifmr->ifm_active |= IFM_2500_SX | IFM_FDX; 1175 break; 1176 case IXGBE_LINK_SPEED_1GB_FULL: 1177 ifmr->ifm_active |= IFM_1000_CX | IFM_FDX; 1178 break; 1179 } 1180 } else if ((layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) || 1181 (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)) { 1182 /* 1183 * XXX: These need to use the proper media types once 1184 * they're added. 1185 */ 1186 switch (sc->link_speed) { 1187 case IXGBE_LINK_SPEED_10GB_FULL: 1188 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 1189 break; 1190 case IXGBE_LINK_SPEED_2_5GB_FULL: 1191 ifmr->ifm_active |= IFM_2500_SX | IFM_FDX; 1192 break; 1193 case IXGBE_LINK_SPEED_1GB_FULL: 1194 ifmr->ifm_active |= IFM_1000_CX | IFM_FDX; 1195 break; 1196 } 1197 } 1198 1199 /* If nothing is recognized... */ 1200 if (IFM_SUBTYPE(ifmr->ifm_active) == 0) 1201 ifmr->ifm_active |= IFM_NONE; 1202 1203 if (sc->ifm_media & IFM_ETH_FORCEPAUSE) 1204 ifmr->ifm_active |= (sc->ifm_media & IFM_ETH_FCMASK); 1205 1206 switch (sc->hw.fc.current_mode) { 1207 case ixgbe_fc_full: 1208 ifmr->ifm_active |= IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE; 1209 break; 1210 case ixgbe_fc_rx_pause: 1211 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 1212 break; 1213 case ixgbe_fc_tx_pause: 1214 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 1215 break; 1216 default: 1217 break; 1218 } 1219 } 1220 1221 static int 1222 ix_media_change(struct ifnet *ifp) 1223 { 1224 struct ix_softc *sc = ifp->if_softc; 1225 struct ifmedia *ifm = &sc->media; 1226 struct ixgbe_hw *hw = &sc->hw; 1227 1228 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1229 return (EINVAL); 1230 1231 if (hw->phy.media_type == ixgbe_media_type_backplane || 1232 hw->mac.ops.setup_link == NULL) { 1233 if ((ifm->ifm_media ^ sc->ifm_media) & IFM_ETH_FCMASK) { 1234 /* Only flow control setting changes are allowed */ 1235 return (EOPNOTSUPP); 1236 } 1237 } 1238 1239 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1240 case IFM_AUTO: 1241 sc->advspeed = IXGBE_LINK_SPEED_UNKNOWN; 1242 break; 1243 1244 case IFM_10G_T: 1245 case IFM_10G_LRM: 1246 case IFM_10G_SR: /* XXX also KR */ 1247 case IFM_10G_LR: 1248 case IFM_10G_CX4: /* XXX also KX4 */ 1249 case IFM_10G_TWINAX: 1250 sc->advspeed = IXGBE_LINK_SPEED_10GB_FULL; 1251 break; 1252 1253 case IFM_1000_T: 1254 case IFM_1000_LX: 1255 case IFM_1000_SX: 1256 case IFM_1000_CX: /* XXX is KX */ 1257 sc->advspeed = IXGBE_LINK_SPEED_1GB_FULL; 1258 break; 1259 1260 case IFM_100_TX: 1261 sc->advspeed = IXGBE_LINK_SPEED_100_FULL; 1262 break; 1263 1264 default: 1265 if (bootverbose) { 1266 if_printf(ifp, "Invalid media type %d!\n", 1267 ifm->ifm_media); 1268 } 1269 return EINVAL; 1270 } 1271 sc->ifm_media = ifm->ifm_media; 1272 1273 #if 0 1274 if (hw->mac.ops.setup_link != NULL) { 1275 hw->mac.autotry_restart = TRUE; 1276 hw->mac.ops.setup_link(hw, sc->advspeed, TRUE); 1277 } 1278 #else 1279 if (ifp->if_flags & IFF_RUNNING) 1280 ix_init(sc); 1281 #endif 1282 return 0; 1283 } 1284 1285 static __inline int 1286 ix_tso_pullup(struct mbuf **mp) 1287 { 1288 int hoff, iphlen, thoff; 1289 struct mbuf *m; 1290 1291 m = *mp; 1292 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable")); 1293 1294 iphlen = m->m_pkthdr.csum_iphlen; 1295 thoff = m->m_pkthdr.csum_thlen; 1296 hoff = m->m_pkthdr.csum_lhlen; 1297 1298 KASSERT(iphlen > 0, ("invalid ip hlen")); 1299 KASSERT(thoff > 0, ("invalid tcp hlen")); 1300 KASSERT(hoff > 0, ("invalid ether hlen")); 1301 1302 if (__predict_false(m->m_len < hoff + iphlen + thoff)) { 1303 m = m_pullup(m, hoff + iphlen + thoff); 1304 if (m == NULL) { 1305 *mp = NULL; 1306 return ENOBUFS; 1307 } 1308 *mp = m; 1309 } 1310 return 0; 1311 } 1312 1313 static int 1314 ix_encap(struct ix_tx_ring *txr, struct mbuf **m_headp, 1315 uint16_t *segs_used, int *idx) 1316 { 1317 uint32_t olinfo_status = 0, cmd_type_len, cmd_rs = 0; 1318 int i, j, error, nsegs, first, maxsegs; 1319 struct mbuf *m_head = *m_headp; 1320 bus_dma_segment_t segs[IX_MAX_SCATTER]; 1321 bus_dmamap_t map; 1322 struct ix_tx_buf *txbuf; 1323 union ixgbe_adv_tx_desc *txd = NULL; 1324 1325 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1326 error = ix_tso_pullup(m_headp); 1327 if (__predict_false(error)) 1328 return error; 1329 m_head = *m_headp; 1330 } 1331 1332 /* Basic descriptor defines */ 1333 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1334 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1335 1336 if (m_head->m_flags & M_VLANTAG) 1337 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1338 1339 /* 1340 * Important to capture the first descriptor 1341 * used because it will contain the index of 1342 * the one we tell the hardware to report back 1343 */ 1344 first = txr->tx_next_avail; 1345 txbuf = &txr->tx_buf[first]; 1346 map = txbuf->map; 1347 1348 /* 1349 * Map the packet for DMA. 1350 */ 1351 maxsegs = txr->tx_avail - IX_TX_RESERVED; 1352 if (maxsegs > IX_MAX_SCATTER) 1353 maxsegs = IX_MAX_SCATTER; 1354 1355 error = bus_dmamap_load_mbuf_defrag(txr->tx_tag, map, m_headp, 1356 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 1357 if (__predict_false(error)) { 1358 m_freem(*m_headp); 1359 *m_headp = NULL; 1360 return error; 1361 } 1362 bus_dmamap_sync(txr->tx_tag, map, BUS_DMASYNC_PREWRITE); 1363 1364 m_head = *m_headp; 1365 1366 /* 1367 * Set up the appropriate offload context if requested, 1368 * this may consume one TX descriptor. 1369 */ 1370 if (ix_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status)) { 1371 (*segs_used)++; 1372 txr->tx_nsegs++; 1373 } 1374 1375 *segs_used += nsegs; 1376 txr->tx_nsegs += nsegs; 1377 if (txr->tx_nsegs >= txr->tx_intr_nsegs) { 1378 /* 1379 * Report Status (RS) is turned on every intr_nsegs 1380 * descriptors (roughly). 1381 */ 1382 txr->tx_nsegs = 0; 1383 cmd_rs = IXGBE_TXD_CMD_RS; 1384 } 1385 1386 i = txr->tx_next_avail; 1387 for (j = 0; j < nsegs; j++) { 1388 bus_size_t seglen; 1389 bus_addr_t segaddr; 1390 1391 txbuf = &txr->tx_buf[i]; 1392 txd = &txr->tx_base[i]; 1393 seglen = segs[j].ds_len; 1394 segaddr = htole64(segs[j].ds_addr); 1395 1396 txd->read.buffer_addr = segaddr; 1397 txd->read.cmd_type_len = htole32(IXGBE_TXD_CMD_IFCS | 1398 cmd_type_len |seglen); 1399 txd->read.olinfo_status = htole32(olinfo_status); 1400 1401 if (++i == txr->tx_ndesc) 1402 i = 0; 1403 } 1404 txd->read.cmd_type_len |= htole32(IXGBE_TXD_CMD_EOP | cmd_rs); 1405 1406 txr->tx_avail -= nsegs; 1407 txr->tx_next_avail = i; 1408 1409 txbuf->m_head = m_head; 1410 txr->tx_buf[first].map = txbuf->map; 1411 txbuf->map = map; 1412 1413 /* 1414 * Defer TDT updating, until enough descrptors are setup 1415 */ 1416 *idx = i; 1417 1418 return 0; 1419 } 1420 1421 static void 1422 ix_set_promisc(struct ix_softc *sc) 1423 { 1424 struct ifnet *ifp = &sc->arpcom.ac_if; 1425 uint32_t reg_rctl; 1426 int mcnt = 0; 1427 1428 reg_rctl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1429 reg_rctl &= ~IXGBE_FCTRL_UPE; 1430 if (ifp->if_flags & IFF_ALLMULTI) { 1431 mcnt = IX_MAX_MCASTADDR; 1432 } else { 1433 struct ifmultiaddr *ifma; 1434 1435 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1436 if (ifma->ifma_addr->sa_family != AF_LINK) 1437 continue; 1438 if (mcnt == IX_MAX_MCASTADDR) 1439 break; 1440 mcnt++; 1441 } 1442 } 1443 if (mcnt < IX_MAX_MCASTADDR) 1444 reg_rctl &= ~IXGBE_FCTRL_MPE; 1445 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 1446 1447 if (ifp->if_flags & IFF_PROMISC) { 1448 reg_rctl |= IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE; 1449 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 1450 } else if (ifp->if_flags & IFF_ALLMULTI) { 1451 reg_rctl |= IXGBE_FCTRL_MPE; 1452 reg_rctl &= ~IXGBE_FCTRL_UPE; 1453 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 1454 } 1455 } 1456 1457 static void 1458 ix_set_multi(struct ix_softc *sc) 1459 { 1460 struct ifnet *ifp = &sc->arpcom.ac_if; 1461 struct ifmultiaddr *ifma; 1462 uint32_t fctrl; 1463 uint8_t *mta; 1464 int mcnt = 0; 1465 1466 mta = sc->mta; 1467 bzero(mta, IXGBE_ETH_LENGTH_OF_ADDRESS * IX_MAX_MCASTADDR); 1468 1469 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1470 if (ifma->ifma_addr->sa_family != AF_LINK) 1471 continue; 1472 if (mcnt == IX_MAX_MCASTADDR) 1473 break; 1474 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1475 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1476 IXGBE_ETH_LENGTH_OF_ADDRESS); 1477 mcnt++; 1478 } 1479 1480 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1481 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1482 if (ifp->if_flags & IFF_PROMISC) { 1483 fctrl |= IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE; 1484 } else if (mcnt >= IX_MAX_MCASTADDR || (ifp->if_flags & IFF_ALLMULTI)) { 1485 fctrl |= IXGBE_FCTRL_MPE; 1486 fctrl &= ~IXGBE_FCTRL_UPE; 1487 } else { 1488 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1489 } 1490 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1491 1492 if (mcnt < IX_MAX_MCASTADDR) { 1493 ixgbe_update_mc_addr_list(&sc->hw, 1494 mta, mcnt, ix_mc_array_itr, TRUE); 1495 } 1496 } 1497 1498 /* 1499 * This is an iterator function now needed by the multicast 1500 * shared code. It simply feeds the shared code routine the 1501 * addresses in the array of ix_set_multi() one by one. 1502 */ 1503 static uint8_t * 1504 ix_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1505 { 1506 uint8_t *addr = *update_ptr; 1507 uint8_t *newptr; 1508 *vmdq = 0; 1509 1510 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1511 *update_ptr = newptr; 1512 return addr; 1513 } 1514 1515 static void 1516 ix_timer(void *arg) 1517 { 1518 struct ix_softc *sc = arg; 1519 1520 lwkt_serialize_enter(&sc->main_serialize); 1521 1522 if ((sc->arpcom.ac_if.if_flags & IFF_RUNNING) == 0) { 1523 lwkt_serialize_exit(&sc->main_serialize); 1524 return; 1525 } 1526 1527 /* Check for pluggable optics */ 1528 if (sc->sfp_probe) { 1529 if (!ix_sfp_probe(sc)) 1530 goto done; /* Nothing to do */ 1531 } 1532 1533 ix_update_link_status(sc); 1534 ix_update_stats(sc); 1535 1536 done: 1537 callout_reset_bycpu(&sc->timer, hz, ix_timer, sc, sc->timer_cpuid); 1538 lwkt_serialize_exit(&sc->main_serialize); 1539 } 1540 1541 static void 1542 ix_update_link_status(struct ix_softc *sc) 1543 { 1544 struct ifnet *ifp = &sc->arpcom.ac_if; 1545 1546 if (sc->link_up) { 1547 if (sc->link_active == FALSE) { 1548 if (bootverbose) { 1549 if_printf(ifp, "Link is up %d Gbps %s\n", 1550 sc->link_speed == 128 ? 10 : 1, 1551 "Full Duplex"); 1552 } 1553 1554 /* 1555 * Update any Flow Control changes 1556 */ 1557 ixgbe_fc_enable(&sc->hw); 1558 /* MUST after ixgbe_fc_enable() */ 1559 if (sc->rx_ring_inuse > 1) { 1560 switch (sc->hw.fc.current_mode) { 1561 case ixgbe_fc_rx_pause: 1562 case ixgbe_fc_tx_pause: 1563 case ixgbe_fc_full: 1564 ix_disable_rx_drop(sc); 1565 break; 1566 1567 case ixgbe_fc_none: 1568 ix_enable_rx_drop(sc); 1569 break; 1570 1571 default: 1572 break; 1573 } 1574 } 1575 1576 /* Update DMA coalescing config */ 1577 ix_config_dmac(sc); 1578 1579 sc->link_active = TRUE; 1580 1581 ifp->if_link_state = LINK_STATE_UP; 1582 if_link_state_change(ifp); 1583 } 1584 } else { /* Link down */ 1585 if (sc->link_active == TRUE) { 1586 if (bootverbose) 1587 if_printf(ifp, "Link is Down\n"); 1588 ifp->if_link_state = LINK_STATE_DOWN; 1589 if_link_state_change(ifp); 1590 1591 sc->link_active = FALSE; 1592 } 1593 } 1594 } 1595 1596 static void 1597 ix_stop(struct ix_softc *sc) 1598 { 1599 struct ixgbe_hw *hw = &sc->hw; 1600 struct ifnet *ifp = &sc->arpcom.ac_if; 1601 int i; 1602 1603 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1604 1605 ix_disable_intr(sc); 1606 callout_stop(&sc->timer); 1607 1608 ifp->if_flags &= ~IFF_RUNNING; 1609 for (i = 0; i < sc->tx_ring_cnt; ++i) { 1610 struct ix_tx_ring *txr = &sc->tx_rings[i]; 1611 1612 ifsq_clr_oactive(txr->tx_ifsq); 1613 ifsq_watchdog_stop(&txr->tx_watchdog); 1614 txr->tx_flags &= ~IX_TXFLAG_ENABLED; 1615 } 1616 1617 ixgbe_reset_hw(hw); 1618 hw->adapter_stopped = FALSE; 1619 ixgbe_stop_adapter(hw); 1620 if (hw->mac.type == ixgbe_mac_82599EB) 1621 ixgbe_stop_mac_link_on_d3_82599(hw); 1622 /* Turn off the laser - noop with no optics */ 1623 ixgbe_disable_tx_laser(hw); 1624 1625 /* Update the stack */ 1626 sc->link_up = FALSE; 1627 ix_update_link_status(sc); 1628 1629 /* Reprogram the RAR[0] in case user changed it. */ 1630 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1631 1632 for (i = 0; i < sc->tx_ring_cnt; ++i) 1633 ix_free_tx_ring(&sc->tx_rings[i]); 1634 1635 for (i = 0; i < sc->rx_ring_cnt; ++i) 1636 ix_free_rx_ring(&sc->rx_rings[i]); 1637 } 1638 1639 static void 1640 ix_setup_ifp(struct ix_softc *sc) 1641 { 1642 struct ixgbe_hw *hw = &sc->hw; 1643 struct ifnet *ifp = &sc->arpcom.ac_if; 1644 int i; 1645 1646 ifp->if_baudrate = IF_Gbps(10UL); 1647 1648 ifp->if_softc = sc; 1649 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1650 ifp->if_init = ix_init; 1651 ifp->if_ioctl = ix_ioctl; 1652 ifp->if_start = ix_start; 1653 ifp->if_serialize = ix_serialize; 1654 ifp->if_deserialize = ix_deserialize; 1655 ifp->if_tryserialize = ix_tryserialize; 1656 #ifdef INVARIANTS 1657 ifp->if_serialize_assert = ix_serialize_assert; 1658 #endif 1659 #ifdef IFPOLL_ENABLE 1660 ifp->if_npoll = ix_npoll; 1661 #endif 1662 1663 /* Increase TSO burst length */ 1664 ifp->if_tsolen = (8 * ETHERMTU); 1665 1666 ifp->if_nmbclusters = sc->rx_ring_cnt * sc->rx_rings[0].rx_ndesc; 1667 ifp->if_nmbjclusters = ifp->if_nmbclusters; 1668 1669 ifq_set_maxlen(&ifp->if_snd, sc->tx_rings[0].tx_ndesc - 2); 1670 ifq_set_ready(&ifp->if_snd); 1671 ifq_set_subq_cnt(&ifp->if_snd, sc->tx_ring_cnt); 1672 1673 ifp->if_mapsubq = ifq_mapsubq_mask; 1674 ifq_set_subq_mask(&ifp->if_snd, 0); 1675 1676 ether_ifattach(ifp, hw->mac.addr, NULL); 1677 1678 ifp->if_capabilities = 1679 IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 1680 if (IX_ENABLE_HWRSS(sc)) 1681 ifp->if_capabilities |= IFCAP_RSS; 1682 ifp->if_capenable = ifp->if_capabilities; 1683 ifp->if_hwassist = CSUM_OFFLOAD | CSUM_TSO; 1684 1685 /* 1686 * Tell the upper layer(s) we support long frames. 1687 */ 1688 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 1689 1690 /* Setup TX rings and subqueues */ 1691 for (i = 0; i < sc->tx_ring_cnt; ++i) { 1692 struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i); 1693 struct ix_tx_ring *txr = &sc->tx_rings[i]; 1694 1695 ifsq_set_cpuid(ifsq, txr->tx_intr_cpuid); 1696 ifsq_set_priv(ifsq, txr); 1697 ifsq_set_hw_serialize(ifsq, &txr->tx_serialize); 1698 txr->tx_ifsq = ifsq; 1699 1700 ifsq_watchdog_init(&txr->tx_watchdog, ifsq, ix_watchdog); 1701 } 1702 1703 /* Specify the media types supported by this adapter */ 1704 ix_init_media(sc); 1705 } 1706 1707 static boolean_t 1708 ix_is_sfp(const struct ixgbe_hw *hw) 1709 { 1710 switch (hw->phy.type) { 1711 case ixgbe_phy_sfp_avago: 1712 case ixgbe_phy_sfp_ftl: 1713 case ixgbe_phy_sfp_intel: 1714 case ixgbe_phy_sfp_unknown: 1715 case ixgbe_phy_sfp_passive_tyco: 1716 case ixgbe_phy_sfp_passive_unknown: 1717 case ixgbe_phy_qsfp_passive_unknown: 1718 case ixgbe_phy_qsfp_active_unknown: 1719 case ixgbe_phy_qsfp_intel: 1720 case ixgbe_phy_qsfp_unknown: 1721 return TRUE; 1722 default: 1723 return FALSE; 1724 } 1725 } 1726 1727 static void 1728 ix_config_link(struct ix_softc *sc) 1729 { 1730 struct ixgbe_hw *hw = &sc->hw; 1731 boolean_t sfp; 1732 1733 sfp = ix_is_sfp(hw); 1734 if (sfp) { 1735 if (hw->phy.multispeed_fiber) { 1736 hw->mac.ops.setup_sfp(hw); 1737 ixgbe_enable_tx_laser(hw); 1738 ix_handle_msf(sc); 1739 } else { 1740 ix_handle_mod(sc); 1741 } 1742 } else { 1743 uint32_t autoneg, err = 0; 1744 1745 if (hw->mac.ops.check_link != NULL) { 1746 err = ixgbe_check_link(hw, &sc->link_speed, 1747 &sc->link_up, FALSE); 1748 if (err) 1749 return; 1750 } 1751 1752 if (sc->advspeed != IXGBE_LINK_SPEED_UNKNOWN) 1753 autoneg = sc->advspeed; 1754 else 1755 autoneg = hw->phy.autoneg_advertised; 1756 if (!autoneg && hw->mac.ops.get_link_capabilities != NULL) { 1757 bool negotiate; 1758 1759 err = hw->mac.ops.get_link_capabilities(hw, 1760 &autoneg, &negotiate); 1761 if (err) 1762 return; 1763 } 1764 1765 if (hw->mac.ops.setup_link != NULL) { 1766 err = hw->mac.ops.setup_link(hw, 1767 autoneg, sc->link_up); 1768 if (err) 1769 return; 1770 } 1771 } 1772 } 1773 1774 static int 1775 ix_alloc_rings(struct ix_softc *sc) 1776 { 1777 int error, i; 1778 1779 /* 1780 * Create top level busdma tag 1781 */ 1782 error = bus_dma_tag_create(NULL, 1, 0, 1783 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 1784 BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 0, 1785 &sc->parent_tag); 1786 if (error) { 1787 device_printf(sc->dev, "could not create top level DMA tag\n"); 1788 return error; 1789 } 1790 1791 /* 1792 * Allocate TX descriptor rings and buffers 1793 */ 1794 sc->tx_rings = kmalloc_cachealign( 1795 sizeof(struct ix_tx_ring) * sc->tx_ring_cnt, 1796 M_DEVBUF, M_WAITOK | M_ZERO); 1797 for (i = 0; i < sc->tx_ring_cnt; ++i) { 1798 struct ix_tx_ring *txr = &sc->tx_rings[i]; 1799 1800 txr->tx_sc = sc; 1801 txr->tx_idx = i; 1802 txr->tx_intr_vec = -1; 1803 lwkt_serialize_init(&txr->tx_serialize); 1804 1805 error = ix_create_tx_ring(txr); 1806 if (error) 1807 return error; 1808 } 1809 1810 /* 1811 * Allocate RX descriptor rings and buffers 1812 */ 1813 sc->rx_rings = kmalloc_cachealign( 1814 sizeof(struct ix_rx_ring) * sc->rx_ring_cnt, 1815 M_DEVBUF, M_WAITOK | M_ZERO); 1816 for (i = 0; i < sc->rx_ring_cnt; ++i) { 1817 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 1818 1819 rxr->rx_sc = sc; 1820 rxr->rx_idx = i; 1821 rxr->rx_intr_vec = -1; 1822 lwkt_serialize_init(&rxr->rx_serialize); 1823 1824 error = ix_create_rx_ring(rxr); 1825 if (error) 1826 return error; 1827 } 1828 1829 return 0; 1830 } 1831 1832 static int 1833 ix_create_tx_ring(struct ix_tx_ring *txr) 1834 { 1835 int error, i, tsize, ntxd; 1836 1837 /* 1838 * Validate number of transmit descriptors. It must not exceed 1839 * hardware maximum, and must be multiple of IX_DBA_ALIGN. 1840 */ 1841 ntxd = device_getenv_int(txr->tx_sc->dev, "txd", ix_txd); 1842 if (((ntxd * sizeof(union ixgbe_adv_tx_desc)) % IX_DBA_ALIGN) != 0 || 1843 ntxd < IX_MIN_TXD || ntxd > IX_MAX_TXD) { 1844 device_printf(txr->tx_sc->dev, 1845 "Using %d TX descriptors instead of %d!\n", 1846 IX_DEF_TXD, ntxd); 1847 txr->tx_ndesc = IX_DEF_TXD; 1848 } else { 1849 txr->tx_ndesc = ntxd; 1850 } 1851 1852 /* 1853 * Allocate TX head write-back buffer 1854 */ 1855 txr->tx_hdr = bus_dmamem_coherent_any(txr->tx_sc->parent_tag, 1856 __VM_CACHELINE_SIZE, __VM_CACHELINE_SIZE, BUS_DMA_WAITOK, 1857 &txr->tx_hdr_dtag, &txr->tx_hdr_map, &txr->tx_hdr_paddr); 1858 if (txr->tx_hdr == NULL) { 1859 device_printf(txr->tx_sc->dev, 1860 "Unable to allocate TX head write-back buffer\n"); 1861 return ENOMEM; 1862 } 1863 1864 /* 1865 * Allocate TX descriptor ring 1866 */ 1867 tsize = roundup2(txr->tx_ndesc * sizeof(union ixgbe_adv_tx_desc), 1868 IX_DBA_ALIGN); 1869 txr->tx_base = bus_dmamem_coherent_any(txr->tx_sc->parent_tag, 1870 IX_DBA_ALIGN, tsize, BUS_DMA_WAITOK | BUS_DMA_ZERO, 1871 &txr->tx_base_dtag, &txr->tx_base_map, &txr->tx_base_paddr); 1872 if (txr->tx_base == NULL) { 1873 device_printf(txr->tx_sc->dev, 1874 "Unable to allocate TX Descriptor memory\n"); 1875 return ENOMEM; 1876 } 1877 1878 tsize = __VM_CACHELINE_ALIGN(sizeof(struct ix_tx_buf) * txr->tx_ndesc); 1879 txr->tx_buf = kmalloc_cachealign(tsize, M_DEVBUF, M_WAITOK | M_ZERO); 1880 1881 /* 1882 * Create DMA tag for TX buffers 1883 */ 1884 error = bus_dma_tag_create(txr->tx_sc->parent_tag, 1885 1, 0, /* alignment, bounds */ 1886 BUS_SPACE_MAXADDR, /* lowaddr */ 1887 BUS_SPACE_MAXADDR, /* highaddr */ 1888 NULL, NULL, /* filter, filterarg */ 1889 IX_TSO_SIZE, /* maxsize */ 1890 IX_MAX_SCATTER, /* nsegments */ 1891 PAGE_SIZE, /* maxsegsize */ 1892 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | 1893 BUS_DMA_ONEBPAGE, /* flags */ 1894 &txr->tx_tag); 1895 if (error) { 1896 device_printf(txr->tx_sc->dev, 1897 "Unable to allocate TX DMA tag\n"); 1898 kfree(txr->tx_buf, M_DEVBUF); 1899 txr->tx_buf = NULL; 1900 return error; 1901 } 1902 1903 /* 1904 * Create DMA maps for TX buffers 1905 */ 1906 for (i = 0; i < txr->tx_ndesc; ++i) { 1907 struct ix_tx_buf *txbuf = &txr->tx_buf[i]; 1908 1909 error = bus_dmamap_create(txr->tx_tag, 1910 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, &txbuf->map); 1911 if (error) { 1912 device_printf(txr->tx_sc->dev, 1913 "Unable to create TX DMA map\n"); 1914 ix_destroy_tx_ring(txr, i); 1915 return error; 1916 } 1917 } 1918 1919 /* 1920 * Initialize various watermark 1921 */ 1922 txr->tx_wreg_nsegs = IX_DEF_TXWREG_NSEGS; 1923 txr->tx_intr_nsegs = txr->tx_ndesc / 16; 1924 1925 return 0; 1926 } 1927 1928 static void 1929 ix_destroy_tx_ring(struct ix_tx_ring *txr, int ndesc) 1930 { 1931 int i; 1932 1933 if (txr->tx_hdr != NULL) { 1934 bus_dmamap_unload(txr->tx_hdr_dtag, txr->tx_hdr_map); 1935 bus_dmamem_free(txr->tx_hdr_dtag, 1936 __DEVOLATILE(void *, txr->tx_hdr), txr->tx_hdr_map); 1937 bus_dma_tag_destroy(txr->tx_hdr_dtag); 1938 txr->tx_hdr = NULL; 1939 } 1940 1941 if (txr->tx_base != NULL) { 1942 bus_dmamap_unload(txr->tx_base_dtag, txr->tx_base_map); 1943 bus_dmamem_free(txr->tx_base_dtag, txr->tx_base, 1944 txr->tx_base_map); 1945 bus_dma_tag_destroy(txr->tx_base_dtag); 1946 txr->tx_base = NULL; 1947 } 1948 1949 if (txr->tx_buf == NULL) 1950 return; 1951 1952 for (i = 0; i < ndesc; ++i) { 1953 struct ix_tx_buf *txbuf = &txr->tx_buf[i]; 1954 1955 KKASSERT(txbuf->m_head == NULL); 1956 bus_dmamap_destroy(txr->tx_tag, txbuf->map); 1957 } 1958 bus_dma_tag_destroy(txr->tx_tag); 1959 1960 kfree(txr->tx_buf, M_DEVBUF); 1961 txr->tx_buf = NULL; 1962 } 1963 1964 static void 1965 ix_init_tx_ring(struct ix_tx_ring *txr) 1966 { 1967 /* Clear the old ring contents */ 1968 bzero(txr->tx_base, sizeof(union ixgbe_adv_tx_desc) * txr->tx_ndesc); 1969 1970 /* Clear TX head write-back buffer */ 1971 *(txr->tx_hdr) = 0; 1972 1973 /* Reset indices */ 1974 txr->tx_next_avail = 0; 1975 txr->tx_next_clean = 0; 1976 txr->tx_nsegs = 0; 1977 1978 /* Set number of descriptors available */ 1979 txr->tx_avail = txr->tx_ndesc; 1980 1981 /* Enable this TX ring */ 1982 txr->tx_flags |= IX_TXFLAG_ENABLED; 1983 } 1984 1985 static void 1986 ix_init_tx_unit(struct ix_softc *sc) 1987 { 1988 struct ixgbe_hw *hw = &sc->hw; 1989 int i; 1990 1991 /* 1992 * Setup the Base and Length of the Tx Descriptor Ring 1993 */ 1994 for (i = 0; i < sc->tx_ring_inuse; ++i) { 1995 struct ix_tx_ring *txr = &sc->tx_rings[i]; 1996 uint64_t tdba = txr->tx_base_paddr; 1997 uint64_t hdr_paddr = txr->tx_hdr_paddr; 1998 uint32_t txctrl; 1999 2000 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), (uint32_t)tdba); 2001 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (uint32_t)(tdba >> 32)); 2002 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 2003 txr->tx_ndesc * sizeof(union ixgbe_adv_tx_desc)); 2004 2005 /* Setup the HW Tx Head and Tail descriptor pointers */ 2006 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 2007 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 2008 2009 /* Disable TX head write-back relax ordering */ 2010 switch (hw->mac.type) { 2011 case ixgbe_mac_82598EB: 2012 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 2013 break; 2014 case ixgbe_mac_82599EB: 2015 case ixgbe_mac_X540: 2016 default: 2017 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 2018 break; 2019 } 2020 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2021 switch (hw->mac.type) { 2022 case ixgbe_mac_82598EB: 2023 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 2024 break; 2025 case ixgbe_mac_82599EB: 2026 case ixgbe_mac_X540: 2027 default: 2028 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 2029 break; 2030 } 2031 2032 /* Enable TX head write-back */ 2033 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(i), 2034 (uint32_t)(hdr_paddr >> 32)); 2035 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(i), 2036 ((uint32_t)hdr_paddr) | IXGBE_TDWBAL_HEAD_WB_ENABLE); 2037 } 2038 2039 if (hw->mac.type != ixgbe_mac_82598EB) { 2040 uint32_t dmatxctl, rttdcs; 2041 2042 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2043 dmatxctl |= IXGBE_DMATXCTL_TE; 2044 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 2045 2046 /* Disable arbiter to set MTQC */ 2047 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2048 rttdcs |= IXGBE_RTTDCS_ARBDIS; 2049 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2050 2051 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2052 2053 /* Reenable aribter */ 2054 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 2055 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2056 } 2057 } 2058 2059 static int 2060 ix_tx_ctx_setup(struct ix_tx_ring *txr, const struct mbuf *mp, 2061 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2062 { 2063 struct ixgbe_adv_tx_context_desc *TXD; 2064 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2065 int ehdrlen, ip_hlen = 0, ctxd; 2066 boolean_t offload = TRUE; 2067 2068 /* First check if TSO is to be used */ 2069 if (mp->m_pkthdr.csum_flags & CSUM_TSO) { 2070 return ix_tso_ctx_setup(txr, mp, 2071 cmd_type_len, olinfo_status); 2072 } 2073 2074 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 2075 offload = FALSE; 2076 2077 /* Indicate the whole packet as payload when not doing TSO */ 2078 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; 2079 2080 /* 2081 * In advanced descriptors the vlan tag must be placed into the 2082 * context descriptor. Hence we need to make one even if not 2083 * doing checksum offloads. 2084 */ 2085 if (mp->m_flags & M_VLANTAG) { 2086 vlan_macip_lens |= htole16(mp->m_pkthdr.ether_vlantag) << 2087 IXGBE_ADVTXD_VLAN_SHIFT; 2088 } else if (!offload) { 2089 /* No TX descriptor is consumed */ 2090 return 0; 2091 } 2092 2093 /* Set the ether header length */ 2094 ehdrlen = mp->m_pkthdr.csum_lhlen; 2095 KASSERT(ehdrlen > 0, ("invalid ether hlen")); 2096 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2097 2098 if (mp->m_pkthdr.csum_flags & CSUM_IP) { 2099 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 2100 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2101 ip_hlen = mp->m_pkthdr.csum_iphlen; 2102 KASSERT(ip_hlen > 0, ("invalid ip hlen")); 2103 } 2104 vlan_macip_lens |= ip_hlen; 2105 2106 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2107 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 2108 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2109 else if (mp->m_pkthdr.csum_flags & CSUM_UDP) 2110 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2111 2112 if (mp->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)) 2113 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2114 2115 /* Now ready a context descriptor */ 2116 ctxd = txr->tx_next_avail; 2117 TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; 2118 2119 /* Now copy bits into descriptor */ 2120 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2121 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2122 TXD->seqnum_seed = htole32(0); 2123 TXD->mss_l4len_idx = htole32(0); 2124 2125 /* We've consumed the first desc, adjust counters */ 2126 if (++ctxd == txr->tx_ndesc) 2127 ctxd = 0; 2128 txr->tx_next_avail = ctxd; 2129 --txr->tx_avail; 2130 2131 /* One TX descriptor is consumed */ 2132 return 1; 2133 } 2134 2135 static int 2136 ix_tso_ctx_setup(struct ix_tx_ring *txr, const struct mbuf *mp, 2137 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2138 { 2139 struct ixgbe_adv_tx_context_desc *TXD; 2140 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2141 uint32_t mss_l4len_idx = 0, paylen; 2142 int ctxd, ehdrlen, ip_hlen, tcp_hlen; 2143 2144 ehdrlen = mp->m_pkthdr.csum_lhlen; 2145 KASSERT(ehdrlen > 0, ("invalid ether hlen")); 2146 2147 ip_hlen = mp->m_pkthdr.csum_iphlen; 2148 KASSERT(ip_hlen > 0, ("invalid ip hlen")); 2149 2150 tcp_hlen = mp->m_pkthdr.csum_thlen; 2151 KASSERT(tcp_hlen > 0, ("invalid tcp hlen")); 2152 2153 ctxd = txr->tx_next_avail; 2154 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2155 2156 if (mp->m_flags & M_VLANTAG) { 2157 vlan_macip_lens |= htole16(mp->m_pkthdr.ether_vlantag) << 2158 IXGBE_ADVTXD_VLAN_SHIFT; 2159 } 2160 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2161 vlan_macip_lens |= ip_hlen; 2162 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2163 2164 /* ADV DTYPE TUCMD */ 2165 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2166 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2167 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2168 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2169 2170 /* MSS L4LEN IDX */ 2171 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2172 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2173 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2174 2175 TXD->seqnum_seed = htole32(0); 2176 2177 if (++ctxd == txr->tx_ndesc) 2178 ctxd = 0; 2179 2180 txr->tx_avail--; 2181 txr->tx_next_avail = ctxd; 2182 2183 *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 2184 2185 /* This is used in the transmit desc in encap */ 2186 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; 2187 2188 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 2189 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2190 *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 2191 2192 /* One TX descriptor is consumed */ 2193 return 1; 2194 } 2195 2196 static void 2197 ix_txeof(struct ix_tx_ring *txr, int hdr) 2198 { 2199 int first, avail; 2200 2201 if (txr->tx_avail == txr->tx_ndesc) 2202 return; 2203 2204 first = txr->tx_next_clean; 2205 if (first == hdr) 2206 return; 2207 2208 avail = txr->tx_avail; 2209 while (first != hdr) { 2210 struct ix_tx_buf *txbuf = &txr->tx_buf[first]; 2211 2212 ++avail; 2213 if (txbuf->m_head) { 2214 bus_dmamap_unload(txr->tx_tag, txbuf->map); 2215 m_freem(txbuf->m_head); 2216 txbuf->m_head = NULL; 2217 } 2218 if (++first == txr->tx_ndesc) 2219 first = 0; 2220 } 2221 txr->tx_next_clean = first; 2222 txr->tx_avail = avail; 2223 2224 if (txr->tx_avail > IX_MAX_SCATTER + IX_TX_RESERVED) { 2225 ifsq_clr_oactive(txr->tx_ifsq); 2226 txr->tx_watchdog.wd_timer = 0; 2227 } 2228 } 2229 2230 static int 2231 ix_create_rx_ring(struct ix_rx_ring *rxr) 2232 { 2233 int i, rsize, error, nrxd; 2234 2235 /* 2236 * Validate number of receive descriptors. It must not exceed 2237 * hardware maximum, and must be multiple of IX_DBA_ALIGN. 2238 */ 2239 nrxd = device_getenv_int(rxr->rx_sc->dev, "rxd", ix_rxd); 2240 if (((nrxd * sizeof(union ixgbe_adv_rx_desc)) % IX_DBA_ALIGN) != 0 || 2241 nrxd < IX_MIN_RXD || nrxd > IX_MAX_RXD) { 2242 device_printf(rxr->rx_sc->dev, 2243 "Using %d RX descriptors instead of %d!\n", 2244 IX_DEF_RXD, nrxd); 2245 rxr->rx_ndesc = IX_DEF_RXD; 2246 } else { 2247 rxr->rx_ndesc = nrxd; 2248 } 2249 2250 /* 2251 * Allocate RX descriptor ring 2252 */ 2253 rsize = roundup2(rxr->rx_ndesc * sizeof(union ixgbe_adv_rx_desc), 2254 IX_DBA_ALIGN); 2255 rxr->rx_base = bus_dmamem_coherent_any(rxr->rx_sc->parent_tag, 2256 IX_DBA_ALIGN, rsize, BUS_DMA_WAITOK | BUS_DMA_ZERO, 2257 &rxr->rx_base_dtag, &rxr->rx_base_map, &rxr->rx_base_paddr); 2258 if (rxr->rx_base == NULL) { 2259 device_printf(rxr->rx_sc->dev, 2260 "Unable to allocate TX Descriptor memory\n"); 2261 return ENOMEM; 2262 } 2263 2264 rsize = __VM_CACHELINE_ALIGN(sizeof(struct ix_rx_buf) * rxr->rx_ndesc); 2265 rxr->rx_buf = kmalloc_cachealign(rsize, M_DEVBUF, M_WAITOK | M_ZERO); 2266 2267 /* 2268 * Create DMA tag for RX buffers 2269 */ 2270 error = bus_dma_tag_create(rxr->rx_sc->parent_tag, 2271 1, 0, /* alignment, bounds */ 2272 BUS_SPACE_MAXADDR, /* lowaddr */ 2273 BUS_SPACE_MAXADDR, /* highaddr */ 2274 NULL, NULL, /* filter, filterarg */ 2275 PAGE_SIZE, /* maxsize */ 2276 1, /* nsegments */ 2277 PAGE_SIZE, /* maxsegsize */ 2278 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, /* flags */ 2279 &rxr->rx_tag); 2280 if (error) { 2281 device_printf(rxr->rx_sc->dev, 2282 "Unable to create RX DMA tag\n"); 2283 kfree(rxr->rx_buf, M_DEVBUF); 2284 rxr->rx_buf = NULL; 2285 return error; 2286 } 2287 2288 /* 2289 * Create spare DMA map for RX buffers 2290 */ 2291 error = bus_dmamap_create(rxr->rx_tag, BUS_DMA_WAITOK, 2292 &rxr->rx_sparemap); 2293 if (error) { 2294 device_printf(rxr->rx_sc->dev, 2295 "Unable to create spare RX DMA map\n"); 2296 bus_dma_tag_destroy(rxr->rx_tag); 2297 kfree(rxr->rx_buf, M_DEVBUF); 2298 rxr->rx_buf = NULL; 2299 return error; 2300 } 2301 2302 /* 2303 * Create DMA maps for RX buffers 2304 */ 2305 for (i = 0; i < rxr->rx_ndesc; ++i) { 2306 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i]; 2307 2308 error = bus_dmamap_create(rxr->rx_tag, 2309 BUS_DMA_WAITOK, &rxbuf->map); 2310 if (error) { 2311 device_printf(rxr->rx_sc->dev, 2312 "Unable to create RX dma map\n"); 2313 ix_destroy_rx_ring(rxr, i); 2314 return error; 2315 } 2316 } 2317 2318 /* 2319 * Initialize various watermark 2320 */ 2321 rxr->rx_wreg_nsegs = IX_DEF_RXWREG_NSEGS; 2322 2323 return 0; 2324 } 2325 2326 static void 2327 ix_destroy_rx_ring(struct ix_rx_ring *rxr, int ndesc) 2328 { 2329 int i; 2330 2331 if (rxr->rx_base != NULL) { 2332 bus_dmamap_unload(rxr->rx_base_dtag, rxr->rx_base_map); 2333 bus_dmamem_free(rxr->rx_base_dtag, rxr->rx_base, 2334 rxr->rx_base_map); 2335 bus_dma_tag_destroy(rxr->rx_base_dtag); 2336 rxr->rx_base = NULL; 2337 } 2338 2339 if (rxr->rx_buf == NULL) 2340 return; 2341 2342 for (i = 0; i < ndesc; ++i) { 2343 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i]; 2344 2345 KKASSERT(rxbuf->m_head == NULL); 2346 bus_dmamap_destroy(rxr->rx_tag, rxbuf->map); 2347 } 2348 bus_dmamap_destroy(rxr->rx_tag, rxr->rx_sparemap); 2349 bus_dma_tag_destroy(rxr->rx_tag); 2350 2351 kfree(rxr->rx_buf, M_DEVBUF); 2352 rxr->rx_buf = NULL; 2353 } 2354 2355 /* 2356 ** Used to detect a descriptor that has 2357 ** been merged by Hardware RSC. 2358 */ 2359 static __inline uint32_t 2360 ix_rsc_count(union ixgbe_adv_rx_desc *rx) 2361 { 2362 return (le32toh(rx->wb.lower.lo_dword.data) & 2363 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; 2364 } 2365 2366 #if 0 2367 /********************************************************************* 2368 * 2369 * Initialize Hardware RSC (LRO) feature on 82599 2370 * for an RX ring, this is toggled by the LRO capability 2371 * even though it is transparent to the stack. 2372 * 2373 * NOTE: since this HW feature only works with IPV4 and 2374 * our testing has shown soft LRO to be as effective 2375 * I have decided to disable this by default. 2376 * 2377 **********************************************************************/ 2378 static void 2379 ix_setup_hw_rsc(struct ix_rx_ring *rxr) 2380 { 2381 struct ix_softc *sc = rxr->rx_sc; 2382 struct ixgbe_hw *hw = &sc->hw; 2383 uint32_t rscctrl, rdrxctl; 2384 2385 #if 0 2386 /* If turning LRO/RSC off we need to disable it */ 2387 if ((sc->arpcom.ac_if.if_capenable & IFCAP_LRO) == 0) { 2388 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 2389 rscctrl &= ~IXGBE_RSCCTL_RSCEN; 2390 return; 2391 } 2392 #endif 2393 2394 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2395 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 2396 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 2397 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 2398 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 2399 2400 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 2401 rscctrl |= IXGBE_RSCCTL_RSCEN; 2402 /* 2403 ** Limit the total number of descriptors that 2404 ** can be combined, so it does not exceed 64K 2405 */ 2406 if (rxr->mbuf_sz == MCLBYTES) 2407 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 2408 else if (rxr->mbuf_sz == MJUMPAGESIZE) 2409 rscctrl |= IXGBE_RSCCTL_MAXDESC_8; 2410 else if (rxr->mbuf_sz == MJUM9BYTES) 2411 rscctrl |= IXGBE_RSCCTL_MAXDESC_4; 2412 else /* Using 16K cluster */ 2413 rscctrl |= IXGBE_RSCCTL_MAXDESC_1; 2414 2415 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); 2416 2417 /* Enable TCP header recognition */ 2418 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 2419 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | 2420 IXGBE_PSRTYPE_TCPHDR)); 2421 2422 /* Disable RSC for ACK packets */ 2423 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 2424 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 2425 2426 rxr->hw_rsc = TRUE; 2427 } 2428 #endif 2429 2430 static int 2431 ix_init_rx_ring(struct ix_rx_ring *rxr) 2432 { 2433 int i; 2434 2435 /* Clear the ring contents */ 2436 bzero(rxr->rx_base, rxr->rx_ndesc * sizeof(union ixgbe_adv_rx_desc)); 2437 2438 /* XXX we need JUMPAGESIZE for RSC too */ 2439 if (rxr->rx_sc->max_frame_size <= MCLBYTES) 2440 rxr->rx_mbuf_sz = MCLBYTES; 2441 else 2442 rxr->rx_mbuf_sz = MJUMPAGESIZE; 2443 2444 /* Now replenish the mbufs */ 2445 for (i = 0; i < rxr->rx_ndesc; ++i) { 2446 int error; 2447 2448 error = ix_newbuf(rxr, i, TRUE); 2449 if (error) 2450 return error; 2451 } 2452 2453 /* Setup our descriptor indices */ 2454 rxr->rx_next_check = 0; 2455 rxr->rx_flags &= ~IX_RXRING_FLAG_DISC; 2456 2457 #if 0 2458 /* 2459 ** Now set up the LRO interface: 2460 */ 2461 if (ixgbe_rsc_enable) 2462 ix_setup_hw_rsc(rxr); 2463 #endif 2464 2465 return 0; 2466 } 2467 2468 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2469 2470 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 2471 2472 static void 2473 ix_init_rx_unit(struct ix_softc *sc) 2474 { 2475 struct ixgbe_hw *hw = &sc->hw; 2476 struct ifnet *ifp = &sc->arpcom.ac_if; 2477 uint32_t bufsz, fctrl, rxcsum, hlreg; 2478 int i; 2479 2480 /* 2481 * Make sure receives are disabled while setting up the descriptor ring 2482 */ 2483 ixgbe_disable_rx(hw); 2484 2485 /* Enable broadcasts */ 2486 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2487 fctrl |= IXGBE_FCTRL_BAM; 2488 if (hw->mac.type == ixgbe_mac_82598EB) { 2489 fctrl |= IXGBE_FCTRL_DPF; 2490 fctrl |= IXGBE_FCTRL_PMCF; 2491 } 2492 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 2493 2494 /* Set for Jumbo Frames? */ 2495 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2496 if (ifp->if_mtu > ETHERMTU) 2497 hlreg |= IXGBE_HLREG0_JUMBOEN; 2498 else 2499 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 2500 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 2501 2502 KKASSERT(sc->rx_rings[0].rx_mbuf_sz >= MCLBYTES); 2503 bufsz = (sc->rx_rings[0].rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 2504 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2505 2506 for (i = 0; i < sc->rx_ring_inuse; ++i) { 2507 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 2508 uint64_t rdba = rxr->rx_base_paddr; 2509 uint32_t srrctl; 2510 2511 /* Setup the Base and Length of the Rx Descriptor Ring */ 2512 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), (uint32_t)rdba); 2513 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (uint32_t)(rdba >> 32)); 2514 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 2515 rxr->rx_ndesc * sizeof(union ixgbe_adv_rx_desc)); 2516 2517 /* 2518 * Set up the SRRCTL register 2519 */ 2520 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 2521 2522 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 2523 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 2524 srrctl |= bufsz; 2525 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2526 if (sc->rx_ring_inuse > 1) { 2527 /* See the commend near ix_enable_rx_drop() */ 2528 if (sc->ifm_media & 2529 (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) { 2530 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 2531 if (i == 0 && bootverbose) { 2532 if_printf(ifp, "flow control %s, " 2533 "disable RX drop\n", 2534 ix_ifmedia2str(sc->ifm_media)); 2535 } 2536 } else { 2537 srrctl |= IXGBE_SRRCTL_DROP_EN; 2538 if (i == 0 && bootverbose) { 2539 if_printf(ifp, "flow control %s, " 2540 "enable RX drop\n", 2541 ix_ifmedia2str(sc->ifm_media)); 2542 } 2543 } 2544 } 2545 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 2546 2547 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2548 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 2549 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 2550 } 2551 2552 if (sc->hw.mac.type != ixgbe_mac_82598EB) 2553 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 0); 2554 2555 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2556 2557 /* 2558 * Setup RSS 2559 */ 2560 if (IX_ENABLE_HWRSS(sc)) { 2561 uint8_t key[IX_NRSSRK * IX_RSSRK_SIZE]; 2562 int j, r, nreta; 2563 2564 /* 2565 * NOTE: 2566 * When we reach here, RSS has already been disabled 2567 * in ix_stop(), so we could safely configure RSS key 2568 * and redirect table. 2569 */ 2570 2571 /* 2572 * Configure RSS key 2573 */ 2574 toeplitz_get_key(key, sizeof(key)); 2575 for (i = 0; i < IX_NRSSRK; ++i) { 2576 uint32_t rssrk; 2577 2578 rssrk = IX_RSSRK_VAL(key, i); 2579 IX_RSS_DPRINTF(sc, 1, "rssrk%d 0x%08x\n", 2580 i, rssrk); 2581 2582 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rssrk); 2583 } 2584 2585 /* Table size will differ based on MAC */ 2586 switch (hw->mac.type) { 2587 case ixgbe_mac_X550: 2588 case ixgbe_mac_X550EM_x: 2589 case ixgbe_mac_X550EM_a: 2590 nreta = IX_NRETA_X550; 2591 break; 2592 default: 2593 nreta = IX_NRETA; 2594 break; 2595 } 2596 2597 /* 2598 * Configure RSS redirect table in following fashion: 2599 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] 2600 */ 2601 r = 0; 2602 for (j = 0; j < nreta; ++j) { 2603 uint32_t reta = 0; 2604 2605 for (i = 0; i < IX_RETA_SIZE; ++i) { 2606 uint32_t q; 2607 2608 q = r % sc->rx_ring_inuse; 2609 reta |= q << (8 * i); 2610 ++r; 2611 } 2612 IX_RSS_DPRINTF(sc, 1, "reta 0x%08x\n", reta); 2613 if (j < IX_NRETA) { 2614 IXGBE_WRITE_REG(hw, IXGBE_RETA(j), reta); 2615 } else { 2616 IXGBE_WRITE_REG(hw, IXGBE_ERETA(j - IX_NRETA), 2617 reta); 2618 } 2619 } 2620 2621 /* 2622 * Enable multiple receive queues. 2623 * Enable IPv4 RSS standard hash functions. 2624 */ 2625 IXGBE_WRITE_REG(hw, IXGBE_MRQC, 2626 IXGBE_MRQC_RSSEN | 2627 IXGBE_MRQC_RSS_FIELD_IPV4 | 2628 IXGBE_MRQC_RSS_FIELD_IPV4_TCP); 2629 2630 /* 2631 * NOTE: 2632 * PCSD must be enabled to enable multiple 2633 * receive queues. 2634 */ 2635 rxcsum |= IXGBE_RXCSUM_PCSD; 2636 } 2637 2638 if (ifp->if_capenable & IFCAP_RXCSUM) 2639 rxcsum |= IXGBE_RXCSUM_PCSD; 2640 2641 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2642 } 2643 2644 static __inline void 2645 ix_rx_refresh(struct ix_rx_ring *rxr, int i) 2646 { 2647 if (--i < 0) 2648 i = rxr->rx_ndesc - 1; 2649 IXGBE_WRITE_REG(&rxr->rx_sc->hw, IXGBE_RDT(rxr->rx_idx), i); 2650 } 2651 2652 static __inline void 2653 ix_rxcsum(uint32_t staterr, struct mbuf *mp, uint32_t ptype) 2654 { 2655 if ((ptype & 2656 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_IPV4_EX)) == 0) { 2657 /* Not IPv4 */ 2658 return; 2659 } 2660 2661 if ((staterr & (IXGBE_RXD_STAT_IPCS | IXGBE_RXDADV_ERR_IPE)) == 2662 IXGBE_RXD_STAT_IPCS) 2663 mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID; 2664 2665 if ((ptype & 2666 (IXGBE_RXDADV_PKTTYPE_TCP | IXGBE_RXDADV_PKTTYPE_UDP)) == 0) { 2667 /* 2668 * - Neither TCP nor UDP 2669 * - IPv4 fragment 2670 */ 2671 return; 2672 } 2673 2674 if ((staterr & (IXGBE_RXD_STAT_L4CS | IXGBE_RXDADV_ERR_TCPE)) == 2675 IXGBE_RXD_STAT_L4CS) { 2676 mp->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR | 2677 CSUM_FRAG_NOT_CHECKED; 2678 mp->m_pkthdr.csum_data = htons(0xffff); 2679 } 2680 } 2681 2682 static __inline struct pktinfo * 2683 ix_rssinfo(struct mbuf *m, struct pktinfo *pi, 2684 uint32_t hash, uint32_t hashtype, uint32_t ptype) 2685 { 2686 switch (hashtype) { 2687 case IXGBE_RXDADV_RSSTYPE_IPV4_TCP: 2688 pi->pi_netisr = NETISR_IP; 2689 pi->pi_flags = 0; 2690 pi->pi_l3proto = IPPROTO_TCP; 2691 break; 2692 2693 case IXGBE_RXDADV_RSSTYPE_IPV4: 2694 if ((ptype & IXGBE_RXDADV_PKTTYPE_UDP) == 0) { 2695 /* Not UDP or is fragment */ 2696 return NULL; 2697 } 2698 pi->pi_netisr = NETISR_IP; 2699 pi->pi_flags = 0; 2700 pi->pi_l3proto = IPPROTO_UDP; 2701 break; 2702 2703 default: 2704 return NULL; 2705 } 2706 2707 m_sethash(m, toeplitz_hash(hash)); 2708 return pi; 2709 } 2710 2711 static __inline void 2712 ix_setup_rxdesc(union ixgbe_adv_rx_desc *rxd, const struct ix_rx_buf *rxbuf) 2713 { 2714 rxd->read.pkt_addr = htole64(rxbuf->paddr); 2715 rxd->wb.upper.status_error = 0; 2716 } 2717 2718 static void 2719 ix_rx_discard(struct ix_rx_ring *rxr, int i, boolean_t eop) 2720 { 2721 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i]; 2722 2723 /* 2724 * XXX discard may not be correct 2725 */ 2726 if (eop) { 2727 IFNET_STAT_INC(&rxr->rx_sc->arpcom.ac_if, ierrors, 1); 2728 rxr->rx_flags &= ~IX_RXRING_FLAG_DISC; 2729 } else { 2730 rxr->rx_flags |= IX_RXRING_FLAG_DISC; 2731 } 2732 if (rxbuf->fmp != NULL) { 2733 m_freem(rxbuf->fmp); 2734 rxbuf->fmp = NULL; 2735 rxbuf->lmp = NULL; 2736 } 2737 ix_setup_rxdesc(&rxr->rx_base[i], rxbuf); 2738 } 2739 2740 static void 2741 ix_rxeof(struct ix_rx_ring *rxr, int count) 2742 { 2743 struct ifnet *ifp = &rxr->rx_sc->arpcom.ac_if; 2744 int i, nsegs = 0, cpuid = mycpuid; 2745 2746 i = rxr->rx_next_check; 2747 while (count != 0) { 2748 struct ix_rx_buf *rxbuf, *nbuf = NULL; 2749 union ixgbe_adv_rx_desc *cur; 2750 struct mbuf *sendmp = NULL, *mp; 2751 struct pktinfo *pi = NULL, pi0; 2752 uint32_t rsc = 0, ptype, staterr, hash, hashtype; 2753 uint16_t len; 2754 boolean_t eop; 2755 2756 cur = &rxr->rx_base[i]; 2757 staterr = le32toh(cur->wb.upper.status_error); 2758 2759 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 2760 break; 2761 ++nsegs; 2762 2763 rxbuf = &rxr->rx_buf[i]; 2764 mp = rxbuf->m_head; 2765 2766 len = le16toh(cur->wb.upper.length); 2767 ptype = le32toh(cur->wb.lower.lo_dword.data) & 2768 IXGBE_RXDADV_PKTTYPE_MASK; 2769 hash = le32toh(cur->wb.lower.hi_dword.rss); 2770 hashtype = le32toh(cur->wb.lower.lo_dword.data) & 2771 IXGBE_RXDADV_RSSTYPE_MASK; 2772 2773 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 2774 if (eop) 2775 --count; 2776 2777 /* 2778 * Make sure bad packets are discarded 2779 */ 2780 if ((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) || 2781 (rxr->rx_flags & IX_RXRING_FLAG_DISC)) { 2782 ix_rx_discard(rxr, i, eop); 2783 goto next_desc; 2784 } 2785 2786 bus_dmamap_sync(rxr->rx_tag, rxbuf->map, BUS_DMASYNC_POSTREAD); 2787 if (ix_newbuf(rxr, i, FALSE) != 0) { 2788 ix_rx_discard(rxr, i, eop); 2789 goto next_desc; 2790 } 2791 2792 /* 2793 * On 82599 which supports a hardware LRO, packets 2794 * need not be fragmented across sequential descriptors, 2795 * rather the next descriptor is indicated in bits 2796 * of the descriptor. This also means that we might 2797 * proceses more than one packet at a time, something 2798 * that has never been true before, it required 2799 * eliminating global chain pointers in favor of what 2800 * we are doing here. 2801 */ 2802 if (!eop) { 2803 int nextp; 2804 2805 /* 2806 * Figure out the next descriptor 2807 * of this frame. 2808 */ 2809 if (rxr->rx_flags & IX_RXRING_FLAG_LRO) 2810 rsc = ix_rsc_count(cur); 2811 if (rsc) { /* Get hardware index */ 2812 nextp = ((staterr & 2813 IXGBE_RXDADV_NEXTP_MASK) >> 2814 IXGBE_RXDADV_NEXTP_SHIFT); 2815 } else { /* Just sequential */ 2816 nextp = i + 1; 2817 if (nextp == rxr->rx_ndesc) 2818 nextp = 0; 2819 } 2820 nbuf = &rxr->rx_buf[nextp]; 2821 prefetch(nbuf); 2822 } 2823 mp->m_len = len; 2824 2825 /* 2826 * Rather than using the fmp/lmp global pointers 2827 * we now keep the head of a packet chain in the 2828 * buffer struct and pass this along from one 2829 * descriptor to the next, until we get EOP. 2830 */ 2831 if (rxbuf->fmp == NULL) { 2832 mp->m_pkthdr.len = len; 2833 rxbuf->fmp = mp; 2834 rxbuf->lmp = mp; 2835 } else { 2836 rxbuf->fmp->m_pkthdr.len += len; 2837 rxbuf->lmp->m_next = mp; 2838 rxbuf->lmp = mp; 2839 } 2840 2841 if (nbuf != NULL) { 2842 /* 2843 * Not the last fragment of this frame, 2844 * pass this fragment list on 2845 */ 2846 nbuf->fmp = rxbuf->fmp; 2847 nbuf->lmp = rxbuf->lmp; 2848 } else { 2849 /* 2850 * Send this frame 2851 */ 2852 sendmp = rxbuf->fmp; 2853 2854 sendmp->m_pkthdr.rcvif = ifp; 2855 IFNET_STAT_INC(ifp, ipackets, 1); 2856 #ifdef IX_RSS_DEBUG 2857 rxr->rx_pkts++; 2858 #endif 2859 2860 /* Process vlan info */ 2861 if (staterr & IXGBE_RXD_STAT_VP) { 2862 sendmp->m_pkthdr.ether_vlantag = 2863 le16toh(cur->wb.upper.vlan); 2864 sendmp->m_flags |= M_VLANTAG; 2865 } 2866 if (ifp->if_capenable & IFCAP_RXCSUM) 2867 ix_rxcsum(staterr, sendmp, ptype); 2868 if (ifp->if_capenable & IFCAP_RSS) { 2869 pi = ix_rssinfo(sendmp, &pi0, 2870 hash, hashtype, ptype); 2871 } 2872 } 2873 rxbuf->fmp = NULL; 2874 rxbuf->lmp = NULL; 2875 next_desc: 2876 /* Advance our pointers to the next descriptor. */ 2877 if (++i == rxr->rx_ndesc) 2878 i = 0; 2879 2880 if (sendmp != NULL) 2881 ifp->if_input(ifp, sendmp, pi, cpuid); 2882 2883 if (nsegs >= rxr->rx_wreg_nsegs) { 2884 ix_rx_refresh(rxr, i); 2885 nsegs = 0; 2886 } 2887 } 2888 rxr->rx_next_check = i; 2889 2890 if (nsegs > 0) 2891 ix_rx_refresh(rxr, i); 2892 } 2893 2894 static void 2895 ix_set_vlan(struct ix_softc *sc) 2896 { 2897 struct ixgbe_hw *hw = &sc->hw; 2898 uint32_t ctrl; 2899 2900 if (hw->mac.type == ixgbe_mac_82598EB) { 2901 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2902 ctrl |= IXGBE_VLNCTRL_VME; 2903 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2904 } else { 2905 int i; 2906 2907 /* 2908 * On 82599 and later chips the VLAN enable is 2909 * per queue in RXDCTL 2910 */ 2911 for (i = 0; i < sc->rx_ring_inuse; ++i) { 2912 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 2913 ctrl |= IXGBE_RXDCTL_VME; 2914 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); 2915 } 2916 } 2917 } 2918 2919 static void 2920 ix_enable_intr(struct ix_softc *sc) 2921 { 2922 struct ixgbe_hw *hw = &sc->hw; 2923 uint32_t fwsm; 2924 int i; 2925 2926 for (i = 0; i < sc->intr_cnt; ++i) 2927 lwkt_serialize_handler_enable(sc->intr_data[i].intr_serialize); 2928 2929 sc->intr_mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 2930 2931 /* Enable Fan Failure detection */ 2932 if (hw->device_id == IXGBE_DEV_ID_82598AT) 2933 sc->intr_mask |= IXGBE_EIMS_GPI_SDP1; 2934 2935 switch (hw->mac.type) { 2936 case ixgbe_mac_82599EB: 2937 sc->intr_mask |= IXGBE_EIMS_ECC; 2938 /* Temperature sensor on some adapters */ 2939 sc->intr_mask |= IXGBE_EIMS_GPI_SDP0; 2940 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 2941 sc->intr_mask |= IXGBE_EIMS_GPI_SDP1; 2942 sc->intr_mask |= IXGBE_EIMS_GPI_SDP2; 2943 break; 2944 2945 case ixgbe_mac_X540: 2946 sc->intr_mask |= IXGBE_EIMS_ECC; 2947 /* Detect if Thermal Sensor is enabled */ 2948 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 2949 if (fwsm & IXGBE_FWSM_TS_ENABLED) 2950 sc->intr_mask |= IXGBE_EIMS_TS; 2951 break; 2952 2953 case ixgbe_mac_X550: 2954 case ixgbe_mac_X550EM_a: 2955 case ixgbe_mac_X550EM_x: 2956 sc->intr_mask |= IXGBE_EIMS_ECC; 2957 /* MAC thermal sensor is automatically enabled */ 2958 sc->intr_mask |= IXGBE_EIMS_TS; 2959 /* Some devices use SDP0 for important information */ 2960 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 2961 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 2962 sc->intr_mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw); 2963 /* FALL THROUGH */ 2964 default: 2965 break; 2966 } 2967 2968 /* With MSI-X we use auto clear for RX and TX rings */ 2969 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 2970 /* 2971 * There are no EIAC1/EIAC2 for newer chips; the related 2972 * bits for TX and RX rings > 16 are always auto clear. 2973 * 2974 * XXX which bits? There are _no_ documented EICR1 and 2975 * EICR2 at all; only EICR. 2976 */ 2977 IXGBE_WRITE_REG(hw, IXGBE_EIAC, IXGBE_EIMS_RTX_QUEUE); 2978 } else { 2979 sc->intr_mask |= IX_TX_INTR_MASK | IX_RX0_INTR_MASK; 2980 2981 KKASSERT(sc->rx_ring_inuse <= IX_MIN_RXRING_RSS); 2982 if (sc->rx_ring_inuse == IX_MIN_RXRING_RSS) 2983 sc->intr_mask |= IX_RX1_INTR_MASK; 2984 } 2985 2986 IXGBE_WRITE_REG(hw, IXGBE_EIMS, sc->intr_mask); 2987 2988 /* 2989 * Enable RX and TX rings for MSI-X 2990 */ 2991 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 2992 for (i = 0; i < sc->tx_ring_inuse; ++i) { 2993 const struct ix_tx_ring *txr = &sc->tx_rings[i]; 2994 2995 if (txr->tx_intr_vec >= 0) { 2996 IXGBE_WRITE_REG(hw, txr->tx_eims, 2997 txr->tx_eims_val); 2998 } 2999 } 3000 for (i = 0; i < sc->rx_ring_inuse; ++i) { 3001 const struct ix_rx_ring *rxr = &sc->rx_rings[i]; 3002 3003 KKASSERT(rxr->rx_intr_vec >= 0); 3004 IXGBE_WRITE_REG(hw, rxr->rx_eims, rxr->rx_eims_val); 3005 } 3006 } 3007 3008 IXGBE_WRITE_FLUSH(hw); 3009 } 3010 3011 static void 3012 ix_disable_intr(struct ix_softc *sc) 3013 { 3014 int i; 3015 3016 if (sc->intr_type == PCI_INTR_TYPE_MSIX) 3017 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 3018 3019 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3020 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 3021 } else { 3022 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000); 3023 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0); 3024 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0); 3025 } 3026 IXGBE_WRITE_FLUSH(&sc->hw); 3027 3028 for (i = 0; i < sc->intr_cnt; ++i) 3029 lwkt_serialize_handler_disable(sc->intr_data[i].intr_serialize); 3030 } 3031 3032 uint16_t 3033 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) 3034 { 3035 return pci_read_config(((struct ixgbe_osdep *)hw->back)->dev, 3036 reg, 2); 3037 } 3038 3039 void 3040 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value) 3041 { 3042 pci_write_config(((struct ixgbe_osdep *)hw->back)->dev, 3043 reg, value, 2); 3044 } 3045 3046 static void 3047 ix_slot_info(struct ix_softc *sc) 3048 { 3049 struct ixgbe_hw *hw = &sc->hw; 3050 device_t dev = sc->dev; 3051 struct ixgbe_mac_info *mac = &hw->mac; 3052 uint16_t link; 3053 uint32_t offset; 3054 3055 /* For most devices simply call the shared code routine */ 3056 if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) { 3057 ixgbe_get_bus_info(hw); 3058 /* These devices don't use PCI-E */ 3059 if (hw->mac.type == ixgbe_mac_X550EM_x || 3060 hw->mac.type == ixgbe_mac_X550EM_a) 3061 return; 3062 goto display; 3063 } 3064 3065 /* 3066 * For the Quad port adapter we need to parse back 3067 * up the PCI tree to find the speed of the expansion 3068 * slot into which this adapter is plugged. A bit more work. 3069 */ 3070 dev = device_get_parent(device_get_parent(dev)); 3071 #ifdef IXGBE_DEBUG 3072 device_printf(dev, "parent pcib = %x,%x,%x\n", 3073 pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev)); 3074 #endif 3075 dev = device_get_parent(device_get_parent(dev)); 3076 #ifdef IXGBE_DEBUG 3077 device_printf(dev, "slot pcib = %x,%x,%x\n", 3078 pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev)); 3079 #endif 3080 /* Now get the PCI Express Capabilities offset */ 3081 offset = pci_get_pciecap_ptr(dev); 3082 /* ...and read the Link Status Register */ 3083 link = pci_read_config(dev, offset + PCIER_LINKSTAT, 2); 3084 switch (link & IXGBE_PCI_LINK_WIDTH) { 3085 case IXGBE_PCI_LINK_WIDTH_1: 3086 hw->bus.width = ixgbe_bus_width_pcie_x1; 3087 break; 3088 case IXGBE_PCI_LINK_WIDTH_2: 3089 hw->bus.width = ixgbe_bus_width_pcie_x2; 3090 break; 3091 case IXGBE_PCI_LINK_WIDTH_4: 3092 hw->bus.width = ixgbe_bus_width_pcie_x4; 3093 break; 3094 case IXGBE_PCI_LINK_WIDTH_8: 3095 hw->bus.width = ixgbe_bus_width_pcie_x8; 3096 break; 3097 default: 3098 hw->bus.width = ixgbe_bus_width_unknown; 3099 break; 3100 } 3101 3102 switch (link & IXGBE_PCI_LINK_SPEED) { 3103 case IXGBE_PCI_LINK_SPEED_2500: 3104 hw->bus.speed = ixgbe_bus_speed_2500; 3105 break; 3106 case IXGBE_PCI_LINK_SPEED_5000: 3107 hw->bus.speed = ixgbe_bus_speed_5000; 3108 break; 3109 case IXGBE_PCI_LINK_SPEED_8000: 3110 hw->bus.speed = ixgbe_bus_speed_8000; 3111 break; 3112 default: 3113 hw->bus.speed = ixgbe_bus_speed_unknown; 3114 break; 3115 } 3116 3117 mac->ops.set_lan_id(hw); 3118 3119 display: 3120 device_printf(dev, "PCI Express Bus: Speed %s %s\n", 3121 hw->bus.speed == ixgbe_bus_speed_8000 ? "8.0GT/s" : 3122 hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0GT/s" : 3123 hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5GT/s" : "Unknown", 3124 hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" : 3125 hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" : 3126 hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" : "Unknown"); 3127 3128 if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP && 3129 hw->bus.width <= ixgbe_bus_width_pcie_x4 && 3130 hw->bus.speed == ixgbe_bus_speed_2500) { 3131 device_printf(dev, "For optimal performance a x8 " 3132 "PCIE, or x4 PCIE Gen2 slot is required.\n"); 3133 } else if (hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP && 3134 hw->bus.width <= ixgbe_bus_width_pcie_x8 && 3135 hw->bus.speed < ixgbe_bus_speed_8000) { 3136 device_printf(dev, "For optimal performance a x8 " 3137 "PCIE Gen3 slot is required.\n"); 3138 } 3139 } 3140 3141 /* 3142 * TODO comment is incorrect 3143 * 3144 * Setup the correct IVAR register for a particular MSIX interrupt 3145 * - entry is the register array entry 3146 * - vector is the MSIX vector for this queue 3147 * - type is RX/TX/MISC 3148 */ 3149 static void 3150 ix_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, 3151 int8_t type) 3152 { 3153 struct ixgbe_hw *hw = &sc->hw; 3154 uint32_t ivar, index; 3155 3156 vector |= IXGBE_IVAR_ALLOC_VAL; 3157 3158 switch (hw->mac.type) { 3159 case ixgbe_mac_82598EB: 3160 if (type == -1) 3161 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3162 else 3163 entry += (type * 64); 3164 index = (entry >> 2) & 0x1F; 3165 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3166 ivar &= ~(0xFF << (8 * (entry & 0x3))); 3167 ivar |= (vector << (8 * (entry & 0x3))); 3168 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 3169 break; 3170 3171 case ixgbe_mac_82599EB: 3172 case ixgbe_mac_X540: 3173 case ixgbe_mac_X550: 3174 case ixgbe_mac_X550EM_a: 3175 case ixgbe_mac_X550EM_x: 3176 if (type == -1) { /* MISC IVAR */ 3177 index = (entry & 1) * 8; 3178 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3179 ivar &= ~(0xFF << index); 3180 ivar |= (vector << index); 3181 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3182 } else { /* RX/TX IVARS */ 3183 index = (16 * (entry & 1)) + (8 * type); 3184 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 3185 ivar &= ~(0xFF << index); 3186 ivar |= (vector << index); 3187 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 3188 } 3189 /* FALL THROUGH */ 3190 default: 3191 break; 3192 } 3193 } 3194 3195 static boolean_t 3196 ix_sfp_probe(struct ix_softc *sc) 3197 { 3198 struct ixgbe_hw *hw = &sc->hw; 3199 3200 if (hw->phy.type == ixgbe_phy_nl && 3201 hw->phy.sfp_type == ixgbe_sfp_type_not_present) { 3202 int32_t ret; 3203 3204 ret = hw->phy.ops.identify_sfp(hw); 3205 if (ret) 3206 return FALSE; 3207 3208 ret = hw->phy.ops.reset(hw); 3209 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3210 if_printf(&sc->arpcom.ac_if, 3211 "Unsupported SFP+ module detected! " 3212 "Reload driver with supported module.\n"); 3213 sc->sfp_probe = FALSE; 3214 return FALSE; 3215 } 3216 if_printf(&sc->arpcom.ac_if, "SFP+ module detected!\n"); 3217 3218 /* We now have supported optics */ 3219 sc->sfp_probe = FALSE; 3220 3221 return TRUE; 3222 } 3223 return FALSE; 3224 } 3225 3226 static void 3227 ix_handle_link(struct ix_softc *sc) 3228 { 3229 ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0); 3230 ix_update_link_status(sc); 3231 } 3232 3233 /* 3234 * Handling SFP module 3235 */ 3236 static void 3237 ix_handle_mod(struct ix_softc *sc) 3238 { 3239 struct ixgbe_hw *hw = &sc->hw; 3240 uint32_t err; 3241 3242 err = hw->phy.ops.identify_sfp(hw); 3243 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3244 if_printf(&sc->arpcom.ac_if, 3245 "Unsupported SFP+ module type was detected.\n"); 3246 return; 3247 } 3248 err = hw->mac.ops.setup_sfp(hw); 3249 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3250 if_printf(&sc->arpcom.ac_if, 3251 "Setup failure - unsupported SFP+ module type.\n"); 3252 return; 3253 } 3254 ix_handle_msf(sc); 3255 } 3256 3257 /* 3258 * Handling MSF (multispeed fiber) 3259 */ 3260 static void 3261 ix_handle_msf(struct ix_softc *sc) 3262 { 3263 struct ixgbe_hw *hw = &sc->hw; 3264 uint32_t autoneg; 3265 3266 hw->phy.ops.identify_sfp(hw); 3267 ix_init_media(sc); 3268 3269 if (sc->advspeed != IXGBE_LINK_SPEED_UNKNOWN) 3270 autoneg = sc->advspeed; 3271 else 3272 autoneg = hw->phy.autoneg_advertised; 3273 if (!autoneg && hw->mac.ops.get_link_capabilities != NULL) { 3274 bool negotiate; 3275 3276 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 3277 } 3278 if (hw->mac.ops.setup_link != NULL) 3279 hw->mac.ops.setup_link(hw, autoneg, TRUE); 3280 } 3281 3282 static void 3283 ix_handle_phy(struct ix_softc *sc) 3284 { 3285 struct ixgbe_hw *hw = &sc->hw; 3286 int error; 3287 3288 error = hw->phy.ops.handle_lasi(hw); 3289 if (error == IXGBE_ERR_OVERTEMP) { 3290 if_printf(&sc->arpcom.ac_if, 3291 "CRITICAL: EXTERNAL PHY OVER TEMP!! " 3292 "PHY will downshift to lower power state!\n"); 3293 } else if (error) { 3294 if_printf(&sc->arpcom.ac_if, 3295 "Error handling LASI interrupt: %d\n", error); 3296 } 3297 } 3298 3299 static void 3300 ix_update_stats(struct ix_softc *sc) 3301 { 3302 struct ifnet *ifp = &sc->arpcom.ac_if; 3303 struct ixgbe_hw *hw = &sc->hw; 3304 uint32_t missed_rx = 0, bprc, lxon, lxoff, total; 3305 uint64_t total_missed_rx = 0; 3306 int i; 3307 3308 sc->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 3309 sc->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 3310 sc->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC); 3311 sc->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC); 3312 3313 for (i = 0; i < 16; i++) { 3314 sc->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 3315 sc->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 3316 sc->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 3317 } 3318 sc->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC); 3319 sc->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC); 3320 sc->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 3321 3322 /* Hardware workaround, gprc counts missed packets */ 3323 sc->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 3324 sc->stats.gprc -= missed_rx; 3325 3326 if (hw->mac.type != ixgbe_mac_82598EB) { 3327 sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) + 3328 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 3329 sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) + 3330 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32); 3331 sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) + 3332 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 3333 sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3334 sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3335 } else { 3336 sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3337 sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3338 /* 82598 only has a counter in the high register */ 3339 sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3340 sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3341 sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3342 } 3343 3344 /* 3345 * Workaround: mprc hardware is incorrectly counting 3346 * broadcasts, so for now we subtract those. 3347 */ 3348 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3349 sc->stats.bprc += bprc; 3350 sc->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3351 if (hw->mac.type == ixgbe_mac_82598EB) 3352 sc->stats.mprc -= bprc; 3353 3354 sc->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3355 sc->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3356 sc->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3357 sc->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3358 sc->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3359 sc->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3360 3361 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3362 sc->stats.lxontxc += lxon; 3363 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3364 sc->stats.lxofftxc += lxoff; 3365 total = lxon + lxoff; 3366 3367 sc->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 3368 sc->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3369 sc->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3370 sc->stats.gptc -= total; 3371 sc->stats.mptc -= total; 3372 sc->stats.ptc64 -= total; 3373 sc->stats.gotc -= total * ETHER_MIN_LEN; 3374 3375 sc->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3376 sc->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3377 sc->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3378 sc->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3379 sc->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 3380 sc->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 3381 sc->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 3382 sc->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3383 sc->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT); 3384 sc->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3385 sc->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3386 sc->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3387 sc->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3388 sc->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3389 sc->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3390 sc->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC); 3391 sc->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC); 3392 sc->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST); 3393 /* Only read FCOE on 82599 */ 3394 if (hw->mac.type != ixgbe_mac_82598EB) { 3395 sc->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 3396 sc->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 3397 sc->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 3398 sc->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 3399 sc->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 3400 } 3401 3402 /* Rx Errors */ 3403 IFNET_STAT_SET(ifp, iqdrops, total_missed_rx); 3404 IFNET_STAT_SET(ifp, ierrors, sc->stats.crcerrs + sc->stats.rlec); 3405 } 3406 3407 #if 0 3408 /* 3409 * Add sysctl variables, one per statistic, to the system. 3410 */ 3411 static void 3412 ix_add_hw_stats(struct ix_softc *sc) 3413 { 3414 3415 device_t dev = sc->dev; 3416 3417 struct ix_tx_ring *txr = sc->tx_rings; 3418 struct ix_rx_ring *rxr = sc->rx_rings; 3419 3420 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 3421 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 3422 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 3423 struct ixgbe_hw_stats *stats = &sc->stats; 3424 3425 struct sysctl_oid *stat_node, *queue_node; 3426 struct sysctl_oid_list *stat_list, *queue_list; 3427 3428 #define QUEUE_NAME_LEN 32 3429 char namebuf[QUEUE_NAME_LEN]; 3430 3431 /* MAC stats get the own sub node */ 3432 3433 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 3434 CTLFLAG_RD, NULL, "MAC Statistics"); 3435 stat_list = SYSCTL_CHILDREN(stat_node); 3436 3437 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs", 3438 CTLFLAG_RD, &stats->crcerrs, 3439 "CRC Errors"); 3440 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs", 3441 CTLFLAG_RD, &stats->illerrc, 3442 "Illegal Byte Errors"); 3443 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs", 3444 CTLFLAG_RD, &stats->errbc, 3445 "Byte Errors"); 3446 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards", 3447 CTLFLAG_RD, &stats->mspdc, 3448 "MAC Short Packets Discarded"); 3449 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults", 3450 CTLFLAG_RD, &stats->mlfc, 3451 "MAC Local Faults"); 3452 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults", 3453 CTLFLAG_RD, &stats->mrfc, 3454 "MAC Remote Faults"); 3455 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs", 3456 CTLFLAG_RD, &stats->rlec, 3457 "Receive Length Errors"); 3458 3459 /* Flow Control stats */ 3460 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd", 3461 CTLFLAG_RD, &stats->lxontxc, 3462 "Link XON Transmitted"); 3463 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd", 3464 CTLFLAG_RD, &stats->lxonrxc, 3465 "Link XON Received"); 3466 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd", 3467 CTLFLAG_RD, &stats->lxofftxc, 3468 "Link XOFF Transmitted"); 3469 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", 3470 CTLFLAG_RD, &stats->lxoffrxc, 3471 "Link XOFF Received"); 3472 3473 /* Packet Reception Stats */ 3474 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd", 3475 CTLFLAG_RD, &stats->tor, 3476 "Total Octets Received"); 3477 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd", 3478 CTLFLAG_RD, &stats->gorc, 3479 "Good Octets Received"); 3480 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd", 3481 CTLFLAG_RD, &stats->tpr, 3482 "Total Packets Received"); 3483 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd", 3484 CTLFLAG_RD, &stats->gprc, 3485 "Good Packets Received"); 3486 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd", 3487 CTLFLAG_RD, &stats->mprc, 3488 "Multicast Packets Received"); 3489 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd", 3490 CTLFLAG_RD, &stats->bprc, 3491 "Broadcast Packets Received"); 3492 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 3493 CTLFLAG_RD, &stats->prc64, 3494 "64 byte frames received "); 3495 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 3496 CTLFLAG_RD, &stats->prc127, 3497 "65-127 byte frames received"); 3498 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 3499 CTLFLAG_RD, &stats->prc255, 3500 "128-255 byte frames received"); 3501 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 3502 CTLFLAG_RD, &stats->prc511, 3503 "256-511 byte frames received"); 3504 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 3505 CTLFLAG_RD, &stats->prc1023, 3506 "512-1023 byte frames received"); 3507 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 3508 CTLFLAG_RD, &stats->prc1522, 3509 "1023-1522 byte frames received"); 3510 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized", 3511 CTLFLAG_RD, &stats->ruc, 3512 "Receive Undersized"); 3513 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 3514 CTLFLAG_RD, &stats->rfc, 3515 "Fragmented Packets Received "); 3516 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized", 3517 CTLFLAG_RD, &stats->roc, 3518 "Oversized Packets Received"); 3519 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd", 3520 CTLFLAG_RD, &stats->rjc, 3521 "Received Jabber"); 3522 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd", 3523 CTLFLAG_RD, &stats->mngprc, 3524 "Management Packets Received"); 3525 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd", 3526 CTLFLAG_RD, &stats->mngptc, 3527 "Management Packets Dropped"); 3528 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs", 3529 CTLFLAG_RD, &stats->xec, 3530 "Checksum Errors"); 3531 3532 /* Packet Transmission Stats */ 3533 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 3534 CTLFLAG_RD, &stats->gotc, 3535 "Good Octets Transmitted"); 3536 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 3537 CTLFLAG_RD, &stats->tpt, 3538 "Total Packets Transmitted"); 3539 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 3540 CTLFLAG_RD, &stats->gptc, 3541 "Good Packets Transmitted"); 3542 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 3543 CTLFLAG_RD, &stats->bptc, 3544 "Broadcast Packets Transmitted"); 3545 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 3546 CTLFLAG_RD, &stats->mptc, 3547 "Multicast Packets Transmitted"); 3548 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd", 3549 CTLFLAG_RD, &stats->mngptc, 3550 "Management Packets Transmitted"); 3551 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 3552 CTLFLAG_RD, &stats->ptc64, 3553 "64 byte frames transmitted "); 3554 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 3555 CTLFLAG_RD, &stats->ptc127, 3556 "65-127 byte frames transmitted"); 3557 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 3558 CTLFLAG_RD, &stats->ptc255, 3559 "128-255 byte frames transmitted"); 3560 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 3561 CTLFLAG_RD, &stats->ptc511, 3562 "256-511 byte frames transmitted"); 3563 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 3564 CTLFLAG_RD, &stats->ptc1023, 3565 "512-1023 byte frames transmitted"); 3566 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 3567 CTLFLAG_RD, &stats->ptc1522, 3568 "1024-1522 byte frames transmitted"); 3569 } 3570 #endif 3571 3572 /* 3573 * Enable the hardware to drop packets when the buffer is full. 3574 * This is useful when multiple RX rings are used, so that no 3575 * single RX ring being full stalls the entire RX engine. We 3576 * only enable this when multiple RX rings are used and when 3577 * flow control is disabled. 3578 */ 3579 static void 3580 ix_enable_rx_drop(struct ix_softc *sc) 3581 { 3582 struct ixgbe_hw *hw = &sc->hw; 3583 int i; 3584 3585 if (bootverbose) { 3586 if_printf(&sc->arpcom.ac_if, 3587 "flow control %s, enable RX drop\n", 3588 ix_fc2str(sc->hw.fc.current_mode)); 3589 } 3590 3591 for (i = 0; i < sc->rx_ring_inuse; ++i) { 3592 uint32_t srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 3593 3594 srrctl |= IXGBE_SRRCTL_DROP_EN; 3595 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 3596 } 3597 } 3598 3599 static void 3600 ix_disable_rx_drop(struct ix_softc *sc) 3601 { 3602 struct ixgbe_hw *hw = &sc->hw; 3603 int i; 3604 3605 if (bootverbose) { 3606 if_printf(&sc->arpcom.ac_if, 3607 "flow control %s, disable RX drop\n", 3608 ix_fc2str(sc->hw.fc.current_mode)); 3609 } 3610 3611 for (i = 0; i < sc->rx_ring_inuse; ++i) { 3612 uint32_t srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 3613 3614 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 3615 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 3616 } 3617 } 3618 3619 static void 3620 ix_setup_serialize(struct ix_softc *sc) 3621 { 3622 int i = 0, j; 3623 3624 /* Main + RX + TX */ 3625 sc->nserialize = 1 + sc->rx_ring_cnt + sc->tx_ring_cnt; 3626 sc->serializes = 3627 kmalloc(sc->nserialize * sizeof(struct lwkt_serialize *), 3628 M_DEVBUF, M_WAITOK | M_ZERO); 3629 3630 /* 3631 * Setup serializes 3632 * 3633 * NOTE: Order is critical 3634 */ 3635 3636 KKASSERT(i < sc->nserialize); 3637 sc->serializes[i++] = &sc->main_serialize; 3638 3639 for (j = 0; j < sc->rx_ring_cnt; ++j) { 3640 KKASSERT(i < sc->nserialize); 3641 sc->serializes[i++] = &sc->rx_rings[j].rx_serialize; 3642 } 3643 3644 for (j = 0; j < sc->tx_ring_cnt; ++j) { 3645 KKASSERT(i < sc->nserialize); 3646 sc->serializes[i++] = &sc->tx_rings[j].tx_serialize; 3647 } 3648 3649 KKASSERT(i == sc->nserialize); 3650 } 3651 3652 static int 3653 ix_alloc_intr(struct ix_softc *sc) 3654 { 3655 struct ix_intr_data *intr; 3656 u_int intr_flags; 3657 3658 ix_alloc_msix(sc); 3659 if (sc->intr_type == PCI_INTR_TYPE_MSIX) { 3660 ix_set_ring_inuse(sc, FALSE); 3661 return 0; 3662 } 3663 3664 if (sc->intr_data != NULL) 3665 kfree(sc->intr_data, M_DEVBUF); 3666 3667 sc->intr_cnt = 1; 3668 sc->intr_data = kmalloc(sizeof(struct ix_intr_data), M_DEVBUF, 3669 M_WAITOK | M_ZERO); 3670 intr = &sc->intr_data[0]; 3671 3672 /* 3673 * Allocate MSI/legacy interrupt resource 3674 */ 3675 sc->intr_type = pci_alloc_1intr(sc->dev, ix_msi_enable, 3676 &intr->intr_rid, &intr_flags); 3677 3678 intr->intr_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 3679 &intr->intr_rid, intr_flags); 3680 if (intr->intr_res == NULL) { 3681 device_printf(sc->dev, "Unable to allocate bus resource: " 3682 "interrupt\n"); 3683 return ENXIO; 3684 } 3685 3686 intr->intr_serialize = &sc->main_serialize; 3687 intr->intr_cpuid = rman_get_cpuid(intr->intr_res); 3688 intr->intr_func = ix_intr; 3689 intr->intr_funcarg = sc; 3690 intr->intr_rate = IX_INTR_RATE; 3691 intr->intr_use = IX_INTR_USE_RXTX; 3692 3693 sc->tx_rings[0].tx_intr_cpuid = intr->intr_cpuid; 3694 sc->tx_rings[0].tx_intr_vec = IX_TX_INTR_VEC; 3695 3696 sc->rx_rings[0].rx_intr_vec = IX_RX0_INTR_VEC; 3697 3698 ix_set_ring_inuse(sc, FALSE); 3699 3700 KKASSERT(sc->rx_ring_inuse <= IX_MIN_RXRING_RSS); 3701 if (sc->rx_ring_inuse == IX_MIN_RXRING_RSS) 3702 sc->rx_rings[1].rx_intr_vec = IX_RX1_INTR_VEC; 3703 3704 return 0; 3705 } 3706 3707 static void 3708 ix_free_intr(struct ix_softc *sc) 3709 { 3710 if (sc->intr_data == NULL) 3711 return; 3712 3713 if (sc->intr_type != PCI_INTR_TYPE_MSIX) { 3714 struct ix_intr_data *intr = &sc->intr_data[0]; 3715 3716 KKASSERT(sc->intr_cnt == 1); 3717 if (intr->intr_res != NULL) { 3718 bus_release_resource(sc->dev, SYS_RES_IRQ, 3719 intr->intr_rid, intr->intr_res); 3720 } 3721 if (sc->intr_type == PCI_INTR_TYPE_MSI) 3722 pci_release_msi(sc->dev); 3723 3724 kfree(sc->intr_data, M_DEVBUF); 3725 } else { 3726 ix_free_msix(sc, TRUE); 3727 } 3728 } 3729 3730 static void 3731 ix_set_ring_inuse(struct ix_softc *sc, boolean_t polling) 3732 { 3733 sc->rx_ring_inuse = ix_get_rxring_inuse(sc, polling); 3734 sc->tx_ring_inuse = ix_get_txring_inuse(sc, polling); 3735 if (bootverbose) { 3736 if_printf(&sc->arpcom.ac_if, 3737 "RX rings %d/%d, TX rings %d/%d\n", 3738 sc->rx_ring_inuse, sc->rx_ring_cnt, 3739 sc->tx_ring_inuse, sc->tx_ring_cnt); 3740 } 3741 } 3742 3743 static int 3744 ix_get_rxring_inuse(const struct ix_softc *sc, boolean_t polling) 3745 { 3746 if (!IX_ENABLE_HWRSS(sc)) 3747 return 1; 3748 3749 if (polling) 3750 return sc->rx_ring_cnt; 3751 else if (sc->intr_type != PCI_INTR_TYPE_MSIX) 3752 return IX_MIN_RXRING_RSS; 3753 else 3754 return sc->rx_ring_msix; 3755 } 3756 3757 static int 3758 ix_get_txring_inuse(const struct ix_softc *sc, boolean_t polling) 3759 { 3760 if (!IX_ENABLE_HWTSS(sc)) 3761 return 1; 3762 3763 if (polling) 3764 return sc->tx_ring_cnt; 3765 else if (sc->intr_type != PCI_INTR_TYPE_MSIX) 3766 return 1; 3767 else 3768 return sc->tx_ring_msix; 3769 } 3770 3771 static int 3772 ix_setup_intr(struct ix_softc *sc) 3773 { 3774 int i; 3775 3776 for (i = 0; i < sc->intr_cnt; ++i) { 3777 struct ix_intr_data *intr = &sc->intr_data[i]; 3778 int error; 3779 3780 error = bus_setup_intr_descr(sc->dev, intr->intr_res, 3781 INTR_MPSAFE, intr->intr_func, intr->intr_funcarg, 3782 &intr->intr_hand, intr->intr_serialize, intr->intr_desc); 3783 if (error) { 3784 device_printf(sc->dev, "can't setup %dth intr\n", i); 3785 ix_teardown_intr(sc, i); 3786 return error; 3787 } 3788 } 3789 return 0; 3790 } 3791 3792 static void 3793 ix_teardown_intr(struct ix_softc *sc, int intr_cnt) 3794 { 3795 int i; 3796 3797 if (sc->intr_data == NULL) 3798 return; 3799 3800 for (i = 0; i < intr_cnt; ++i) { 3801 struct ix_intr_data *intr = &sc->intr_data[i]; 3802 3803 bus_teardown_intr(sc->dev, intr->intr_res, intr->intr_hand); 3804 } 3805 } 3806 3807 static void 3808 ix_serialize(struct ifnet *ifp, enum ifnet_serialize slz) 3809 { 3810 struct ix_softc *sc = ifp->if_softc; 3811 3812 ifnet_serialize_array_enter(sc->serializes, sc->nserialize, slz); 3813 } 3814 3815 static void 3816 ix_deserialize(struct ifnet *ifp, enum ifnet_serialize slz) 3817 { 3818 struct ix_softc *sc = ifp->if_softc; 3819 3820 ifnet_serialize_array_exit(sc->serializes, sc->nserialize, slz); 3821 } 3822 3823 static int 3824 ix_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz) 3825 { 3826 struct ix_softc *sc = ifp->if_softc; 3827 3828 return ifnet_serialize_array_try(sc->serializes, sc->nserialize, slz); 3829 } 3830 3831 #ifdef INVARIANTS 3832 3833 static void 3834 ix_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz, 3835 boolean_t serialized) 3836 { 3837 struct ix_softc *sc = ifp->if_softc; 3838 3839 ifnet_serialize_array_assert(sc->serializes, sc->nserialize, slz, 3840 serialized); 3841 } 3842 3843 #endif /* INVARIANTS */ 3844 3845 static void 3846 ix_free_rings(struct ix_softc *sc) 3847 { 3848 int i; 3849 3850 if (sc->tx_rings != NULL) { 3851 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3852 struct ix_tx_ring *txr = &sc->tx_rings[i]; 3853 3854 ix_destroy_tx_ring(txr, txr->tx_ndesc); 3855 } 3856 kfree(sc->tx_rings, M_DEVBUF); 3857 } 3858 3859 if (sc->rx_rings != NULL) { 3860 for (i =0; i < sc->rx_ring_cnt; ++i) { 3861 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 3862 3863 ix_destroy_rx_ring(rxr, rxr->rx_ndesc); 3864 } 3865 kfree(sc->rx_rings, M_DEVBUF); 3866 } 3867 3868 if (sc->parent_tag != NULL) 3869 bus_dma_tag_destroy(sc->parent_tag); 3870 } 3871 3872 static void 3873 ix_watchdog(struct ifaltq_subque *ifsq) 3874 { 3875 struct ix_tx_ring *txr = ifsq_get_priv(ifsq); 3876 struct ifnet *ifp = ifsq_get_ifp(ifsq); 3877 struct ix_softc *sc = ifp->if_softc; 3878 int i; 3879 3880 KKASSERT(txr->tx_ifsq == ifsq); 3881 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3882 3883 /* 3884 * If the interface has been paused then don't do the watchdog check 3885 */ 3886 if (IXGBE_READ_REG(&sc->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) { 3887 txr->tx_watchdog.wd_timer = 5; 3888 return; 3889 } 3890 3891 if_printf(ifp, "Watchdog timeout -- resetting\n"); 3892 if_printf(ifp, "Queue(%d) tdh = %d, hw tdt = %d\n", txr->tx_idx, 3893 IXGBE_READ_REG(&sc->hw, IXGBE_TDH(txr->tx_idx)), 3894 IXGBE_READ_REG(&sc->hw, IXGBE_TDT(txr->tx_idx))); 3895 if_printf(ifp, "TX(%d) desc avail = %d, next TX to Clean = %d\n", 3896 txr->tx_idx, txr->tx_avail, txr->tx_next_clean); 3897 3898 ix_init(sc); 3899 for (i = 0; i < sc->tx_ring_inuse; ++i) 3900 ifsq_devstart_sched(sc->tx_rings[i].tx_ifsq); 3901 } 3902 3903 static void 3904 ix_free_tx_ring(struct ix_tx_ring *txr) 3905 { 3906 int i; 3907 3908 for (i = 0; i < txr->tx_ndesc; ++i) { 3909 struct ix_tx_buf *txbuf = &txr->tx_buf[i]; 3910 3911 if (txbuf->m_head != NULL) { 3912 bus_dmamap_unload(txr->tx_tag, txbuf->map); 3913 m_freem(txbuf->m_head); 3914 txbuf->m_head = NULL; 3915 } 3916 } 3917 } 3918 3919 static void 3920 ix_free_rx_ring(struct ix_rx_ring *rxr) 3921 { 3922 int i; 3923 3924 for (i = 0; i < rxr->rx_ndesc; ++i) { 3925 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i]; 3926 3927 if (rxbuf->fmp != NULL) { 3928 m_freem(rxbuf->fmp); 3929 rxbuf->fmp = NULL; 3930 rxbuf->lmp = NULL; 3931 } else { 3932 KKASSERT(rxbuf->lmp == NULL); 3933 } 3934 if (rxbuf->m_head != NULL) { 3935 bus_dmamap_unload(rxr->rx_tag, rxbuf->map); 3936 m_freem(rxbuf->m_head); 3937 rxbuf->m_head = NULL; 3938 } 3939 } 3940 } 3941 3942 static int 3943 ix_newbuf(struct ix_rx_ring *rxr, int i, boolean_t wait) 3944 { 3945 struct mbuf *m; 3946 bus_dma_segment_t seg; 3947 bus_dmamap_t map; 3948 struct ix_rx_buf *rxbuf; 3949 int flags, error, nseg; 3950 3951 flags = M_NOWAIT; 3952 if (__predict_false(wait)) 3953 flags = M_WAITOK; 3954 3955 m = m_getjcl(flags, MT_DATA, M_PKTHDR, rxr->rx_mbuf_sz); 3956 if (m == NULL) { 3957 if (wait) { 3958 if_printf(&rxr->rx_sc->arpcom.ac_if, 3959 "Unable to allocate RX mbuf\n"); 3960 } 3961 return ENOBUFS; 3962 } 3963 m->m_len = m->m_pkthdr.len = rxr->rx_mbuf_sz; 3964 3965 error = bus_dmamap_load_mbuf_segment(rxr->rx_tag, 3966 rxr->rx_sparemap, m, &seg, 1, &nseg, BUS_DMA_NOWAIT); 3967 if (error) { 3968 m_freem(m); 3969 if (wait) { 3970 if_printf(&rxr->rx_sc->arpcom.ac_if, 3971 "Unable to load RX mbuf\n"); 3972 } 3973 return error; 3974 } 3975 3976 rxbuf = &rxr->rx_buf[i]; 3977 if (rxbuf->m_head != NULL) 3978 bus_dmamap_unload(rxr->rx_tag, rxbuf->map); 3979 3980 map = rxbuf->map; 3981 rxbuf->map = rxr->rx_sparemap; 3982 rxr->rx_sparemap = map; 3983 3984 rxbuf->m_head = m; 3985 rxbuf->paddr = seg.ds_addr; 3986 3987 ix_setup_rxdesc(&rxr->rx_base[i], rxbuf); 3988 return 0; 3989 } 3990 3991 static void 3992 ix_add_sysctl(struct ix_softc *sc) 3993 { 3994 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 3995 struct sysctl_oid *tree = device_get_sysctl_tree(sc->dev); 3996 #ifdef IX_RSS_DEBUG 3997 char node[32]; 3998 int i; 3999 #endif 4000 4001 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4002 OID_AUTO, "rxr", CTLFLAG_RD, &sc->rx_ring_cnt, 0, "# of RX rings"); 4003 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4004 OID_AUTO, "rxr_inuse", CTLFLAG_RD, &sc->rx_ring_inuse, 0, 4005 "# of RX rings used"); 4006 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4007 OID_AUTO, "txr", CTLFLAG_RD, &sc->tx_ring_cnt, 0, "# of TX rings"); 4008 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4009 OID_AUTO, "txr_inuse", CTLFLAG_RD, &sc->tx_ring_inuse, 0, 4010 "# of TX rings used"); 4011 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4012 OID_AUTO, "rxd", CTLTYPE_INT | CTLFLAG_RD, 4013 sc, 0, ix_sysctl_rxd, "I", 4014 "# of RX descs"); 4015 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4016 OID_AUTO, "txd", CTLTYPE_INT | CTLFLAG_RD, 4017 sc, 0, ix_sysctl_txd, "I", 4018 "# of TX descs"); 4019 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4020 OID_AUTO, "tx_wreg_nsegs", CTLTYPE_INT | CTLFLAG_RW, 4021 sc, 0, ix_sysctl_tx_wreg_nsegs, "I", 4022 "# of segments sent before write to hardware register"); 4023 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4024 OID_AUTO, "rx_wreg_nsegs", CTLTYPE_INT | CTLFLAG_RW, 4025 sc, 0, ix_sysctl_rx_wreg_nsegs, "I", 4026 "# of received segments sent before write to hardware register"); 4027 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4028 OID_AUTO, "tx_intr_nsegs", CTLTYPE_INT | CTLFLAG_RW, 4029 sc, 0, ix_sysctl_tx_intr_nsegs, "I", 4030 "# of segments per TX interrupt"); 4031 4032 #ifdef IFPOLL_ENABLE 4033 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4034 OID_AUTO, "npoll_rxoff", CTLTYPE_INT|CTLFLAG_RW, 4035 sc, 0, ix_sysctl_npoll_rxoff, "I", "NPOLLING RX cpu offset"); 4036 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4037 OID_AUTO, "npoll_txoff", CTLTYPE_INT|CTLFLAG_RW, 4038 sc, 0, ix_sysctl_npoll_txoff, "I", "NPOLLING TX cpu offset"); 4039 #endif 4040 4041 #define IX_ADD_INTR_RATE_SYSCTL(sc, use, name) \ 4042 do { \ 4043 ix_add_intr_rate_sysctl(sc, IX_INTR_USE_##use, #name, \ 4044 ix_sysctl_##name, #use " interrupt rate"); \ 4045 } while (0) 4046 4047 IX_ADD_INTR_RATE_SYSCTL(sc, RXTX, rxtx_intr_rate); 4048 IX_ADD_INTR_RATE_SYSCTL(sc, RX, rx_intr_rate); 4049 IX_ADD_INTR_RATE_SYSCTL(sc, TX, tx_intr_rate); 4050 IX_ADD_INTR_RATE_SYSCTL(sc, STATUS, sts_intr_rate); 4051 4052 #undef IX_ADD_INTR_RATE_SYSCTL 4053 4054 #ifdef IX_RSS_DEBUG 4055 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4056 OID_AUTO, "rss_debug", CTLFLAG_RW, &sc->rss_debug, 0, 4057 "RSS debug level"); 4058 for (i = 0; i < sc->rx_ring_cnt; ++i) { 4059 ksnprintf(node, sizeof(node), "rx%d_pkt", i); 4060 SYSCTL_ADD_ULONG(ctx, 4061 SYSCTL_CHILDREN(tree), OID_AUTO, node, 4062 CTLFLAG_RW, &sc->rx_rings[i].rx_pkts, "RXed packets"); 4063 } 4064 #endif 4065 4066 #if 0 4067 ix_add_hw_stats(sc); 4068 #endif 4069 4070 } 4071 4072 static int 4073 ix_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS) 4074 { 4075 struct ix_softc *sc = (void *)arg1; 4076 struct ifnet *ifp = &sc->arpcom.ac_if; 4077 int error, nsegs, i; 4078 4079 nsegs = sc->tx_rings[0].tx_wreg_nsegs; 4080 error = sysctl_handle_int(oidp, &nsegs, 0, req); 4081 if (error || req->newptr == NULL) 4082 return error; 4083 if (nsegs < 0) 4084 return EINVAL; 4085 4086 ifnet_serialize_all(ifp); 4087 for (i = 0; i < sc->tx_ring_cnt; ++i) 4088 sc->tx_rings[i].tx_wreg_nsegs = nsegs; 4089 ifnet_deserialize_all(ifp); 4090 4091 return 0; 4092 } 4093 4094 static int 4095 ix_sysctl_rx_wreg_nsegs(SYSCTL_HANDLER_ARGS) 4096 { 4097 struct ix_softc *sc = (void *)arg1; 4098 struct ifnet *ifp = &sc->arpcom.ac_if; 4099 int error, nsegs, i; 4100 4101 nsegs = sc->rx_rings[0].rx_wreg_nsegs; 4102 error = sysctl_handle_int(oidp, &nsegs, 0, req); 4103 if (error || req->newptr == NULL) 4104 return error; 4105 if (nsegs < 0) 4106 return EINVAL; 4107 4108 ifnet_serialize_all(ifp); 4109 for (i = 0; i < sc->rx_ring_cnt; ++i) 4110 sc->rx_rings[i].rx_wreg_nsegs =nsegs; 4111 ifnet_deserialize_all(ifp); 4112 4113 return 0; 4114 } 4115 4116 static int 4117 ix_sysctl_txd(SYSCTL_HANDLER_ARGS) 4118 { 4119 struct ix_softc *sc = (void *)arg1; 4120 int txd; 4121 4122 txd = sc->tx_rings[0].tx_ndesc; 4123 return sysctl_handle_int(oidp, &txd, 0, req); 4124 } 4125 4126 static int 4127 ix_sysctl_rxd(SYSCTL_HANDLER_ARGS) 4128 { 4129 struct ix_softc *sc = (void *)arg1; 4130 int rxd; 4131 4132 rxd = sc->rx_rings[0].rx_ndesc; 4133 return sysctl_handle_int(oidp, &rxd, 0, req); 4134 } 4135 4136 static int 4137 ix_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS) 4138 { 4139 struct ix_softc *sc = (void *)arg1; 4140 struct ifnet *ifp = &sc->arpcom.ac_if; 4141 struct ix_tx_ring *txr = &sc->tx_rings[0]; 4142 int error, nsegs; 4143 4144 nsegs = txr->tx_intr_nsegs; 4145 error = sysctl_handle_int(oidp, &nsegs, 0, req); 4146 if (error || req->newptr == NULL) 4147 return error; 4148 if (nsegs < 0) 4149 return EINVAL; 4150 4151 ifnet_serialize_all(ifp); 4152 4153 if (nsegs >= txr->tx_ndesc - IX_MAX_SCATTER - IX_TX_RESERVED) { 4154 error = EINVAL; 4155 } else { 4156 int i; 4157 4158 error = 0; 4159 for (i = 0; i < sc->tx_ring_cnt; ++i) 4160 sc->tx_rings[i].tx_intr_nsegs = nsegs; 4161 } 4162 4163 ifnet_deserialize_all(ifp); 4164 4165 return error; 4166 } 4167 4168 static void 4169 ix_set_eitr(struct ix_softc *sc, int idx, int rate) 4170 { 4171 uint32_t eitr, eitr_intvl; 4172 4173 eitr = IXGBE_READ_REG(&sc->hw, IXGBE_EITR(idx)); 4174 eitr_intvl = 1000000000 / 256 / rate; 4175 4176 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 4177 eitr &= ~IX_EITR_INTVL_MASK_82598; 4178 if (eitr_intvl == 0) 4179 eitr_intvl = 1; 4180 else if (eitr_intvl > IX_EITR_INTVL_MASK_82598) 4181 eitr_intvl = IX_EITR_INTVL_MASK_82598; 4182 } else { 4183 eitr &= ~IX_EITR_INTVL_MASK; 4184 4185 eitr_intvl &= ~IX_EITR_INTVL_RSVD_MASK; 4186 if (eitr_intvl == 0) 4187 eitr_intvl = IX_EITR_INTVL_MIN; 4188 else if (eitr_intvl > IX_EITR_INTVL_MAX) 4189 eitr_intvl = IX_EITR_INTVL_MAX; 4190 } 4191 eitr |= eitr_intvl; 4192 4193 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(idx), eitr); 4194 } 4195 4196 static int 4197 ix_sysctl_rxtx_intr_rate(SYSCTL_HANDLER_ARGS) 4198 { 4199 return ix_sysctl_intr_rate(oidp, arg1, arg2, req, IX_INTR_USE_RXTX); 4200 } 4201 4202 static int 4203 ix_sysctl_rx_intr_rate(SYSCTL_HANDLER_ARGS) 4204 { 4205 return ix_sysctl_intr_rate(oidp, arg1, arg2, req, IX_INTR_USE_RX); 4206 } 4207 4208 static int 4209 ix_sysctl_tx_intr_rate(SYSCTL_HANDLER_ARGS) 4210 { 4211 return ix_sysctl_intr_rate(oidp, arg1, arg2, req, IX_INTR_USE_TX); 4212 } 4213 4214 static int 4215 ix_sysctl_sts_intr_rate(SYSCTL_HANDLER_ARGS) 4216 { 4217 return ix_sysctl_intr_rate(oidp, arg1, arg2, req, IX_INTR_USE_STATUS); 4218 } 4219 4220 static int 4221 ix_sysctl_intr_rate(SYSCTL_HANDLER_ARGS, int use) 4222 { 4223 struct ix_softc *sc = (void *)arg1; 4224 struct ifnet *ifp = &sc->arpcom.ac_if; 4225 int error, rate, i; 4226 4227 rate = 0; 4228 for (i = 0; i < sc->intr_cnt; ++i) { 4229 if (sc->intr_data[i].intr_use == use) { 4230 rate = sc->intr_data[i].intr_rate; 4231 break; 4232 } 4233 } 4234 4235 error = sysctl_handle_int(oidp, &rate, 0, req); 4236 if (error || req->newptr == NULL) 4237 return error; 4238 if (rate <= 0) 4239 return EINVAL; 4240 4241 ifnet_serialize_all(ifp); 4242 4243 for (i = 0; i < sc->intr_cnt; ++i) { 4244 if (sc->intr_data[i].intr_use == use) { 4245 sc->intr_data[i].intr_rate = rate; 4246 if (ifp->if_flags & IFF_RUNNING) 4247 ix_set_eitr(sc, i, rate); 4248 } 4249 } 4250 4251 ifnet_deserialize_all(ifp); 4252 4253 return error; 4254 } 4255 4256 static void 4257 ix_add_intr_rate_sysctl(struct ix_softc *sc, int use, 4258 const char *name, int (*handler)(SYSCTL_HANDLER_ARGS), const char *desc) 4259 { 4260 int i; 4261 4262 for (i = 0; i < sc->intr_cnt; ++i) { 4263 if (sc->intr_data[i].intr_use == use) { 4264 SYSCTL_ADD_PROC(device_get_sysctl_ctx(sc->dev), 4265 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), 4266 OID_AUTO, name, CTLTYPE_INT | CTLFLAG_RW, 4267 sc, 0, handler, "I", desc); 4268 break; 4269 } 4270 } 4271 } 4272 4273 static void 4274 ix_set_timer_cpuid(struct ix_softc *sc, boolean_t polling) 4275 { 4276 if (polling || sc->intr_type == PCI_INTR_TYPE_MSIX) 4277 sc->timer_cpuid = 0; /* XXX fixed */ 4278 else 4279 sc->timer_cpuid = rman_get_cpuid(sc->intr_data[0].intr_res); 4280 } 4281 4282 static void 4283 ix_alloc_msix(struct ix_softc *sc) 4284 { 4285 int msix_enable, msix_cnt, msix_cnt2, alloc_cnt; 4286 struct ix_intr_data *intr; 4287 int i, x, error; 4288 int offset, offset_def, agg_rxtx, ring_max; 4289 boolean_t aggregate, setup = FALSE; 4290 4291 msix_enable = ix_msix_enable; 4292 /* 4293 * Don't enable MSI-X on 82598 by default, see: 4294 * 82598 specification update errata #38 4295 */ 4296 if (sc->hw.mac.type == ixgbe_mac_82598EB) 4297 msix_enable = 0; 4298 msix_enable = device_getenv_int(sc->dev, "msix.enable", msix_enable); 4299 if (!msix_enable) 4300 return; 4301 4302 msix_cnt = pci_msix_count(sc->dev); 4303 #ifdef IX_MSIX_DEBUG 4304 msix_cnt = device_getenv_int(sc->dev, "msix.count", msix_cnt); 4305 #endif 4306 if (msix_cnt <= 1) { 4307 /* One MSI-X model does not make sense */ 4308 return; 4309 } 4310 4311 i = 0; 4312 while ((1 << (i + 1)) <= msix_cnt) 4313 ++i; 4314 msix_cnt2 = 1 << i; 4315 4316 if (bootverbose) { 4317 device_printf(sc->dev, "MSI-X count %d/%d\n", 4318 msix_cnt2, msix_cnt); 4319 } 4320 4321 KKASSERT(msix_cnt >= msix_cnt2); 4322 if (msix_cnt == msix_cnt2) { 4323 /* We need at least one MSI-X for link status */ 4324 msix_cnt2 >>= 1; 4325 if (msix_cnt2 <= 1) { 4326 /* One MSI-X for RX/TX does not make sense */ 4327 device_printf(sc->dev, "not enough MSI-X for TX/RX, " 4328 "MSI-X count %d/%d\n", msix_cnt2, msix_cnt); 4329 return; 4330 } 4331 KKASSERT(msix_cnt > msix_cnt2); 4332 4333 if (bootverbose) { 4334 device_printf(sc->dev, "MSI-X count eq fixup %d/%d\n", 4335 msix_cnt2, msix_cnt); 4336 } 4337 } 4338 4339 /* 4340 * Make sure that we don't break interrupt related registers 4341 * (EIMS, etc) limitation. 4342 * 4343 * NOTE: msix_cnt > msix_cnt2, when we reach here 4344 */ 4345 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 4346 if (msix_cnt2 > IX_MAX_MSIX_82598) 4347 msix_cnt2 = IX_MAX_MSIX_82598; 4348 } else { 4349 if (msix_cnt2 > IX_MAX_MSIX) 4350 msix_cnt2 = IX_MAX_MSIX; 4351 } 4352 msix_cnt = msix_cnt2 + 1; /* +1 for status */ 4353 4354 if (bootverbose) { 4355 device_printf(sc->dev, "MSI-X count max fixup %d/%d\n", 4356 msix_cnt2, msix_cnt); 4357 } 4358 4359 sc->rx_ring_msix = sc->rx_ring_cnt; 4360 if (sc->rx_ring_msix > msix_cnt2) 4361 sc->rx_ring_msix = msix_cnt2; 4362 4363 sc->tx_ring_msix = sc->tx_ring_cnt; 4364 if (sc->tx_ring_msix > msix_cnt2) 4365 sc->tx_ring_msix = msix_cnt2; 4366 4367 ring_max = sc->rx_ring_msix; 4368 if (ring_max < sc->tx_ring_msix) 4369 ring_max = sc->tx_ring_msix; 4370 4371 /* Allow user to force independent RX/TX MSI-X handling */ 4372 agg_rxtx = device_getenv_int(sc->dev, "msix.agg_rxtx", 4373 ix_msix_agg_rxtx); 4374 4375 if (!agg_rxtx && msix_cnt >= sc->tx_ring_msix + sc->rx_ring_msix + 1) { 4376 /* 4377 * Independent TX/RX MSI-X 4378 */ 4379 aggregate = FALSE; 4380 if (bootverbose) 4381 device_printf(sc->dev, "independent TX/RX MSI-X\n"); 4382 alloc_cnt = sc->tx_ring_msix + sc->rx_ring_msix; 4383 } else { 4384 /* 4385 * Aggregate TX/RX MSI-X 4386 */ 4387 aggregate = TRUE; 4388 if (bootverbose) 4389 device_printf(sc->dev, "aggregate TX/RX MSI-X\n"); 4390 alloc_cnt = msix_cnt2; 4391 if (alloc_cnt > ring_max) 4392 alloc_cnt = ring_max; 4393 KKASSERT(alloc_cnt >= sc->rx_ring_msix && 4394 alloc_cnt >= sc->tx_ring_msix); 4395 } 4396 ++alloc_cnt; /* For status */ 4397 4398 if (bootverbose) { 4399 device_printf(sc->dev, "MSI-X alloc %d, " 4400 "RX ring %d, TX ring %d\n", alloc_cnt, 4401 sc->rx_ring_msix, sc->tx_ring_msix); 4402 } 4403 4404 sc->msix_mem_rid = PCIR_BAR(IX_MSIX_BAR_82598); 4405 sc->msix_mem_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 4406 &sc->msix_mem_rid, RF_ACTIVE); 4407 if (sc->msix_mem_res == NULL) { 4408 sc->msix_mem_rid = PCIR_BAR(IX_MSIX_BAR_82599); 4409 sc->msix_mem_res = bus_alloc_resource_any(sc->dev, 4410 SYS_RES_MEMORY, &sc->msix_mem_rid, RF_ACTIVE); 4411 if (sc->msix_mem_res == NULL) { 4412 device_printf(sc->dev, "Unable to map MSI-X table\n"); 4413 return; 4414 } 4415 } 4416 4417 sc->intr_cnt = alloc_cnt; 4418 sc->intr_data = kmalloc(sizeof(struct ix_intr_data) * sc->intr_cnt, 4419 M_DEVBUF, M_WAITOK | M_ZERO); 4420 for (x = 0; x < sc->intr_cnt; ++x) { 4421 intr = &sc->intr_data[x]; 4422 intr->intr_rid = -1; 4423 intr->intr_rate = IX_INTR_RATE; 4424 } 4425 4426 x = 0; 4427 if (!aggregate) { 4428 /* 4429 * RX rings 4430 */ 4431 if (sc->rx_ring_msix == ncpus2) { 4432 offset = 0; 4433 } else { 4434 offset_def = (sc->rx_ring_msix * 4435 device_get_unit(sc->dev)) % ncpus2; 4436 4437 offset = device_getenv_int(sc->dev, 4438 "msix.rxoff", offset_def); 4439 if (offset >= ncpus2 || 4440 offset % sc->rx_ring_msix != 0) { 4441 device_printf(sc->dev, 4442 "invalid msix.rxoff %d, use %d\n", 4443 offset, offset_def); 4444 offset = offset_def; 4445 } 4446 } 4447 ix_conf_rx_msix(sc, 0, &x, offset); 4448 4449 /* 4450 * TX rings 4451 */ 4452 if (sc->tx_ring_msix == ncpus2) { 4453 offset = 0; 4454 } else { 4455 offset_def = (sc->tx_ring_msix * 4456 device_get_unit(sc->dev)) % ncpus2; 4457 4458 offset = device_getenv_int(sc->dev, 4459 "msix.txoff", offset_def); 4460 if (offset >= ncpus2 || 4461 offset % sc->tx_ring_msix != 0) { 4462 device_printf(sc->dev, 4463 "invalid msix.txoff %d, use %d\n", 4464 offset, offset_def); 4465 offset = offset_def; 4466 } 4467 } 4468 ix_conf_tx_msix(sc, 0, &x, offset); 4469 } else { 4470 int ring_agg; 4471 4472 ring_agg = sc->rx_ring_msix; 4473 if (ring_agg > sc->tx_ring_msix) 4474 ring_agg = sc->tx_ring_msix; 4475 4476 if (ring_max == ncpus2) { 4477 offset = 0; 4478 } else { 4479 offset_def = (ring_max * device_get_unit(sc->dev)) % 4480 ncpus2; 4481 4482 offset = device_getenv_int(sc->dev, "msix.off", 4483 offset_def); 4484 if (offset >= ncpus2 || offset % ring_max != 0) { 4485 device_printf(sc->dev, 4486 "invalid msix.off %d, use %d\n", 4487 offset, offset_def); 4488 offset = offset_def; 4489 } 4490 } 4491 4492 for (i = 0; i < ring_agg; ++i) { 4493 struct ix_tx_ring *txr = &sc->tx_rings[i]; 4494 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 4495 4496 KKASSERT(x < sc->intr_cnt); 4497 rxr->rx_intr_vec = x; 4498 ix_setup_msix_eims(sc, x, 4499 &rxr->rx_eims, &rxr->rx_eims_val); 4500 rxr->rx_txr = txr; 4501 /* NOTE: Leave TX ring's intr_vec negative */ 4502 4503 intr = &sc->intr_data[x++]; 4504 4505 intr->intr_serialize = &rxr->rx_serialize; 4506 intr->intr_func = ix_msix_rxtx; 4507 intr->intr_funcarg = rxr; 4508 intr->intr_use = IX_INTR_USE_RXTX; 4509 4510 intr->intr_cpuid = i + offset; 4511 KKASSERT(intr->intr_cpuid < ncpus2); 4512 txr->tx_intr_cpuid = intr->intr_cpuid; 4513 4514 ksnprintf(intr->intr_desc0, sizeof(intr->intr_desc0), 4515 "%s rxtx%d", device_get_nameunit(sc->dev), i); 4516 intr->intr_desc = intr->intr_desc0; 4517 } 4518 4519 if (ring_agg != ring_max) { 4520 if (ring_max == sc->tx_ring_msix) 4521 ix_conf_tx_msix(sc, i, &x, offset); 4522 else 4523 ix_conf_rx_msix(sc, i, &x, offset); 4524 } 4525 } 4526 4527 /* 4528 * Status MSI-X 4529 */ 4530 KKASSERT(x < sc->intr_cnt); 4531 sc->sts_msix_vec = x; 4532 4533 intr = &sc->intr_data[x++]; 4534 4535 intr->intr_serialize = &sc->main_serialize; 4536 intr->intr_func = ix_msix_status; 4537 intr->intr_funcarg = sc; 4538 intr->intr_cpuid = 0; 4539 intr->intr_use = IX_INTR_USE_STATUS; 4540 4541 ksnprintf(intr->intr_desc0, sizeof(intr->intr_desc0), "%s sts", 4542 device_get_nameunit(sc->dev)); 4543 intr->intr_desc = intr->intr_desc0; 4544 4545 KKASSERT(x == sc->intr_cnt); 4546 4547 error = pci_setup_msix(sc->dev); 4548 if (error) { 4549 device_printf(sc->dev, "Setup MSI-X failed\n"); 4550 goto back; 4551 } 4552 setup = TRUE; 4553 4554 for (i = 0; i < sc->intr_cnt; ++i) { 4555 intr = &sc->intr_data[i]; 4556 4557 error = pci_alloc_msix_vector(sc->dev, i, &intr->intr_rid, 4558 intr->intr_cpuid); 4559 if (error) { 4560 device_printf(sc->dev, 4561 "Unable to allocate MSI-X %d on cpu%d\n", i, 4562 intr->intr_cpuid); 4563 goto back; 4564 } 4565 4566 intr->intr_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 4567 &intr->intr_rid, RF_ACTIVE); 4568 if (intr->intr_res == NULL) { 4569 device_printf(sc->dev, 4570 "Unable to allocate MSI-X %d resource\n", i); 4571 error = ENOMEM; 4572 goto back; 4573 } 4574 } 4575 4576 pci_enable_msix(sc->dev); 4577 sc->intr_type = PCI_INTR_TYPE_MSIX; 4578 back: 4579 if (error) 4580 ix_free_msix(sc, setup); 4581 } 4582 4583 static void 4584 ix_free_msix(struct ix_softc *sc, boolean_t setup) 4585 { 4586 int i; 4587 4588 KKASSERT(sc->intr_cnt > 1); 4589 4590 for (i = 0; i < sc->intr_cnt; ++i) { 4591 struct ix_intr_data *intr = &sc->intr_data[i]; 4592 4593 if (intr->intr_res != NULL) { 4594 bus_release_resource(sc->dev, SYS_RES_IRQ, 4595 intr->intr_rid, intr->intr_res); 4596 } 4597 if (intr->intr_rid >= 0) 4598 pci_release_msix_vector(sc->dev, intr->intr_rid); 4599 } 4600 if (setup) 4601 pci_teardown_msix(sc->dev); 4602 4603 sc->intr_cnt = 0; 4604 kfree(sc->intr_data, M_DEVBUF); 4605 sc->intr_data = NULL; 4606 } 4607 4608 static void 4609 ix_conf_rx_msix(struct ix_softc *sc, int i, int *x0, int offset) 4610 { 4611 int x = *x0; 4612 4613 for (; i < sc->rx_ring_msix; ++i) { 4614 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 4615 struct ix_intr_data *intr; 4616 4617 KKASSERT(x < sc->intr_cnt); 4618 rxr->rx_intr_vec = x; 4619 ix_setup_msix_eims(sc, x, &rxr->rx_eims, &rxr->rx_eims_val); 4620 4621 intr = &sc->intr_data[x++]; 4622 4623 intr->intr_serialize = &rxr->rx_serialize; 4624 intr->intr_func = ix_msix_rx; 4625 intr->intr_funcarg = rxr; 4626 intr->intr_rate = IX_MSIX_RX_RATE; 4627 intr->intr_use = IX_INTR_USE_RX; 4628 4629 intr->intr_cpuid = i + offset; 4630 KKASSERT(intr->intr_cpuid < ncpus2); 4631 4632 ksnprintf(intr->intr_desc0, sizeof(intr->intr_desc0), "%s rx%d", 4633 device_get_nameunit(sc->dev), i); 4634 intr->intr_desc = intr->intr_desc0; 4635 } 4636 *x0 = x; 4637 } 4638 4639 static void 4640 ix_conf_tx_msix(struct ix_softc *sc, int i, int *x0, int offset) 4641 { 4642 int x = *x0; 4643 4644 for (; i < sc->tx_ring_msix; ++i) { 4645 struct ix_tx_ring *txr = &sc->tx_rings[i]; 4646 struct ix_intr_data *intr; 4647 4648 KKASSERT(x < sc->intr_cnt); 4649 txr->tx_intr_vec = x; 4650 ix_setup_msix_eims(sc, x, &txr->tx_eims, &txr->tx_eims_val); 4651 4652 intr = &sc->intr_data[x++]; 4653 4654 intr->intr_serialize = &txr->tx_serialize; 4655 intr->intr_func = ix_msix_tx; 4656 intr->intr_funcarg = txr; 4657 intr->intr_rate = IX_MSIX_TX_RATE; 4658 intr->intr_use = IX_INTR_USE_TX; 4659 4660 intr->intr_cpuid = i + offset; 4661 KKASSERT(intr->intr_cpuid < ncpus2); 4662 txr->tx_intr_cpuid = intr->intr_cpuid; 4663 4664 ksnprintf(intr->intr_desc0, sizeof(intr->intr_desc0), "%s tx%d", 4665 device_get_nameunit(sc->dev), i); 4666 intr->intr_desc = intr->intr_desc0; 4667 } 4668 *x0 = x; 4669 } 4670 4671 static void 4672 ix_msix_rx(void *xrxr) 4673 { 4674 struct ix_rx_ring *rxr = xrxr; 4675 4676 ASSERT_SERIALIZED(&rxr->rx_serialize); 4677 4678 ix_rxeof(rxr, -1); 4679 IXGBE_WRITE_REG(&rxr->rx_sc->hw, rxr->rx_eims, rxr->rx_eims_val); 4680 } 4681 4682 static void 4683 ix_msix_tx(void *xtxr) 4684 { 4685 struct ix_tx_ring *txr = xtxr; 4686 4687 ASSERT_SERIALIZED(&txr->tx_serialize); 4688 4689 ix_txeof(txr, *(txr->tx_hdr)); 4690 if (!ifsq_is_empty(txr->tx_ifsq)) 4691 ifsq_devstart(txr->tx_ifsq); 4692 IXGBE_WRITE_REG(&txr->tx_sc->hw, txr->tx_eims, txr->tx_eims_val); 4693 } 4694 4695 static void 4696 ix_msix_rxtx(void *xrxr) 4697 { 4698 struct ix_rx_ring *rxr = xrxr; 4699 struct ix_tx_ring *txr; 4700 int hdr; 4701 4702 ASSERT_SERIALIZED(&rxr->rx_serialize); 4703 4704 ix_rxeof(rxr, -1); 4705 4706 /* 4707 * NOTE: 4708 * Since tx_next_clean is only changed by ix_txeof(), 4709 * which is called only in interrupt handler, the 4710 * check w/o holding tx serializer is MPSAFE. 4711 */ 4712 txr = rxr->rx_txr; 4713 hdr = *(txr->tx_hdr); 4714 if (hdr != txr->tx_next_clean) { 4715 lwkt_serialize_enter(&txr->tx_serialize); 4716 ix_txeof(txr, hdr); 4717 if (!ifsq_is_empty(txr->tx_ifsq)) 4718 ifsq_devstart(txr->tx_ifsq); 4719 lwkt_serialize_exit(&txr->tx_serialize); 4720 } 4721 4722 IXGBE_WRITE_REG(&rxr->rx_sc->hw, rxr->rx_eims, rxr->rx_eims_val); 4723 } 4724 4725 static void 4726 ix_intr_status(struct ix_softc *sc, uint32_t eicr) 4727 { 4728 struct ixgbe_hw *hw = &sc->hw; 4729 4730 /* Link status change */ 4731 if (eicr & IXGBE_EICR_LSC) 4732 ix_handle_link(sc); 4733 4734 if (hw->mac.type != ixgbe_mac_82598EB) { 4735 if (eicr & IXGBE_EICR_ECC) 4736 if_printf(&sc->arpcom.ac_if, "ECC ERROR!! Reboot!!\n"); 4737 4738 /* Check for over temp condition */ 4739 if (eicr & IXGBE_EICR_TS) { 4740 if_printf(&sc->arpcom.ac_if, "CRITICAL: OVER TEMP!! " 4741 "PHY IS SHUT DOWN!! Shutdown!!\n"); 4742 } 4743 } 4744 4745 if (ix_is_sfp(hw)) { 4746 uint32_t mod_mask; 4747 4748 /* Pluggable optics-related interrupt */ 4749 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP) 4750 mod_mask = IXGBE_EICR_GPI_SDP0_X540; 4751 else 4752 mod_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 4753 if (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) 4754 ix_handle_msf(sc); 4755 else if (eicr & mod_mask) 4756 ix_handle_mod(sc); 4757 } 4758 4759 /* Check for fan failure */ 4760 if (hw->device_id == IXGBE_DEV_ID_82598AT && 4761 (eicr & IXGBE_EICR_GPI_SDP1)) 4762 if_printf(&sc->arpcom.ac_if, "FAN FAILURE!! Replace!!\n"); 4763 4764 /* External PHY interrupt */ 4765 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 4766 (eicr & IXGBE_EICR_GPI_SDP0_X540)) 4767 ix_handle_phy(sc); 4768 } 4769 4770 static void 4771 ix_msix_status(void *xsc) 4772 { 4773 struct ix_softc *sc = xsc; 4774 uint32_t eicr; 4775 4776 ASSERT_SERIALIZED(&sc->main_serialize); 4777 4778 eicr = IXGBE_READ_REG(&sc->hw, IXGBE_EICR); 4779 ix_intr_status(sc, eicr); 4780 4781 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, sc->intr_mask); 4782 } 4783 4784 static void 4785 ix_setup_msix_eims(const struct ix_softc *sc, int x, 4786 uint32_t *eims, uint32_t *eims_val) 4787 { 4788 if (x < 32) { 4789 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 4790 KASSERT(x < IX_MAX_MSIX_82598, 4791 ("%s: invalid vector %d for 82598", 4792 device_get_nameunit(sc->dev), x)); 4793 *eims = IXGBE_EIMS; 4794 } else { 4795 *eims = IXGBE_EIMS_EX(0); 4796 } 4797 *eims_val = 1 << x; 4798 } else { 4799 KASSERT(x < IX_MAX_MSIX, ("%s: invalid vector %d", 4800 device_get_nameunit(sc->dev), x)); 4801 KASSERT(sc->hw.mac.type != ixgbe_mac_82598EB, 4802 ("%s: invalid vector %d for 82598", 4803 device_get_nameunit(sc->dev), x)); 4804 *eims = IXGBE_EIMS_EX(1); 4805 *eims_val = 1 << (x - 32); 4806 } 4807 } 4808 4809 #ifdef IFPOLL_ENABLE 4810 4811 static void 4812 ix_npoll_status(struct ifnet *ifp) 4813 { 4814 struct ix_softc *sc = ifp->if_softc; 4815 uint32_t eicr; 4816 4817 ASSERT_SERIALIZED(&sc->main_serialize); 4818 4819 eicr = IXGBE_READ_REG(&sc->hw, IXGBE_EICR); 4820 ix_intr_status(sc, eicr); 4821 } 4822 4823 static void 4824 ix_npoll_tx(struct ifnet *ifp, void *arg, int cycle __unused) 4825 { 4826 struct ix_tx_ring *txr = arg; 4827 4828 ASSERT_SERIALIZED(&txr->tx_serialize); 4829 4830 ix_txeof(txr, *(txr->tx_hdr)); 4831 if (!ifsq_is_empty(txr->tx_ifsq)) 4832 ifsq_devstart(txr->tx_ifsq); 4833 } 4834 4835 static void 4836 ix_npoll_rx(struct ifnet *ifp __unused, void *arg, int cycle) 4837 { 4838 struct ix_rx_ring *rxr = arg; 4839 4840 ASSERT_SERIALIZED(&rxr->rx_serialize); 4841 4842 ix_rxeof(rxr, cycle); 4843 } 4844 4845 static void 4846 ix_npoll(struct ifnet *ifp, struct ifpoll_info *info) 4847 { 4848 struct ix_softc *sc = ifp->if_softc; 4849 int i, txr_cnt, rxr_cnt; 4850 4851 ASSERT_IFNET_SERIALIZED_ALL(ifp); 4852 4853 if (info) { 4854 int off; 4855 4856 info->ifpi_status.status_func = ix_npoll_status; 4857 info->ifpi_status.serializer = &sc->main_serialize; 4858 4859 txr_cnt = ix_get_txring_inuse(sc, TRUE); 4860 off = sc->tx_npoll_off; 4861 for (i = 0; i < txr_cnt; ++i) { 4862 struct ix_tx_ring *txr = &sc->tx_rings[i]; 4863 int idx = i + off; 4864 4865 KKASSERT(idx < ncpus2); 4866 info->ifpi_tx[idx].poll_func = ix_npoll_tx; 4867 info->ifpi_tx[idx].arg = txr; 4868 info->ifpi_tx[idx].serializer = &txr->tx_serialize; 4869 ifsq_set_cpuid(txr->tx_ifsq, idx); 4870 } 4871 4872 rxr_cnt = ix_get_rxring_inuse(sc, TRUE); 4873 off = sc->rx_npoll_off; 4874 for (i = 0; i < rxr_cnt; ++i) { 4875 struct ix_rx_ring *rxr = &sc->rx_rings[i]; 4876 int idx = i + off; 4877 4878 KKASSERT(idx < ncpus2); 4879 info->ifpi_rx[idx].poll_func = ix_npoll_rx; 4880 info->ifpi_rx[idx].arg = rxr; 4881 info->ifpi_rx[idx].serializer = &rxr->rx_serialize; 4882 } 4883 4884 if (ifp->if_flags & IFF_RUNNING) { 4885 if (rxr_cnt == sc->rx_ring_inuse && 4886 txr_cnt == sc->tx_ring_inuse) { 4887 ix_set_timer_cpuid(sc, TRUE); 4888 ix_disable_intr(sc); 4889 } else { 4890 ix_init(sc); 4891 } 4892 } 4893 } else { 4894 for (i = 0; i < sc->tx_ring_cnt; ++i) { 4895 struct ix_tx_ring *txr = &sc->tx_rings[i]; 4896 4897 ifsq_set_cpuid(txr->tx_ifsq, txr->tx_intr_cpuid); 4898 } 4899 4900 if (ifp->if_flags & IFF_RUNNING) { 4901 txr_cnt = ix_get_txring_inuse(sc, FALSE); 4902 rxr_cnt = ix_get_rxring_inuse(sc, FALSE); 4903 4904 if (rxr_cnt == sc->rx_ring_inuse && 4905 txr_cnt == sc->tx_ring_inuse) { 4906 ix_set_timer_cpuid(sc, FALSE); 4907 ix_enable_intr(sc); 4908 } else { 4909 ix_init(sc); 4910 } 4911 } 4912 } 4913 } 4914 4915 static int 4916 ix_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS) 4917 { 4918 struct ix_softc *sc = (void *)arg1; 4919 struct ifnet *ifp = &sc->arpcom.ac_if; 4920 int error, off; 4921 4922 off = sc->rx_npoll_off; 4923 error = sysctl_handle_int(oidp, &off, 0, req); 4924 if (error || req->newptr == NULL) 4925 return error; 4926 if (off < 0) 4927 return EINVAL; 4928 4929 ifnet_serialize_all(ifp); 4930 if (off >= ncpus2 || off % sc->rx_ring_cnt != 0) { 4931 error = EINVAL; 4932 } else { 4933 error = 0; 4934 sc->rx_npoll_off = off; 4935 } 4936 ifnet_deserialize_all(ifp); 4937 4938 return error; 4939 } 4940 4941 static int 4942 ix_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS) 4943 { 4944 struct ix_softc *sc = (void *)arg1; 4945 struct ifnet *ifp = &sc->arpcom.ac_if; 4946 int error, off; 4947 4948 off = sc->tx_npoll_off; 4949 error = sysctl_handle_int(oidp, &off, 0, req); 4950 if (error || req->newptr == NULL) 4951 return error; 4952 if (off < 0) 4953 return EINVAL; 4954 4955 ifnet_serialize_all(ifp); 4956 if (off >= ncpus2 || off % sc->tx_ring_cnt != 0) { 4957 error = EINVAL; 4958 } else { 4959 error = 0; 4960 sc->tx_npoll_off = off; 4961 } 4962 ifnet_deserialize_all(ifp); 4963 4964 return error; 4965 } 4966 4967 #endif /* IFPOLL_ENABLE */ 4968 4969 static enum ixgbe_fc_mode 4970 ix_ifmedia2fc(int ifm) 4971 { 4972 int fc_opt = ifm & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE); 4973 4974 switch (fc_opt) { 4975 case (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE): 4976 return ixgbe_fc_full; 4977 4978 case IFM_ETH_RXPAUSE: 4979 return ixgbe_fc_rx_pause; 4980 4981 case IFM_ETH_TXPAUSE: 4982 return ixgbe_fc_tx_pause; 4983 4984 default: 4985 return ixgbe_fc_none; 4986 } 4987 } 4988 4989 static const char * 4990 ix_ifmedia2str(int ifm) 4991 { 4992 int fc_opt = ifm & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE); 4993 4994 switch (fc_opt) { 4995 case (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE): 4996 return IFM_ETH_FC_FULL; 4997 4998 case IFM_ETH_RXPAUSE: 4999 return IFM_ETH_FC_RXPAUSE; 5000 5001 case IFM_ETH_TXPAUSE: 5002 return IFM_ETH_FC_TXPAUSE; 5003 5004 default: 5005 return IFM_ETH_FC_NONE; 5006 } 5007 } 5008 5009 static const char * 5010 ix_fc2str(enum ixgbe_fc_mode fc) 5011 { 5012 switch (fc) { 5013 case ixgbe_fc_full: 5014 return IFM_ETH_FC_FULL; 5015 5016 case ixgbe_fc_rx_pause: 5017 return IFM_ETH_FC_RXPAUSE; 5018 5019 case ixgbe_fc_tx_pause: 5020 return IFM_ETH_FC_TXPAUSE; 5021 5022 default: 5023 return IFM_ETH_FC_NONE; 5024 } 5025 } 5026 5027 static int 5028 ix_powerdown(struct ix_softc *sc) 5029 { 5030 struct ixgbe_hw *hw = &sc->hw; 5031 int error = 0; 5032 5033 /* Limit power managment flow to X550EM baseT */ 5034 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 5035 hw->phy.ops.enter_lplu) { 5036 /* Turn off support for APM wakeup. (Using ACPI instead) */ 5037 IXGBE_WRITE_REG(hw, IXGBE_GRC, 5038 IXGBE_READ_REG(hw, IXGBE_GRC) & ~(uint32_t)2); 5039 5040 /* 5041 * Clear Wake Up Status register to prevent any previous wakeup 5042 * events from waking us up immediately after we suspend. 5043 */ 5044 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 5045 5046 /* 5047 * Program the Wakeup Filter Control register with user filter 5048 * settings 5049 */ 5050 IXGBE_WRITE_REG(hw, IXGBE_WUFC, sc->wufc); 5051 5052 /* Enable wakeups and power management in Wakeup Control */ 5053 IXGBE_WRITE_REG(hw, IXGBE_WUC, 5054 IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN); 5055 5056 /* X550EM baseT adapters need a special LPLU flow */ 5057 hw->phy.reset_disable = true; 5058 ix_stop(sc); 5059 error = hw->phy.ops.enter_lplu(hw); 5060 if (error) { 5061 if_printf(&sc->arpcom.ac_if, 5062 "Error entering LPLU: %d\n", error); 5063 } 5064 hw->phy.reset_disable = false; 5065 } else { 5066 /* Just stop for other adapters */ 5067 ix_stop(sc); 5068 } 5069 return error; 5070 } 5071 5072 static void 5073 ix_config_flowctrl(struct ix_softc *sc) 5074 { 5075 struct ixgbe_hw *hw = &sc->hw; 5076 uint32_t rxpb, frame, size, tmp; 5077 5078 frame = sc->max_frame_size; 5079 5080 /* Calculate High Water */ 5081 switch (hw->mac.type) { 5082 case ixgbe_mac_X540: 5083 case ixgbe_mac_X550: 5084 case ixgbe_mac_X550EM_a: 5085 case ixgbe_mac_X550EM_x: 5086 tmp = IXGBE_DV_X540(frame, frame); 5087 break; 5088 default: 5089 tmp = IXGBE_DV(frame, frame); 5090 break; 5091 } 5092 size = IXGBE_BT2KB(tmp); 5093 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 5094 hw->fc.high_water[0] = rxpb - size; 5095 5096 /* Now calculate Low Water */ 5097 switch (hw->mac.type) { 5098 case ixgbe_mac_X540: 5099 case ixgbe_mac_X550: 5100 case ixgbe_mac_X550EM_a: 5101 case ixgbe_mac_X550EM_x: 5102 tmp = IXGBE_LOW_DV_X540(frame); 5103 break; 5104 default: 5105 tmp = IXGBE_LOW_DV(frame); 5106 break; 5107 } 5108 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 5109 5110 hw->fc.requested_mode = ix_ifmedia2fc(sc->ifm_media); 5111 if (sc->ifm_media & IFM_ETH_FORCEPAUSE) 5112 hw->fc.disable_fc_autoneg = TRUE; 5113 else 5114 hw->fc.disable_fc_autoneg = FALSE; 5115 hw->fc.pause_time = IX_FC_PAUSE; 5116 hw->fc.send_xon = TRUE; 5117 } 5118 5119 static void 5120 ix_config_dmac(struct ix_softc *sc) 5121 { 5122 struct ixgbe_hw *hw = &sc->hw; 5123 struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config; 5124 5125 if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config) 5126 return; 5127 5128 if ((dcfg->watchdog_timer ^ sc->dmac) || 5129 (dcfg->link_speed ^ sc->link_speed)) { 5130 dcfg->watchdog_timer = sc->dmac; 5131 dcfg->fcoe_en = false; 5132 dcfg->link_speed = sc->link_speed; 5133 dcfg->num_tcs = 1; 5134 5135 if (bootverbose) { 5136 if_printf(&sc->arpcom.ac_if, "dmac settings: " 5137 "watchdog %d, link speed %d\n", 5138 dcfg->watchdog_timer, dcfg->link_speed); 5139 } 5140 5141 hw->mac.ops.dmac_config(hw); 5142 } 5143 } 5144 5145 static void 5146 ix_init_media(struct ix_softc *sc) 5147 { 5148 struct ixgbe_hw *hw = &sc->hw; 5149 int layer, msf_ifm = IFM_NONE; 5150 5151 ifmedia_removeall(&sc->media); 5152 5153 layer = ixgbe_get_supported_physical_layer(hw); 5154 5155 /* 5156 * Media types with matching DragonFlyBSD media defines 5157 */ 5158 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) { 5159 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX, 5160 0, NULL); 5161 } 5162 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) { 5163 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 5164 0, NULL); 5165 } 5166 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) { 5167 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 5168 0, NULL); 5169 /* No half-duplex support */ 5170 } 5171 5172 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 5173 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_LR | IFM_FDX, 5174 0, NULL); 5175 msf_ifm = IFM_1000_LX; 5176 } 5177 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM) { 5178 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_LRM | IFM_FDX, 5179 0, NULL); 5180 msf_ifm = IFM_1000_LX; 5181 } 5182 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 5183 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_SR | IFM_FDX, 5184 0, NULL); 5185 msf_ifm = IFM_1000_SX; 5186 } 5187 5188 /* Add media for multispeed fiber */ 5189 if (ix_is_sfp(hw) && hw->phy.multispeed_fiber && msf_ifm != IFM_NONE) { 5190 uint32_t linkcap; 5191 bool autoneg; 5192 5193 hw->mac.ops.get_link_capabilities(hw, &linkcap, &autoneg); 5194 if (linkcap & IXGBE_LINK_SPEED_1GB_FULL) 5195 ifmedia_add_nodup(&sc->media, 5196 IFM_ETHER | msf_ifm | IFM_FDX, 0, NULL); 5197 } 5198 5199 if ((layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) || 5200 (layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)) { 5201 ifmedia_add_nodup(&sc->media, 5202 IFM_ETHER | IFM_10G_TWINAX | IFM_FDX, 0, NULL); 5203 } 5204 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 5205 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_CX4 | IFM_FDX, 5206 0, NULL); 5207 } 5208 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 5209 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 5210 0, NULL); 5211 } 5212 5213 /* 5214 * XXX Other (no matching DragonFlyBSD media type): 5215 * To workaround this, we'll assign these completely 5216 * inappropriate media types. 5217 */ 5218 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 5219 if_printf(&sc->arpcom.ac_if, "Media supported: 10GbaseKR\n"); 5220 if_printf(&sc->arpcom.ac_if, "10GbaseKR mapped to 10GbaseSR\n"); 5221 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_SR | IFM_FDX, 5222 0, NULL); 5223 } 5224 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) { 5225 if_printf(&sc->arpcom.ac_if, "Media supported: 10GbaseKX4\n"); 5226 if_printf(&sc->arpcom.ac_if, 5227 "10GbaseKX4 mapped to 10GbaseCX4\n"); 5228 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_CX4 | IFM_FDX, 5229 0, NULL); 5230 } 5231 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 5232 if_printf(&sc->arpcom.ac_if, "Media supported: 1000baseKX\n"); 5233 if_printf(&sc->arpcom.ac_if, 5234 "1000baseKX mapped to 1000baseCX\n"); 5235 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_1000_CX | IFM_FDX, 5236 0, NULL); 5237 } 5238 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX) { 5239 /* Someday, someone will care about you... */ 5240 if_printf(&sc->arpcom.ac_if, 5241 "Media supported: 1000baseBX, ignored\n"); 5242 } 5243 5244 /* XXX we probably don't need this */ 5245 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 5246 ifmedia_add_nodup(&sc->media, 5247 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 5248 } 5249 5250 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 5251 5252 if (ifmedia_tryset(&sc->media, sc->ifm_media)) { 5253 int flowctrl = (sc->ifm_media & IFM_ETH_FCMASK); 5254 5255 sc->advspeed = IXGBE_LINK_SPEED_UNKNOWN; 5256 sc->ifm_media = IX_IFM_DEFAULT | flowctrl; 5257 ifmedia_set(&sc->media, sc->ifm_media); 5258 } 5259 } 5260