1 /* $OpenBSD: if_ix.c,v 1.178 2020/12/22 23:25:37 dlg Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2013, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /* FreeBSD: src/sys/dev/ixgbe/ixgbe.c 251964 Jun 18 21:28:19 2013 UTC */ 36 37 #include <dev/pci/if_ix.h> 38 #include <dev/pci/ixgbe_type.h> 39 40 /********************************************************************* 41 * Driver version 42 *********************************************************************/ 43 /* char ixgbe_driver_version[] = "2.5.13"; */ 44 45 /********************************************************************* 46 * PCI Device ID Table 47 * 48 * Used by probe to select devices to load on 49 *********************************************************************/ 50 51 const struct pci_matchid ixgbe_devices[] = { 52 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598 }, 53 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_BX }, 54 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF_DUAL }, 55 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF }, 56 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT }, 57 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT2 }, 58 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT_DUAL }, 59 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4 }, 60 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4_DUAL }, 61 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_XF_LR }, 62 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_SFP }, 63 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_SR_DUAL_EM }, 64 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_DA_DUAL }, 65 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4 }, 66 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4_MEZZ }, 67 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_XAUI }, 68 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_COMBO_BP }, 69 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_BPLANE_FCOE }, 70 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_CX4 }, 71 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_T3_LOM }, 72 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP }, 73 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_EM }, 74 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF_QP }, 75 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF2 }, 76 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_FCOE }, 77 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599EN_SFP }, 78 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_QSFP_SF_QP }, 79 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T }, 80 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T1 }, 81 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T }, 82 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T1 }, 83 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KX4 }, 84 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KR }, 85 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_SFP }, 86 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_10G_T }, 87 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_1G_T }, 88 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR }, 89 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR_L }, 90 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP_N }, 91 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP }, 92 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII }, 93 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII_L }, 94 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_10G_T }, 95 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T }, 96 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T_L } 97 }; 98 99 /********************************************************************* 100 * Function prototypes 101 *********************************************************************/ 102 int ixgbe_probe(struct device *, void *, void *); 103 void ixgbe_attach(struct device *, struct device *, void *); 104 int ixgbe_detach(struct device *, int); 105 void ixgbe_start(struct ifqueue *); 106 int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); 107 int ixgbe_rxrinfo(struct ix_softc *, struct if_rxrinfo *); 108 int ixgbe_get_sffpage(struct ix_softc *, struct if_sffpage *); 109 void ixgbe_watchdog(struct ifnet *); 110 void ixgbe_init(void *); 111 void ixgbe_stop(void *); 112 void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 113 int ixgbe_media_change(struct ifnet *); 114 void ixgbe_identify_hardware(struct ix_softc *); 115 int ixgbe_allocate_pci_resources(struct ix_softc *); 116 int ixgbe_allocate_legacy(struct ix_softc *); 117 int ixgbe_allocate_msix(struct ix_softc *); 118 void ixgbe_setup_msix(struct ix_softc *); 119 int ixgbe_allocate_queues(struct ix_softc *); 120 void ixgbe_free_pci_resources(struct ix_softc *); 121 void ixgbe_local_timer(void *); 122 void ixgbe_setup_interface(struct ix_softc *); 123 void ixgbe_config_gpie(struct ix_softc *); 124 void ixgbe_config_delay_values(struct ix_softc *); 125 void ixgbe_add_media_types(struct ix_softc *); 126 void ixgbe_config_link(struct ix_softc *); 127 128 int ixgbe_allocate_transmit_buffers(struct tx_ring *); 129 int ixgbe_setup_transmit_structures(struct ix_softc *); 130 int ixgbe_setup_transmit_ring(struct tx_ring *); 131 void ixgbe_initialize_transmit_units(struct ix_softc *); 132 void ixgbe_free_transmit_structures(struct ix_softc *); 133 void ixgbe_free_transmit_buffers(struct tx_ring *); 134 135 int ixgbe_allocate_receive_buffers(struct rx_ring *); 136 int ixgbe_setup_receive_structures(struct ix_softc *); 137 int ixgbe_setup_receive_ring(struct rx_ring *); 138 void ixgbe_initialize_receive_units(struct ix_softc *); 139 void ixgbe_free_receive_structures(struct ix_softc *); 140 void ixgbe_free_receive_buffers(struct rx_ring *); 141 void ixgbe_initialize_rss_mapping(struct ix_softc *); 142 int ixgbe_rxfill(struct rx_ring *); 143 void ixgbe_rxrefill(void *); 144 145 int ixgbe_intr(struct ix_softc *sc); 146 void ixgbe_enable_intr(struct ix_softc *); 147 void ixgbe_disable_intr(struct ix_softc *); 148 int ixgbe_txeof(struct tx_ring *); 149 int ixgbe_rxeof(struct rx_ring *); 150 void ixgbe_rx_checksum(uint32_t, struct mbuf *, uint32_t); 151 void ixgbe_iff(struct ix_softc *); 152 void ixgbe_map_queue_statistics(struct ix_softc *); 153 void ixgbe_update_link_status(struct ix_softc *); 154 int ixgbe_get_buf(struct rx_ring *, int); 155 int ixgbe_encap(struct tx_ring *, struct mbuf *); 156 int ixgbe_dma_malloc(struct ix_softc *, bus_size_t, 157 struct ixgbe_dma_alloc *, int); 158 void ixgbe_dma_free(struct ix_softc *, struct ixgbe_dma_alloc *); 159 int ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *, uint32_t *, 160 uint32_t *); 161 int ixgbe_tso_setup(struct tx_ring *, struct mbuf *, uint32_t *, 162 uint32_t *); 163 void ixgbe_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t); 164 void ixgbe_configure_ivars(struct ix_softc *); 165 uint8_t *ixgbe_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 166 167 void ixgbe_setup_vlan_hw_support(struct ix_softc *); 168 169 /* Support for pluggable optic modules */ 170 void ixgbe_handle_mod(struct ix_softc *); 171 void ixgbe_handle_msf(struct ix_softc *); 172 void ixgbe_handle_phy(struct ix_softc *); 173 174 /* Legacy (single vector interrupt handler */ 175 int ixgbe_legacy_intr(void *); 176 void ixgbe_enable_queue(struct ix_softc *, uint32_t); 177 void ixgbe_enable_queues(struct ix_softc *); 178 void ixgbe_disable_queue(struct ix_softc *, uint32_t); 179 void ixgbe_rearm_queue(struct ix_softc *, uint32_t); 180 181 /* MSI-X (multiple vectors interrupt handlers) */ 182 int ixgbe_link_intr(void *); 183 int ixgbe_queue_intr(void *); 184 185 #if NKSTAT > 0 186 static void ix_kstats(struct ix_softc *); 187 static void ix_rxq_kstats(struct ix_softc *, struct rx_ring *); 188 static void ix_txq_kstats(struct ix_softc *, struct tx_ring *); 189 static void ix_kstats_tick(void *); 190 #endif 191 192 /********************************************************************* 193 * OpenBSD Device Interface Entry Points 194 *********************************************************************/ 195 196 struct cfdriver ix_cd = { 197 NULL, "ix", DV_IFNET 198 }; 199 200 struct cfattach ix_ca = { 201 sizeof(struct ix_softc), ixgbe_probe, ixgbe_attach, ixgbe_detach 202 }; 203 204 int ixgbe_smart_speed = ixgbe_smart_speed_on; 205 int ixgbe_enable_msix = 1; 206 207 /********************************************************************* 208 * Device identification routine 209 * 210 * ixgbe_probe determines if the driver should be loaded on 211 * adapter based on PCI vendor/device id of the adapter. 212 * 213 * return 0 on success, positive on failure 214 *********************************************************************/ 215 216 int 217 ixgbe_probe(struct device *parent, void *match, void *aux) 218 { 219 INIT_DEBUGOUT("ixgbe_probe: begin"); 220 221 return (pci_matchbyid((struct pci_attach_args *)aux, ixgbe_devices, 222 nitems(ixgbe_devices))); 223 } 224 225 /********************************************************************* 226 * Device initialization routine 227 * 228 * The attach entry point is called when the driver is being loaded. 229 * This routine identifies the type of hardware, allocates all resources 230 * and initializes the hardware. 231 * 232 * return 0 on success, positive on failure 233 *********************************************************************/ 234 235 void 236 ixgbe_attach(struct device *parent, struct device *self, void *aux) 237 { 238 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 239 struct ix_softc *sc = (struct ix_softc *)self; 240 int error = 0; 241 uint16_t csum; 242 uint32_t ctrl_ext; 243 struct ixgbe_hw *hw = &sc->hw; 244 245 INIT_DEBUGOUT("ixgbe_attach: begin"); 246 247 sc->osdep.os_sc = sc; 248 sc->osdep.os_pa = *pa; 249 250 rw_init(&sc->sfflock, "ixsff"); 251 252 #if NKSTAT > 0 253 ix_kstats(sc); 254 #endif 255 256 /* Determine hardware revision */ 257 ixgbe_identify_hardware(sc); 258 259 /* Indicate to RX setup to use Jumbo Clusters */ 260 sc->num_tx_desc = DEFAULT_TXD; 261 sc->num_rx_desc = DEFAULT_RXD; 262 263 /* Do base PCI setup - map BAR0 */ 264 if (ixgbe_allocate_pci_resources(sc)) 265 goto err_out; 266 267 /* Allocate our TX/RX Queues */ 268 if (ixgbe_allocate_queues(sc)) 269 goto err_out; 270 271 /* Allocate multicast array memory. */ 272 sc->mta = mallocarray(IXGBE_ETH_LENGTH_OF_ADDRESS, 273 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 274 if (sc->mta == NULL) { 275 printf(": Can not allocate multicast setup array\n"); 276 goto err_late; 277 } 278 279 /* Initialize the shared code */ 280 error = ixgbe_init_shared_code(hw); 281 if (error) { 282 printf(": Unable to initialize the shared code\n"); 283 goto err_late; 284 } 285 286 /* Make sure we have a good EEPROM before we read from it */ 287 if (sc->hw.eeprom.ops.validate_checksum(&sc->hw, &csum) < 0) { 288 printf(": The EEPROM Checksum Is Not Valid\n"); 289 goto err_late; 290 } 291 292 error = ixgbe_init_hw(hw); 293 if (error == IXGBE_ERR_EEPROM_VERSION) { 294 printf(": This device is a pre-production adapter/" 295 "LOM. Please be aware there may be issues associated " 296 "with your hardware.\nIf you are experiencing problems " 297 "please contact your Intel or hardware representative " 298 "who provided you with this hardware.\n"); 299 } else if (error && (error != IXGBE_ERR_SFP_NOT_PRESENT && 300 error != IXGBE_ERR_SFP_NOT_SUPPORTED)) { 301 printf(": Hardware Initialization Failure\n"); 302 goto err_late; 303 } 304 305 bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr, 306 IXGBE_ETH_LENGTH_OF_ADDRESS); 307 308 if (sc->sc_intrmap) 309 error = ixgbe_allocate_msix(sc); 310 else 311 error = ixgbe_allocate_legacy(sc); 312 if (error) 313 goto err_late; 314 315 /* Enable the optics for 82599 SFP+ fiber */ 316 if (sc->hw.mac.ops.enable_tx_laser) 317 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 318 319 /* Enable power to the phy */ 320 if (hw->phy.ops.set_phy_power) 321 hw->phy.ops.set_phy_power(&sc->hw, TRUE); 322 323 /* Setup OS specific network interface */ 324 ixgbe_setup_interface(sc); 325 326 /* Get the PCI-E bus info and determine LAN ID */ 327 hw->mac.ops.get_bus_info(hw); 328 329 /* Set an initial default flow control value */ 330 sc->fc = ixgbe_fc_full; 331 332 /* let hardware know driver is loaded */ 333 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 334 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 335 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 336 337 printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); 338 339 INIT_DEBUGOUT("ixgbe_attach: end"); 340 return; 341 342 err_late: 343 ixgbe_free_transmit_structures(sc); 344 ixgbe_free_receive_structures(sc); 345 err_out: 346 ixgbe_free_pci_resources(sc); 347 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 348 MAX_NUM_MULTICAST_ADDRESSES); 349 } 350 351 /********************************************************************* 352 * Device removal routine 353 * 354 * The detach entry point is called when the driver is being removed. 355 * This routine stops the adapter and deallocates all the resources 356 * that were allocated for driver operation. 357 * 358 * return 0 on success, positive on failure 359 *********************************************************************/ 360 361 int 362 ixgbe_detach(struct device *self, int flags) 363 { 364 struct ix_softc *sc = (struct ix_softc *)self; 365 struct ifnet *ifp = &sc->arpcom.ac_if; 366 uint32_t ctrl_ext; 367 368 INIT_DEBUGOUT("ixgbe_detach: begin"); 369 370 ixgbe_stop(sc); 371 372 /* let hardware know driver is unloading */ 373 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 374 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 375 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 376 377 ether_ifdetach(ifp); 378 if_detach(ifp); 379 380 ixgbe_free_pci_resources(sc); 381 382 ixgbe_free_transmit_structures(sc); 383 ixgbe_free_receive_structures(sc); 384 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 385 MAX_NUM_MULTICAST_ADDRESSES); 386 387 /* XXX kstat */ 388 389 return (0); 390 } 391 392 /********************************************************************* 393 * Transmit entry point 394 * 395 * ixgbe_start is called by the stack to initiate a transmit. 396 * The driver will remain in this routine as long as there are 397 * packets to transmit and transmit resources are available. 398 * In case resources are not available stack is notified and 399 * the packet is requeued. 400 **********************************************************************/ 401 402 void 403 ixgbe_start(struct ifqueue *ifq) 404 { 405 struct ifnet *ifp = ifq->ifq_if; 406 struct ix_softc *sc = ifp->if_softc; 407 struct tx_ring *txr = ifq->ifq_softc; 408 struct mbuf *m_head; 409 unsigned int head, free, used; 410 int post = 0; 411 412 if (!sc->link_up) 413 return; 414 415 head = txr->next_avail_desc; 416 free = txr->next_to_clean; 417 if (free <= head) 418 free += sc->num_tx_desc; 419 free -= head; 420 421 membar_consumer(); 422 423 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 424 0, txr->txdma.dma_map->dm_mapsize, 425 BUS_DMASYNC_POSTWRITE); 426 427 for (;;) { 428 /* Check that we have the minimal number of TX descriptors. */ 429 if (free <= IXGBE_TX_OP_THRESHOLD) { 430 ifq_set_oactive(ifq); 431 break; 432 } 433 434 m_head = ifq_dequeue(ifq); 435 if (m_head == NULL) 436 break; 437 438 used = ixgbe_encap(txr, m_head); 439 if (used == 0) { 440 m_freem(m_head); 441 continue; 442 } 443 444 free -= used; 445 446 #if NBPFILTER > 0 447 if (ifp->if_bpf) 448 bpf_mtap_ether(ifp->if_bpf, m_head, BPF_DIRECTION_OUT); 449 #endif 450 451 /* Set timeout in case hardware has problems transmitting */ 452 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 453 ifp->if_timer = IXGBE_TX_TIMEOUT; 454 455 post = 1; 456 } 457 458 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 459 0, txr->txdma.dma_map->dm_mapsize, 460 BUS_DMASYNC_PREWRITE); 461 462 /* 463 * Advance the Transmit Descriptor Tail (Tdt), this tells the 464 * hardware that this frame is available to transmit. 465 */ 466 if (post) 467 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->me), 468 txr->next_avail_desc); 469 } 470 471 /********************************************************************* 472 * Ioctl entry point 473 * 474 * ixgbe_ioctl is called when the user wants to configure the 475 * interface. 476 * 477 * return 0 on success, positive on failure 478 **********************************************************************/ 479 480 int 481 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 482 { 483 struct ix_softc *sc = ifp->if_softc; 484 struct ifreq *ifr = (struct ifreq *) data; 485 int s, error = 0; 486 487 s = splnet(); 488 489 switch (command) { 490 case SIOCSIFADDR: 491 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); 492 ifp->if_flags |= IFF_UP; 493 if (!(ifp->if_flags & IFF_RUNNING)) 494 ixgbe_init(sc); 495 break; 496 497 case SIOCSIFFLAGS: 498 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 499 if (ifp->if_flags & IFF_UP) { 500 if (ifp->if_flags & IFF_RUNNING) 501 error = ENETRESET; 502 else 503 ixgbe_init(sc); 504 } else { 505 if (ifp->if_flags & IFF_RUNNING) 506 ixgbe_stop(sc); 507 } 508 break; 509 510 case SIOCSIFMEDIA: 511 case SIOCGIFMEDIA: 512 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 513 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 514 break; 515 516 case SIOCGIFRXR: 517 error = ixgbe_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 518 break; 519 520 case SIOCGIFSFFPAGE: 521 error = rw_enter(&sc->sfflock, RW_WRITE|RW_INTR); 522 if (error != 0) 523 break; 524 525 error = ixgbe_get_sffpage(sc, (struct if_sffpage *)data); 526 rw_exit(&sc->sfflock); 527 break; 528 529 default: 530 error = ether_ioctl(ifp, &sc->arpcom, command, data); 531 } 532 533 if (error == ENETRESET) { 534 if (ifp->if_flags & IFF_RUNNING) { 535 ixgbe_disable_intr(sc); 536 ixgbe_iff(sc); 537 ixgbe_enable_intr(sc); 538 ixgbe_enable_queues(sc); 539 } 540 error = 0; 541 } 542 543 splx(s); 544 return (error); 545 } 546 547 int 548 ixgbe_get_sffpage(struct ix_softc *sc, struct if_sffpage *sff) 549 { 550 struct ixgbe_hw *hw = &sc->hw; 551 uint32_t swfw_mask = hw->phy.phy_semaphore_mask; 552 uint8_t page; 553 size_t i; 554 int error = EIO; 555 556 if (hw->phy.type == ixgbe_phy_fw) 557 return (ENODEV); 558 559 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)) 560 return (EBUSY); /* XXX */ 561 562 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 563 if (hw->phy.ops.read_i2c_byte_unlocked(hw, 127, 564 IFSFF_ADDR_EEPROM, &page)) 565 goto error; 566 if (page != sff->sff_page && 567 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 568 IFSFF_ADDR_EEPROM, sff->sff_page)) 569 goto error; 570 } 571 572 for (i = 0; i < sizeof(sff->sff_data); i++) { 573 if (hw->phy.ops.read_i2c_byte_unlocked(hw, i, 574 sff->sff_addr, &sff->sff_data[i])) 575 goto error; 576 } 577 578 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 579 if (page != sff->sff_page && 580 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 581 IFSFF_ADDR_EEPROM, page)) 582 goto error; 583 } 584 585 error = 0; 586 error: 587 hw->mac.ops.release_swfw_sync(hw, swfw_mask); 588 return (error); 589 } 590 591 int 592 ixgbe_rxrinfo(struct ix_softc *sc, struct if_rxrinfo *ifri) 593 { 594 struct if_rxring_info *ifr, ifr1; 595 struct rx_ring *rxr; 596 int error, i; 597 u_int n = 0; 598 599 if (sc->num_queues > 1) { 600 if ((ifr = mallocarray(sc->num_queues, sizeof(*ifr), M_DEVBUF, 601 M_WAITOK | M_ZERO)) == NULL) 602 return (ENOMEM); 603 } else 604 ifr = &ifr1; 605 606 for (i = 0; i < sc->num_queues; i++) { 607 rxr = &sc->rx_rings[i]; 608 ifr[n].ifr_size = MCLBYTES; 609 snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "%d", i); 610 ifr[n].ifr_info = rxr->rx_ring; 611 n++; 612 } 613 614 error = if_rxr_info_ioctl(ifri, sc->num_queues, ifr); 615 616 if (sc->num_queues > 1) 617 free(ifr, M_DEVBUF, sc->num_queues * sizeof(*ifr)); 618 return (error); 619 } 620 621 /********************************************************************* 622 * Watchdog entry point 623 * 624 **********************************************************************/ 625 626 void 627 ixgbe_watchdog(struct ifnet * ifp) 628 { 629 struct ix_softc *sc = (struct ix_softc *)ifp->if_softc; 630 struct tx_ring *txr = sc->tx_rings; 631 struct ixgbe_hw *hw = &sc->hw; 632 int tx_hang = FALSE; 633 int i; 634 635 /* 636 * The timer is set to 5 every time ixgbe_start() queues a packet. 637 * Anytime all descriptors are clean the timer is set to 0. 638 */ 639 for (i = 0; i < sc->num_queues; i++, txr++) { 640 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) 641 continue; 642 else { 643 tx_hang = TRUE; 644 break; 645 } 646 } 647 if (tx_hang == FALSE) 648 return; 649 650 /* 651 * If we are in this routine because of pause frames, then don't 652 * reset the hardware. 653 */ 654 if (!(IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXON)) { 655 for (i = 0; i < sc->num_queues; i++, txr++) 656 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 657 ifp->if_timer = IXGBE_TX_TIMEOUT; 658 return; 659 } 660 661 662 printf("%s: Watchdog timeout -- resetting\n", ifp->if_xname); 663 for (i = 0; i < sc->num_queues; i++, txr++) { 664 printf("%s: Queue(%d) tdh = %d, hw tdt = %d\n", ifp->if_xname, i, 665 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 666 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 667 printf("%s: TX(%d) Next TX to Clean = %d\n", ifp->if_xname, 668 i, txr->next_to_clean); 669 } 670 ifp->if_flags &= ~IFF_RUNNING; 671 672 ixgbe_init(sc); 673 } 674 675 /********************************************************************* 676 * Init entry point 677 * 678 * This routine is used in two ways. It is used by the stack as 679 * init entry point in network interface structure. It is also used 680 * by the driver as a hw/sw initialization routine to get to a 681 * consistent state. 682 * 683 * return 0 on success, positive on failure 684 **********************************************************************/ 685 #define IXGBE_MHADD_MFS_SHIFT 16 686 687 void 688 ixgbe_init(void *arg) 689 { 690 struct ix_softc *sc = (struct ix_softc *)arg; 691 struct ifnet *ifp = &sc->arpcom.ac_if; 692 struct rx_ring *rxr = sc->rx_rings; 693 uint32_t k, txdctl, rxdctl, rxctrl, mhadd, itr; 694 int i, s, err; 695 696 INIT_DEBUGOUT("ixgbe_init: begin"); 697 698 s = splnet(); 699 700 ixgbe_stop(sc); 701 702 /* reprogram the RAR[0] in case user changed it. */ 703 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 704 705 /* Get the latest mac address, User can use a LAA */ 706 bcopy(sc->arpcom.ac_enaddr, sc->hw.mac.addr, 707 IXGBE_ETH_LENGTH_OF_ADDRESS); 708 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, 1); 709 sc->hw.addr_ctrl.rar_used_count = 1; 710 711 /* Prepare transmit descriptors and buffers */ 712 if (ixgbe_setup_transmit_structures(sc)) { 713 printf("%s: Could not setup transmit structures\n", 714 ifp->if_xname); 715 ixgbe_stop(sc); 716 splx(s); 717 return; 718 } 719 720 ixgbe_init_hw(&sc->hw); 721 ixgbe_initialize_transmit_units(sc); 722 723 /* Use 2k clusters, even for jumbo frames */ 724 sc->rx_mbuf_sz = MCLBYTES + ETHER_ALIGN; 725 726 /* Prepare receive descriptors and buffers */ 727 if (ixgbe_setup_receive_structures(sc)) { 728 printf("%s: Could not setup receive structures\n", 729 ifp->if_xname); 730 ixgbe_stop(sc); 731 splx(s); 732 return; 733 } 734 735 /* Configure RX settings */ 736 ixgbe_initialize_receive_units(sc); 737 738 /* Enable SDP & MSIX interrupts based on adapter */ 739 ixgbe_config_gpie(sc); 740 741 /* Program promiscuous mode and multicast filters. */ 742 ixgbe_iff(sc); 743 744 /* Set MRU size */ 745 mhadd = IXGBE_READ_REG(&sc->hw, IXGBE_MHADD); 746 mhadd &= ~IXGBE_MHADD_MFS_MASK; 747 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 748 IXGBE_WRITE_REG(&sc->hw, IXGBE_MHADD, mhadd); 749 750 /* Now enable all the queues */ 751 for (i = 0; i < sc->num_queues; i++) { 752 txdctl = IXGBE_READ_REG(&sc->hw, IXGBE_TXDCTL(i)); 753 txdctl |= IXGBE_TXDCTL_ENABLE; 754 /* Set WTHRESH to 8, burst writeback */ 755 txdctl |= (8 << 16); 756 /* 757 * When the internal queue falls below PTHRESH (16), 758 * start prefetching as long as there are at least 759 * HTHRESH (1) buffers ready. 760 */ 761 txdctl |= (16 << 0) | (1 << 8); 762 IXGBE_WRITE_REG(&sc->hw, IXGBE_TXDCTL(i), txdctl); 763 } 764 765 for (i = 0; i < sc->num_queues; i++) { 766 rxdctl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 767 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 768 /* 769 * PTHRESH = 21 770 * HTHRESH = 4 771 * WTHRESH = 8 772 */ 773 rxdctl &= ~0x3FFFFF; 774 rxdctl |= 0x080420; 775 } 776 rxdctl |= IXGBE_RXDCTL_ENABLE; 777 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), rxdctl); 778 for (k = 0; k < 10; k++) { 779 if (IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)) & 780 IXGBE_RXDCTL_ENABLE) 781 break; 782 else 783 msec_delay(1); 784 } 785 IXGBE_WRITE_FLUSH(&sc->hw); 786 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), rxr->last_desc_filled); 787 } 788 789 /* Set up VLAN support and filter */ 790 ixgbe_setup_vlan_hw_support(sc); 791 792 /* Enable Receive engine */ 793 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 794 if (sc->hw.mac.type == ixgbe_mac_82598EB) 795 rxctrl |= IXGBE_RXCTRL_DMBYPS; 796 rxctrl |= IXGBE_RXCTRL_RXEN; 797 sc->hw.mac.ops.enable_rx_dma(&sc->hw, rxctrl); 798 799 /* Set up MSI/X routing */ 800 if (sc->sc_intrmap) { 801 ixgbe_configure_ivars(sc); 802 /* Set up auto-mask */ 803 if (sc->hw.mac.type == ixgbe_mac_82598EB) 804 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 805 else { 806 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 807 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 808 } 809 } else { /* Simple settings for Legacy/MSI */ 810 ixgbe_set_ivar(sc, 0, 0, 0); 811 ixgbe_set_ivar(sc, 0, 0, 1); 812 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 813 } 814 815 /* Check on any SFP devices that need to be kick-started */ 816 if (sc->hw.phy.type == ixgbe_phy_none) { 817 err = sc->hw.phy.ops.identify(&sc->hw); 818 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 819 printf("Unsupported SFP+ module type was detected.\n"); 820 splx(s); 821 return; 822 } 823 } 824 825 /* Setup interrupt moderation */ 826 itr = (4000000 / IXGBE_INTS_PER_SEC) & 0xff8; 827 if (sc->hw.mac.type != ixgbe_mac_82598EB) 828 itr |= IXGBE_EITR_LLI_MOD | IXGBE_EITR_CNT_WDIS; 829 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(0), itr); 830 831 if (sc->sc_intrmap) { 832 /* Set moderation on the Link interrupt */ 833 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(sc->linkvec), 834 IXGBE_LINK_ITR); 835 } 836 837 /* Enable power to the phy */ 838 if (sc->hw.phy.ops.set_phy_power) 839 sc->hw.phy.ops.set_phy_power(&sc->hw, TRUE); 840 841 /* Config/Enable Link */ 842 ixgbe_config_link(sc); 843 844 /* Hardware Packet Buffer & Flow Control setup */ 845 ixgbe_config_delay_values(sc); 846 847 /* Initialize the FC settings */ 848 sc->hw.mac.ops.start_hw(&sc->hw); 849 850 /* And now turn on interrupts */ 851 ixgbe_enable_intr(sc); 852 ixgbe_enable_queues(sc); 853 854 /* Now inform the stack we're ready */ 855 ifp->if_flags |= IFF_RUNNING; 856 for (i = 0; i < sc->num_queues; i++) 857 ifq_clr_oactive(ifp->if_ifqs[i]); 858 859 #if NKSTAT > 0 860 ix_kstats_tick(sc); 861 #endif 862 863 splx(s); 864 } 865 866 void 867 ixgbe_config_gpie(struct ix_softc *sc) 868 { 869 struct ixgbe_hw *hw = &sc->hw; 870 uint32_t gpie; 871 872 gpie = IXGBE_READ_REG(&sc->hw, IXGBE_GPIE); 873 874 /* Fan Failure Interrupt */ 875 if (hw->device_id == IXGBE_DEV_ID_82598AT) 876 gpie |= IXGBE_SDP1_GPIEN; 877 878 if (sc->hw.mac.type == ixgbe_mac_82599EB) { 879 /* Add for Module detection */ 880 gpie |= IXGBE_SDP2_GPIEN; 881 882 /* Media ready */ 883 if (hw->device_id != IXGBE_DEV_ID_82599_QSFP_SF_QP) 884 gpie |= IXGBE_SDP1_GPIEN; 885 886 /* 887 * Set LL interval to max to reduce the number of low latency 888 * interrupts hitting the card when the ring is getting full. 889 */ 890 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 891 } 892 893 if (sc->hw.mac.type == ixgbe_mac_X540 || 894 sc->hw.mac.type == ixgbe_mac_X550EM_x || 895 sc->hw.mac.type == ixgbe_mac_X550EM_a) { 896 /* 897 * Thermal Failure Detection (X540) 898 * Link Detection (X552 SFP+, X552/X557-AT) 899 */ 900 gpie |= IXGBE_SDP0_GPIEN_X540; 901 902 /* 903 * Set LL interval to max to reduce the number of low latency 904 * interrupts hitting the card when the ring is getting full. 905 */ 906 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 907 } 908 909 if (sc->sc_intrmap) { 910 /* Enable Enhanced MSIX mode */ 911 gpie |= IXGBE_GPIE_MSIX_MODE; 912 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 913 IXGBE_GPIE_OCD; 914 } 915 916 IXGBE_WRITE_REG(&sc->hw, IXGBE_GPIE, gpie); 917 } 918 919 /* 920 * Requires sc->max_frame_size to be set. 921 */ 922 void 923 ixgbe_config_delay_values(struct ix_softc *sc) 924 { 925 struct ixgbe_hw *hw = &sc->hw; 926 uint32_t rxpb, frame, size, tmp; 927 928 frame = sc->max_frame_size; 929 930 /* Calculate High Water */ 931 switch (hw->mac.type) { 932 case ixgbe_mac_X540: 933 case ixgbe_mac_X550: 934 case ixgbe_mac_X550EM_x: 935 case ixgbe_mac_X550EM_a: 936 tmp = IXGBE_DV_X540(frame, frame); 937 break; 938 default: 939 tmp = IXGBE_DV(frame, frame); 940 break; 941 } 942 size = IXGBE_BT2KB(tmp); 943 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 944 hw->fc.high_water[0] = rxpb - size; 945 946 /* Now calculate Low Water */ 947 switch (hw->mac.type) { 948 case ixgbe_mac_X540: 949 case ixgbe_mac_X550: 950 case ixgbe_mac_X550EM_x: 951 case ixgbe_mac_X550EM_a: 952 tmp = IXGBE_LOW_DV_X540(frame); 953 break; 954 default: 955 tmp = IXGBE_LOW_DV(frame); 956 break; 957 } 958 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 959 960 hw->fc.requested_mode = sc->fc; 961 hw->fc.pause_time = IXGBE_FC_PAUSE; 962 hw->fc.send_xon = TRUE; 963 } 964 965 /* 966 * MSIX Interrupt Handlers 967 */ 968 void 969 ixgbe_enable_queue(struct ix_softc *sc, uint32_t vector) 970 { 971 uint64_t queue = 1ULL << vector; 972 uint32_t mask; 973 974 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 975 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 976 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, mask); 977 } else { 978 mask = (queue & 0xFFFFFFFF); 979 if (mask) 980 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(0), mask); 981 mask = (queue >> 32); 982 if (mask) 983 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(1), mask); 984 } 985 } 986 987 void 988 ixgbe_enable_queues(struct ix_softc *sc) 989 { 990 struct ix_queue *que; 991 int i; 992 993 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 994 ixgbe_enable_queue(sc, que->msix); 995 } 996 997 void 998 ixgbe_disable_queue(struct ix_softc *sc, uint32_t vector) 999 { 1000 uint64_t queue = 1ULL << vector; 1001 uint32_t mask; 1002 1003 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 1004 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1005 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, mask); 1006 } else { 1007 mask = (queue & 0xFFFFFFFF); 1008 if (mask) 1009 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), mask); 1010 mask = (queue >> 32); 1011 if (mask) 1012 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), mask); 1013 } 1014 } 1015 1016 /* 1017 * MSIX Interrupt Handlers 1018 */ 1019 int 1020 ixgbe_link_intr(void *vsc) 1021 { 1022 struct ix_softc *sc = (struct ix_softc *)vsc; 1023 1024 return ixgbe_intr(sc); 1025 } 1026 1027 int 1028 ixgbe_queue_intr(void *vque) 1029 { 1030 struct ix_queue *que = vque; 1031 struct ix_softc *sc = que->sc; 1032 struct ifnet *ifp = &sc->arpcom.ac_if; 1033 struct rx_ring *rxr = que->rxr; 1034 struct tx_ring *txr = que->txr; 1035 1036 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1037 ixgbe_rxeof(rxr); 1038 ixgbe_txeof(txr); 1039 ixgbe_rxrefill(rxr); 1040 } 1041 1042 ixgbe_enable_queue(sc, que->msix); 1043 1044 return (1); 1045 } 1046 1047 /********************************************************************* 1048 * 1049 * Legacy Interrupt Service routine 1050 * 1051 **********************************************************************/ 1052 1053 int 1054 ixgbe_legacy_intr(void *arg) 1055 { 1056 struct ix_softc *sc = (struct ix_softc *)arg; 1057 struct ifnet *ifp = &sc->arpcom.ac_if; 1058 struct rx_ring *rxr = sc->rx_rings; 1059 struct tx_ring *txr = sc->tx_rings; 1060 int rv; 1061 1062 rv = ixgbe_intr(sc); 1063 if (rv == 0) { 1064 return (0); 1065 } 1066 1067 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1068 ixgbe_rxeof(rxr); 1069 ixgbe_txeof(txr); 1070 ixgbe_rxrefill(rxr); 1071 } 1072 1073 ixgbe_enable_queues(sc); 1074 return (rv); 1075 } 1076 1077 int 1078 ixgbe_intr(struct ix_softc *sc) 1079 { 1080 struct ifnet *ifp = &sc->arpcom.ac_if; 1081 struct ixgbe_hw *hw = &sc->hw; 1082 uint32_t reg_eicr, mod_mask, msf_mask; 1083 1084 if (sc->sc_intrmap) { 1085 /* Pause other interrupts */ 1086 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_OTHER); 1087 /* First get the cause */ 1088 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1089 /* Be sure the queue bits are not cleared */ 1090 reg_eicr &= ~IXGBE_EICR_RTX_QUEUE; 1091 /* Clear interrupt with write */ 1092 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1093 } else { 1094 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1095 if (reg_eicr == 0) { 1096 ixgbe_enable_intr(sc); 1097 ixgbe_enable_queues(sc); 1098 return (0); 1099 } 1100 } 1101 1102 /* Link status change */ 1103 if (reg_eicr & IXGBE_EICR_LSC) { 1104 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC); 1105 KERNEL_LOCK(); 1106 ixgbe_update_link_status(sc); 1107 KERNEL_UNLOCK(); 1108 } 1109 1110 if (hw->mac.type != ixgbe_mac_82598EB) { 1111 if (reg_eicr & IXGBE_EICR_ECC) { 1112 printf("%s: CRITICAL: ECC ERROR!! " 1113 "Please Reboot!!\n", sc->dev.dv_xname); 1114 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1115 } 1116 /* Check for over temp condition */ 1117 if (reg_eicr & IXGBE_EICR_TS) { 1118 printf("%s: CRITICAL: OVER TEMP!! " 1119 "PHY IS SHUT DOWN!!\n", ifp->if_xname); 1120 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS); 1121 } 1122 } 1123 1124 /* Pluggable optics-related interrupt */ 1125 if (ixgbe_is_sfp(hw)) { 1126 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP) { 1127 mod_mask = IXGBE_EICR_GPI_SDP0_X540; 1128 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1129 } else if (hw->mac.type == ixgbe_mac_X540 || 1130 hw->mac.type == ixgbe_mac_X550 || 1131 hw->mac.type == ixgbe_mac_X550EM_x) { 1132 mod_mask = IXGBE_EICR_GPI_SDP2_X540; 1133 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1134 } else { 1135 mod_mask = IXGBE_EICR_GPI_SDP2; 1136 msf_mask = IXGBE_EICR_GPI_SDP1; 1137 } 1138 if (reg_eicr & mod_mask) { 1139 /* Clear the interrupt */ 1140 IXGBE_WRITE_REG(hw, IXGBE_EICR, mod_mask); 1141 KERNEL_LOCK(); 1142 ixgbe_handle_mod(sc); 1143 KERNEL_UNLOCK(); 1144 } else if ((hw->phy.media_type != ixgbe_media_type_copper) && 1145 (reg_eicr & msf_mask)) { 1146 /* Clear the interrupt */ 1147 IXGBE_WRITE_REG(hw, IXGBE_EICR, msf_mask); 1148 KERNEL_LOCK(); 1149 ixgbe_handle_msf(sc); 1150 KERNEL_UNLOCK(); 1151 } 1152 } 1153 1154 /* Check for fan failure */ 1155 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1156 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1157 printf("%s: CRITICAL: FAN FAILURE!! " 1158 "REPLACE IMMEDIATELY!!\n", ifp->if_xname); 1159 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1160 } 1161 1162 /* External PHY interrupt */ 1163 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 1164 (reg_eicr & IXGBE_EICR_GPI_SDP0_X540)) { 1165 /* Clear the interrupt */ 1166 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0_X540); 1167 KERNEL_LOCK(); 1168 ixgbe_handle_phy(sc); 1169 KERNEL_UNLOCK(); 1170 } 1171 1172 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 1173 1174 return (1); 1175 } 1176 1177 /********************************************************************* 1178 * 1179 * Media Ioctl callback 1180 * 1181 * This routine is called whenever the user queries the status of 1182 * the interface using ifconfig. 1183 * 1184 **********************************************************************/ 1185 void 1186 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq *ifmr) 1187 { 1188 struct ix_softc *sc = ifp->if_softc; 1189 uint64_t layer; 1190 1191 ifmr->ifm_active = IFM_ETHER; 1192 ifmr->ifm_status = IFM_AVALID; 1193 1194 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1195 ixgbe_update_link_status(sc); 1196 1197 if (!LINK_STATE_IS_UP(ifp->if_link_state)) 1198 return; 1199 1200 ifmr->ifm_status |= IFM_ACTIVE; 1201 layer = sc->phy_layer; 1202 1203 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 1204 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 1205 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 1206 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) { 1207 switch (sc->link_speed) { 1208 case IXGBE_LINK_SPEED_10GB_FULL: 1209 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1210 break; 1211 case IXGBE_LINK_SPEED_1GB_FULL: 1212 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1213 break; 1214 case IXGBE_LINK_SPEED_100_FULL: 1215 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1216 break; 1217 case IXGBE_LINK_SPEED_10_FULL: 1218 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1219 break; 1220 } 1221 } 1222 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1223 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) { 1224 switch (sc->link_speed) { 1225 case IXGBE_LINK_SPEED_10GB_FULL: 1226 ifmr->ifm_active |= IFM_10G_SFP_CU | IFM_FDX; 1227 break; 1228 } 1229 } 1230 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1231 switch (sc->link_speed) { 1232 case IXGBE_LINK_SPEED_10GB_FULL: 1233 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 1234 break; 1235 case IXGBE_LINK_SPEED_1GB_FULL: 1236 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 1237 break; 1238 } 1239 } 1240 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 1241 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 1242 switch (sc->link_speed) { 1243 case IXGBE_LINK_SPEED_10GB_FULL: 1244 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 1245 break; 1246 case IXGBE_LINK_SPEED_1GB_FULL: 1247 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 1248 break; 1249 } 1250 } 1251 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1252 switch (sc->link_speed) { 1253 case IXGBE_LINK_SPEED_10GB_FULL: 1254 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 1255 break; 1256 } 1257 } 1258 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1259 switch (sc->link_speed) { 1260 case IXGBE_LINK_SPEED_10GB_FULL: 1261 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 1262 break; 1263 case IXGBE_LINK_SPEED_2_5GB_FULL: 1264 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1265 break; 1266 case IXGBE_LINK_SPEED_1GB_FULL: 1267 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1268 break; 1269 } 1270 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 1271 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 1272 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 1273 switch (sc->link_speed) { 1274 case IXGBE_LINK_SPEED_10GB_FULL: 1275 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 1276 break; 1277 case IXGBE_LINK_SPEED_2_5GB_FULL: 1278 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1279 break; 1280 case IXGBE_LINK_SPEED_1GB_FULL: 1281 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1282 break; 1283 } 1284 } 1285 1286 switch (sc->hw.fc.current_mode) { 1287 case ixgbe_fc_tx_pause: 1288 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 1289 break; 1290 case ixgbe_fc_rx_pause: 1291 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 1292 break; 1293 case ixgbe_fc_full: 1294 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE | 1295 IFM_ETH_TXPAUSE; 1296 break; 1297 default: 1298 ifmr->ifm_active &= ~(IFM_FLOW | IFM_ETH_RXPAUSE | 1299 IFM_ETH_TXPAUSE); 1300 break; 1301 } 1302 } 1303 1304 /********************************************************************* 1305 * 1306 * Media Ioctl callback 1307 * 1308 * This routine is called when the user changes speed/duplex using 1309 * media/mediopt option with ifconfig. 1310 * 1311 **********************************************************************/ 1312 int 1313 ixgbe_media_change(struct ifnet *ifp) 1314 { 1315 struct ix_softc *sc = ifp->if_softc; 1316 struct ixgbe_hw *hw = &sc->hw; 1317 struct ifmedia *ifm = &sc->media; 1318 ixgbe_link_speed speed = 0; 1319 1320 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1321 return (EINVAL); 1322 1323 if (hw->phy.media_type == ixgbe_media_type_backplane) 1324 return (ENODEV); 1325 1326 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1327 case IFM_AUTO: 1328 case IFM_10G_T: 1329 speed |= IXGBE_LINK_SPEED_100_FULL; 1330 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1331 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1332 break; 1333 case IFM_10G_SR: 1334 case IFM_10G_KR: 1335 case IFM_10G_LR: 1336 case IFM_10G_LRM: 1337 case IFM_10G_CX4: 1338 case IFM_10G_KX4: 1339 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1340 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1341 break; 1342 case IFM_10G_SFP_CU: 1343 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1344 break; 1345 case IFM_1000_T: 1346 speed |= IXGBE_LINK_SPEED_100_FULL; 1347 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1348 break; 1349 case IFM_1000_LX: 1350 case IFM_1000_SX: 1351 case IFM_1000_CX: 1352 case IFM_1000_KX: 1353 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1354 break; 1355 case IFM_100_TX: 1356 speed |= IXGBE_LINK_SPEED_100_FULL; 1357 break; 1358 case IFM_10_T: 1359 speed |= IXGBE_LINK_SPEED_10_FULL; 1360 break; 1361 default: 1362 return (EINVAL); 1363 } 1364 1365 hw->mac.autotry_restart = TRUE; 1366 hw->mac.ops.setup_link(hw, speed, TRUE); 1367 1368 return (0); 1369 } 1370 1371 /********************************************************************* 1372 * 1373 * This routine maps the mbufs to tx descriptors, allowing the 1374 * TX engine to transmit the packets. 1375 * - return 0 on success, positive on failure 1376 * 1377 **********************************************************************/ 1378 1379 int 1380 ixgbe_encap(struct tx_ring *txr, struct mbuf *m_head) 1381 { 1382 struct ix_softc *sc = txr->sc; 1383 uint32_t olinfo_status = 0, cmd_type_len; 1384 int i, j, ntxc; 1385 int first, last = 0; 1386 bus_dmamap_t map; 1387 struct ixgbe_tx_buf *txbuf; 1388 union ixgbe_adv_tx_desc *txd = NULL; 1389 1390 /* Basic descriptor defines */ 1391 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1392 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1393 1394 #if NVLAN > 0 1395 if (m_head->m_flags & M_VLANTAG) 1396 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1397 #endif 1398 1399 /* 1400 * Important to capture the first descriptor 1401 * used because it will contain the index of 1402 * the one we tell the hardware to report back 1403 */ 1404 first = txr->next_avail_desc; 1405 txbuf = &txr->tx_buffers[first]; 1406 map = txbuf->map; 1407 1408 /* 1409 * Map the packet for DMA. 1410 */ 1411 switch (bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1412 m_head, BUS_DMA_NOWAIT)) { 1413 case 0: 1414 break; 1415 case EFBIG: 1416 if (m_defrag(m_head, M_NOWAIT) == 0 && 1417 bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1418 m_head, BUS_DMA_NOWAIT) == 0) 1419 break; 1420 /* FALLTHROUGH */ 1421 default: 1422 return (0); 1423 } 1424 1425 /* 1426 * Set the appropriate offload context 1427 * this will becomes the first descriptor. 1428 */ 1429 ntxc = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); 1430 if (ntxc == -1) 1431 goto xmit_fail; 1432 1433 i = txr->next_avail_desc + ntxc; 1434 if (i >= sc->num_tx_desc) 1435 i -= sc->num_tx_desc; 1436 1437 for (j = 0; j < map->dm_nsegs; j++) { 1438 txd = &txr->tx_base[i]; 1439 1440 txd->read.buffer_addr = htole64(map->dm_segs[j].ds_addr); 1441 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1442 cmd_type_len | map->dm_segs[j].ds_len); 1443 txd->read.olinfo_status = htole32(olinfo_status); 1444 last = i; /* descriptor that will get completion IRQ */ 1445 1446 if (++i == sc->num_tx_desc) 1447 i = 0; 1448 } 1449 1450 txd->read.cmd_type_len |= 1451 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1452 1453 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1454 BUS_DMASYNC_PREWRITE); 1455 1456 /* Set the index of the descriptor that will be marked done */ 1457 txbuf->m_head = m_head; 1458 txbuf->eop_index = last; 1459 1460 membar_producer(); 1461 1462 txr->next_avail_desc = i; 1463 1464 return (ntxc + j); 1465 1466 xmit_fail: 1467 bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map); 1468 return (0); 1469 } 1470 1471 void 1472 ixgbe_iff(struct ix_softc *sc) 1473 { 1474 struct ifnet *ifp = &sc->arpcom.ac_if; 1475 struct arpcom *ac = &sc->arpcom; 1476 uint32_t fctrl; 1477 uint8_t *mta; 1478 uint8_t *update_ptr; 1479 struct ether_multi *enm; 1480 struct ether_multistep step; 1481 int mcnt = 0; 1482 1483 IOCTL_DEBUGOUT("ixgbe_iff: begin"); 1484 1485 mta = sc->mta; 1486 bzero(mta, sizeof(uint8_t) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1487 MAX_NUM_MULTICAST_ADDRESSES); 1488 1489 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1490 fctrl &= ~(IXGBE_FCTRL_MPE | IXGBE_FCTRL_UPE); 1491 ifp->if_flags &= ~IFF_ALLMULTI; 1492 1493 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 || 1494 ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) { 1495 ifp->if_flags |= IFF_ALLMULTI; 1496 fctrl |= IXGBE_FCTRL_MPE; 1497 if (ifp->if_flags & IFF_PROMISC) 1498 fctrl |= IXGBE_FCTRL_UPE; 1499 } else { 1500 ETHER_FIRST_MULTI(step, &sc->arpcom, enm); 1501 while (enm != NULL) { 1502 bcopy(enm->enm_addrlo, 1503 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1504 IXGBE_ETH_LENGTH_OF_ADDRESS); 1505 mcnt++; 1506 1507 ETHER_NEXT_MULTI(step, enm); 1508 } 1509 1510 update_ptr = mta; 1511 sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt, 1512 ixgbe_mc_array_itr, TRUE); 1513 } 1514 1515 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1516 } 1517 1518 /* 1519 * This is an iterator function now needed by the multicast 1520 * shared code. It simply feeds the shared code routine the 1521 * addresses in the array of ixgbe_iff() one by one. 1522 */ 1523 uint8_t * 1524 ixgbe_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1525 { 1526 uint8_t *addr = *update_ptr; 1527 uint8_t *newptr; 1528 *vmdq = 0; 1529 1530 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1531 *update_ptr = newptr; 1532 return addr; 1533 } 1534 1535 void 1536 ixgbe_update_link_status(struct ix_softc *sc) 1537 { 1538 struct ifnet *ifp = &sc->arpcom.ac_if; 1539 int link_state = LINK_STATE_DOWN; 1540 1541 ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0); 1542 1543 ifp->if_baudrate = 0; 1544 if (sc->link_up) { 1545 link_state = LINK_STATE_FULL_DUPLEX; 1546 1547 switch (sc->link_speed) { 1548 case IXGBE_LINK_SPEED_UNKNOWN: 1549 ifp->if_baudrate = 0; 1550 break; 1551 case IXGBE_LINK_SPEED_100_FULL: 1552 ifp->if_baudrate = IF_Mbps(100); 1553 break; 1554 case IXGBE_LINK_SPEED_1GB_FULL: 1555 ifp->if_baudrate = IF_Gbps(1); 1556 break; 1557 case IXGBE_LINK_SPEED_10GB_FULL: 1558 ifp->if_baudrate = IF_Gbps(10); 1559 break; 1560 } 1561 1562 /* Update any Flow Control changes */ 1563 sc->hw.mac.ops.fc_enable(&sc->hw); 1564 } 1565 if (ifp->if_link_state != link_state) { 1566 ifp->if_link_state = link_state; 1567 if_link_state_change(ifp); 1568 } 1569 } 1570 1571 1572 /********************************************************************* 1573 * 1574 * This routine disables all traffic on the adapter by issuing a 1575 * global reset on the MAC and deallocates TX/RX buffers. 1576 * 1577 **********************************************************************/ 1578 1579 void 1580 ixgbe_stop(void *arg) 1581 { 1582 struct ix_softc *sc = arg; 1583 struct ifnet *ifp = &sc->arpcom.ac_if; 1584 int i; 1585 1586 /* Tell the stack that the interface is no longer active */ 1587 ifp->if_flags &= ~IFF_RUNNING; 1588 1589 #if NKSTAT > 0 1590 timeout_del(&sc->sc_kstat_tmo); 1591 #endif 1592 1593 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1594 ixgbe_disable_intr(sc); 1595 1596 sc->hw.mac.ops.reset_hw(&sc->hw); 1597 sc->hw.adapter_stopped = FALSE; 1598 sc->hw.mac.ops.stop_adapter(&sc->hw); 1599 if (sc->hw.mac.type == ixgbe_mac_82599EB) 1600 sc->hw.mac.ops.stop_mac_link_on_d3(&sc->hw); 1601 /* Turn off the laser */ 1602 if (sc->hw.mac.ops.disable_tx_laser) 1603 sc->hw.mac.ops.disable_tx_laser(&sc->hw); 1604 1605 /* reprogram the RAR[0] in case user changed it. */ 1606 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 1607 1608 intr_barrier(sc->tag); 1609 for (i = 0; i < sc->num_queues; i++) { 1610 struct ifqueue *ifq = ifp->if_ifqs[i]; 1611 ifq_barrier(ifq); 1612 ifq_clr_oactive(ifq); 1613 1614 if (sc->queues[i].tag != NULL) 1615 intr_barrier(sc->queues[i].tag); 1616 timeout_del(&sc->rx_rings[i].rx_refill); 1617 } 1618 1619 KASSERT((ifp->if_flags & IFF_RUNNING) == 0); 1620 1621 /* Should we really clear all structures on stop? */ 1622 ixgbe_free_transmit_structures(sc); 1623 ixgbe_free_receive_structures(sc); 1624 1625 ixgbe_update_link_status(sc); 1626 } 1627 1628 1629 /********************************************************************* 1630 * 1631 * Determine hardware revision. 1632 * 1633 **********************************************************************/ 1634 void 1635 ixgbe_identify_hardware(struct ix_softc *sc) 1636 { 1637 struct ixgbe_osdep *os = &sc->osdep; 1638 struct pci_attach_args *pa = &os->os_pa; 1639 uint32_t reg; 1640 1641 /* Save off the information about this board */ 1642 sc->hw.vendor_id = PCI_VENDOR(pa->pa_id); 1643 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 1644 1645 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG); 1646 sc->hw.revision_id = PCI_REVISION(reg); 1647 1648 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 1649 sc->hw.subsystem_vendor_id = PCI_VENDOR(reg); 1650 sc->hw.subsystem_device_id = PCI_PRODUCT(reg); 1651 1652 /* We need this here to set the num_segs below */ 1653 ixgbe_set_mac_type(&sc->hw); 1654 1655 /* Pick up the 82599 and VF settings */ 1656 if (sc->hw.mac.type != ixgbe_mac_82598EB) 1657 sc->hw.phy.smart_speed = ixgbe_smart_speed; 1658 sc->num_segs = IXGBE_82599_SCATTER; 1659 } 1660 1661 /********************************************************************* 1662 * 1663 * Setup the Legacy or MSI Interrupt handler 1664 * 1665 **********************************************************************/ 1666 int 1667 ixgbe_allocate_legacy(struct ix_softc *sc) 1668 { 1669 struct ixgbe_osdep *os = &sc->osdep; 1670 struct pci_attach_args *pa = &os->os_pa; 1671 const char *intrstr = NULL; 1672 pci_chipset_tag_t pc = pa->pa_pc; 1673 pci_intr_handle_t ih; 1674 1675 /* We allocate a single interrupt resource */ 1676 if (pci_intr_map_msi(pa, &ih) != 0 && 1677 pci_intr_map(pa, &ih) != 0) { 1678 printf(": couldn't map interrupt\n"); 1679 return (ENXIO); 1680 } 1681 1682 #if 0 1683 /* XXX */ 1684 /* Tasklets for Link, SFP and Multispeed Fiber */ 1685 TASK_INIT(&sc->link_task, 0, ixgbe_handle_link, sc); 1686 TASK_INIT(&sc->mod_task, 0, ixgbe_handle_mod, sc); 1687 TASK_INIT(&sc->msf_task, 0, ixgbe_handle_msf, sc); 1688 #endif 1689 1690 intrstr = pci_intr_string(pc, ih); 1691 sc->tag = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE, 1692 ixgbe_legacy_intr, sc, sc->dev.dv_xname); 1693 if (sc->tag == NULL) { 1694 printf(": couldn't establish interrupt"); 1695 if (intrstr != NULL) 1696 printf(" at %s", intrstr); 1697 printf("\n"); 1698 return (ENXIO); 1699 } 1700 printf(": %s", intrstr); 1701 1702 /* For simplicity in the handlers */ 1703 sc->que_mask = IXGBE_EIMS_ENABLE_MASK; 1704 1705 return (0); 1706 } 1707 1708 /********************************************************************* 1709 * 1710 * Setup the MSI-X Interrupt handlers 1711 * 1712 **********************************************************************/ 1713 int 1714 ixgbe_allocate_msix(struct ix_softc *sc) 1715 { 1716 struct ixgbe_osdep *os = &sc->osdep; 1717 struct pci_attach_args *pa = &os->os_pa; 1718 int i = 0, error = 0; 1719 struct ix_queue *que; 1720 pci_intr_handle_t ih; 1721 1722 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) { 1723 if (pci_intr_map_msix(pa, i, &ih)) { 1724 printf("ixgbe_allocate_msix: " 1725 "pci_intr_map_msix vec %d failed\n", i); 1726 error = ENOMEM; 1727 goto fail; 1728 } 1729 1730 que->tag = pci_intr_establish_cpu(pa->pa_pc, ih, 1731 IPL_NET | IPL_MPSAFE, intrmap_cpu(sc->sc_intrmap, i), 1732 ixgbe_queue_intr, que, que->name); 1733 if (que->tag == NULL) { 1734 printf("ixgbe_allocate_msix: " 1735 "pci_intr_establish vec %d failed\n", i); 1736 error = ENOMEM; 1737 goto fail; 1738 } 1739 1740 que->msix = i; 1741 } 1742 1743 /* Now the link status/control last MSI-X vector */ 1744 if (pci_intr_map_msix(pa, i, &ih)) { 1745 printf("ixgbe_allocate_msix: " 1746 "pci_intr_map_msix link vector failed\n"); 1747 error = ENOMEM; 1748 goto fail; 1749 } 1750 1751 sc->tag = pci_intr_establish(pa->pa_pc, ih, IPL_NET | IPL_MPSAFE, 1752 ixgbe_link_intr, sc, sc->dev.dv_xname); 1753 if (sc->tag == NULL) { 1754 printf("ixgbe_allocate_msix: " 1755 "pci_intr_establish link vector failed\n"); 1756 error = ENOMEM; 1757 goto fail; 1758 } 1759 sc->linkvec = i; 1760 printf(", %s, %d queue%s", pci_intr_string(pa->pa_pc, ih), 1761 i, (i > 1) ? "s" : ""); 1762 1763 return (0); 1764 fail: 1765 for (que = sc->queues; i > 0; i--, que++) { 1766 if (que->tag == NULL) 1767 continue; 1768 pci_intr_disestablish(pa->pa_pc, que->tag); 1769 que->tag = NULL; 1770 } 1771 1772 return (error); 1773 } 1774 1775 void 1776 ixgbe_setup_msix(struct ix_softc *sc) 1777 { 1778 struct ixgbe_osdep *os = &sc->osdep; 1779 struct pci_attach_args *pa = &os->os_pa; 1780 int nmsix; 1781 unsigned int maxq; 1782 1783 if (!ixgbe_enable_msix) 1784 return; 1785 1786 nmsix = pci_intr_msix_count(pa->pa_pc, pa->pa_tag); 1787 if (nmsix <= 1) 1788 return; 1789 1790 /* give one vector to events */ 1791 nmsix--; 1792 1793 /* XXX the number of queues is limited to what we can keep stats on */ 1794 maxq = (sc->hw.mac.type == ixgbe_mac_82598EB) ? 8 : 16; 1795 1796 sc->sc_intrmap = intrmap_create(&sc->dev, nmsix, maxq, 0); 1797 sc->num_queues = intrmap_count(sc->sc_intrmap); 1798 } 1799 1800 int 1801 ixgbe_allocate_pci_resources(struct ix_softc *sc) 1802 { 1803 struct ixgbe_osdep *os = &sc->osdep; 1804 struct pci_attach_args *pa = &os->os_pa; 1805 int val; 1806 1807 val = pci_conf_read(pa->pa_pc, pa->pa_tag, PCIR_BAR(0)); 1808 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM) { 1809 printf(": mmba is not mem space\n"); 1810 return (ENXIO); 1811 } 1812 1813 if (pci_mapreg_map(pa, PCIR_BAR(0), PCI_MAPREG_MEM_TYPE(val), 0, 1814 &os->os_memt, &os->os_memh, &os->os_membase, &os->os_memsize, 0)) { 1815 printf(": cannot find mem space\n"); 1816 return (ENXIO); 1817 } 1818 sc->hw.hw_addr = (uint8_t *)os->os_membase; 1819 1820 /* Legacy defaults */ 1821 sc->num_queues = 1; 1822 sc->hw.back = os; 1823 1824 /* Now setup MSI or MSI/X, return us the number of supported vectors. */ 1825 ixgbe_setup_msix(sc); 1826 1827 return (0); 1828 } 1829 1830 void 1831 ixgbe_free_pci_resources(struct ix_softc * sc) 1832 { 1833 struct ixgbe_osdep *os = &sc->osdep; 1834 struct pci_attach_args *pa = &os->os_pa; 1835 struct ix_queue *que = sc->queues; 1836 int i; 1837 1838 /* Release all msix queue resources: */ 1839 for (i = 0; i < sc->num_queues; i++, que++) { 1840 if (que->tag) 1841 pci_intr_disestablish(pa->pa_pc, que->tag); 1842 que->tag = NULL; 1843 } 1844 1845 if (sc->tag) 1846 pci_intr_disestablish(pa->pa_pc, sc->tag); 1847 sc->tag = NULL; 1848 if (os->os_membase != 0) 1849 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 1850 os->os_membase = 0; 1851 } 1852 1853 /********************************************************************* 1854 * 1855 * Setup networking device structure and register an interface. 1856 * 1857 **********************************************************************/ 1858 void 1859 ixgbe_setup_interface(struct ix_softc *sc) 1860 { 1861 struct ifnet *ifp = &sc->arpcom.ac_if; 1862 int i; 1863 1864 strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ); 1865 ifp->if_softc = sc; 1866 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1867 ifp->if_xflags = IFXF_MPSAFE; 1868 ifp->if_ioctl = ixgbe_ioctl; 1869 ifp->if_qstart = ixgbe_start; 1870 ifp->if_timer = 0; 1871 ifp->if_watchdog = ixgbe_watchdog; 1872 ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE - 1873 ETHER_HDR_LEN - ETHER_CRC_LEN; 1874 ifq_set_maxlen(&ifp->if_snd, sc->num_tx_desc - 1); 1875 1876 ifp->if_capabilities = IFCAP_VLAN_MTU; 1877 1878 #if NVLAN > 0 1879 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 1880 #endif 1881 1882 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4; 1883 1884 /* 1885 * Specify the media types supported by this sc and register 1886 * callbacks to update media and link information 1887 */ 1888 ifmedia_init(&sc->media, IFM_IMASK, ixgbe_media_change, 1889 ixgbe_media_status); 1890 ixgbe_add_media_types(sc); 1891 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1892 1893 if_attach(ifp); 1894 ether_ifattach(ifp); 1895 1896 if_attach_queues(ifp, sc->num_queues); 1897 if_attach_iqueues(ifp, sc->num_queues); 1898 for (i = 0; i < sc->num_queues; i++) { 1899 struct ifqueue *ifq = ifp->if_ifqs[i]; 1900 struct ifiqueue *ifiq = ifp->if_iqs[i]; 1901 struct tx_ring *txr = &sc->tx_rings[i]; 1902 struct rx_ring *rxr = &sc->rx_rings[i]; 1903 1904 ifq->ifq_softc = txr; 1905 txr->ifq = ifq; 1906 1907 ifiq->ifiq_softc = rxr; 1908 rxr->ifiq = ifiq; 1909 1910 #if NKSTAT > 0 1911 ix_txq_kstats(sc, txr); 1912 ix_rxq_kstats(sc, rxr); 1913 #endif 1914 } 1915 1916 sc->max_frame_size = IXGBE_MAX_FRAME_SIZE; 1917 } 1918 1919 void 1920 ixgbe_add_media_types(struct ix_softc *sc) 1921 { 1922 struct ixgbe_hw *hw = &sc->hw; 1923 uint64_t layer; 1924 1925 sc->phy_layer = hw->mac.ops.get_supported_physical_layer(hw); 1926 layer = sc->phy_layer; 1927 1928 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) 1929 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T, 0, NULL); 1930 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) 1931 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 1932 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) 1933 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1934 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1935 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 1936 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SFP_CU, 0, NULL); 1937 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1938 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_LR, 0, NULL); 1939 if (hw->phy.multispeed_fiber) 1940 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_LX, 0, 1941 NULL); 1942 } 1943 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1944 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL); 1945 if (hw->phy.multispeed_fiber) 1946 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, 1947 NULL); 1948 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 1949 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 1950 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 1951 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL); 1952 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 1953 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KR, 0, NULL); 1954 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) 1955 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); 1956 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 1957 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_KX, 0, NULL); 1958 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) 1959 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_KX, 0, NULL); 1960 1961 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 1962 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, 1963 NULL); 1964 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 1965 } 1966 1967 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1968 } 1969 1970 void 1971 ixgbe_config_link(struct ix_softc *sc) 1972 { 1973 uint32_t autoneg, err = 0; 1974 bool negotiate; 1975 1976 if (ixgbe_is_sfp(&sc->hw)) { 1977 if (sc->hw.phy.multispeed_fiber) { 1978 sc->hw.mac.ops.setup_sfp(&sc->hw); 1979 if (sc->hw.mac.ops.enable_tx_laser) 1980 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 1981 ixgbe_handle_msf(sc); 1982 } else 1983 ixgbe_handle_mod(sc); 1984 } else { 1985 if (sc->hw.mac.ops.check_link) 1986 err = sc->hw.mac.ops.check_link(&sc->hw, &autoneg, 1987 &sc->link_up, FALSE); 1988 if (err) 1989 return; 1990 autoneg = sc->hw.phy.autoneg_advertised; 1991 if ((!autoneg) && (sc->hw.mac.ops.get_link_capabilities)) 1992 err = sc->hw.mac.ops.get_link_capabilities(&sc->hw, 1993 &autoneg, &negotiate); 1994 if (err) 1995 return; 1996 if (sc->hw.mac.ops.setup_link) 1997 sc->hw.mac.ops.setup_link(&sc->hw, 1998 autoneg, sc->link_up); 1999 } 2000 } 2001 2002 /******************************************************************** 2003 * Manage DMA'able memory. 2004 *******************************************************************/ 2005 int 2006 ixgbe_dma_malloc(struct ix_softc *sc, bus_size_t size, 2007 struct ixgbe_dma_alloc *dma, int mapflags) 2008 { 2009 struct ifnet *ifp = &sc->arpcom.ac_if; 2010 struct ixgbe_osdep *os = &sc->osdep; 2011 int r; 2012 2013 dma->dma_tag = os->os_pa.pa_dmat; 2014 r = bus_dmamap_create(dma->dma_tag, size, 1, 2015 size, 0, BUS_DMA_NOWAIT, &dma->dma_map); 2016 if (r != 0) { 2017 printf("%s: ixgbe_dma_malloc: bus_dmamap_create failed; " 2018 "error %u\n", ifp->if_xname, r); 2019 goto fail_0; 2020 } 2021 2022 r = bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 2023 1, &dma->dma_nseg, BUS_DMA_NOWAIT); 2024 if (r != 0) { 2025 printf("%s: ixgbe_dma_malloc: bus_dmamem_alloc failed; " 2026 "error %u\n", ifp->if_xname, r); 2027 goto fail_1; 2028 } 2029 2030 r = bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 2031 &dma->dma_vaddr, BUS_DMA_NOWAIT); 2032 if (r != 0) { 2033 printf("%s: ixgbe_dma_malloc: bus_dmamem_map failed; " 2034 "error %u\n", ifp->if_xname, r); 2035 goto fail_2; 2036 } 2037 2038 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2039 size, NULL, mapflags | BUS_DMA_NOWAIT); 2040 if (r != 0) { 2041 printf("%s: ixgbe_dma_malloc: bus_dmamap_load failed; " 2042 "error %u\n", ifp->if_xname, r); 2043 goto fail_3; 2044 } 2045 2046 dma->dma_size = size; 2047 return (0); 2048 fail_3: 2049 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 2050 fail_2: 2051 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2052 fail_1: 2053 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2054 fail_0: 2055 dma->dma_map = NULL; 2056 dma->dma_tag = NULL; 2057 return (r); 2058 } 2059 2060 void 2061 ixgbe_dma_free(struct ix_softc *sc, struct ixgbe_dma_alloc *dma) 2062 { 2063 if (dma->dma_tag == NULL) 2064 return; 2065 2066 if (dma->dma_map != NULL) { 2067 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 2068 dma->dma_map->dm_mapsize, 2069 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2070 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2071 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 2072 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2073 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2074 dma->dma_map = NULL; 2075 } 2076 } 2077 2078 2079 /********************************************************************* 2080 * 2081 * Allocate memory for the transmit and receive rings, and then 2082 * the descriptors associated with each, called only once at attach. 2083 * 2084 **********************************************************************/ 2085 int 2086 ixgbe_allocate_queues(struct ix_softc *sc) 2087 { 2088 struct ifnet *ifp = &sc->arpcom.ac_if; 2089 struct ix_queue *que; 2090 struct tx_ring *txr; 2091 struct rx_ring *rxr; 2092 int rsize, tsize; 2093 int txconf = 0, rxconf = 0, i; 2094 2095 /* First allocate the top level queue structs */ 2096 if (!(sc->queues = mallocarray(sc->num_queues, 2097 sizeof(struct ix_queue), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2098 printf("%s: Unable to allocate queue memory\n", ifp->if_xname); 2099 goto fail; 2100 } 2101 2102 /* Then allocate the TX ring struct memory */ 2103 if (!(sc->tx_rings = mallocarray(sc->num_queues, 2104 sizeof(struct tx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2105 printf("%s: Unable to allocate TX ring memory\n", ifp->if_xname); 2106 goto fail; 2107 } 2108 2109 /* Next allocate the RX */ 2110 if (!(sc->rx_rings = mallocarray(sc->num_queues, 2111 sizeof(struct rx_ring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2112 printf("%s: Unable to allocate RX ring memory\n", ifp->if_xname); 2113 goto rx_fail; 2114 } 2115 2116 /* For the ring itself */ 2117 tsize = roundup2(sc->num_tx_desc * 2118 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2119 2120 /* 2121 * Now set up the TX queues, txconf is needed to handle the 2122 * possibility that things fail midcourse and we need to 2123 * undo memory gracefully 2124 */ 2125 for (i = 0; i < sc->num_queues; i++, txconf++) { 2126 /* Set up some basics */ 2127 txr = &sc->tx_rings[i]; 2128 txr->sc = sc; 2129 txr->me = i; 2130 2131 if (ixgbe_dma_malloc(sc, tsize, 2132 &txr->txdma, BUS_DMA_NOWAIT)) { 2133 printf("%s: Unable to allocate TX Descriptor memory\n", 2134 ifp->if_xname); 2135 goto err_tx_desc; 2136 } 2137 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2138 bzero((void *)txr->tx_base, tsize); 2139 } 2140 2141 /* 2142 * Next the RX queues... 2143 */ 2144 rsize = roundup2(sc->num_rx_desc * 2145 sizeof(union ixgbe_adv_rx_desc), 4096); 2146 for (i = 0; i < sc->num_queues; i++, rxconf++) { 2147 rxr = &sc->rx_rings[i]; 2148 /* Set up some basics */ 2149 rxr->sc = sc; 2150 rxr->me = i; 2151 timeout_set(&rxr->rx_refill, ixgbe_rxrefill, rxr); 2152 2153 if (ixgbe_dma_malloc(sc, rsize, 2154 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2155 printf("%s: Unable to allocate RxDescriptor memory\n", 2156 ifp->if_xname); 2157 goto err_rx_desc; 2158 } 2159 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2160 bzero((void *)rxr->rx_base, rsize); 2161 } 2162 2163 /* 2164 * Finally set up the queue holding structs 2165 */ 2166 for (i = 0; i < sc->num_queues; i++) { 2167 que = &sc->queues[i]; 2168 que->sc = sc; 2169 que->txr = &sc->tx_rings[i]; 2170 que->rxr = &sc->rx_rings[i]; 2171 snprintf(que->name, sizeof(que->name), "%s:%d", 2172 sc->dev.dv_xname, i); 2173 } 2174 2175 return (0); 2176 2177 err_rx_desc: 2178 for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 2179 ixgbe_dma_free(sc, &rxr->rxdma); 2180 err_tx_desc: 2181 for (txr = sc->tx_rings; txconf > 0; txr++, txconf--) 2182 ixgbe_dma_free(sc, &txr->txdma); 2183 free(sc->rx_rings, M_DEVBUF, sc->num_queues * sizeof(struct rx_ring)); 2184 sc->rx_rings = NULL; 2185 rx_fail: 2186 free(sc->tx_rings, M_DEVBUF, sc->num_queues * sizeof(struct tx_ring)); 2187 sc->tx_rings = NULL; 2188 fail: 2189 return (ENOMEM); 2190 } 2191 2192 /********************************************************************* 2193 * 2194 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2195 * the information needed to transmit a packet on the wire. This is 2196 * called only once at attach, setup is done every reset. 2197 * 2198 **********************************************************************/ 2199 int 2200 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 2201 { 2202 struct ix_softc *sc = txr->sc; 2203 struct ifnet *ifp = &sc->arpcom.ac_if; 2204 struct ixgbe_tx_buf *txbuf; 2205 int error, i; 2206 2207 if (!(txr->tx_buffers = mallocarray(sc->num_tx_desc, 2208 sizeof(struct ixgbe_tx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2209 printf("%s: Unable to allocate tx_buffer memory\n", 2210 ifp->if_xname); 2211 error = ENOMEM; 2212 goto fail; 2213 } 2214 txr->txtag = txr->txdma.dma_tag; 2215 2216 /* Create the descriptor buffer dma maps */ 2217 for (i = 0; i < sc->num_tx_desc; i++) { 2218 txbuf = &txr->tx_buffers[i]; 2219 error = bus_dmamap_create(txr->txdma.dma_tag, IXGBE_TSO_SIZE, 2220 sc->num_segs, PAGE_SIZE, 0, 2221 BUS_DMA_NOWAIT, &txbuf->map); 2222 2223 if (error != 0) { 2224 printf("%s: Unable to create TX DMA map\n", 2225 ifp->if_xname); 2226 goto fail; 2227 } 2228 } 2229 2230 return 0; 2231 fail: 2232 return (error); 2233 } 2234 2235 /********************************************************************* 2236 * 2237 * Initialize a transmit ring. 2238 * 2239 **********************************************************************/ 2240 int 2241 ixgbe_setup_transmit_ring(struct tx_ring *txr) 2242 { 2243 struct ix_softc *sc = txr->sc; 2244 int error; 2245 2246 /* Now allocate transmit buffers for the ring */ 2247 if ((error = ixgbe_allocate_transmit_buffers(txr)) != 0) 2248 return (error); 2249 2250 /* Clear the old ring contents */ 2251 bzero((void *)txr->tx_base, 2252 (sizeof(union ixgbe_adv_tx_desc)) * sc->num_tx_desc); 2253 2254 /* Reset indices */ 2255 txr->next_avail_desc = 0; 2256 txr->next_to_clean = 0; 2257 2258 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2259 0, txr->txdma.dma_map->dm_mapsize, 2260 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2261 2262 return (0); 2263 } 2264 2265 /********************************************************************* 2266 * 2267 * Initialize all transmit rings. 2268 * 2269 **********************************************************************/ 2270 int 2271 ixgbe_setup_transmit_structures(struct ix_softc *sc) 2272 { 2273 struct tx_ring *txr = sc->tx_rings; 2274 int i, error; 2275 2276 for (i = 0; i < sc->num_queues; i++, txr++) { 2277 if ((error = ixgbe_setup_transmit_ring(txr)) != 0) 2278 goto fail; 2279 } 2280 2281 return (0); 2282 fail: 2283 ixgbe_free_transmit_structures(sc); 2284 return (error); 2285 } 2286 2287 /********************************************************************* 2288 * 2289 * Enable transmit unit. 2290 * 2291 **********************************************************************/ 2292 void 2293 ixgbe_initialize_transmit_units(struct ix_softc *sc) 2294 { 2295 struct ifnet *ifp = &sc->arpcom.ac_if; 2296 struct tx_ring *txr; 2297 struct ixgbe_hw *hw = &sc->hw; 2298 int i; 2299 uint64_t tdba; 2300 uint32_t txctrl; 2301 2302 /* Setup the Base and Length of the Tx Descriptor Ring */ 2303 2304 for (i = 0; i < sc->num_queues; i++) { 2305 txr = &sc->tx_rings[i]; 2306 2307 /* Setup descriptor base address */ 2308 tdba = txr->txdma.dma_map->dm_segs[0].ds_addr; 2309 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 2310 (tdba & 0x00000000ffffffffULL)); 2311 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 2312 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 2313 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 2314 2315 /* Setup the HW Tx Head and Tail descriptor pointers */ 2316 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 2317 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 2318 2319 /* Setup Transmit Descriptor Cmd Settings */ 2320 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2321 txr->queue_status = IXGBE_QUEUE_IDLE; 2322 txr->watchdog_timer = 0; 2323 2324 /* Disable Head Writeback */ 2325 switch (hw->mac.type) { 2326 case ixgbe_mac_82598EB: 2327 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 2328 break; 2329 case ixgbe_mac_82599EB: 2330 case ixgbe_mac_X540: 2331 default: 2332 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 2333 break; 2334 } 2335 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2336 switch (hw->mac.type) { 2337 case ixgbe_mac_82598EB: 2338 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 2339 break; 2340 case ixgbe_mac_82599EB: 2341 case ixgbe_mac_X540: 2342 default: 2343 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 2344 break; 2345 } 2346 } 2347 ifp->if_timer = 0; 2348 2349 if (hw->mac.type != ixgbe_mac_82598EB) { 2350 uint32_t dmatxctl, rttdcs; 2351 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2352 dmatxctl |= IXGBE_DMATXCTL_TE; 2353 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 2354 /* Disable arbiter to set MTQC */ 2355 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2356 rttdcs |= IXGBE_RTTDCS_ARBDIS; 2357 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2358 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2359 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 2360 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2361 } 2362 } 2363 2364 /********************************************************************* 2365 * 2366 * Free all transmit rings. 2367 * 2368 **********************************************************************/ 2369 void 2370 ixgbe_free_transmit_structures(struct ix_softc *sc) 2371 { 2372 struct tx_ring *txr = sc->tx_rings; 2373 int i; 2374 2375 for (i = 0; i < sc->num_queues; i++, txr++) 2376 ixgbe_free_transmit_buffers(txr); 2377 } 2378 2379 /********************************************************************* 2380 * 2381 * Free transmit ring related data structures. 2382 * 2383 **********************************************************************/ 2384 void 2385 ixgbe_free_transmit_buffers(struct tx_ring *txr) 2386 { 2387 struct ix_softc *sc = txr->sc; 2388 struct ixgbe_tx_buf *tx_buffer; 2389 int i; 2390 2391 INIT_DEBUGOUT("free_transmit_ring: begin"); 2392 2393 if (txr->tx_buffers == NULL) 2394 return; 2395 2396 tx_buffer = txr->tx_buffers; 2397 for (i = 0; i < sc->num_tx_desc; i++, tx_buffer++) { 2398 if (tx_buffer->map != NULL && tx_buffer->map->dm_nsegs > 0) { 2399 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2400 0, tx_buffer->map->dm_mapsize, 2401 BUS_DMASYNC_POSTWRITE); 2402 bus_dmamap_unload(txr->txdma.dma_tag, 2403 tx_buffer->map); 2404 } 2405 if (tx_buffer->m_head != NULL) { 2406 m_freem(tx_buffer->m_head); 2407 tx_buffer->m_head = NULL; 2408 } 2409 if (tx_buffer->map != NULL) { 2410 bus_dmamap_destroy(txr->txdma.dma_tag, 2411 tx_buffer->map); 2412 tx_buffer->map = NULL; 2413 } 2414 } 2415 2416 if (txr->tx_buffers != NULL) 2417 free(txr->tx_buffers, M_DEVBUF, 2418 sc->num_tx_desc * sizeof(struct ixgbe_tx_buf)); 2419 txr->tx_buffers = NULL; 2420 txr->txtag = NULL; 2421 } 2422 2423 /********************************************************************* 2424 * 2425 * Advanced Context Descriptor setup for VLAN or CSUM 2426 * 2427 **********************************************************************/ 2428 2429 int 2430 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, 2431 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2432 { 2433 struct ixgbe_adv_tx_context_desc *TXD; 2434 struct ixgbe_tx_buf *tx_buffer; 2435 #if NVLAN > 0 2436 struct ether_vlan_header *eh; 2437 #else 2438 struct ether_header *eh; 2439 #endif 2440 struct ip *ip; 2441 #ifdef notyet 2442 struct ip6_hdr *ip6; 2443 #endif 2444 struct mbuf *m; 2445 int ipoff; 2446 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2447 int ehdrlen, ip_hlen = 0; 2448 uint16_t etype; 2449 uint8_t ipproto = 0; 2450 int offload = TRUE; 2451 int ctxd = txr->next_avail_desc; 2452 #if NVLAN > 0 2453 uint16_t vtag = 0; 2454 #endif 2455 2456 #if notyet 2457 /* First check if TSO is to be used */ 2458 if (mp->m_pkthdr.csum_flags & CSUM_TSO) 2459 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status)); 2460 #endif 2461 2462 if ((mp->m_pkthdr.csum_flags & (M_TCP_CSUM_OUT | M_UDP_CSUM_OUT)) == 0) 2463 offload = FALSE; 2464 2465 /* Indicate the whole packet as payload when not doing TSO */ 2466 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; 2467 2468 /* Now ready a context descriptor */ 2469 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2470 tx_buffer = &txr->tx_buffers[ctxd]; 2471 2472 /* 2473 * In advanced descriptors the vlan tag must 2474 * be placed into the descriptor itself. Hence 2475 * we need to make one even if not doing offloads. 2476 */ 2477 #if NVLAN > 0 2478 if (mp->m_flags & M_VLANTAG) { 2479 vtag = mp->m_pkthdr.ether_vtag; 2480 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2481 } else 2482 #endif 2483 if (offload == FALSE) 2484 return (0); /* No need for CTX */ 2485 2486 /* 2487 * Determine where frame payload starts. 2488 * Jump over vlan headers if already present, 2489 * helpful for QinQ too. 2490 */ 2491 if (mp->m_len < sizeof(struct ether_header)) 2492 return (-1); 2493 #if NVLAN > 0 2494 eh = mtod(mp, struct ether_vlan_header *); 2495 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2496 if (mp->m_len < sizeof(struct ether_vlan_header)) 2497 return (-1); 2498 etype = ntohs(eh->evl_proto); 2499 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2500 } else { 2501 etype = ntohs(eh->evl_encap_proto); 2502 ehdrlen = ETHER_HDR_LEN; 2503 } 2504 #else 2505 eh = mtod(mp, struct ether_header *); 2506 etype = ntohs(eh->ether_type); 2507 ehdrlen = ETHER_HDR_LEN; 2508 #endif 2509 2510 /* Set the ether header length */ 2511 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2512 2513 switch (etype) { 2514 case ETHERTYPE_IP: 2515 if (mp->m_pkthdr.len < ehdrlen + sizeof(*ip)) 2516 return (-1); 2517 m = m_getptr(mp, ehdrlen, &ipoff); 2518 KASSERT(m != NULL && m->m_len - ipoff >= sizeof(*ip)); 2519 ip = (struct ip *)(m->m_data + ipoff); 2520 ip_hlen = ip->ip_hl << 2; 2521 ipproto = ip->ip_p; 2522 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2523 break; 2524 #ifdef notyet 2525 case ETHERTYPE_IPV6: 2526 if (mp->m_pkthdr.len < ehdrlen + sizeof(*ip6)) 2527 return (-1); 2528 m = m_getptr(mp, ehdrlen, &ipoff); 2529 KASSERT(m != NULL && m->m_len - ipoff >= sizeof(*ip6)); 2530 ip6 = (struct ip6 *)(m->m_data + ipoff); 2531 ip_hlen = sizeof(*ip6); 2532 /* XXX-BZ this will go badly in case of ext hdrs. */ 2533 ipproto = ip6->ip6_nxt; 2534 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2535 break; 2536 #endif 2537 default: 2538 offload = FALSE; 2539 break; 2540 } 2541 2542 vlan_macip_lens |= ip_hlen; 2543 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2544 2545 switch (ipproto) { 2546 case IPPROTO_TCP: 2547 if (mp->m_pkthdr.csum_flags & M_TCP_CSUM_OUT) 2548 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2549 break; 2550 case IPPROTO_UDP: 2551 if (mp->m_pkthdr.csum_flags & M_UDP_CSUM_OUT) 2552 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2553 break; 2554 default: 2555 offload = FALSE; 2556 break; 2557 } 2558 2559 if (offload) /* For the TX descriptor setup */ 2560 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2561 2562 /* Now copy bits into descriptor */ 2563 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2564 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2565 TXD->seqnum_seed = htole32(0); 2566 TXD->mss_l4len_idx = htole32(0); 2567 2568 tx_buffer->m_head = NULL; 2569 tx_buffer->eop_index = -1; 2570 2571 return (1); 2572 } 2573 2574 /********************************************************************** 2575 * 2576 * Examine each tx_buffer in the used queue. If the hardware is done 2577 * processing the packet then free associated resources. The 2578 * tx_buffer is put back on the free queue. 2579 * 2580 **********************************************************************/ 2581 int 2582 ixgbe_txeof(struct tx_ring *txr) 2583 { 2584 struct ix_softc *sc = txr->sc; 2585 struct ifqueue *ifq = txr->ifq; 2586 struct ifnet *ifp = &sc->arpcom.ac_if; 2587 unsigned int head, tail, last; 2588 struct ixgbe_tx_buf *tx_buffer; 2589 struct ixgbe_legacy_tx_desc *tx_desc; 2590 2591 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2592 return FALSE; 2593 2594 head = txr->next_avail_desc; 2595 tail = txr->next_to_clean; 2596 2597 membar_consumer(); 2598 2599 if (head == tail) 2600 return (FALSE); 2601 2602 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2603 0, txr->txdma.dma_map->dm_mapsize, 2604 BUS_DMASYNC_POSTREAD); 2605 2606 for (;;) { 2607 tx_buffer = &txr->tx_buffers[tail]; 2608 last = tx_buffer->eop_index; 2609 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2610 2611 if (!ISSET(tx_desc->upper.fields.status, IXGBE_TXD_STAT_DD)) 2612 break; 2613 2614 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2615 0, tx_buffer->map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2616 bus_dmamap_unload(txr->txdma.dma_tag, tx_buffer->map); 2617 m_freem(tx_buffer->m_head); 2618 2619 tx_buffer->m_head = NULL; 2620 tx_buffer->eop_index = -1; 2621 2622 tail = last + 1; 2623 if (tail == sc->num_tx_desc) 2624 tail = 0; 2625 if (head == tail) { 2626 /* All clean, turn off the timer */ 2627 ifp->if_timer = 0; 2628 break; 2629 } 2630 } 2631 2632 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2633 0, txr->txdma.dma_map->dm_mapsize, 2634 BUS_DMASYNC_PREREAD); 2635 2636 membar_producer(); 2637 2638 txr->next_to_clean = tail; 2639 2640 if (ifq_is_oactive(ifq)) 2641 ifq_restart(ifq); 2642 2643 return TRUE; 2644 } 2645 2646 /********************************************************************* 2647 * 2648 * Get a buffer from system mbuf buffer pool. 2649 * 2650 **********************************************************************/ 2651 int 2652 ixgbe_get_buf(struct rx_ring *rxr, int i) 2653 { 2654 struct ix_softc *sc = rxr->sc; 2655 struct ixgbe_rx_buf *rxbuf; 2656 struct mbuf *mp; 2657 int error; 2658 union ixgbe_adv_rx_desc *rxdesc; 2659 2660 rxbuf = &rxr->rx_buffers[i]; 2661 rxdesc = &rxr->rx_base[i]; 2662 if (rxbuf->buf) { 2663 printf("%s: ixgbe_get_buf: slot %d already has an mbuf\n", 2664 sc->dev.dv_xname, i); 2665 return (ENOBUFS); 2666 } 2667 2668 /* needed in any case so prealocate since this one will fail for sure */ 2669 mp = MCLGETL(NULL, M_DONTWAIT, sc->rx_mbuf_sz); 2670 if (!mp) 2671 return (ENOBUFS); 2672 2673 mp->m_data += (mp->m_ext.ext_size - sc->rx_mbuf_sz); 2674 mp->m_len = mp->m_pkthdr.len = sc->rx_mbuf_sz; 2675 2676 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map, 2677 mp, BUS_DMA_NOWAIT); 2678 if (error) { 2679 m_freem(mp); 2680 return (error); 2681 } 2682 2683 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2684 0, rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD); 2685 rxbuf->buf = mp; 2686 2687 rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr); 2688 2689 return (0); 2690 } 2691 2692 /********************************************************************* 2693 * 2694 * Allocate memory for rx_buffer structures. Since we use one 2695 * rx_buffer per received packet, the maximum number of rx_buffer's 2696 * that we'll need is equal to the number of receive descriptors 2697 * that we've allocated. 2698 * 2699 **********************************************************************/ 2700 int 2701 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 2702 { 2703 struct ix_softc *sc = rxr->sc; 2704 struct ifnet *ifp = &sc->arpcom.ac_if; 2705 struct ixgbe_rx_buf *rxbuf; 2706 int i, error; 2707 2708 if (!(rxr->rx_buffers = mallocarray(sc->num_rx_desc, 2709 sizeof(struct ixgbe_rx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2710 printf("%s: Unable to allocate rx_buffer memory\n", 2711 ifp->if_xname); 2712 error = ENOMEM; 2713 goto fail; 2714 } 2715 2716 rxbuf = rxr->rx_buffers; 2717 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2718 error = bus_dmamap_create(rxr->rxdma.dma_tag, 16 * 1024, 1, 2719 16 * 1024, 0, BUS_DMA_NOWAIT, &rxbuf->map); 2720 if (error) { 2721 printf("%s: Unable to create Pack DMA map\n", 2722 ifp->if_xname); 2723 goto fail; 2724 } 2725 } 2726 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 2727 rxr->rxdma.dma_map->dm_mapsize, 2728 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2729 2730 return (0); 2731 2732 fail: 2733 return (error); 2734 } 2735 2736 /********************************************************************* 2737 * 2738 * Initialize a receive ring and its buffers. 2739 * 2740 **********************************************************************/ 2741 int 2742 ixgbe_setup_receive_ring(struct rx_ring *rxr) 2743 { 2744 struct ix_softc *sc = rxr->sc; 2745 struct ifnet *ifp = &sc->arpcom.ac_if; 2746 int rsize, error; 2747 2748 rsize = roundup2(sc->num_rx_desc * 2749 sizeof(union ixgbe_adv_rx_desc), 4096); 2750 /* Clear the ring contents */ 2751 bzero((void *)rxr->rx_base, rsize); 2752 2753 if ((error = ixgbe_allocate_receive_buffers(rxr)) != 0) 2754 return (error); 2755 2756 /* Setup our descriptor indices */ 2757 rxr->next_to_check = 0; 2758 rxr->last_desc_filled = sc->num_rx_desc - 1; 2759 2760 if_rxr_init(&rxr->rx_ring, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1), 2761 sc->num_rx_desc - 1); 2762 2763 ixgbe_rxfill(rxr); 2764 if (if_rxr_inuse(&rxr->rx_ring) == 0) { 2765 printf("%s: unable to fill any rx descriptors\n", 2766 sc->dev.dv_xname); 2767 return (ENOBUFS); 2768 } 2769 2770 return (0); 2771 } 2772 2773 int 2774 ixgbe_rxfill(struct rx_ring *rxr) 2775 { 2776 struct ix_softc *sc = rxr->sc; 2777 int post = 0; 2778 u_int slots; 2779 int i; 2780 2781 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2782 0, rxr->rxdma.dma_map->dm_mapsize, 2783 BUS_DMASYNC_POSTWRITE); 2784 2785 i = rxr->last_desc_filled; 2786 for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc); 2787 slots > 0; slots--) { 2788 if (++i == sc->num_rx_desc) 2789 i = 0; 2790 2791 if (ixgbe_get_buf(rxr, i) != 0) 2792 break; 2793 2794 rxr->last_desc_filled = i; 2795 post = 1; 2796 } 2797 2798 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2799 0, rxr->rxdma.dma_map->dm_mapsize, 2800 BUS_DMASYNC_PREWRITE); 2801 2802 if_rxr_put(&rxr->rx_ring, slots); 2803 2804 return (post); 2805 } 2806 2807 void 2808 ixgbe_rxrefill(void *xrxr) 2809 { 2810 struct rx_ring *rxr = xrxr; 2811 struct ix_softc *sc = rxr->sc; 2812 2813 if (ixgbe_rxfill(rxr)) { 2814 /* Advance the Rx Queue "Tail Pointer" */ 2815 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(rxr->me), 2816 rxr->last_desc_filled); 2817 } else if (if_rxr_inuse(&rxr->rx_ring) == 0) 2818 timeout_add(&rxr->rx_refill, 1); 2819 2820 } 2821 2822 /********************************************************************* 2823 * 2824 * Initialize all receive rings. 2825 * 2826 **********************************************************************/ 2827 int 2828 ixgbe_setup_receive_structures(struct ix_softc *sc) 2829 { 2830 struct rx_ring *rxr = sc->rx_rings; 2831 int i; 2832 2833 for (i = 0; i < sc->num_queues; i++, rxr++) 2834 if (ixgbe_setup_receive_ring(rxr)) 2835 goto fail; 2836 2837 return (0); 2838 fail: 2839 ixgbe_free_receive_structures(sc); 2840 return (ENOBUFS); 2841 } 2842 2843 /********************************************************************* 2844 * 2845 * Setup receive registers and features. 2846 * 2847 **********************************************************************/ 2848 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2849 2850 void 2851 ixgbe_initialize_receive_units(struct ix_softc *sc) 2852 { 2853 struct rx_ring *rxr = sc->rx_rings; 2854 struct ixgbe_hw *hw = &sc->hw; 2855 uint32_t bufsz, fctrl, srrctl, rxcsum; 2856 uint32_t hlreg; 2857 int i; 2858 2859 /* 2860 * Make sure receives are disabled while 2861 * setting up the descriptor ring 2862 */ 2863 ixgbe_disable_rx(hw); 2864 2865 /* Enable broadcasts */ 2866 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2867 fctrl |= IXGBE_FCTRL_BAM; 2868 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 2869 fctrl |= IXGBE_FCTRL_DPF; 2870 fctrl |= IXGBE_FCTRL_PMCF; 2871 } 2872 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 2873 2874 /* Always enable jumbo frame reception */ 2875 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2876 hlreg |= IXGBE_HLREG0_JUMBOEN; 2877 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 2878 2879 bufsz = (sc->rx_mbuf_sz - ETHER_ALIGN) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2880 2881 for (i = 0; i < sc->num_queues; i++, rxr++) { 2882 uint64_t rdba = rxr->rxdma.dma_map->dm_segs[0].ds_addr; 2883 2884 /* Setup the Base and Length of the Rx Descriptor Ring */ 2885 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 2886 (rdba & 0x00000000ffffffffULL)); 2887 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 2888 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 2889 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 2890 2891 /* Set up the SRRCTL register */ 2892 srrctl = bufsz | IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2893 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 2894 2895 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2896 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 2897 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 2898 } 2899 2900 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 2901 uint32_t psrtype = IXGBE_PSRTYPE_TCPHDR | 2902 IXGBE_PSRTYPE_UDPHDR | 2903 IXGBE_PSRTYPE_IPV4HDR | 2904 IXGBE_PSRTYPE_IPV6HDR; 2905 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 2906 } 2907 2908 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2909 rxcsum &= ~IXGBE_RXCSUM_PCSD; 2910 2911 ixgbe_initialize_rss_mapping(sc); 2912 2913 /* Setup RSS */ 2914 if (sc->num_queues > 1) { 2915 /* RSS and RX IPP Checksum are mutually exclusive */ 2916 rxcsum |= IXGBE_RXCSUM_PCSD; 2917 } 2918 2919 /* Map QPRC/QPRDC/QPTC on a per queue basis */ 2920 ixgbe_map_queue_statistics(sc); 2921 2922 /* This is useful for calculating UDP/IP fragment checksums */ 2923 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 2924 rxcsum |= IXGBE_RXCSUM_IPPCSE; 2925 2926 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2927 } 2928 2929 void 2930 ixgbe_initialize_rss_mapping(struct ix_softc *sc) 2931 { 2932 struct ixgbe_hw *hw = &sc->hw; 2933 uint32_t reta = 0, mrqc, rss_key[10]; 2934 int i, j, queue_id, table_size, index_mult; 2935 2936 /* set up random bits */ 2937 stoeplitz_to_key(&rss_key, sizeof(rss_key)); 2938 2939 /* Set multiplier for RETA setup and table size based on MAC */ 2940 index_mult = 0x1; 2941 table_size = 128; 2942 switch (sc->hw.mac.type) { 2943 case ixgbe_mac_82598EB: 2944 index_mult = 0x11; 2945 break; 2946 case ixgbe_mac_X550: 2947 case ixgbe_mac_X550EM_x: 2948 case ixgbe_mac_X550EM_a: 2949 table_size = 512; 2950 break; 2951 default: 2952 break; 2953 } 2954 2955 /* Set up the redirection table */ 2956 for (i = 0, j = 0; i < table_size; i++, j++) { 2957 if (j == sc->num_queues) j = 0; 2958 queue_id = (j * index_mult); 2959 /* 2960 * The low 8 bits are for hash value (n+0); 2961 * The next 8 bits are for hash value (n+1), etc. 2962 */ 2963 reta = reta >> 8; 2964 reta = reta | ( ((uint32_t) queue_id) << 24); 2965 if ((i & 3) == 3) { 2966 if (i < 128) 2967 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2968 else 2969 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 2970 reta); 2971 reta = 0; 2972 } 2973 } 2974 2975 /* Now fill our hash function seeds */ 2976 for (i = 0; i < 10; i++) 2977 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 2978 2979 /* 2980 * Disable UDP - IP fragments aren't currently being handled 2981 * and so we end up with a mix of 2-tuple and 4-tuple 2982 * traffic. 2983 */ 2984 mrqc = IXGBE_MRQC_RSSEN 2985 | IXGBE_MRQC_RSS_FIELD_IPV4 2986 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 2987 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 2988 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 2989 | IXGBE_MRQC_RSS_FIELD_IPV6 2990 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 2991 ; 2992 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2993 } 2994 2995 /********************************************************************* 2996 * 2997 * Free all receive rings. 2998 * 2999 **********************************************************************/ 3000 void 3001 ixgbe_free_receive_structures(struct ix_softc *sc) 3002 { 3003 struct rx_ring *rxr; 3004 int i; 3005 3006 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 3007 if_rxr_init(&rxr->rx_ring, 0, 0); 3008 3009 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 3010 ixgbe_free_receive_buffers(rxr); 3011 } 3012 3013 /********************************************************************* 3014 * 3015 * Free receive ring data structures 3016 * 3017 **********************************************************************/ 3018 void 3019 ixgbe_free_receive_buffers(struct rx_ring *rxr) 3020 { 3021 struct ix_softc *sc; 3022 struct ixgbe_rx_buf *rxbuf; 3023 int i; 3024 3025 sc = rxr->sc; 3026 if (rxr->rx_buffers != NULL) { 3027 for (i = 0; i < sc->num_rx_desc; i++) { 3028 rxbuf = &rxr->rx_buffers[i]; 3029 if (rxbuf->buf != NULL) { 3030 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 3031 0, rxbuf->map->dm_mapsize, 3032 BUS_DMASYNC_POSTREAD); 3033 bus_dmamap_unload(rxr->rxdma.dma_tag, 3034 rxbuf->map); 3035 m_freem(rxbuf->buf); 3036 rxbuf->buf = NULL; 3037 } 3038 bus_dmamap_destroy(rxr->rxdma.dma_tag, rxbuf->map); 3039 rxbuf->map = NULL; 3040 } 3041 free(rxr->rx_buffers, M_DEVBUF, 3042 sc->num_rx_desc * sizeof(struct ixgbe_rx_buf)); 3043 rxr->rx_buffers = NULL; 3044 } 3045 } 3046 3047 /********************************************************************* 3048 * 3049 * This routine executes in interrupt context. It replenishes 3050 * the mbufs in the descriptor and sends data which has been 3051 * dma'ed into host memory to upper layer. 3052 * 3053 *********************************************************************/ 3054 int 3055 ixgbe_rxeof(struct rx_ring *rxr) 3056 { 3057 struct ix_softc *sc = rxr->sc; 3058 struct ifnet *ifp = &sc->arpcom.ac_if; 3059 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 3060 struct mbuf *mp, *sendmp; 3061 uint8_t eop = 0; 3062 uint16_t len, vtag; 3063 uint32_t staterr = 0, ptype; 3064 struct ixgbe_rx_buf *rxbuf, *nxbuf; 3065 union ixgbe_adv_rx_desc *rxdesc; 3066 size_t dsize = sizeof(union ixgbe_adv_rx_desc); 3067 int i, nextp; 3068 3069 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3070 return FALSE; 3071 3072 i = rxr->next_to_check; 3073 while (if_rxr_inuse(&rxr->rx_ring) > 0) { 3074 uint32_t hash, hashtype; 3075 3076 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3077 dsize * i, dsize, BUS_DMASYNC_POSTREAD); 3078 3079 rxdesc = &rxr->rx_base[i]; 3080 staterr = letoh32(rxdesc->wb.upper.status_error); 3081 if (!ISSET(staterr, IXGBE_RXD_STAT_DD)) { 3082 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3083 dsize * i, dsize, 3084 BUS_DMASYNC_PREREAD); 3085 break; 3086 } 3087 3088 /* Zero out the receive descriptors status */ 3089 rxdesc->wb.upper.status_error = 0; 3090 rxbuf = &rxr->rx_buffers[i]; 3091 3092 /* pull the mbuf off the ring */ 3093 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 3094 rxbuf->map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3095 bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map); 3096 3097 mp = rxbuf->buf; 3098 len = letoh16(rxdesc->wb.upper.length); 3099 ptype = letoh32(rxdesc->wb.lower.lo_dword.data) & 3100 IXGBE_RXDADV_PKTTYPE_MASK; 3101 vtag = letoh16(rxdesc->wb.upper.vlan); 3102 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3103 hash = lemtoh32(&rxdesc->wb.lower.hi_dword.rss); 3104 hashtype = lemtoh32(&rxdesc->wb.lower.lo_dword.data) & 3105 IXGBE_RXDADV_RSSTYPE_MASK; 3106 3107 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) { 3108 if (rxbuf->fmp) { 3109 m_freem(rxbuf->fmp); 3110 rxbuf->fmp = NULL; 3111 } 3112 3113 m_freem(mp); 3114 rxbuf->buf = NULL; 3115 goto next_desc; 3116 } 3117 3118 if (mp == NULL) { 3119 panic("%s: ixgbe_rxeof: NULL mbuf in slot %d " 3120 "(nrx %d, filled %d)", sc->dev.dv_xname, 3121 i, if_rxr_inuse(&rxr->rx_ring), 3122 rxr->last_desc_filled); 3123 } 3124 3125 /* Currently no HW RSC support of 82599 */ 3126 if (!eop) { 3127 /* 3128 * Figure out the next descriptor of this frame. 3129 */ 3130 nextp = i + 1; 3131 if (nextp == sc->num_rx_desc) 3132 nextp = 0; 3133 nxbuf = &rxr->rx_buffers[nextp]; 3134 /* prefetch(nxbuf); */ 3135 } 3136 3137 /* 3138 * Rather than using the fmp/lmp global pointers 3139 * we now keep the head of a packet chain in the 3140 * buffer struct and pass this along from one 3141 * descriptor to the next, until we get EOP. 3142 */ 3143 mp->m_len = len; 3144 /* 3145 * See if there is a stored head 3146 * that determines what we are 3147 */ 3148 sendmp = rxbuf->fmp; 3149 rxbuf->buf = rxbuf->fmp = NULL; 3150 3151 if (sendmp != NULL) /* secondary frag */ 3152 sendmp->m_pkthdr.len += mp->m_len; 3153 else { 3154 /* first desc of a non-ps chain */ 3155 sendmp = mp; 3156 sendmp->m_pkthdr.len = mp->m_len; 3157 #if NVLAN > 0 3158 if (staterr & IXGBE_RXD_STAT_VP) { 3159 sendmp->m_pkthdr.ether_vtag = vtag; 3160 sendmp->m_flags |= M_VLANTAG; 3161 } 3162 #endif 3163 } 3164 3165 /* Pass the head pointer on */ 3166 if (eop == 0) { 3167 nxbuf->fmp = sendmp; 3168 sendmp = NULL; 3169 mp->m_next = nxbuf->buf; 3170 } else { /* Sending this frame? */ 3171 ixgbe_rx_checksum(staterr, sendmp, ptype); 3172 3173 if (hashtype != IXGBE_RXDADV_RSSTYPE_NONE) { 3174 sendmp->m_pkthdr.ph_flowid = hash; 3175 SET(sendmp->m_pkthdr.csum_flags, M_FLOWID); 3176 } 3177 3178 ml_enqueue(&ml, sendmp); 3179 } 3180 next_desc: 3181 if_rxr_put(&rxr->rx_ring, 1); 3182 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3183 dsize * i, dsize, 3184 BUS_DMASYNC_PREREAD); 3185 3186 /* Advance our pointers to the next descriptor. */ 3187 if (++i == sc->num_rx_desc) 3188 i = 0; 3189 } 3190 rxr->next_to_check = i; 3191 3192 if (ifiq_input(rxr->ifiq, &ml)) 3193 if_rxr_livelocked(&rxr->rx_ring); 3194 3195 if (!(staterr & IXGBE_RXD_STAT_DD)) 3196 return FALSE; 3197 3198 return TRUE; 3199 } 3200 3201 /********************************************************************* 3202 * 3203 * Verify that the hardware indicated that the checksum is valid. 3204 * Inform the stack about the status of checksum so that stack 3205 * doesn't spend time verifying the checksum. 3206 * 3207 *********************************************************************/ 3208 void 3209 ixgbe_rx_checksum(uint32_t staterr, struct mbuf * mp, uint32_t ptype) 3210 { 3211 uint16_t status = (uint16_t) staterr; 3212 uint8_t errors = (uint8_t) (staterr >> 24); 3213 3214 if (status & IXGBE_RXD_STAT_IPCS) { 3215 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3216 /* IP Checksum Good */ 3217 mp->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 3218 } else 3219 mp->m_pkthdr.csum_flags = 0; 3220 } 3221 if (status & IXGBE_RXD_STAT_L4CS) { 3222 if (!(errors & IXGBE_RXD_ERR_TCPE)) 3223 mp->m_pkthdr.csum_flags |= 3224 M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK; 3225 } 3226 } 3227 3228 void 3229 ixgbe_setup_vlan_hw_support(struct ix_softc *sc) 3230 { 3231 uint32_t ctrl; 3232 int i; 3233 3234 /* 3235 * A soft reset zero's out the VFTA, so 3236 * we need to repopulate it now. 3237 */ 3238 for (i = 0; i < IXGBE_VFTA_SIZE; i++) { 3239 if (sc->shadow_vfta[i] != 0) 3240 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTA(i), 3241 sc->shadow_vfta[i]); 3242 } 3243 3244 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_VLNCTRL); 3245 #if 0 3246 /* Enable the Filter Table if enabled */ 3247 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3248 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 3249 ctrl |= IXGBE_VLNCTRL_VFE; 3250 } 3251 #endif 3252 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3253 ctrl |= IXGBE_VLNCTRL_VME; 3254 IXGBE_WRITE_REG(&sc->hw, IXGBE_VLNCTRL, ctrl); 3255 3256 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 3257 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 3258 for (i = 0; i < sc->num_queues; i++) { 3259 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 3260 ctrl |= IXGBE_RXDCTL_VME; 3261 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), ctrl); 3262 } 3263 } 3264 } 3265 3266 void 3267 ixgbe_enable_intr(struct ix_softc *sc) 3268 { 3269 struct ixgbe_hw *hw = &sc->hw; 3270 uint32_t mask, fwsm; 3271 3272 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3273 /* Enable Fan Failure detection */ 3274 if (hw->device_id == IXGBE_DEV_ID_82598AT) 3275 mask |= IXGBE_EIMS_GPI_SDP1; 3276 3277 switch (sc->hw.mac.type) { 3278 case ixgbe_mac_82599EB: 3279 mask |= IXGBE_EIMS_ECC; 3280 /* Temperature sensor on some adapters */ 3281 mask |= IXGBE_EIMS_GPI_SDP0; 3282 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 3283 mask |= IXGBE_EIMS_GPI_SDP1; 3284 mask |= IXGBE_EIMS_GPI_SDP2; 3285 break; 3286 case ixgbe_mac_X540: 3287 mask |= IXGBE_EIMS_ECC; 3288 /* Detect if Thermal Sensor is enabled */ 3289 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 3290 if (fwsm & IXGBE_FWSM_TS_ENABLED) 3291 mask |= IXGBE_EIMS_TS; 3292 break; 3293 case ixgbe_mac_X550: 3294 case ixgbe_mac_X550EM_x: 3295 case ixgbe_mac_X550EM_a: 3296 mask |= IXGBE_EIMS_ECC; 3297 /* MAC thermal sensor is automatically enabled */ 3298 mask |= IXGBE_EIMS_TS; 3299 /* Some devices use SDP0 for important information */ 3300 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 3301 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 3302 mask |= IXGBE_EIMS_GPI_SDP0_X540; 3303 default: 3304 break; 3305 } 3306 3307 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 3308 3309 /* With MSI-X we use auto clear */ 3310 if (sc->sc_intrmap) { 3311 mask = IXGBE_EIMS_ENABLE_MASK; 3312 /* Don't autoclear Link */ 3313 mask &= ~IXGBE_EIMS_OTHER; 3314 mask &= ~IXGBE_EIMS_LSC; 3315 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 3316 } 3317 3318 IXGBE_WRITE_FLUSH(hw); 3319 } 3320 3321 void 3322 ixgbe_disable_intr(struct ix_softc *sc) 3323 { 3324 if (sc->sc_intrmap) 3325 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 3326 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3327 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 3328 } else { 3329 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000); 3330 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0); 3331 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0); 3332 } 3333 IXGBE_WRITE_FLUSH(&sc->hw); 3334 } 3335 3336 uint16_t 3337 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) 3338 { 3339 struct pci_attach_args *pa; 3340 uint32_t value; 3341 int high = 0; 3342 3343 if (reg & 0x2) { 3344 high = 1; 3345 reg &= ~0x2; 3346 } 3347 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3348 value = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3349 3350 if (high) 3351 value >>= 16; 3352 3353 return (value & 0xffff); 3354 } 3355 3356 void 3357 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value) 3358 { 3359 struct pci_attach_args *pa; 3360 uint32_t rv; 3361 int high = 0; 3362 3363 /* Need to do read/mask/write... because 16 vs 32 bit!!! */ 3364 if (reg & 0x2) { 3365 high = 1; 3366 reg &= ~0x2; 3367 } 3368 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3369 rv = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3370 if (!high) 3371 rv = (rv & 0xffff0000) | value; 3372 else 3373 rv = (rv & 0xffff) | ((uint32_t)value << 16); 3374 pci_conf_write(pa->pa_pc, pa->pa_tag, reg, rv); 3375 } 3376 3377 /* 3378 * Setup the correct IVAR register for a particular MSIX interrupt 3379 * (yes this is all very magic and confusing :) 3380 * - entry is the register array entry 3381 * - vector is the MSIX vector for this queue 3382 * - type is RX/TX/MISC 3383 */ 3384 void 3385 ixgbe_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, int8_t type) 3386 { 3387 struct ixgbe_hw *hw = &sc->hw; 3388 uint32_t ivar, index; 3389 3390 vector |= IXGBE_IVAR_ALLOC_VAL; 3391 3392 switch (hw->mac.type) { 3393 3394 case ixgbe_mac_82598EB: 3395 if (type == -1) 3396 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3397 else 3398 entry += (type * 64); 3399 index = (entry >> 2) & 0x1F; 3400 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3401 ivar &= ~(0xFF << (8 * (entry & 0x3))); 3402 ivar |= (vector << (8 * (entry & 0x3))); 3403 IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar); 3404 break; 3405 3406 case ixgbe_mac_82599EB: 3407 case ixgbe_mac_X540: 3408 case ixgbe_mac_X550: 3409 case ixgbe_mac_X550EM_x: 3410 case ixgbe_mac_X550EM_a: 3411 if (type == -1) { /* MISC IVAR */ 3412 index = (entry & 1) * 8; 3413 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3414 ivar &= ~(0xFF << index); 3415 ivar |= (vector << index); 3416 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3417 } else { /* RX/TX IVARS */ 3418 index = (16 * (entry & 1)) + (8 * type); 3419 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 3420 ivar &= ~(0xFF << index); 3421 ivar |= (vector << index); 3422 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 3423 } 3424 3425 default: 3426 break; 3427 } 3428 } 3429 3430 void 3431 ixgbe_configure_ivars(struct ix_softc *sc) 3432 { 3433 struct ix_queue *que = sc->queues; 3434 uint32_t newitr; 3435 int i; 3436 3437 newitr = (4000000 / IXGBE_INTS_PER_SEC) & 0x0FF8; 3438 3439 for (i = 0; i < sc->num_queues; i++, que++) { 3440 /* First the RX queue entry */ 3441 ixgbe_set_ivar(sc, i, que->msix, 0); 3442 /* ... and the TX */ 3443 ixgbe_set_ivar(sc, i, que->msix, 1); 3444 /* Set an Initial EITR value */ 3445 IXGBE_WRITE_REG(&sc->hw, 3446 IXGBE_EITR(que->msix), newitr); 3447 } 3448 3449 /* For the Link interrupt */ 3450 ixgbe_set_ivar(sc, 1, sc->linkvec, -1); 3451 } 3452 3453 /* 3454 * SFP module interrupts handler 3455 */ 3456 void 3457 ixgbe_handle_mod(struct ix_softc *sc) 3458 { 3459 struct ixgbe_hw *hw = &sc->hw; 3460 uint32_t err; 3461 3462 err = hw->phy.ops.identify_sfp(hw); 3463 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3464 printf("%s: Unsupported SFP+ module type was detected!\n", 3465 sc->dev.dv_xname); 3466 return; 3467 } 3468 err = hw->mac.ops.setup_sfp(hw); 3469 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3470 printf("%s: Setup failure - unsupported SFP+ module type!\n", 3471 sc->dev.dv_xname); 3472 return; 3473 } 3474 3475 ixgbe_handle_msf(sc); 3476 } 3477 3478 3479 /* 3480 * MSF (multispeed fiber) interrupts handler 3481 */ 3482 void 3483 ixgbe_handle_msf(struct ix_softc *sc) 3484 { 3485 struct ixgbe_hw *hw = &sc->hw; 3486 uint32_t autoneg; 3487 bool negotiate; 3488 3489 autoneg = hw->phy.autoneg_advertised; 3490 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) { 3491 if (hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate)) 3492 return; 3493 } 3494 if (hw->mac.ops.setup_link) 3495 hw->mac.ops.setup_link(hw, autoneg, TRUE); 3496 3497 ifmedia_delete_instance(&sc->media, IFM_INST_ANY); 3498 ixgbe_add_media_types(sc); 3499 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 3500 } 3501 3502 /* 3503 * External PHY interrupts handler 3504 */ 3505 void 3506 ixgbe_handle_phy(struct ix_softc *sc) 3507 { 3508 struct ixgbe_hw *hw = &sc->hw; 3509 int error; 3510 3511 error = hw->phy.ops.handle_lasi(hw); 3512 if (error == IXGBE_ERR_OVERTEMP) 3513 printf("%s: CRITICAL: EXTERNAL PHY OVER TEMP!! " 3514 " PHY will downshift to lower power state!\n", 3515 sc->dev.dv_xname); 3516 else if (error) 3517 printf("%s: Error handling LASI interrupt: %d\n", 3518 sc->dev.dv_xname, error); 3519 3520 } 3521 3522 #if NKSTAT > 0 3523 enum ix_counter_idx { 3524 ix_counter_crcerrs, 3525 ix_counter_lxontxc, 3526 ix_counter_lxonrxc, 3527 ix_counter_lxofftxc, 3528 ix_counter_lxoffrxc, 3529 ix_counter_prc64, 3530 ix_counter_prc127, 3531 ix_counter_prc255, 3532 ix_counter_prc511, 3533 ix_counter_prc1023, 3534 ix_counter_prc1522, 3535 ix_counter_gptc, 3536 ix_counter_gorc, 3537 ix_counter_gotc, 3538 ix_counter_ruc, 3539 ix_counter_rfc, 3540 ix_counter_roc, 3541 ix_counter_rjc, 3542 ix_counter_tor, 3543 ix_counter_tpr, 3544 ix_counter_tpt, 3545 ix_counter_gprc, 3546 ix_counter_bprc, 3547 ix_counter_mprc, 3548 ix_counter_ptc64, 3549 ix_counter_ptc127, 3550 ix_counter_ptc255, 3551 ix_counter_ptc511, 3552 ix_counter_ptc1023, 3553 ix_counter_ptc1522, 3554 ix_counter_mptc, 3555 ix_counter_bptc, 3556 3557 ix_counter_num, 3558 }; 3559 3560 CTASSERT(KSTAT_KV_U_PACKETS <= 0xff); 3561 CTASSERT(KSTAT_KV_U_BYTES <= 0xff); 3562 3563 struct ix_counter { 3564 char name[KSTAT_KV_NAMELEN]; 3565 uint32_t reg; 3566 uint8_t width; 3567 uint8_t unit; 3568 }; 3569 3570 static const struct ix_counter ix_counters[ix_counter_num] = { 3571 [ix_counter_crcerrs] = { "crc errs", IXGBE_CRCERRS, 32, 3572 KSTAT_KV_U_PACKETS }, 3573 [ix_counter_lxontxc] = { "tx link xon", IXGBE_LXONTXC, 32, 3574 KSTAT_KV_U_PACKETS }, 3575 [ix_counter_lxonrxc] = { "rx link xon", 0, 32, 3576 KSTAT_KV_U_PACKETS }, 3577 [ix_counter_lxofftxc] = { "tx link xoff", IXGBE_LXOFFTXC, 32, 3578 KSTAT_KV_U_PACKETS }, 3579 [ix_counter_lxoffrxc] = { "rx link xoff", 0, 32, 3580 KSTAT_KV_U_PACKETS }, 3581 [ix_counter_prc64] = { "rx 64B", IXGBE_PRC64, 32, 3582 KSTAT_KV_U_PACKETS }, 3583 [ix_counter_prc127] = { "rx 65-127B", IXGBE_PRC127, 32, 3584 KSTAT_KV_U_PACKETS }, 3585 [ix_counter_prc255] = { "rx 128-255B", IXGBE_PRC255, 32, 3586 KSTAT_KV_U_PACKETS }, 3587 [ix_counter_prc511] = { "rx 256-511B", IXGBE_PRC511, 32, 3588 KSTAT_KV_U_PACKETS }, 3589 [ix_counter_prc1023] = { "rx 512-1023B", IXGBE_PRC1023, 32, 3590 KSTAT_KV_U_PACKETS }, 3591 [ix_counter_prc1522] = { "rx 1024-maxB", IXGBE_PRC1522, 32, 3592 KSTAT_KV_U_PACKETS }, 3593 [ix_counter_gptc] = { "tx good", IXGBE_GPTC, 32, 3594 KSTAT_KV_U_PACKETS }, 3595 [ix_counter_gorc] = { "rx good", IXGBE_GORCL, 36, 3596 KSTAT_KV_U_BYTES }, 3597 [ix_counter_gotc] = { "tx good", IXGBE_GOTCL, 36, 3598 KSTAT_KV_U_BYTES }, 3599 [ix_counter_ruc] = { "rx undersize", IXGBE_RUC, 32, 3600 KSTAT_KV_U_PACKETS }, 3601 [ix_counter_rfc] = { "rx fragment", IXGBE_RFC, 32, 3602 KSTAT_KV_U_PACKETS }, 3603 [ix_counter_roc] = { "rx oversize", IXGBE_ROC, 32, 3604 KSTAT_KV_U_PACKETS }, 3605 [ix_counter_rjc] = { "rx jabber", IXGBE_RJC, 32, 3606 KSTAT_KV_U_PACKETS }, 3607 [ix_counter_tor] = { "rx total", IXGBE_TORL, 36, 3608 KSTAT_KV_U_BYTES }, 3609 [ix_counter_tpr] = { "rx total", IXGBE_TPR, 32, 3610 KSTAT_KV_U_PACKETS }, 3611 [ix_counter_tpt] = { "tx total", IXGBE_TPT, 32, 3612 KSTAT_KV_U_PACKETS }, 3613 [ix_counter_gprc] = { "rx good", IXGBE_GPRC, 32, 3614 KSTAT_KV_U_PACKETS }, 3615 [ix_counter_bprc] = { "rx bcast", IXGBE_BPRC, 32, 3616 KSTAT_KV_U_PACKETS }, 3617 [ix_counter_mprc] = { "rx mcast", IXGBE_MPRC, 32, 3618 KSTAT_KV_U_PACKETS }, 3619 [ix_counter_ptc64] = { "tx 64B", IXGBE_PTC64, 32, 3620 KSTAT_KV_U_PACKETS }, 3621 [ix_counter_ptc127] = { "tx 65-127B", IXGBE_PTC127, 32, 3622 KSTAT_KV_U_PACKETS }, 3623 [ix_counter_ptc255] = { "tx 128-255B", IXGBE_PTC255, 32, 3624 KSTAT_KV_U_PACKETS }, 3625 [ix_counter_ptc511] = { "tx 256-511B", IXGBE_PTC511, 32, 3626 KSTAT_KV_U_PACKETS }, 3627 [ix_counter_ptc1023] = { "tx 512-1023B", IXGBE_PTC1023, 32, 3628 KSTAT_KV_U_PACKETS }, 3629 [ix_counter_ptc1522] = { "tx 1024-maxB", IXGBE_PTC1522, 32, 3630 KSTAT_KV_U_PACKETS }, 3631 [ix_counter_mptc] = { "tx mcast", IXGBE_MPTC, 32, 3632 KSTAT_KV_U_PACKETS }, 3633 [ix_counter_bptc] = { "tx bcast", IXGBE_BPTC, 32, 3634 KSTAT_KV_U_PACKETS }, 3635 }; 3636 3637 struct ix_rxq_kstats { 3638 struct kstat_kv qprc; 3639 struct kstat_kv qbrc; 3640 struct kstat_kv qprdc; 3641 }; 3642 3643 static const struct ix_rxq_kstats ix_rxq_kstats_tpl = { 3644 KSTAT_KV_UNIT_INITIALIZER("packets", 3645 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3646 KSTAT_KV_UNIT_INITIALIZER("bytes", 3647 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), 3648 KSTAT_KV_UNIT_INITIALIZER("qdrops", 3649 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3650 }; 3651 3652 struct ix_txq_kstats { 3653 struct kstat_kv qptc; 3654 struct kstat_kv qbtc; 3655 }; 3656 3657 static const struct ix_txq_kstats ix_txq_kstats_tpl = { 3658 KSTAT_KV_UNIT_INITIALIZER("packets", 3659 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3660 KSTAT_KV_UNIT_INITIALIZER("bytes", 3661 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), 3662 }; 3663 3664 static int ix_kstats_read(struct kstat *ks); 3665 static int ix_rxq_kstats_read(struct kstat *ks); 3666 static int ix_txq_kstats_read(struct kstat *ks); 3667 3668 static void 3669 ix_kstats(struct ix_softc *sc) 3670 { 3671 struct kstat *ks; 3672 struct kstat_kv *kvs; 3673 unsigned int i; 3674 3675 mtx_init(&sc->sc_kstat_mtx, IPL_SOFTCLOCK); 3676 timeout_set(&sc->sc_kstat_tmo, ix_kstats_tick, sc); 3677 3678 ks = kstat_create(sc->dev.dv_xname, 0, "ix-stats", 0, 3679 KSTAT_T_KV, 0); 3680 if (ks == NULL) 3681 return; 3682 3683 kvs = mallocarray(nitems(ix_counters), sizeof(*kvs), 3684 M_DEVBUF, M_WAITOK|M_ZERO); 3685 3686 for (i = 0; i < nitems(ix_counters); i++) { 3687 const struct ix_counter *ixc = &ix_counters[i]; 3688 3689 kstat_kv_unit_init(&kvs[i], ixc->name, 3690 KSTAT_KV_T_COUNTER64, ixc->unit); 3691 } 3692 3693 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3694 ks->ks_softc = sc; 3695 ks->ks_data = kvs; 3696 ks->ks_datalen = nitems(ix_counters) * sizeof(*kvs); 3697 ks->ks_read = ix_kstats_read; 3698 3699 sc->sc_kstat = ks; 3700 kstat_install(ks); 3701 } 3702 3703 static void 3704 ix_rxq_kstats(struct ix_softc *sc, struct rx_ring *rxr) 3705 { 3706 struct ix_rxq_kstats *stats; 3707 struct kstat *ks; 3708 3709 ks = kstat_create(sc->dev.dv_xname, 0, "ix-rxq", rxr->me, 3710 KSTAT_T_KV, 0); 3711 if (ks == NULL) 3712 return; 3713 3714 stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO); 3715 *stats = ix_rxq_kstats_tpl; 3716 3717 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3718 ks->ks_softc = rxr; 3719 ks->ks_data = stats; 3720 ks->ks_datalen = sizeof(*stats); 3721 ks->ks_read = ix_rxq_kstats_read; 3722 3723 rxr->kstat = ks; 3724 kstat_install(ks); 3725 } 3726 3727 static void 3728 ix_txq_kstats(struct ix_softc *sc, struct tx_ring *txr) 3729 { 3730 struct ix_txq_kstats *stats; 3731 struct kstat *ks; 3732 3733 ks = kstat_create(sc->dev.dv_xname, 0, "ix-txq", txr->me, 3734 KSTAT_T_KV, 0); 3735 if (ks == NULL) 3736 return; 3737 3738 stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO); 3739 *stats = ix_txq_kstats_tpl; 3740 3741 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3742 ks->ks_softc = txr; 3743 ks->ks_data = stats; 3744 ks->ks_datalen = sizeof(*stats); 3745 ks->ks_read = ix_txq_kstats_read; 3746 3747 txr->kstat = ks; 3748 kstat_install(ks); 3749 } 3750 3751 /********************************************************************** 3752 * 3753 * Update the board statistics counters. 3754 * 3755 **********************************************************************/ 3756 3757 static void 3758 ix_kstats_tick(void *arg) 3759 { 3760 struct ix_softc *sc = arg; 3761 int i; 3762 3763 timeout_add_sec(&sc->sc_kstat_tmo, 1); 3764 3765 mtx_enter(&sc->sc_kstat_mtx); 3766 ix_kstats_read(sc->sc_kstat); 3767 for (i = 0; i < sc->num_queues; i++) { 3768 ix_rxq_kstats_read(sc->rx_rings[i].kstat); 3769 ix_txq_kstats_read(sc->tx_rings[i].kstat); 3770 } 3771 mtx_leave(&sc->sc_kstat_mtx); 3772 } 3773 3774 static uint64_t 3775 ix_read36(struct ixgbe_hw *hw, bus_size_t loreg, bus_size_t hireg) 3776 { 3777 uint64_t lo, hi; 3778 3779 lo = IXGBE_READ_REG(hw, loreg); 3780 hi = IXGBE_READ_REG(hw, hireg); 3781 3782 return (((hi & 0xf) << 32) | lo); 3783 } 3784 3785 static int 3786 ix_kstats_read(struct kstat *ks) 3787 { 3788 struct ix_softc *sc = ks->ks_softc; 3789 struct kstat_kv *kvs = ks->ks_data; 3790 struct ixgbe_hw *hw = &sc->hw; 3791 unsigned int i; 3792 3793 for (i = 0; i < nitems(ix_counters); i++) { 3794 const struct ix_counter *ixc = &ix_counters[i]; 3795 uint32_t reg = ixc->reg; 3796 uint64_t v; 3797 3798 if (reg == 0) 3799 continue; 3800 3801 if (ixc->width > 32) { 3802 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3803 v = IXGBE_READ_REG(hw, reg + 4); 3804 else 3805 v = ix_read36(hw, reg, reg + 4); 3806 } else 3807 v = IXGBE_READ_REG(hw, reg); 3808 3809 kstat_kv_u64(&kvs[i]) += v; 3810 } 3811 3812 /* handle the exceptions */ 3813 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3814 kstat_kv_u64(&kvs[ix_counter_lxonrxc]) += 3815 IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3816 kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) += 3817 IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3818 } else { 3819 kstat_kv_u64(&kvs[ix_counter_lxonrxc]) += 3820 IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3821 kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) += 3822 IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3823 } 3824 3825 getnanouptime(&ks->ks_updated); 3826 3827 return (0); 3828 } 3829 3830 int 3831 ix_rxq_kstats_read(struct kstat *ks) 3832 { 3833 struct ix_rxq_kstats *stats = ks->ks_data; 3834 struct rx_ring *rxr = ks->ks_softc; 3835 struct ix_softc *sc = rxr->sc; 3836 struct ixgbe_hw *hw = &sc->hw; 3837 uint32_t i = rxr->me; 3838 3839 kstat_kv_u64(&stats->qprc) += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 3840 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3841 kstat_kv_u64(&stats->qprdc) += 3842 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3843 kstat_kv_u64(&stats->qbrc) += 3844 IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 3845 } else { 3846 kstat_kv_u64(&stats->qprdc) += 3847 IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 3848 kstat_kv_u64(&stats->qbrc) += 3849 ix_read36(hw, IXGBE_QBRC_L(i), IXGBE_QBRC_H(i)); 3850 } 3851 3852 getnanouptime(&ks->ks_updated); 3853 3854 return (0); 3855 } 3856 3857 int 3858 ix_txq_kstats_read(struct kstat *ks) 3859 { 3860 struct ix_txq_kstats *stats = ks->ks_data; 3861 struct tx_ring *txr = ks->ks_softc; 3862 struct ix_softc *sc = txr->sc; 3863 struct ixgbe_hw *hw = &sc->hw; 3864 uint32_t i = txr->me; 3865 3866 kstat_kv_u64(&stats->qptc) += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 3867 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3868 kstat_kv_u64(&stats->qbtc) += 3869 IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 3870 } else { 3871 kstat_kv_u64(&stats->qbtc) += 3872 ix_read36(hw, IXGBE_QBTC_L(i), IXGBE_QBTC_H(i)); 3873 } 3874 3875 getnanouptime(&ks->ks_updated); 3876 3877 return (0); 3878 } 3879 #endif /* NKVSTAT > 0 */ 3880 3881 void 3882 ixgbe_map_queue_statistics(struct ix_softc *sc) 3883 { 3884 int i; 3885 uint32_t r; 3886 3887 for (i = 0; i < 32; i++) { 3888 /* 3889 * Queues 0-15 are mapped 1:1 3890 * Queue 0 -> Counter 0 3891 * Queue 1 -> Counter 1 3892 * Queue 2 -> Counter 2.... 3893 * Queues 16-127 are mapped to Counter 0 3894 */ 3895 if (i < 4) { 3896 r = (i * 4 + 0); 3897 r |= (i * 4 + 1) << 8; 3898 r |= (i * 4 + 2) << 16; 3899 r |= (i * 4 + 3) << 24; 3900 } else 3901 r = 0; 3902 3903 IXGBE_WRITE_REG(&sc->hw, IXGBE_RQSMR(i), r); 3904 IXGBE_WRITE_REG(&sc->hw, IXGBE_TQSM(i), r); 3905 } 3906 } 3907