1 /* $OpenBSD: if_ix.c,v 1.51 2011/04/15 15:12:27 chl Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2008, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.5 2008/05/16 18:46:30 jfv Exp $*/ 36 37 #include <dev/pci/if_ix.h> 38 #include <dev/pci/ixgbe_type.h> 39 40 /********************************************************************* 41 * Driver version 42 *********************************************************************/ 43 44 #define IXGBE_DRIVER_VERSION "1.4.4" 45 46 /********************************************************************* 47 * PCI Device ID Table 48 * 49 * Used by probe to select devices to load on 50 *********************************************************************/ 51 52 const struct pci_matchid ixgbe_devices[] = { 53 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598 }, 54 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_BX }, 55 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF_DUAL }, 56 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF }, 57 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT }, 58 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT_DUAL }, 59 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4 }, 60 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4_DUAL }, 61 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_XF_LR }, 62 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_SFP }, 63 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_SR_DUAL_EM }, 64 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_DA_DUAL }, 65 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4 }, 66 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4_MEZZ }, 67 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_XAUI }, 68 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_COMBO_BACKPLANE }, 69 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_CX4 }, 70 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP }, 71 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_EM } 72 }; 73 74 /********************************************************************* 75 * Function prototypes 76 *********************************************************************/ 77 int ixgbe_probe(struct device *, void *, void *); 78 void ixgbe_attach(struct device *, struct device *, void *); 79 int ixgbe_detach(struct device *, int); 80 void ixgbe_start(struct ifnet *); 81 void ixgbe_start_locked(struct tx_ring *, struct ifnet *); 82 int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); 83 void ixgbe_watchdog(struct ifnet *); 84 void ixgbe_init(void *); 85 void ixgbe_stop(void *); 86 void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 87 int ixgbe_media_change(struct ifnet *); 88 void ixgbe_identify_hardware(struct ix_softc *); 89 int ixgbe_allocate_pci_resources(struct ix_softc *); 90 int ixgbe_allocate_legacy(struct ix_softc *); 91 int ixgbe_allocate_queues(struct ix_softc *); 92 void ixgbe_free_pci_resources(struct ix_softc *); 93 void ixgbe_local_timer(void *); 94 int ixgbe_hardware_init(struct ix_softc *); 95 void ixgbe_setup_interface(struct ix_softc *); 96 97 int ixgbe_allocate_transmit_buffers(struct tx_ring *); 98 int ixgbe_setup_transmit_structures(struct ix_softc *); 99 int ixgbe_setup_transmit_ring(struct tx_ring *); 100 void ixgbe_initialize_transmit_units(struct ix_softc *); 101 void ixgbe_free_transmit_structures(struct ix_softc *); 102 void ixgbe_free_transmit_buffers(struct tx_ring *); 103 104 int ixgbe_allocate_receive_buffers(struct rx_ring *); 105 int ixgbe_setup_receive_structures(struct ix_softc *); 106 int ixgbe_setup_receive_ring(struct rx_ring *); 107 void ixgbe_initialize_receive_units(struct ix_softc *); 108 void ixgbe_free_receive_structures(struct ix_softc *); 109 void ixgbe_free_receive_buffers(struct rx_ring *); 110 int ixgbe_rxfill(struct rx_ring *); 111 112 void ixgbe_enable_intr(struct ix_softc *); 113 void ixgbe_disable_intr(struct ix_softc *); 114 void ixgbe_update_stats_counters(struct ix_softc *); 115 int ixgbe_txeof(struct tx_ring *); 116 int ixgbe_rxeof(struct rx_ring *, int); 117 void ixgbe_rx_checksum(struct ix_softc *, uint32_t, struct mbuf *); 118 void ixgbe_set_promisc(struct ix_softc *); 119 void ixgbe_disable_promisc(struct ix_softc *); 120 void ixgbe_set_multi(struct ix_softc *); 121 #ifdef IX_DEBUG 122 void ixgbe_print_hw_stats(struct ix_softc *); 123 #endif 124 void ixgbe_update_link_status(struct ix_softc *); 125 int ixgbe_get_buf(struct rx_ring *, int); 126 int ixgbe_encap(struct tx_ring *, struct mbuf *); 127 void ixgbe_enable_hw_vlans(struct ix_softc * sc); 128 int ixgbe_dma_malloc(struct ix_softc *, bus_size_t, 129 struct ixgbe_dma_alloc *, int); 130 void ixgbe_dma_free(struct ix_softc *, struct ixgbe_dma_alloc *); 131 int ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); 132 int ixgbe_tso_setup(struct tx_ring *, struct mbuf *, uint32_t *); 133 void ixgbe_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t); 134 void ixgbe_configure_ivars(struct ix_softc *); 135 uint8_t *ixgbe_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 136 137 /* Legacy (single vector interrupt handler */ 138 int ixgbe_legacy_irq(void *); 139 140 /********************************************************************* 141 * OpenBSD Device Interface Entry Points 142 *********************************************************************/ 143 144 struct cfdriver ix_cd = { 145 NULL, "ix", DV_IFNET 146 }; 147 148 struct cfattach ix_ca = { 149 sizeof(struct ix_softc), ixgbe_probe, ixgbe_attach, ixgbe_detach 150 }; 151 152 int ixgbe_smart_speed = ixgbe_smart_speed_on; 153 154 /********************************************************************* 155 * Device identification routine 156 * 157 * ixgbe_probe determines if the driver should be loaded on 158 * sc based on PCI vendor/device id of the sc. 159 * 160 * return 0 on success, positive on failure 161 *********************************************************************/ 162 163 int 164 ixgbe_probe(struct device *parent, void *match, void *aux) 165 { 166 INIT_DEBUGOUT("ixgbe_probe: begin"); 167 168 return (pci_matchbyid((struct pci_attach_args *)aux, ixgbe_devices, 169 nitems(ixgbe_devices))); 170 } 171 172 /********************************************************************* 173 * Device initialization routine 174 * 175 * The attach entry point is called when the driver is being loaded. 176 * This routine identifies the type of hardware, allocates all resources 177 * and initializes the hardware. 178 * 179 * return 0 on success, positive on failure 180 *********************************************************************/ 181 182 void 183 ixgbe_attach(struct device *parent, struct device *self, void *aux) 184 { 185 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 186 struct ix_softc *sc = (struct ix_softc *)self; 187 int error = 0; 188 uint16_t csum; 189 uint32_t ctrl_ext; 190 struct ixgbe_hw *hw = &sc->hw; 191 192 INIT_DEBUGOUT("ixgbe_attach: begin"); 193 194 sc->osdep.os_sc = sc; 195 sc->osdep.os_pa = pa; 196 197 /* Core Lock Init*/ 198 mtx_init(&sc->core_mtx, IPL_NET); 199 200 /* Set up the timer callout */ 201 timeout_set(&sc->timer, ixgbe_local_timer, sc); 202 203 /* Determine hardware revision */ 204 ixgbe_identify_hardware(sc); 205 206 /* Indicate to RX setup to use Jumbo Clusters */ 207 sc->bigbufs = FALSE; 208 sc->num_tx_desc = DEFAULT_TXD; 209 sc->num_rx_desc = DEFAULT_RXD; 210 sc->rx_process_limit = 100; // XXX 211 212 /* Do base PCI setup - map BAR0 */ 213 if (ixgbe_allocate_pci_resources(sc)) 214 goto err_out; 215 216 /* Allocate our TX/RX Queues */ 217 if (ixgbe_allocate_queues(sc)) 218 goto err_out; 219 220 /* Initialize the shared code */ 221 if (hw->mac.type == ixgbe_mac_82598EB) 222 error = ixgbe_init_ops_82598(&sc->hw); 223 else 224 error = ixgbe_init_ops_82599(&sc->hw); 225 if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 226 /* 227 * No optics in this port, set up 228 * so the timer routine will probe 229 * for later insertion. 230 */ 231 sc->sfp_probe = TRUE; 232 error = 0; 233 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 234 printf(": Unsupported SFP+ module detected!\n"); 235 error = EIO; 236 goto err_late; 237 } else if (error) { 238 printf(": Unable to initialize the shared code\n"); 239 error = EIO; 240 goto err_late; 241 } 242 243 /* Make sure we have a good EEPROM before we read from it */ 244 if (sc->hw.eeprom.ops.validate_checksum(&sc->hw, &csum) < 0) { 245 printf(": The EEPROM Checksum Is Not Valid\n"); 246 error = EIO; 247 goto err_late; 248 } 249 250 /* Pick up the smart speed setting */ 251 if (sc->hw.mac.type == ixgbe_mac_82599EB) 252 sc->hw.phy.smart_speed = ixgbe_smart_speed; 253 254 /* Get Hardware Flow Control setting */ 255 hw->fc.requested_mode = ixgbe_fc_full; 256 hw->fc.pause_time = IXGBE_FC_PAUSE; 257 hw->fc.low_water = IXGBE_FC_LO; 258 hw->fc.high_water = IXGBE_FC_HI; 259 hw->fc.send_xon = TRUE; 260 261 error = sc->hw.mac.ops.init_hw(hw); 262 if (error == IXGBE_ERR_EEPROM_VERSION) { 263 printf(": This device is a pre-production adapter/" 264 "LOM. Please be aware there may be issues associated " 265 "with your hardware.\n If you are experiencing problems " 266 "please contact your Intel or hardware representative " 267 "who provided you with this hardware.\n"); 268 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 269 printf("Unsupported SFP+ Module\n"); 270 } 271 272 if (error) { 273 printf(": Hardware Initialization Failure\n"); 274 goto err_late; 275 } 276 277 bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr, 278 IXGBE_ETH_LENGTH_OF_ADDRESS); 279 280 /* XXX sc->msix > 1 && ixgbe_allocate_msix() */ 281 error = ixgbe_allocate_legacy(sc); 282 if (error) 283 goto err_late; 284 285 /* Setup OS specific network interface */ 286 ixgbe_setup_interface(sc); 287 288 /* Initialize statistics */ 289 ixgbe_update_stats_counters(sc); 290 291 /* let hardware know driver is loaded */ 292 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 293 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 294 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 295 296 printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); 297 298 INIT_DEBUGOUT("ixgbe_attach: end"); 299 return; 300 301 err_late: 302 ixgbe_free_transmit_structures(sc); 303 ixgbe_free_receive_structures(sc); 304 err_out: 305 ixgbe_free_pci_resources(sc); 306 } 307 308 /********************************************************************* 309 * Device removal routine 310 * 311 * The detach entry point is called when the driver is being removed. 312 * This routine stops the adapter and deallocates all the resources 313 * that were allocated for driver operation. 314 * 315 * return 0 on success, positive on failure 316 *********************************************************************/ 317 318 int 319 ixgbe_detach(struct device *self, int flags) 320 { 321 struct ix_softc *sc = (struct ix_softc *)self; 322 struct ifnet *ifp = &sc->arpcom.ac_if; 323 uint32_t ctrl_ext; 324 325 INIT_DEBUGOUT("ixgbe_detach: begin"); 326 327 ixgbe_stop(sc); 328 329 /* let hardware know driver is unloading */ 330 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 331 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 332 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 333 334 ether_ifdetach(ifp); 335 if_detach(ifp); 336 337 timeout_del(&sc->timer); 338 ixgbe_free_pci_resources(sc); 339 340 ixgbe_free_transmit_structures(sc); 341 ixgbe_free_receive_structures(sc); 342 343 return (0); 344 } 345 346 /********************************************************************* 347 * Transmit entry point 348 * 349 * ixgbe_start is called by the stack to initiate a transmit. 350 * The driver will remain in this routine as long as there are 351 * packets to transmit and transmit resources are available. 352 * In case resources are not available stack is notified and 353 * the packet is requeued. 354 **********************************************************************/ 355 356 void 357 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) 358 { 359 struct mbuf *m_head; 360 struct ix_softc *sc = txr->sc; 361 int post = 0; 362 363 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING) 364 return; 365 366 if (!sc->link_active) 367 return; 368 369 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 370 txr->txdma.dma_map->dm_mapsize, 371 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 372 373 for (;;) { 374 IFQ_POLL(&ifp->if_snd, m_head); 375 if (m_head == NULL) 376 break; 377 378 if (ixgbe_encap(txr, m_head)) { 379 ifp->if_flags |= IFF_OACTIVE; 380 break; 381 } 382 383 IFQ_DEQUEUE(&ifp->if_snd, m_head); 384 385 #if NBPFILTER > 0 386 if (ifp->if_bpf) 387 bpf_mtap_ether(ifp->if_bpf, m_head, BPF_DIRECTION_OUT); 388 #endif 389 390 /* Set timeout in case hardware has problems transmitting */ 391 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 392 ifp->if_timer = IXGBE_TX_TIMEOUT; 393 394 post = 1; 395 } 396 397 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 398 0, txr->txdma.dma_map->dm_mapsize, 399 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 400 401 /* 402 * Advance the Transmit Descriptor Tail (Tdt), this tells the 403 * hardware that this frame is available to transmit. 404 */ 405 if (post) 406 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->me), 407 txr->next_avail_tx_desc); 408 } 409 410 411 void 412 ixgbe_start(struct ifnet *ifp) 413 { 414 struct ix_softc *sc = ifp->if_softc; 415 struct tx_ring *txr = sc->tx_rings; 416 uint32_t queue = 0; 417 418 #if 0 419 /* 420 * This is really just here for testing 421 * TX multiqueue, ultimately what is 422 * needed is the flow support in the stack 423 * and appropriate logic here to deal with 424 * it. -jfv 425 */ 426 if (sc->num_tx_queues > 1) 427 queue = (curcpu % sc->num_tx_queues); 428 #endif 429 430 txr = &sc->tx_rings[queue]; 431 432 if (ifp->if_flags & IFF_RUNNING) 433 ixgbe_start_locked(txr, ifp); 434 435 return; 436 } 437 438 /********************************************************************* 439 * Ioctl entry point 440 * 441 * ixgbe_ioctl is called when the user wants to configure the 442 * interface. 443 * 444 * return 0 on success, positive on failure 445 **********************************************************************/ 446 447 int 448 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 449 { 450 struct ix_softc *sc = ifp->if_softc; 451 struct ifaddr *ifa = (struct ifaddr *) data; 452 struct ifreq *ifr = (struct ifreq *) data; 453 int s, error = 0; 454 455 s = splnet(); 456 457 switch (command) { 458 case SIOCSIFADDR: 459 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); 460 ifp->if_flags |= IFF_UP; 461 if (!(ifp->if_flags & IFF_RUNNING)) 462 ixgbe_init(sc); 463 #ifdef INET 464 if (ifa->ifa_addr->sa_family == AF_INET) 465 arp_ifinit(&sc->arpcom, ifa); 466 #endif 467 break; 468 469 case SIOCSIFMTU: 470 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 471 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > ifp->if_hardmtu) 472 error = EINVAL; 473 else if (ifp->if_mtu != ifr->ifr_mtu) { 474 ifp->if_mtu = ifr->ifr_mtu; 475 sc->max_frame_size = 476 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 477 ixgbe_init(sc); 478 } 479 break; 480 481 case SIOCSIFFLAGS: 482 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 483 if (ifp->if_flags & IFF_UP) { 484 if ((ifp->if_flags & IFF_RUNNING)) { 485 if ((ifp->if_flags ^ sc->if_flags) & 486 (IFF_PROMISC | IFF_ALLMULTI)) { 487 ixgbe_disable_promisc(sc); 488 ixgbe_set_promisc(sc); 489 } 490 } else 491 ixgbe_init(sc); 492 } else 493 if (ifp->if_flags & IFF_RUNNING) 494 ixgbe_stop(sc); 495 sc->if_flags = ifp->if_flags; 496 break; 497 498 case SIOCSIFMEDIA: 499 case SIOCGIFMEDIA: 500 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 501 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 502 break; 503 504 default: 505 error = ether_ioctl(ifp, &sc->arpcom, command, data); 506 } 507 508 if (error == ENETRESET) { 509 if (ifp->if_flags & IFF_RUNNING) { 510 ixgbe_disable_intr(sc); 511 ixgbe_set_multi(sc); 512 ixgbe_enable_intr(sc); 513 } 514 error = 0; 515 } 516 517 splx(s); 518 return (error); 519 } 520 521 /********************************************************************* 522 * Watchdog entry point 523 * 524 * This routine is called by the local timer 525 * to detect hardware hangs . 526 * 527 **********************************************************************/ 528 529 void 530 ixgbe_watchdog(struct ifnet * ifp) 531 { 532 struct ix_softc *sc = (struct ix_softc *)ifp->if_softc; 533 struct tx_ring *txr = sc->tx_rings; 534 struct ixgbe_hw *hw = &sc->hw; 535 int tx_hang = FALSE; 536 int i; 537 538 /* 539 * The timer is set to 5 every time ixgbe_start() queues a packet. 540 * Then ixgbe_txeof() keeps resetting to 5 as long as it cleans at 541 * least one descriptor. 542 * Finally, anytime all descriptors are clean the timer is 543 * set to 0. 544 */ 545 for (i = 0; i < sc->num_tx_queues; i++, txr++) { 546 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) 547 continue; 548 else { 549 tx_hang = TRUE; 550 break; 551 } 552 } 553 if (tx_hang == FALSE) 554 return; 555 556 /* 557 * If we are in this routine because of pause frames, then don't 558 * reset the hardware. 559 */ 560 if (IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) { 561 for (i = 0; i < sc->num_tx_queues; i++, txr++) 562 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 563 ifp->if_timer = IXGBE_TX_TIMEOUT; 564 return; 565 } 566 567 568 printf("%s: Watchdog timeout -- resetting\n", ifp->if_xname); 569 for (i = 0; i < sc->num_tx_queues; i++, txr++) { 570 printf("%s: Queue(%d) tdh = %d, hw tdt = %d\n", ifp->if_xname, i, 571 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 572 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 573 printf("%s: TX(%d) desc avail = %d, Next TX to Clean = %d\n", ifp->if_xname, 574 i, txr->tx_avail, txr->next_tx_to_clean); 575 } 576 ifp->if_flags &= ~IFF_RUNNING; 577 sc->watchdog_events++; 578 579 ixgbe_init(sc); 580 return; 581 } 582 583 /********************************************************************* 584 * Init entry point 585 * 586 * This routine is used in two ways. It is used by the stack as 587 * init entry point in network interface structure. It is also used 588 * by the driver as a hw/sw initialization routine to get to a 589 * consistent state. 590 * 591 * return 0 on success, positive on failure 592 **********************************************************************/ 593 #define IXGBE_MHADD_MFS_SHIFT 16 594 595 void 596 ixgbe_init(void *arg) 597 { 598 struct ix_softc *sc = (struct ix_softc *)arg; 599 struct ifnet *ifp = &sc->arpcom.ac_if; 600 struct rx_ring *rxr = sc->rx_rings; 601 uint32_t k, txdctl, rxdctl, mhadd, gpie; 602 int err; 603 int i, s; 604 605 INIT_DEBUGOUT("ixgbe_init: begin"); 606 607 s = splnet(); 608 609 ixgbe_stop(sc); 610 611 /* Get the latest mac address, User can use a LAA */ 612 bcopy(sc->arpcom.ac_enaddr, sc->hw.mac.addr, 613 IXGBE_ETH_LENGTH_OF_ADDRESS); 614 ixgbe_hw(&sc->hw, set_rar, 0, sc->hw.mac.addr, 0, 1); 615 sc->hw.addr_ctrl.rar_used_count = 1; 616 617 /* Do a warm reset */ 618 sc->hw.mac.ops.reset_hw(&sc->hw); 619 620 if (ifp->if_capabilities & IFCAP_VLAN_HWTAGGING) 621 ixgbe_enable_hw_vlans(sc); 622 623 /* Prepare transmit descriptors and buffers */ 624 if (ixgbe_setup_transmit_structures(sc)) { 625 printf("%s: Could not setup transmit structures\n", 626 ifp->if_xname); 627 ixgbe_stop(sc); 628 splx(s); 629 return; 630 } 631 632 ixgbe_initialize_transmit_units(sc); 633 634 /* Setup Multicast table */ 635 ixgbe_set_multi(sc); 636 637 /* 638 * If we are resetting MTU smaller than 2K 639 * drop to small RX buffers 640 */ 641 if (sc->max_frame_size <= MCLBYTES) 642 sc->bigbufs = FALSE; 643 644 /* Prepare receive descriptors and buffers */ 645 if (ixgbe_setup_receive_structures(sc)) { 646 printf("%s: Could not setup receive structures\n", ifp->if_xname); 647 ixgbe_stop(sc); 648 splx(s); 649 return; 650 } 651 652 /* Configure RX settings */ 653 ixgbe_initialize_receive_units(sc); 654 655 gpie = IXGBE_READ_REG(&sc->hw, IXGBE_GPIE); 656 657 if (sc->hw.mac.type == ixgbe_mac_82599EB) { 658 gpie |= IXGBE_SDP1_GPIEN; 659 gpie |= IXGBE_SDP2_GPIEN; 660 /* 661 * Set LL interval to max to reduce the number of low latency 662 * interrupts hitting the card when the ring is getting full. 663 */ 664 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 665 } 666 667 /* Enable Fan Failure Interrupt */ 668 if (sc->hw.phy.media_type == ixgbe_media_type_copper) 669 gpie |= IXGBE_SDP1_GPIEN; 670 if (sc->msix) { 671 /* Enable Enhanced MSIX mode */ 672 gpie |= IXGBE_GPIE_MSIX_MODE; 673 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 674 IXGBE_GPIE_OCD; 675 } 676 IXGBE_WRITE_REG(&sc->hw, IXGBE_GPIE, gpie); 677 678 /* Set MTU size */ 679 if (ifp->if_mtu > ETHERMTU) { 680 mhadd = IXGBE_READ_REG(&sc->hw, IXGBE_MHADD); 681 mhadd &= ~IXGBE_MHADD_MFS_MASK; 682 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 683 IXGBE_WRITE_REG(&sc->hw, IXGBE_MHADD, mhadd); 684 } 685 686 /* Now enable all the queues */ 687 688 for (i = 0; i < sc->num_tx_queues; i++) { 689 txdctl = IXGBE_READ_REG(&sc->hw, IXGBE_TXDCTL(i)); 690 txdctl |= IXGBE_TXDCTL_ENABLE; 691 /* Set WTHRESH to 8, burst writeback */ 692 txdctl |= (8 << 16); 693 IXGBE_WRITE_REG(&sc->hw, IXGBE_TXDCTL(i), txdctl); 694 } 695 696 for (i = 0; i < sc->num_rx_queues; i++) { 697 rxdctl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 698 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 699 /* PTHRESH set to 32 */ 700 rxdctl |= 0x0020; 701 } 702 rxdctl |= IXGBE_RXDCTL_ENABLE; 703 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), rxdctl); 704 for (k = 0; k < 10; k++) { 705 if (IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)) & 706 IXGBE_RXDCTL_ENABLE) 707 break; 708 else 709 msec_delay(1); 710 } 711 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), rxr->last_rx_desc_filled); 712 } 713 714 timeout_add_sec(&sc->timer, 1); 715 716 #ifdef MSI 717 /* Set up MSI/X routing */ 718 if (ixgbe_enable_msix) 719 ixgbe_configure_ivars(sc); 720 else /* Simple settings for Legacy/MSI */ 721 #else 722 { 723 ixgbe_set_ivar(sc, 0, 0, 0); 724 ixgbe_set_ivar(sc, 0, 0, 1); 725 } 726 #endif 727 728 /* 729 * Check on any SFP devices that 730 * need to be kick-started 731 */ 732 err = sc->hw.phy.ops.identify(&sc->hw); 733 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 734 printf("Unsupported SFP+ module type was detected.\n"); 735 splx(s); 736 return; 737 } 738 739 ixgbe_enable_intr(sc); 740 741 /* Now inform the stack we're ready */ 742 ifp->if_flags |= IFF_RUNNING; 743 ifp->if_flags &= ~IFF_OACTIVE; 744 745 splx(s); 746 } 747 748 /********************************************************************* 749 * 750 * Legacy Interrupt Service routine 751 * 752 **********************************************************************/ 753 754 int 755 ixgbe_legacy_irq(void *arg) 756 { 757 struct ix_softc *sc = (struct ix_softc *)arg; 758 struct ifnet *ifp = &sc->arpcom.ac_if; 759 struct tx_ring *txr = sc->tx_rings; 760 struct rx_ring *rxr = sc->rx_rings; 761 struct ixgbe_hw *hw = &sc->hw; 762 uint32_t reg_eicr; 763 int refill = 0; 764 765 reg_eicr = IXGBE_READ_REG(&sc->hw, IXGBE_EICR); 766 if (reg_eicr == 0) 767 return (0); 768 769 if (ifp->if_flags & IFF_RUNNING) { 770 ixgbe_rxeof(rxr, -1); 771 ixgbe_txeof(txr); 772 refill = 1; 773 } 774 775 /* Check for fan failure */ 776 if ((hw->phy.media_type == ixgbe_media_type_copper) && 777 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 778 printf("%s: \nCRITICAL: FAN FAILURE!! " 779 "REPLACE IMMEDIATELY!!\n", ifp->if_xname); 780 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, 781 IXGBE_EICR_GPI_SDP1); 782 } 783 784 /* Link status change */ 785 if (reg_eicr & IXGBE_EICR_LSC) { 786 timeout_del(&sc->timer); 787 ixgbe_update_link_status(sc); 788 timeout_add_sec(&sc->timer, 1); 789 } 790 791 if (refill && ixgbe_rxfill(rxr)) { 792 /* Advance the Rx Queue "Tail Pointer" */ 793 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(rxr->me), 794 rxr->last_rx_desc_filled); 795 } 796 797 if (ifp->if_flags & IFF_RUNNING && !IFQ_IS_EMPTY(&ifp->if_snd)) 798 ixgbe_start_locked(txr, ifp); 799 800 return (1); 801 } 802 803 /********************************************************************* 804 * 805 * Media Ioctl callback 806 * 807 * This routine is called whenever the user queries the status of 808 * the interface using ifconfig. 809 * 810 **********************************************************************/ 811 void 812 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 813 { 814 struct ix_softc *sc = ifp->if_softc; 815 816 ifmr->ifm_active = IFM_ETHER; 817 ifmr->ifm_status = IFM_AVALID; 818 819 INIT_DEBUGOUT("ixgbe_media_status: begin"); 820 ixgbe_update_link_status(sc); 821 822 if (LINK_STATE_IS_UP(ifp->if_link_state)) { 823 ifmr->ifm_status |= IFM_ACTIVE; 824 825 switch (sc->link_speed) { 826 case IXGBE_LINK_SPEED_1GB_FULL: 827 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 828 break; 829 case IXGBE_LINK_SPEED_10GB_FULL: 830 ifmr->ifm_active |= sc->optics | IFM_FDX; 831 break; 832 } 833 } 834 } 835 836 /********************************************************************* 837 * 838 * Media Ioctl callback 839 * 840 * This routine is called when the user changes speed/duplex using 841 * media/mediopt option with ifconfig. 842 * 843 **********************************************************************/ 844 int 845 ixgbe_media_change(struct ifnet * ifp) 846 { 847 /* ignore */ 848 return (0); 849 } 850 851 /********************************************************************* 852 * 853 * This routine maps the mbufs to tx descriptors. 854 * WARNING: while this code is using an MQ style infrastructure, 855 * it would NOT work as is with more than 1 queue. 856 * 857 * return 0 on success, positive on failure 858 **********************************************************************/ 859 860 int 861 ixgbe_encap(struct tx_ring *txr, struct mbuf *m_head) 862 { 863 struct ix_softc *sc = txr->sc; 864 uint32_t olinfo_status = 0, cmd_type_len = 0; 865 int i, j, error; 866 int first; 867 bus_dmamap_t map; 868 struct ixgbe_tx_buf *txbuf, *txbuf_mapped; 869 union ixgbe_adv_tx_desc *txd = NULL; 870 uint32_t paylen = 0; 871 872 /* Basic descriptor defines */ 873 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA; 874 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; 875 876 #if NVLAN > 0 877 if (m_head->m_flags & M_VLANTAG) 878 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 879 #endif 880 881 /* 882 * Force a cleanup if number of TX descriptors 883 * available is below the threshold. If it fails 884 * to get above, then abort transmit. 885 */ 886 if (txr->tx_avail <= IXGBE_TX_CLEANUP_THRESHOLD) { 887 ixgbe_txeof(txr); 888 /* Make sure things have improved */ 889 if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) { 890 txr->no_tx_desc_avail++; 891 return (ENOBUFS); 892 } 893 } 894 895 /* 896 * Important to capture the first descriptor 897 * used because it will contain the index of 898 * the one we tell the hardware to report back 899 */ 900 first = txr->next_avail_tx_desc; 901 txbuf = &txr->tx_buffers[first]; 902 txbuf_mapped = txbuf; 903 map = txbuf->map; 904 905 /* 906 * Map the packet for DMA. 907 */ 908 error = bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 909 m_head, BUS_DMA_NOWAIT); 910 911 if (error == ENOMEM) { 912 sc->no_tx_dma_setup++; 913 return (error); 914 } else if (error != 0) { 915 sc->no_tx_dma_setup++; 916 return (error); 917 } 918 919 /* Make certain there are enough descriptors */ 920 if (map->dm_nsegs > txr->tx_avail - 2) { 921 txr->no_tx_desc_avail++; 922 error = ENOBUFS; 923 goto xmit_fail; 924 } 925 926 /* 927 * Set the appropriate offload context 928 * this becomes the first descriptor of 929 * a packet. 930 */ 931 #ifdef notyet 932 if (ixgbe_tso_setup(txr, m_head, &paylen)) { 933 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 934 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 935 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 936 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 937 ++sc->tso_tx; 938 } else 939 #endif 940 if (ixgbe_tx_ctx_setup(txr, m_head)) 941 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 942 943 /* Record payload length */ 944 if (paylen == 0) 945 olinfo_status |= m_head->m_pkthdr.len << 946 IXGBE_ADVTXD_PAYLEN_SHIFT; 947 948 i = txr->next_avail_tx_desc; 949 for (j = 0; j < map->dm_nsegs; j++) { 950 txbuf = &txr->tx_buffers[i]; 951 txd = &txr->tx_base[i]; 952 953 txd->read.buffer_addr = htole64(map->dm_segs[j].ds_addr); 954 txd->read.cmd_type_len = htole32(txr->txd_cmd | 955 cmd_type_len | map->dm_segs[j].ds_len); 956 txd->read.olinfo_status = htole32(olinfo_status); 957 958 if (++i == sc->num_tx_desc) 959 i = 0; 960 961 txbuf->m_head = NULL; 962 } 963 964 txd->read.cmd_type_len |= 965 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 966 txr->tx_avail -= map->dm_nsegs; 967 txr->next_avail_tx_desc = i; 968 969 txbuf->m_head = m_head; 970 txbuf_mapped->map = txbuf->map; 971 txbuf->map = map; 972 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 973 BUS_DMASYNC_PREWRITE); 974 975 /* Set the index of the descriptor that will be marked done */ 976 txbuf = &txr->tx_buffers[first]; 977 978 ++txr->tx_packets; 979 return (0); 980 981 xmit_fail: 982 bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map); 983 return (error); 984 985 } 986 987 void 988 ixgbe_set_promisc(struct ix_softc *sc) 989 { 990 991 uint32_t reg_rctl; 992 struct ifnet *ifp = &sc->arpcom.ac_if; 993 994 reg_rctl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 995 996 if (ifp->if_flags & IFF_PROMISC) { 997 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 998 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 999 } else if (ifp->if_flags & IFF_ALLMULTI) { 1000 reg_rctl |= IXGBE_FCTRL_MPE; 1001 reg_rctl &= ~IXGBE_FCTRL_UPE; 1002 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 1003 } 1004 return; 1005 } 1006 1007 void 1008 ixgbe_disable_promisc(struct ix_softc * sc) 1009 { 1010 uint32_t reg_rctl; 1011 1012 reg_rctl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1013 1014 reg_rctl &= (~IXGBE_FCTRL_UPE); 1015 reg_rctl &= (~IXGBE_FCTRL_MPE); 1016 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl); 1017 1018 return; 1019 } 1020 1021 1022 /********************************************************************* 1023 * Multicast Update 1024 * 1025 * This routine is called whenever multicast address list is updated. 1026 * 1027 **********************************************************************/ 1028 #define IXGBE_RAR_ENTRIES 16 1029 1030 void 1031 ixgbe_set_multi(struct ix_softc *sc) 1032 { 1033 uint32_t fctrl; 1034 uint8_t mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1035 uint8_t *update_ptr; 1036 struct ether_multi *enm; 1037 struct ether_multistep step; 1038 int mcnt = 0; 1039 struct ifnet *ifp = &sc->arpcom.ac_if; 1040 1041 IOCTL_DEBUGOUT("ixgbe_set_multi: begin"); 1042 1043 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1044 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1045 if (ifp->if_flags & IFF_PROMISC) 1046 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1047 else if (ifp->if_flags & IFF_ALLMULTI) { 1048 fctrl |= IXGBE_FCTRL_MPE; 1049 fctrl &= ~IXGBE_FCTRL_UPE; 1050 } else 1051 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1052 1053 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1054 1055 ETHER_FIRST_MULTI(step, &sc->arpcom, enm); 1056 while (enm != NULL) { 1057 if (bcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) { 1058 ifp->if_flags |= IFF_ALLMULTI; 1059 mcnt = MAX_NUM_MULTICAST_ADDRESSES; 1060 } 1061 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 1062 break; 1063 bcopy(enm->enm_addrlo, 1064 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1065 IXGBE_ETH_LENGTH_OF_ADDRESS); 1066 mcnt++; 1067 ETHER_NEXT_MULTI(step, enm); 1068 } 1069 1070 update_ptr = mta; 1071 ixgbe_hw(&sc->hw, update_mc_addr_list, 1072 update_ptr, mcnt, ixgbe_mc_array_itr); 1073 1074 return; 1075 } 1076 1077 /* 1078 * This is an iterator function now needed by the multicast 1079 * shared code. It simply feeds the shared code routine the 1080 * addresses in the array of ixgbe_set_multi() one by one. 1081 */ 1082 uint8_t * 1083 ixgbe_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1084 { 1085 uint8_t *addr = *update_ptr; 1086 uint8_t *newptr; 1087 *vmdq = 0; 1088 1089 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1090 *update_ptr = newptr; 1091 return addr; 1092 } 1093 1094 1095 /********************************************************************* 1096 * Timer routine 1097 * 1098 * This routine checks for link status,updates statistics, 1099 * and runs the watchdog timer. 1100 * 1101 **********************************************************************/ 1102 1103 void 1104 ixgbe_local_timer(void *arg) 1105 { 1106 struct ix_softc *sc = arg; 1107 #ifdef IX_DEBUG 1108 struct ifnet *ifp = &sc->arpcom.ac_if; 1109 #endif 1110 int s; 1111 1112 s = splnet(); 1113 1114 ixgbe_update_link_status(sc); 1115 ixgbe_update_stats_counters(sc); 1116 1117 #ifdef IX_DEBUG 1118 if ((ifp->if_flags & (IFF_RUNNING|IFF_DEBUG)) == 1119 (IFF_RUNNING|IFF_DEBUG)) 1120 ixgbe_print_hw_stats(sc); 1121 #endif 1122 1123 timeout_add_sec(&sc->timer, 1); 1124 1125 splx(s); 1126 } 1127 1128 void 1129 ixgbe_update_link_status(struct ix_softc *sc) 1130 { 1131 int link_up = FALSE; 1132 struct ifnet *ifp = &sc->arpcom.ac_if; 1133 struct tx_ring *txr = sc->tx_rings; 1134 int link_state; 1135 int i; 1136 1137 ixgbe_hw(&sc->hw, check_link, &sc->link_speed, &link_up, 0); 1138 1139 link_state = link_up ? LINK_STATE_FULL_DUPLEX : LINK_STATE_DOWN; 1140 1141 if (ifp->if_link_state != link_state) { 1142 sc->link_active = link_up; 1143 ifp->if_link_state = link_state; 1144 if_link_state_change(ifp); 1145 } 1146 1147 if (LINK_STATE_IS_UP(ifp->if_link_state)) { 1148 switch (sc->link_speed) { 1149 case IXGBE_LINK_SPEED_UNKNOWN: 1150 ifp->if_baudrate = 0; 1151 break; 1152 case IXGBE_LINK_SPEED_100_FULL: 1153 ifp->if_baudrate = IF_Mbps(100); 1154 break; 1155 case IXGBE_LINK_SPEED_1GB_FULL: 1156 ifp->if_baudrate = IF_Gbps(1); 1157 break; 1158 case IXGBE_LINK_SPEED_10GB_FULL: 1159 ifp->if_baudrate = IF_Gbps(10); 1160 break; 1161 } 1162 } else { 1163 ifp->if_baudrate = 0; 1164 ifp->if_timer = 0; 1165 for (i = 0; i < sc->num_tx_queues; i++) 1166 txr[i].watchdog_timer = FALSE; 1167 } 1168 1169 1170 return; 1171 } 1172 1173 1174 1175 /********************************************************************* 1176 * 1177 * This routine disables all traffic on the sc by issuing a 1178 * global reset on the MAC and deallocates TX/RX buffers. 1179 * 1180 **********************************************************************/ 1181 1182 void 1183 ixgbe_stop(void *arg) 1184 { 1185 struct ix_softc *sc = arg; 1186 struct ifnet *ifp = &sc->arpcom.ac_if; 1187 1188 /* Tell the stack that the interface is no longer active */ 1189 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1190 1191 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1192 ixgbe_disable_intr(sc); 1193 1194 ixgbe_hw0(&sc->hw, reset_hw); 1195 sc->hw.adapter_stopped = FALSE; 1196 ixgbe_hw0(&sc->hw, stop_adapter); 1197 timeout_del(&sc->timer); 1198 1199 /* reprogram the RAR[0] in case user changed it. */ 1200 ixgbe_hw(&sc->hw, set_rar, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 1201 1202 ixgbe_free_transmit_structures(sc); 1203 ixgbe_free_receive_structures(sc); 1204 } 1205 1206 1207 /********************************************************************* 1208 * 1209 * Determine hardware revision. 1210 * 1211 **********************************************************************/ 1212 void 1213 ixgbe_identify_hardware(struct ix_softc *sc) 1214 { 1215 struct ixgbe_osdep *os = &sc->osdep; 1216 struct pci_attach_args *pa = os->os_pa; 1217 uint32_t reg; 1218 1219 /* Save off the information about this board */ 1220 sc->hw.vendor_id = PCI_VENDOR(pa->pa_id); 1221 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 1222 1223 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG); 1224 sc->hw.revision_id = PCI_REVISION(reg); 1225 1226 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 1227 sc->hw.subsystem_vendor_id = PCI_VENDOR(reg); 1228 sc->hw.subsystem_device_id = PCI_PRODUCT(reg); 1229 1230 switch (sc->hw.device_id) { 1231 case PCI_PRODUCT_INTEL_82598: 1232 case PCI_PRODUCT_INTEL_82598AF_DUAL: 1233 case PCI_PRODUCT_INTEL_82598_DA_DUAL: 1234 case PCI_PRODUCT_INTEL_82598AF: 1235 case PCI_PRODUCT_INTEL_82598_SR_DUAL_EM: 1236 case PCI_PRODUCT_INTEL_82598EB_SFP: 1237 sc->hw.mac.type = ixgbe_mac_82598EB; 1238 sc->optics = IFM_10G_SR; 1239 break; 1240 case PCI_PRODUCT_INTEL_82598EB_CX4_DUAL: 1241 case PCI_PRODUCT_INTEL_82598EB_CX4: 1242 sc->hw.mac.type = ixgbe_mac_82598EB; 1243 sc->optics = IFM_10G_CX4; 1244 break; 1245 case PCI_PRODUCT_INTEL_82598EB_XF_LR: 1246 sc->hw.mac.type = ixgbe_mac_82598EB; 1247 sc->optics = IFM_10G_LR; 1248 break; 1249 case PCI_PRODUCT_INTEL_82598AT_DUAL: 1250 case PCI_PRODUCT_INTEL_82598AT: 1251 sc->hw.mac.type = ixgbe_mac_82598EB; 1252 sc->optics = IFM_10G_T; 1253 break; 1254 case PCI_PRODUCT_INTEL_82598_BX: 1255 sc->hw.mac.type = ixgbe_mac_82598EB; 1256 sc->optics = IFM_AUTO; 1257 break; 1258 case PCI_PRODUCT_INTEL_82599_SFP: 1259 case PCI_PRODUCT_INTEL_82599_SFP_EM: 1260 sc->hw.mac.type = ixgbe_mac_82599EB; 1261 sc->optics = IFM_10G_SR; 1262 break; 1263 case PCI_PRODUCT_INTEL_82599_KX4: 1264 case PCI_PRODUCT_INTEL_82599_KX4_MEZZ: 1265 case PCI_PRODUCT_INTEL_82599_CX4: 1266 sc->hw.mac.type = ixgbe_mac_82599EB; 1267 sc->optics = IFM_10G_CX4; 1268 break; 1269 case PCI_PRODUCT_INTEL_82599_XAUI: 1270 case PCI_PRODUCT_INTEL_82599_COMBO_BACKPLANE: 1271 sc->hw.mac.type = ixgbe_mac_82599EB; 1272 sc->optics = IFM_AUTO; 1273 break; 1274 default: 1275 sc->optics = IFM_AUTO; 1276 break; 1277 } 1278 } 1279 1280 /********************************************************************* 1281 * 1282 * Setup the Legacy or MSI Interrupt handler 1283 * 1284 **********************************************************************/ 1285 int 1286 ixgbe_allocate_legacy(struct ix_softc *sc) 1287 { 1288 struct ifnet *ifp = &sc->arpcom.ac_if; 1289 struct ixgbe_osdep *os = &sc->osdep; 1290 struct pci_attach_args *pa = os->os_pa; 1291 const char *intrstr = NULL; 1292 pci_chipset_tag_t pc = pa->pa_pc; 1293 pci_intr_handle_t ih; 1294 1295 /* Legacy RID at 0 */ 1296 if (sc->msix == 0) 1297 sc->rid[0] = 0; 1298 1299 /* We allocate a single interrupt resource */ 1300 if (pci_intr_map(pa, &ih)) { 1301 printf(": couldn't map interrupt\n"); 1302 return (ENXIO); 1303 } 1304 1305 intrstr = pci_intr_string(pc, ih); 1306 sc->tag[0] = pci_intr_establish(pc, ih, IPL_NET, 1307 ixgbe_legacy_irq, sc, ifp->if_xname); 1308 if (sc->tag[0] == NULL) { 1309 printf(": couldn't establish interrupt"); 1310 if (intrstr != NULL) 1311 printf(" at %s", intrstr); 1312 printf("\n"); 1313 return (ENXIO); 1314 } 1315 printf(": %s", intrstr); 1316 1317 return (0); 1318 } 1319 1320 int 1321 ixgbe_allocate_pci_resources(struct ix_softc *sc) 1322 { 1323 struct ixgbe_osdep *os = &sc->osdep; 1324 struct pci_attach_args *pa = os->os_pa; 1325 int val, i; 1326 1327 val = pci_conf_read(pa->pa_pc, pa->pa_tag, PCIR_BAR(0)); 1328 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM && 1329 PCI_MAPREG_TYPE(val) != PCI_MAPREG_MEM_TYPE_64BIT) { 1330 printf(": mmba is not mem space\n"); 1331 return (ENXIO); 1332 } 1333 1334 if (pci_mapreg_map(pa, PCIR_BAR(0), PCI_MAPREG_MEM_TYPE(val), 0, 1335 &os->os_memt, &os->os_memh, &os->os_membase, &os->os_memsize, 0)) { 1336 printf(": cannot find mem space\n"); 1337 return (ENXIO); 1338 } 1339 sc->hw.hw_addr = (uint8_t *)os->os_membase; 1340 1341 /* 1342 * Init the resource arrays 1343 */ 1344 for (i = 0; i < IXGBE_MSGS; i++) { 1345 sc->rid[i] = i + 1; /* MSI/X RID starts at 1 */ 1346 sc->tag[i] = NULL; 1347 sc->res[i] = NULL; 1348 } 1349 1350 /* Legacy defaults */ 1351 sc->num_tx_queues = 1; 1352 sc->num_rx_queues = 1; 1353 1354 #ifdef notyet 1355 /* Now setup MSI or MSI/X */ 1356 sc->msix = ixgbe_setup_msix(sc); 1357 #endif 1358 sc->hw.back = os; 1359 1360 return (0); 1361 } 1362 1363 void 1364 ixgbe_free_pci_resources(struct ix_softc * sc) 1365 { 1366 struct ixgbe_osdep *os = &sc->osdep; 1367 struct pci_attach_args *pa = os->os_pa; 1368 1369 if (sc->tag[0]) 1370 pci_intr_disestablish(pa->pa_pc, sc->tag[0]); 1371 sc->tag[0] = NULL; 1372 if (os->os_membase != 0) 1373 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 1374 os->os_membase = 0; 1375 1376 return; 1377 } 1378 1379 /********************************************************************* 1380 * 1381 * Initialize the hardware to a configuration as specified by the 1382 * sc structure. The controller is reset, the EEPROM is 1383 * verified, the MAC address is set, then the shared initialization 1384 * routines are called. 1385 * 1386 **********************************************************************/ 1387 int 1388 ixgbe_hardware_init(struct ix_softc *sc) 1389 { 1390 struct ifnet *ifp = &sc->arpcom.ac_if; 1391 uint16_t csum; 1392 1393 csum = 0; 1394 /* Issue a global reset */ 1395 sc->hw.adapter_stopped = FALSE; 1396 ixgbe_hw0(&sc->hw, stop_adapter); 1397 1398 /* Make sure we have a good EEPROM before we read from it */ 1399 if (ixgbe_ee(&sc->hw, validate_checksum, &csum) < 0) { 1400 printf("%s: The EEPROM Checksum Is Not Valid\n", ifp->if_xname); 1401 return (EIO); 1402 } 1403 1404 /* Pick up the smart speed setting */ 1405 if (sc->hw.mac.type == ixgbe_mac_82599EB) 1406 sc->hw.phy.smart_speed = ixgbe_smart_speed; 1407 1408 /* Get Hardware Flow Control setting */ 1409 sc->hw.fc.requested_mode = ixgbe_fc_full; 1410 sc->hw.fc.pause_time = IXGBE_FC_PAUSE; 1411 sc->hw.fc.low_water = IXGBE_FC_LO; 1412 sc->hw.fc.high_water = IXGBE_FC_HI; 1413 sc->hw.fc.send_xon = TRUE; 1414 1415 if (ixgbe_hw0(&sc->hw, init_hw) != 0) { 1416 printf("%s: Hardware Initialization Failed", ifp->if_xname); 1417 return (EIO); 1418 } 1419 bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr, 1420 IXGBE_ETH_LENGTH_OF_ADDRESS); 1421 1422 return (0); 1423 } 1424 1425 /********************************************************************* 1426 * 1427 * Setup networking device structure and register an interface. 1428 * 1429 **********************************************************************/ 1430 void 1431 ixgbe_setup_interface(struct ix_softc *sc) 1432 { 1433 struct ixgbe_hw *hw = &sc->hw; 1434 struct ifnet *ifp = &sc->arpcom.ac_if; 1435 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 1436 1437 strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ); 1438 ifp->if_baudrate = IF_Gbps(10); 1439 ifp->if_softc = sc; 1440 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1441 ifp->if_ioctl = ixgbe_ioctl; 1442 ifp->if_start = ixgbe_start; 1443 ifp->if_timer = 0; 1444 ifp->if_watchdog = ixgbe_watchdog; 1445 ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE - 1446 ETHER_HDR_LEN - ETHER_CRC_LEN; 1447 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1448 IFQ_SET_READY(&ifp->if_snd); 1449 1450 m_clsetwms(ifp, MCLBYTES, 4, sc->num_rx_desc); 1451 1452 ifp->if_capabilities = IFCAP_VLAN_MTU; 1453 1454 #if NVLAN > 0 1455 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 1456 #endif 1457 1458 #ifdef IX_CSUM_OFFLOAD 1459 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4 | 1460 IFCAP_CSUM_IPv4; 1461 #endif 1462 1463 sc->max_frame_size = 1464 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1465 1466 /* 1467 * Specify the media types supported by this sc and register 1468 * callbacks to update media and link information 1469 */ 1470 ifmedia_init(&sc->media, IFM_IMASK, ixgbe_media_change, 1471 ixgbe_media_status); 1472 ifmedia_add(&sc->media, IFM_ETHER | sc->optics | 1473 IFM_FDX, 0, NULL); 1474 if ((hw->device_id == PCI_PRODUCT_INTEL_82598AT) || 1475 (hw->device_id == PCI_PRODUCT_INTEL_82598AT_DUAL)) { 1476 ifmedia_add(&sc->media, 1477 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1478 ifmedia_add(&sc->media, 1479 IFM_ETHER | IFM_1000_T, 0, NULL); 1480 } 1481 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1482 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1483 1484 if_attach(ifp); 1485 ether_ifattach(ifp); 1486 1487 1488 return; 1489 } 1490 1491 int 1492 ixgbe_dma_malloc(struct ix_softc *sc, bus_size_t size, 1493 struct ixgbe_dma_alloc *dma, int mapflags) 1494 { 1495 struct ifnet *ifp = &sc->arpcom.ac_if; 1496 struct ixgbe_osdep *os = &sc->osdep; 1497 int r; 1498 1499 dma->dma_tag = os->os_pa->pa_dmat; 1500 r = bus_dmamap_create(dma->dma_tag, size, 1, 1501 size, 0, BUS_DMA_NOWAIT, &dma->dma_map); 1502 if (r != 0) { 1503 printf("%s: ixgbe_dma_malloc: bus_dma_tag_create failed; " 1504 "error %u\n", ifp->if_xname, r); 1505 goto fail_0; 1506 } 1507 1508 r = bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 1509 1, &dma->dma_nseg, BUS_DMA_NOWAIT); 1510 if (r != 0) { 1511 printf("%s: ixgbe_dma_malloc: bus_dmamem_alloc failed; " 1512 "error %u\n", ifp->if_xname, r); 1513 goto fail_1; 1514 } 1515 1516 r = bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1517 &dma->dma_vaddr, BUS_DMA_NOWAIT); 1518 if (r != 0) { 1519 printf("%s: ixgbe_dma_malloc: bus_dmamem_map failed; " 1520 "error %u\n", ifp->if_xname, r); 1521 goto fail_2; 1522 } 1523 1524 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, 1525 dma->dma_vaddr, size, NULL, 1526 mapflags | BUS_DMA_NOWAIT); 1527 if (r != 0) { 1528 printf("%s: ixgbe_dma_malloc: bus_dmamap_load failed; " 1529 "error %u\n", ifp->if_xname, r); 1530 goto fail_3; 1531 } 1532 1533 dma->dma_size = size; 1534 return (0); 1535 fail_3: 1536 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1537 fail_2: 1538 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1539 fail_1: 1540 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1541 fail_0: 1542 dma->dma_map = NULL; 1543 dma->dma_tag = NULL; 1544 return (r); 1545 } 1546 1547 void 1548 ixgbe_dma_free(struct ix_softc *sc, struct ixgbe_dma_alloc *dma) 1549 { 1550 if (dma->dma_tag == NULL) 1551 return; 1552 1553 if (dma->dma_map != NULL) { 1554 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1555 dma->dma_map->dm_mapsize, 1556 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1557 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1558 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1559 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1560 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1561 dma->dma_map = NULL; 1562 } 1563 } 1564 1565 1566 /********************************************************************* 1567 * 1568 * Allocate memory for the transmit and receive rings, and then 1569 * the descriptors associated with each, called only once at attach. 1570 * 1571 **********************************************************************/ 1572 int 1573 ixgbe_allocate_queues(struct ix_softc *sc) 1574 { 1575 struct ifnet *ifp = &sc->arpcom.ac_if; 1576 struct tx_ring *txr; 1577 struct rx_ring *rxr; 1578 int rsize, tsize, error = IXGBE_SUCCESS; 1579 int txconf = 0, rxconf = 0, i; 1580 1581 /* First allocate the TX ring struct memory */ 1582 if (!(sc->tx_rings = 1583 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 1584 sc->num_tx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 1585 printf("%s: Unable to allocate TX ring memory\n", ifp->if_xname); 1586 error = ENOMEM; 1587 goto fail; 1588 } 1589 1590 /* Next allocate the RX */ 1591 if (!(sc->rx_rings = 1592 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 1593 sc->num_rx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 1594 printf("%s: Unable to allocate RX ring memory\n", ifp->if_xname); 1595 error = ENOMEM; 1596 goto rx_fail; 1597 } 1598 1599 /* For the ring itself */ 1600 tsize = roundup2(sc->num_tx_desc * 1601 sizeof(union ixgbe_adv_tx_desc), 4096); 1602 1603 /* 1604 * Now set up the TX queues, txconf is needed to handle the 1605 * possibility that things fail midcourse and we need to 1606 * undo memory gracefully 1607 */ 1608 for (i = 0; i < sc->num_tx_queues; i++, txconf++) { 1609 /* Set up some basics */ 1610 txr = &sc->tx_rings[i]; 1611 txr->sc = sc; 1612 txr->me = i; 1613 1614 /* Initialize the TX side lock */ 1615 mtx_init(&txr->tx_mtx, IPL_NET); 1616 1617 if (ixgbe_dma_malloc(sc, tsize, 1618 &txr->txdma, BUS_DMA_NOWAIT)) { 1619 printf("%s: Unable to allocate TX Descriptor memory\n", 1620 ifp->if_xname); 1621 error = ENOMEM; 1622 goto err_tx_desc; 1623 } 1624 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 1625 bzero((void *)txr->tx_base, tsize); 1626 1627 if (ixgbe_dma_malloc(sc, sizeof(uint32_t), 1628 &txr->txwbdma, BUS_DMA_NOWAIT)) { 1629 printf("%s: Unable to allocate TX Write Back memory\n", 1630 ifp->if_xname); 1631 error = ENOMEM; 1632 goto err_tx_desc; 1633 } 1634 txr->tx_hwb = (uint32_t *)txr->txwbdma.dma_vaddr; 1635 *txr->tx_hwb = 0; 1636 } 1637 1638 /* 1639 * Next the RX queues... 1640 */ 1641 rsize = roundup2(sc->num_rx_desc * 1642 sizeof(union ixgbe_adv_rx_desc), 4096); 1643 for (i = 0; i < sc->num_rx_queues; i++, rxconf++) { 1644 rxr = &sc->rx_rings[i]; 1645 /* Set up some basics */ 1646 rxr->sc = sc; 1647 rxr->me = i; 1648 1649 /* Initialize the TX side lock */ 1650 mtx_init(&rxr->rx_mtx, IPL_NET); 1651 1652 if (ixgbe_dma_malloc(sc, rsize, 1653 &rxr->rxdma, BUS_DMA_NOWAIT)) { 1654 printf("%s: Unable to allocate RxDescriptor memory\n", 1655 ifp->if_xname); 1656 error = ENOMEM; 1657 goto err_rx_desc; 1658 } 1659 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 1660 bzero((void *)rxr->rx_base, rsize); 1661 } 1662 1663 return (0); 1664 1665 err_rx_desc: 1666 for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 1667 ixgbe_dma_free(sc, &rxr->rxdma); 1668 err_tx_desc: 1669 for (txr = sc->tx_rings; txconf > 0; txr++, txconf--) { 1670 ixgbe_dma_free(sc, &txr->txdma); 1671 ixgbe_dma_free(sc, &txr->txwbdma); 1672 } 1673 free(sc->rx_rings, M_DEVBUF); 1674 sc->rx_rings = NULL; 1675 rx_fail: 1676 free(sc->tx_rings, M_DEVBUF); 1677 sc->tx_rings = NULL; 1678 fail: 1679 return (error); 1680 } 1681 1682 /********************************************************************* 1683 * 1684 * Allocate memory for tx_buffer structures. The tx_buffer stores all 1685 * the information needed to transmit a packet on the wire. This is 1686 * called only once at attach, setup is done every reset. 1687 * 1688 **********************************************************************/ 1689 int 1690 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 1691 { 1692 struct ix_softc *sc; 1693 struct ixgbe_osdep *os; 1694 struct ifnet *ifp; 1695 struct ixgbe_tx_buf *txbuf; 1696 int error, i; 1697 int max_segs; 1698 1699 sc = txr->sc; 1700 os = &sc->osdep; 1701 ifp = &sc->arpcom.ac_if; 1702 1703 if (sc->hw.mac.type == ixgbe_mac_82598EB) 1704 max_segs = IXGBE_82598_SCATTER; 1705 else 1706 max_segs = IXGBE_82599_SCATTER; 1707 1708 if (!(txr->tx_buffers = 1709 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) * 1710 sc->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 1711 printf("%s: Unable to allocate tx_buffer memory\n", 1712 ifp->if_xname); 1713 error = ENOMEM; 1714 goto fail; 1715 } 1716 txr->txtag = txr->txdma.dma_tag; 1717 1718 /* Create the descriptor buffer dma maps */ 1719 for (i = 0; i < sc->num_tx_desc; i++) { 1720 txbuf = &txr->tx_buffers[i]; 1721 error = bus_dmamap_create(txr->txdma.dma_tag, IXGBE_TSO_SIZE, 1722 max_segs, PAGE_SIZE, 0, 1723 BUS_DMA_NOWAIT, &txbuf->map); 1724 1725 if (error != 0) { 1726 printf("%s: Unable to create TX DMA map\n", 1727 ifp->if_xname); 1728 goto fail; 1729 } 1730 } 1731 1732 return 0; 1733 fail: 1734 return (error); 1735 } 1736 1737 /********************************************************************* 1738 * 1739 * Initialize a transmit ring. 1740 * 1741 **********************************************************************/ 1742 int 1743 ixgbe_setup_transmit_ring(struct tx_ring *txr) 1744 { 1745 struct ix_softc *sc = txr->sc; 1746 int error; 1747 1748 /* Now allocate transmit buffers for the ring */ 1749 if ((error = ixgbe_allocate_transmit_buffers(txr)) != 0) 1750 return (error); 1751 1752 /* Clear the old ring contents */ 1753 bzero((void *)txr->tx_base, 1754 (sizeof(union ixgbe_adv_tx_desc)) * sc->num_tx_desc); 1755 1756 /* Reset indices */ 1757 txr->next_avail_tx_desc = 0; 1758 txr->next_tx_to_clean = 0; 1759 1760 /* Set number of descriptors available */ 1761 txr->tx_avail = sc->num_tx_desc; 1762 1763 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1764 0, txr->txdma.dma_map->dm_mapsize, 1765 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1766 1767 return (0); 1768 } 1769 1770 /********************************************************************* 1771 * 1772 * Initialize all transmit rings. 1773 * 1774 **********************************************************************/ 1775 int 1776 ixgbe_setup_transmit_structures(struct ix_softc *sc) 1777 { 1778 struct tx_ring *txr = sc->tx_rings; 1779 int i, error; 1780 1781 for (i = 0; i < sc->num_tx_queues; i++, txr++) { 1782 if ((error = ixgbe_setup_transmit_ring(txr)) != 0) 1783 goto fail; 1784 } 1785 1786 return (0); 1787 fail: 1788 ixgbe_free_transmit_structures(sc); 1789 return (error); 1790 } 1791 1792 /********************************************************************* 1793 * 1794 * Enable transmit unit. 1795 * 1796 **********************************************************************/ 1797 void 1798 ixgbe_initialize_transmit_units(struct ix_softc *sc) 1799 { 1800 struct ifnet *ifp = &sc->arpcom.ac_if; 1801 struct tx_ring *txr; 1802 struct ixgbe_hw *hw = &sc->hw; 1803 int i; 1804 uint64_t tdba, txhwb; 1805 uint32_t txctrl; 1806 1807 /* Setup the Base and Length of the Tx Descriptor Ring */ 1808 1809 for (i = 0; i < sc->num_tx_queues; i++) { 1810 txr = &sc->tx_rings[i]; 1811 1812 /* Setup descriptor base address */ 1813 tdba = txr->txdma.dma_map->dm_segs[0].ds_addr; 1814 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 1815 (tdba & 0x00000000ffffffffULL)); 1816 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 1817 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 1818 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1819 1820 /* Setup for Head WriteBack */ 1821 txhwb = txr->txwbdma.dma_map->dm_segs[0].ds_addr; 1822 txhwb |= IXGBE_TDWBAL_HEAD_WB_ENABLE; 1823 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(i), 1824 (txhwb & 0x00000000ffffffffULL)); 1825 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(i), 1826 (txhwb >> 32)); 1827 1828 /* Disable Head Writeback */ 1829 switch (hw->mac.type) { 1830 case ixgbe_mac_82598EB: 1831 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 1832 break; 1833 case ixgbe_mac_82599EB: 1834 default: 1835 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 1836 break; 1837 } 1838 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 1839 switch (hw->mac.type) { 1840 case ixgbe_mac_82598EB: 1841 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 1842 break; 1843 case ixgbe_mac_82599EB: 1844 default: 1845 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 1846 break; 1847 } 1848 1849 /* Setup the HW Tx Head and Tail descriptor pointers */ 1850 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 1851 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 1852 1853 /* Setup Transmit Descriptor Cmd Settings */ 1854 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 1855 1856 txr->watchdog_timer = 0; 1857 } 1858 ifp->if_timer = 0; 1859 1860 if (hw->mac.type == ixgbe_mac_82599EB) { 1861 uint32_t dmatxctl, rttdcs; 1862 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1863 dmatxctl |= IXGBE_DMATXCTL_TE; 1864 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 1865 /* Disable arbiter to set MTQC */ 1866 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 1867 rttdcs |= IXGBE_RTTDCS_ARBDIS; 1868 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 1869 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 1870 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 1871 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 1872 } 1873 1874 return; 1875 } 1876 1877 /********************************************************************* 1878 * 1879 * Free all transmit rings. 1880 * 1881 **********************************************************************/ 1882 void 1883 ixgbe_free_transmit_structures(struct ix_softc *sc) 1884 { 1885 struct tx_ring *txr = sc->tx_rings; 1886 int i; 1887 1888 for (i = 0; i < sc->num_tx_queues; i++, txr++) { 1889 ixgbe_free_transmit_buffers(txr); 1890 } 1891 } 1892 1893 /********************************************************************* 1894 * 1895 * Free transmit ring related data structures. 1896 * 1897 **********************************************************************/ 1898 void 1899 ixgbe_free_transmit_buffers(struct tx_ring *txr) 1900 { 1901 struct ix_softc *sc = txr->sc; 1902 struct ixgbe_tx_buf *tx_buffer; 1903 int i; 1904 1905 INIT_DEBUGOUT("free_transmit_ring: begin"); 1906 1907 if (txr->tx_buffers == NULL) 1908 return; 1909 1910 tx_buffer = txr->tx_buffers; 1911 for (i = 0; i < sc->num_tx_desc; i++, tx_buffer++) { 1912 if (tx_buffer->map != NULL && tx_buffer->map->dm_nsegs > 0) { 1913 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 1914 0, tx_buffer->map->dm_mapsize, 1915 BUS_DMASYNC_POSTWRITE); 1916 bus_dmamap_unload(txr->txdma.dma_tag, 1917 tx_buffer->map); 1918 } 1919 if (tx_buffer->m_head != NULL) { 1920 m_freem(tx_buffer->m_head); 1921 tx_buffer->m_head = NULL; 1922 } 1923 if (tx_buffer->map != NULL) { 1924 bus_dmamap_destroy(txr->txdma.dma_tag, 1925 tx_buffer->map); 1926 tx_buffer->map = NULL; 1927 } 1928 } 1929 1930 if (txr->tx_buffers != NULL) { 1931 free(txr->tx_buffers, M_DEVBUF); 1932 txr->tx_buffers = NULL; 1933 } 1934 txr->tx_buffers = NULL; 1935 txr->txtag = NULL; 1936 } 1937 1938 /********************************************************************* 1939 * 1940 * Advanced Context Descriptor setup for VLAN or CSUM 1941 * 1942 **********************************************************************/ 1943 1944 int 1945 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 1946 { 1947 struct ix_softc *sc = txr->sc; 1948 struct ifnet *ifp = &sc->arpcom.ac_if; 1949 struct ixgbe_adv_tx_context_desc *TXD; 1950 struct ixgbe_tx_buf *tx_buffer; 1951 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 1952 struct ip *ip; 1953 #ifdef notyet 1954 struct ip6_hdr *ip6; 1955 #endif 1956 uint8_t ipproto = 0; 1957 int ehdrlen, ip_hlen = 0; 1958 uint16_t etype; 1959 int offload = TRUE; 1960 int ctxd = txr->next_avail_tx_desc; 1961 #if NVLAN > 0 1962 struct ether_vlan_header *eh; 1963 #else 1964 struct ether_header *eh; 1965 #endif 1966 1967 if ((ifp->if_capabilities & IFCAP_CSUM_IPv4) == 0) 1968 offload = FALSE; 1969 1970 tx_buffer = &txr->tx_buffers[ctxd]; 1971 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 1972 1973 /* 1974 * In advanced descriptors the vlan tag must 1975 * be placed into the descriptor itself. 1976 */ 1977 #if NVLAN > 0 1978 if (mp->m_flags & M_VLANTAG) { 1979 vlan_macip_lens |= 1980 htole16(mp->m_pkthdr.ether_vtag) << IXGBE_ADVTXD_VLAN_SHIFT; 1981 } else 1982 #endif 1983 if (offload == FALSE) 1984 return FALSE; /* No need for CTX */ 1985 1986 /* 1987 * Determine where frame payload starts. 1988 * Jump over vlan headers if already present, 1989 * helpful for QinQ too. 1990 */ 1991 #if NVLAN > 0 1992 eh = mtod(mp, struct ether_vlan_header *); 1993 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1994 etype = ntohs(eh->evl_proto); 1995 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1996 } else { 1997 etype = ntohs(eh->evl_encap_proto); 1998 ehdrlen = ETHER_HDR_LEN; 1999 } 2000 #else 2001 eh = mtod(mp, struct ether_header *); 2002 etype = ntohs(eh->ether_type); 2003 ehdrlen = ETHER_HDR_LEN; 2004 #endif 2005 2006 /* Set the ether header length */ 2007 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2008 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2009 2010 if (offload == TRUE) { 2011 switch (etype) { 2012 case ETHERTYPE_IP: 2013 ip = (struct ip *)(mp->m_data + ehdrlen); 2014 ip_hlen = ip->ip_hl << 2; 2015 if (mp->m_len < ehdrlen + ip_hlen) 2016 return FALSE; /* failure */ 2017 ipproto = ip->ip_p; 2018 if (mp->m_pkthdr.csum_flags & M_IPV4_CSUM_OUT) 2019 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2020 break; 2021 #ifdef notyet 2022 case ETHERTYPE_IPV6: 2023 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2024 ip_hlen = sizeof(struct ip6_hdr); 2025 if (mp->m_len < ehdrlen + ip_hlen) 2026 return FALSE; /* failure */ 2027 ipproto = ip6->ip6_nxt; 2028 if (mp->m_pkthdr.csum_flags & M_IPV4_CSUM_OUT) 2029 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2030 break; 2031 #endif 2032 default: 2033 offload = FALSE; 2034 break; 2035 } 2036 2037 vlan_macip_lens |= ip_hlen; 2038 2039 switch (ipproto) { 2040 case IPPROTO_TCP: 2041 if (mp->m_pkthdr.csum_flags & M_TCP_CSUM_OUT) 2042 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2043 break; 2044 case IPPROTO_UDP: 2045 if (mp->m_pkthdr.csum_flags & M_UDP_CSUM_OUT) 2046 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2047 break; 2048 } 2049 } 2050 2051 /* Now copy bits into descriptor */ 2052 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2053 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2054 TXD->seqnum_seed = htole32(0); 2055 TXD->mss_l4len_idx = htole32(0); 2056 2057 tx_buffer->m_head = NULL; 2058 2059 /* We've consumed the first desc, adjust counters */ 2060 if (++ctxd == sc->num_tx_desc) 2061 ctxd = 0; 2062 txr->next_avail_tx_desc = ctxd; 2063 --txr->tx_avail; 2064 2065 return (offload); 2066 } 2067 2068 #ifdef notyet 2069 /********************************************************************** 2070 * 2071 * Setup work for hardware segmentation offload (TSO) on 2072 * scs using advanced tx descriptors 2073 * 2074 **********************************************************************/ 2075 int 2076 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, uint32_t *paylen) 2077 { 2078 struct ix_softc *sc = txr->sc; 2079 struct ixgbe_adv_tx_context_desc *TXD; 2080 struct ixgbe_tx_buf *tx_buffer; 2081 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2082 uint32_t mss_l4len_idx = 0; 2083 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 2084 #if NVLAN > 0 2085 uint16_t vtag = 0; 2086 struct ether_vlan_header *eh; 2087 #else 2088 struct ether_header *eh; 2089 #endif 2090 struct ip *ip; 2091 struct tcphdr *th; 2092 2093 if (((mp->m_pkthdr.csum_flags & CSUM_TSO) == 0) || 2094 (mp->m_pkthdr.len <= IXGBE_TX_BUFFER_SIZE)) 2095 return FALSE; 2096 2097 /* 2098 * Determine where frame payload starts. 2099 * Jump over vlan headers if already present 2100 */ 2101 #if NVLAN > 0 2102 eh = mtod(mp, struct ether_vlan_header *); 2103 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 2104 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2105 else 2106 ehdrlen = ETHER_HDR_LEN; 2107 #else 2108 eh = mtod(mp, struct ether_header *); 2109 ehdrlen = ETHER_HDR_LEN; 2110 #endif 2111 2112 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 2113 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 2114 return FALSE; 2115 2116 ctxd = txr->next_avail_tx_desc; 2117 tx_buffer = &txr->tx_buffers[ctxd]; 2118 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2119 2120 ip = (struct ip *)(mp->m_data + ehdrlen); 2121 if (ip->ip_p != IPPROTO_TCP) 2122 return FALSE; /* 0 */ 2123 ip->ip_len = 0; 2124 ip->ip_sum = 0; 2125 ip_hlen = ip->ip_hl << 2; 2126 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 2127 th->th_sum = in_pseudo(ip->ip_src.s_addr, 2128 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2129 tcp_hlen = th->th_off << 2; 2130 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 2131 /* This is used in the transmit desc in encap */ 2132 *paylen = mp->m_pkthdr.len - hdrlen; 2133 2134 #if NVLAN > 0 2135 /* VLAN MACLEN IPLEN */ 2136 if (mp->m_flags & M_VLANTAG) { 2137 vtag = htole16(mp->m_pkthdr.ether_vtag); 2138 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2139 } 2140 #endif 2141 2142 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2143 vlan_macip_lens |= ip_hlen; 2144 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2145 2146 /* ADV DTYPE TUCMD */ 2147 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2148 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2149 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2150 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2151 2152 2153 /* MSS L4LEN IDX */ 2154 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2155 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2156 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2157 2158 TXD->seqnum_seed = htole32(0); 2159 tx_buffer->m_head = NULL; 2160 2161 if (++ctxd == sc->num_tx_desc) 2162 ctxd = 0; 2163 2164 txr->tx_avail--; 2165 txr->next_avail_tx_desc = ctxd; 2166 return TRUE; 2167 } 2168 2169 #else 2170 /* This makes it easy to keep the code common */ 2171 int 2172 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, uint32_t *paylen) 2173 { 2174 return (FALSE); 2175 } 2176 #endif 2177 2178 /********************************************************************** 2179 * 2180 * Examine each tx_buffer in the used queue. If the hardware is done 2181 * processing the packet then free associated resources. The 2182 * tx_buffer is put back on the free queue. 2183 * 2184 **********************************************************************/ 2185 int 2186 ixgbe_txeof(struct tx_ring *txr) 2187 { 2188 struct ix_softc *sc = txr->sc; 2189 struct ifnet *ifp = &sc->arpcom.ac_if; 2190 uint first, last, done, num_avail; 2191 struct ixgbe_tx_buf *tx_buffer; 2192 struct ixgbe_legacy_tx_desc *tx_desc; 2193 2194 if (txr->tx_avail == sc->num_tx_desc) 2195 return FALSE; 2196 2197 num_avail = txr->tx_avail; 2198 first = txr->next_tx_to_clean; 2199 2200 tx_buffer = &txr->tx_buffers[first]; 2201 2202 /* For cleanup we just use legacy struct */ 2203 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2204 2205 /* Get the HWB */ 2206 bus_dmamap_sync(txr->txwbdma.dma_tag, txr->txwbdma.dma_map, 2207 0, txr->txwbdma.dma_map->dm_mapsize, 2208 BUS_DMASYNC_POSTREAD); 2209 done = *txr->tx_hwb; 2210 2211 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2212 0, txr->txdma.dma_map->dm_mapsize, 2213 BUS_DMASYNC_POSTREAD); 2214 2215 while (TRUE) { 2216 /* We clean the range til last head write back */ 2217 while (first != done) { 2218 tx_desc->upper.data = 0; 2219 tx_desc->lower.data = 0; 2220 tx_desc->buffer_addr = 0; 2221 num_avail++; 2222 2223 if (tx_buffer->m_head) { 2224 ifp->if_opackets++; 2225 bus_dmamap_sync(txr->txdma.dma_tag, 2226 tx_buffer->map, 2227 0, tx_buffer->map->dm_mapsize, 2228 BUS_DMASYNC_POSTWRITE); 2229 bus_dmamap_unload(txr->txdma.dma_tag, 2230 tx_buffer->map); 2231 m_freem(tx_buffer->m_head); 2232 tx_buffer->m_head = NULL; 2233 } 2234 2235 if (++first == sc->num_tx_desc) 2236 first = 0; 2237 2238 tx_buffer = &txr->tx_buffers[first]; 2239 tx_desc = (struct ixgbe_legacy_tx_desc *) 2240 &txr->tx_base[first]; 2241 } 2242 /* See if there is more work now */ 2243 last = done; 2244 bus_dmamap_sync(txr->txwbdma.dma_tag, txr->txwbdma.dma_map, 2245 0, txr->txwbdma.dma_map->dm_mapsize, 2246 BUS_DMASYNC_POSTREAD); 2247 done = *txr->tx_hwb; 2248 if (last == done) 2249 break; 2250 } 2251 2252 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2253 0, txr->txdma.dma_map->dm_mapsize, 2254 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2255 2256 txr->next_tx_to_clean = first; 2257 2258 /* 2259 * If we have enough room, clear IFF_OACTIVE to tell the stack that 2260 * it is OK to send packets. If there are no pending descriptors, 2261 * clear the timeout. Otherwise, if some descriptors have been freed, 2262 * restart the timeout. 2263 */ 2264 if (num_avail > IXGBE_TX_CLEANUP_THRESHOLD) { 2265 ifp->if_flags &= ~IFF_OACTIVE; 2266 2267 /* If all are clean turn off the timer */ 2268 if (num_avail == sc->num_tx_desc) { 2269 ifp->if_timer = 0; 2270 txr->watchdog_timer = 0; 2271 txr->tx_avail = num_avail; 2272 return FALSE; 2273 } 2274 /* Some were cleaned, so reset timer */ 2275 else if (num_avail != txr->tx_avail) { 2276 ifp->if_timer = IXGBE_TX_TIMEOUT; 2277 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 2278 } 2279 } 2280 2281 txr->tx_avail = num_avail; 2282 2283 return TRUE; 2284 } 2285 2286 /********************************************************************* 2287 * 2288 * Get a buffer from system mbuf buffer pool. 2289 * 2290 **********************************************************************/ 2291 int 2292 ixgbe_get_buf(struct rx_ring *rxr, int i) 2293 { 2294 struct ix_softc *sc = rxr->sc; 2295 struct mbuf *m; 2296 int error; 2297 int size = MCLBYTES; 2298 struct ixgbe_rx_buf *rxbuf; 2299 union ixgbe_adv_rx_desc *rxdesc; 2300 size_t dsize = sizeof(union ixgbe_adv_rx_desc); 2301 2302 rxbuf = &rxr->rx_buffers[i]; 2303 rxdesc = &rxr->rx_base[i]; 2304 2305 if (rxbuf->m_head != NULL) { 2306 printf("%s: ixgbe_get_buf: slot %d already has an mbuf\n", 2307 sc->dev.dv_xname, i); 2308 return (ENOBUFS); 2309 } 2310 2311 m = MCLGETI(NULL, M_DONTWAIT, &sc->arpcom.ac_if, size); 2312 if (!m) { 2313 sc->mbuf_cluster_failed++; 2314 return (ENOBUFS); 2315 } 2316 m->m_len = m->m_pkthdr.len = size; 2317 if (sc->max_frame_size <= (size - ETHER_ALIGN)) 2318 m_adj(m, ETHER_ALIGN); 2319 2320 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map, 2321 m, BUS_DMA_NOWAIT); 2322 if (error) { 2323 m_freem(m); 2324 return (error); 2325 } 2326 2327 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2328 0, rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD); 2329 rxbuf->m_head = m; 2330 2331 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2332 dsize * i, dsize, BUS_DMASYNC_POSTWRITE); 2333 2334 bzero(rxdesc, dsize); 2335 rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr); 2336 2337 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2338 dsize * i, dsize, BUS_DMASYNC_PREWRITE); 2339 2340 rxr->rx_ndescs++; 2341 2342 return (0); 2343 } 2344 2345 /********************************************************************* 2346 * 2347 * Allocate memory for rx_buffer structures. Since we use one 2348 * rx_buffer per received packet, the maximum number of rx_buffer's 2349 * that we'll need is equal to the number of receive descriptors 2350 * that we've allocated. 2351 * 2352 **********************************************************************/ 2353 int 2354 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 2355 { 2356 struct ix_softc *sc = rxr->sc; 2357 struct ifnet *ifp = &sc->arpcom.ac_if; 2358 struct ixgbe_rx_buf *rxbuf; 2359 int i, bsize, error, size = MCLBYTES; 2360 2361 bsize = sizeof(struct ixgbe_rx_buf) * sc->num_rx_desc; 2362 if (!(rxr->rx_buffers = (struct ixgbe_rx_buf *) malloc(bsize, 2363 M_DEVBUF, M_NOWAIT | M_ZERO))) { 2364 printf("%s: Unable to allocate rx_buffer memory\n", 2365 ifp->if_xname); 2366 error = ENOMEM; 2367 goto fail; 2368 } 2369 rxr->rxtag = rxr->rxdma.dma_tag; 2370 2371 rxbuf = rxr->rx_buffers; 2372 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2373 error = bus_dmamap_create(rxr->rxdma.dma_tag, size, 1, 2374 size, 0, BUS_DMA_NOWAIT, &rxbuf->map); 2375 if (error) { 2376 printf("%s: Unable to create Rx DMA map\n", 2377 ifp->if_xname); 2378 goto fail; 2379 } 2380 rxbuf->m_head = NULL; 2381 } 2382 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 2383 rxr->rxdma.dma_map->dm_mapsize, 2384 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2385 2386 return (0); 2387 2388 fail: 2389 return (error); 2390 } 2391 2392 /********************************************************************* 2393 * 2394 * Initialize a receive ring and its buffers. 2395 * 2396 **********************************************************************/ 2397 int 2398 ixgbe_setup_receive_ring(struct rx_ring *rxr) 2399 { 2400 struct ix_softc *sc = rxr->sc; 2401 int rsize, error; 2402 2403 rsize = roundup2(sc->num_rx_desc * 2404 sizeof(union ixgbe_adv_rx_desc), 4096); 2405 /* Clear the ring contents */ 2406 bzero((void *)rxr->rx_base, rsize); 2407 2408 if ((error = ixgbe_allocate_receive_buffers(rxr)) != 0) 2409 return (error); 2410 2411 /* Setup our descriptor indices */ 2412 rxr->next_to_check = 0; 2413 rxr->last_rx_desc_filled = sc->num_rx_desc - 1; 2414 rxr->rx_ndescs = 0; 2415 2416 ixgbe_rxfill(rxr); 2417 if (rxr->rx_ndescs < 1) { 2418 printf("%s: unable to fill any rx descriptors\n", 2419 sc->dev.dv_xname); 2420 return (ENOBUFS); 2421 } 2422 2423 return (0); 2424 } 2425 2426 int 2427 ixgbe_rxfill(struct rx_ring *rxr) 2428 { 2429 struct ix_softc *sc = rxr->sc; 2430 int post = 0; 2431 int i; 2432 2433 i = rxr->last_rx_desc_filled; 2434 while (rxr->rx_ndescs < sc->num_rx_desc) { 2435 if (++i == sc->num_rx_desc) 2436 i = 0; 2437 2438 if (ixgbe_get_buf(rxr, i) != 0) 2439 break; 2440 2441 rxr->last_rx_desc_filled = i; 2442 post = 1; 2443 } 2444 2445 return (post); 2446 } 2447 2448 /********************************************************************* 2449 * 2450 * Initialize all receive rings. 2451 * 2452 **********************************************************************/ 2453 int 2454 ixgbe_setup_receive_structures(struct ix_softc *sc) 2455 { 2456 struct rx_ring *rxr = sc->rx_rings; 2457 int i; 2458 2459 for (i = 0; i < sc->num_rx_queues; i++, rxr++) 2460 if (ixgbe_setup_receive_ring(rxr)) 2461 goto fail; 2462 2463 return (0); 2464 2465 fail: 2466 ixgbe_free_receive_structures(sc); 2467 return (ENOBUFS); 2468 } 2469 2470 /********************************************************************* 2471 * 2472 * Enable receive unit. 2473 * 2474 **********************************************************************/ 2475 void 2476 ixgbe_initialize_receive_units(struct ix_softc *sc) 2477 { 2478 struct rx_ring *rxr = sc->rx_rings; 2479 struct ifnet *ifp = &sc->arpcom.ac_if; 2480 uint32_t rxctrl, fctrl, srrctl, rxcsum; 2481 uint32_t reta, mrqc, hlreg, linkvec; 2482 uint32_t random[10]; 2483 uint32_t llimod = 0; 2484 int i; 2485 2486 /* 2487 * Make sure receives are disabled while 2488 * setting up the descriptor ring 2489 */ 2490 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 2491 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXCTRL, 2492 rxctrl & ~IXGBE_RXCTRL_RXEN); 2493 2494 /* Enable broadcasts */ 2495 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 2496 fctrl |= IXGBE_FCTRL_BAM; 2497 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 2498 2499 hlreg = IXGBE_READ_REG(&sc->hw, IXGBE_HLREG0); 2500 if (ifp->if_mtu > ETHERMTU) 2501 hlreg |= IXGBE_HLREG0_JUMBOEN; 2502 else 2503 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 2504 IXGBE_WRITE_REG(&sc->hw, IXGBE_HLREG0, hlreg); 2505 2506 srrctl = IXGBE_READ_REG(&sc->hw, IXGBE_SRRCTL(0)); 2507 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 2508 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 2509 if (sc->bigbufs) 2510 srrctl |= 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2511 else 2512 srrctl |= 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2513 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2514 IXGBE_WRITE_REG(&sc->hw, IXGBE_SRRCTL(0), srrctl); 2515 2516 /* Set Queue moderation rate */ 2517 if (sc->hw.mac.type == ixgbe_mac_82599EB) 2518 llimod = IXGBE_EITR_LLI_MOD; 2519 for (i = 0; i < IXGBE_MSGS; i++) 2520 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(i), DEFAULT_ITR | llimod); 2521 2522 /* Set Link moderation lower */ 2523 linkvec = sc->num_tx_queues + sc->num_rx_queues; 2524 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(linkvec), LINK_ITR); 2525 2526 for (i = 0; i < sc->num_rx_queues; i++, rxr++) { 2527 uint64_t rdba = rxr->rxdma.dma_map->dm_segs[0].ds_addr; 2528 /* Setup the Base and Length of the Rx Descriptor Ring */ 2529 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDBAL(i), 2530 (rdba & 0x00000000ffffffffULL)); 2531 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDBAH(i), (rdba >> 32)); 2532 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDLEN(i), 2533 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 2534 2535 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2536 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDH(i), 0); 2537 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), 2538 rxr->last_rx_desc_filled); 2539 } 2540 2541 rxcsum = IXGBE_READ_REG(&sc->hw, IXGBE_RXCSUM); 2542 2543 if (sc->num_rx_queues > 1) { 2544 /* set up random bits */ 2545 arc4random_buf(&random, sizeof(random)); 2546 switch (sc->num_rx_queues) { 2547 case 8: 2548 case 4: 2549 reta = 0x00010203; 2550 break; 2551 case 2: 2552 reta = 0x00010001; 2553 break; 2554 default: 2555 reta = 0x00000000; 2556 } 2557 2558 /* Set up the redirection table */ 2559 for (i = 0; i < 32; i++) { 2560 IXGBE_WRITE_REG(&sc->hw, IXGBE_RETA(i), reta); 2561 if (sc->num_rx_queues > 4) { 2562 ++i; 2563 IXGBE_WRITE_REG(&sc->hw, 2564 IXGBE_RETA(i), 0x04050607); 2565 } 2566 } 2567 2568 /* Now fill our hash function seeds */ 2569 for (i = 0; i < 10; i++) 2570 IXGBE_WRITE_REG_ARRAY(&sc->hw, 2571 IXGBE_RSSRK(0), i, random[i]); 2572 2573 mrqc = IXGBE_MRQC_RSSEN 2574 /* Perform hash on these packet types */ 2575 | IXGBE_MRQC_RSS_FIELD_IPV4 2576 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 2577 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 2578 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 2579 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 2580 | IXGBE_MRQC_RSS_FIELD_IPV6 2581 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 2582 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 2583 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2584 IXGBE_WRITE_REG(&sc->hw, IXGBE_MRQC, mrqc); 2585 2586 /* RSS and RX IPP Checksum are mutually exclusive */ 2587 rxcsum |= IXGBE_RXCSUM_PCSD; 2588 } 2589 2590 if (ifp->if_capabilities & IFCAP_CSUM_IPv4) 2591 rxcsum |= IXGBE_RXCSUM_PCSD; 2592 2593 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 2594 rxcsum |= IXGBE_RXCSUM_IPPCSE; 2595 2596 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXCSUM, rxcsum); 2597 2598 /* Enable Receive engine */ 2599 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 2600 if (sc->hw.mac.type == ixgbe_mac_82598EB) 2601 rxctrl |= IXGBE_RXCTRL_DMBYPS; 2602 rxctrl |= IXGBE_RXCTRL_RXEN; 2603 sc->hw.mac.ops.enable_rx_dma(&sc->hw, rxctrl); 2604 2605 return; 2606 } 2607 2608 /********************************************************************* 2609 * 2610 * Free all receive rings. 2611 * 2612 **********************************************************************/ 2613 void 2614 ixgbe_free_receive_structures(struct ix_softc *sc) 2615 { 2616 struct rx_ring *rxr = sc->rx_rings; 2617 int i; 2618 2619 for (i = 0; i < sc->num_rx_queues; i++, rxr++) { 2620 ixgbe_free_receive_buffers(rxr); 2621 } 2622 } 2623 2624 /********************************************************************* 2625 * 2626 * Free receive ring data structures 2627 * 2628 **********************************************************************/ 2629 void 2630 ixgbe_free_receive_buffers(struct rx_ring *rxr) 2631 { 2632 struct ix_softc *sc = NULL; 2633 struct ixgbe_rx_buf *rxbuf = NULL; 2634 int i; 2635 2636 INIT_DEBUGOUT("free_receive_buffers: begin"); 2637 sc = rxr->sc; 2638 if (rxr->rx_buffers != NULL) { 2639 rxbuf = rxr->rx_buffers; 2640 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2641 if (rxbuf->m_head != NULL) { 2642 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2643 0, rxbuf->map->dm_mapsize, 2644 BUS_DMASYNC_POSTREAD); 2645 bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map); 2646 m_freem(rxbuf->m_head); 2647 rxbuf->m_head = NULL; 2648 } 2649 bus_dmamap_destroy(rxr->rxdma.dma_tag, rxbuf->map); 2650 rxbuf->map = NULL; 2651 } 2652 } 2653 if (rxr->rx_buffers != NULL) { 2654 free(rxr->rx_buffers, M_DEVBUF); 2655 rxr->rx_buffers = NULL; 2656 } 2657 2658 if (rxr->rxtag != NULL) 2659 rxr->rxtag = NULL; 2660 2661 if (rxr->fmp != NULL) { 2662 m_freem(rxr->fmp); 2663 rxr->fmp = NULL; 2664 rxr->lmp = NULL; 2665 } 2666 } 2667 2668 /********************************************************************* 2669 * 2670 * This routine executes in interrupt context. It replenishes 2671 * the mbufs in the descriptor and sends data which has been 2672 * dma'ed into host memory to upper layer. 2673 * 2674 * We loop at most count times if count is > 0, or until done if 2675 * count < 0. 2676 * 2677 *********************************************************************/ 2678 int 2679 ixgbe_rxeof(struct rx_ring *rxr, int count) 2680 { 2681 struct ix_softc *sc = rxr->sc; 2682 struct ifnet *ifp = &sc->arpcom.ac_if; 2683 struct mbuf *m; 2684 uint8_t accept_frame = 0; 2685 uint8_t eop = 0; 2686 uint16_t len, desc_len, prev_len_adj; 2687 uint32_t staterr; 2688 struct ixgbe_rx_buf *rxbuf; 2689 union ixgbe_adv_rx_desc *rxdesc; 2690 size_t dsize = sizeof(union ixgbe_adv_rx_desc); 2691 int i; 2692 2693 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2694 return FALSE; 2695 2696 i = rxr->next_to_check; 2697 2698 while (count != 0 && rxr->rx_ndescs > 0) { 2699 m = NULL; 2700 2701 rxdesc = &rxr->rx_base[i]; 2702 rxbuf = &rxr->rx_buffers[i]; 2703 2704 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2705 dsize * i, dsize, 2706 BUS_DMASYNC_POSTREAD); 2707 2708 staterr = letoh32(rxdesc->wb.upper.status_error); 2709 if (!ISSET(staterr, IXGBE_RXD_STAT_DD)) { 2710 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2711 dsize * i, dsize, 2712 BUS_DMASYNC_PREREAD); 2713 break; 2714 } 2715 2716 /* pull the mbuf off the ring */ 2717 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 2718 rxbuf->map->dm_mapsize, 2719 BUS_DMASYNC_POSTREAD); 2720 bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map); 2721 m = rxbuf->m_head; 2722 rxbuf->m_head = NULL; 2723 2724 if (m == NULL) { 2725 panic("%s: ixgbe_rxeof: NULL mbuf in slot %d " 2726 "(nrx %d, filled %d)", sc->dev.dv_xname, 2727 i, rxr->rx_ndescs, 2728 rxr->last_rx_desc_filled); 2729 } 2730 2731 m_cluncount(m, 1); 2732 rxr->rx_ndescs--; 2733 2734 accept_frame = 1; 2735 prev_len_adj = 0; 2736 desc_len = letoh16(rxdesc->wb.upper.length); 2737 2738 if (staterr & IXGBE_RXD_STAT_EOP) { 2739 count--; 2740 eop = 1; 2741 } else { 2742 eop = 0; 2743 } 2744 len = desc_len; 2745 2746 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) 2747 accept_frame = 0; 2748 2749 if (accept_frame) { 2750 m->m_len = len; 2751 2752 /* XXX ixgbe_realign() STRICT_ALIGN */ 2753 2754 if (rxr->fmp == NULL) { 2755 m->m_pkthdr.len = m->m_len; 2756 rxr->fmp = m; /* Store the first mbuf */ 2757 rxr->lmp = m; 2758 } else { 2759 /* Chain mbuf's together */ 2760 m->m_flags &= ~M_PKTHDR; 2761 #if 0 2762 /* 2763 * Adjust length of previous mbuf in chain if 2764 * we received less than 4 bytes in the last 2765 * descriptor. 2766 */ 2767 if (prev_len_adj > 0) { 2768 rxr->lmp->m_len -= prev_len_adj; 2769 rxr->fmp->m_pkthdr.len -= prev_len_adj; 2770 } 2771 #endif 2772 rxr->lmp->m_next = m; 2773 rxr->lmp = m; 2774 rxr->fmp->m_pkthdr.len += m->m_len; 2775 } 2776 2777 if (eop) { 2778 ifp->if_ipackets++; 2779 2780 m = rxr->fmp; 2781 m->m_pkthdr.rcvif = ifp; 2782 2783 rxr->packet_count++; 2784 rxr->byte_count += rxr->fmp->m_pkthdr.len; 2785 2786 ixgbe_rx_checksum(sc, staterr, m); 2787 2788 #if NVLAN > 0 2789 if (staterr & IXGBE_RXD_STAT_VP) { 2790 m->m_pkthdr.ether_vtag = 2791 letoh16(rxdesc->wb.upper.vlan); 2792 m->m_flags |= M_VLANTAG; 2793 } 2794 #endif 2795 #if NBPFILTER > 0 2796 if (ifp->if_bpf) 2797 bpf_mtap_ether(ifp->if_bpf, m, 2798 BPF_DIRECTION_IN); 2799 #endif 2800 2801 ether_input_mbuf(ifp, m); 2802 2803 rxr->fmp = NULL; 2804 rxr->lmp = NULL; 2805 } 2806 } else { 2807 sc->dropped_pkts++; 2808 2809 if (rxr->fmp != NULL) { 2810 m_freem(rxr->fmp); 2811 rxr->fmp = NULL; 2812 rxr->lmp = NULL; 2813 } 2814 2815 m_freem(m); 2816 } 2817 2818 /* Zero out the receive descriptors status */ 2819 rxdesc->wb.upper.status_error = 0; 2820 2821 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2822 dsize * i, dsize, 2823 BUS_DMASYNC_PREREAD); 2824 2825 /* Advance our pointers to the next descriptor. */ 2826 if (++i == sc->num_rx_desc) 2827 i = 0; 2828 } 2829 rxr->next_to_check = i; 2830 2831 if (!(staterr & IXGBE_RXD_STAT_DD)) 2832 return FALSE; 2833 2834 return TRUE; 2835 } 2836 2837 /********************************************************************* 2838 * 2839 * Verify that the hardware indicated that the checksum is valid. 2840 * Inform the stack about the status of checksum so that stack 2841 * doesn't spend time verifying the checksum. 2842 * 2843 *********************************************************************/ 2844 void 2845 ixgbe_rx_checksum(struct ix_softc *sc, 2846 uint32_t staterr, struct mbuf * mp) 2847 { 2848 uint16_t status = (uint16_t) staterr; 2849 uint8_t errors = (uint8_t) (staterr >> 24); 2850 2851 if (status & IXGBE_RXD_STAT_IPCS) { 2852 /* Did it pass? */ 2853 if (!(errors & IXGBE_RXD_ERR_IPE)) { 2854 /* IP Checksum Good */ 2855 mp->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 2856 } else 2857 mp->m_pkthdr.csum_flags = 0; 2858 } 2859 2860 if (status & IXGBE_RXD_STAT_L4CS) { 2861 /* Did it pass? */ 2862 if (!(errors & IXGBE_RXD_ERR_TCPE)) 2863 mp->m_pkthdr.csum_flags |= 2864 M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK; 2865 } 2866 2867 } 2868 2869 void 2870 ixgbe_enable_hw_vlans(struct ix_softc *sc) 2871 { 2872 uint32_t ctrl; 2873 2874 ixgbe_disable_intr(sc); 2875 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_VLNCTRL); 2876 if (sc->hw.mac.type == ixgbe_mac_82598EB) 2877 ctrl |= IXGBE_VLNCTRL_VME; 2878 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 2879 ctrl &= ~IXGBE_VLNCTRL_VFE; 2880 IXGBE_WRITE_REG(&sc->hw, IXGBE_VLNCTRL, ctrl); 2881 ixgbe_enable_intr(sc); 2882 } 2883 2884 void 2885 ixgbe_enable_intr(struct ix_softc *sc) 2886 { 2887 struct ixgbe_hw *hw = &sc->hw; 2888 uint32_t mask = IXGBE_EIMS_ENABLE_MASK; 2889 2890 /* Enable Fan Failure detection */ 2891 if (hw->phy.media_type == ixgbe_media_type_copper) 2892 mask |= IXGBE_EIMS_GPI_SDP1; 2893 2894 /* 82599 specific interrupts */ 2895 if (sc->hw.mac.type == ixgbe_mac_82599EB) { 2896 mask |= IXGBE_EIMS_ECC; 2897 mask |= IXGBE_EIMS_GPI_SDP1; 2898 mask |= IXGBE_EIMS_GPI_SDP2; 2899 } 2900 2901 /* With RSS we use auto clear */ 2902 if (sc->msix_mem) { 2903 /* Dont autoclear Link */ 2904 mask &= ~IXGBE_EIMS_OTHER; 2905 mask &= ~IXGBE_EIMS_LSC; 2906 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 2907 sc->eims_mask | mask); 2908 } 2909 2910 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 2911 IXGBE_WRITE_FLUSH(hw); 2912 2913 return; 2914 } 2915 2916 void 2917 ixgbe_disable_intr(struct ix_softc *sc) 2918 { 2919 if (sc->msix_mem) 2920 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 2921 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 2922 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 2923 } else { 2924 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000); 2925 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0); 2926 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0); 2927 } 2928 IXGBE_WRITE_FLUSH(&sc->hw); 2929 return; 2930 } 2931 2932 uint16_t 2933 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) 2934 { 2935 struct pci_attach_args *pa; 2936 uint16_t value; 2937 2938 pa = ((struct ixgbe_osdep *)hw->back)->os_pa; 2939 2940 /* Should we do read/mask/write...? 16 vs 32 bit!!! */ 2941 value = pci_conf_read(pa->pa_pc, pa->pa_tag, reg) & 0xffff; 2942 2943 return (value); 2944 } 2945 2946 void 2947 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value) 2948 { 2949 struct pci_attach_args *pa; 2950 2951 pa = ((struct ixgbe_osdep *)hw->back)->os_pa; 2952 2953 /* Should we do read/mask/write...? 16 vs 32 bit!!! */ 2954 pci_conf_write(pa->pa_pc, pa->pa_tag, reg, value); 2955 } 2956 2957 /* 2958 * Setup the correct IVAR register for a particular MSIX interrupt 2959 * (yes this is all very magic and confusing :) 2960 * - entry is the register array entry 2961 * - vector is the MSIX vector for this queue 2962 * - type is RX/TX/MISC 2963 */ 2964 void 2965 ixgbe_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, int8_t type) 2966 { 2967 struct ixgbe_hw *hw = &sc->hw; 2968 uint32_t ivar, index; 2969 2970 vector |= IXGBE_IVAR_ALLOC_VAL; 2971 2972 switch (hw->mac.type) { 2973 2974 case ixgbe_mac_82598EB: 2975 if (type == -1) 2976 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 2977 else 2978 entry += (type * 64); 2979 index = (entry >> 2) & 0x1F; 2980 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 2981 ivar &= ~(0xFF << (8 * (entry & 0x3))); 2982 ivar |= (vector << (8 * (entry & 0x3))); 2983 IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar); 2984 break; 2985 2986 case ixgbe_mac_82599EB: 2987 if (type == -1) { /* MISC IVAR */ 2988 index = (entry & 1) * 8; 2989 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 2990 ivar &= ~(0xFF << index); 2991 ivar |= (vector << index); 2992 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 2993 } else { /* RX/TX IVARS */ 2994 index = (16 * (entry & 1)) + (8 * type); 2995 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 2996 ivar &= ~(0xFF << index); 2997 ivar |= (vector << index); 2998 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 2999 } 3000 3001 default: 3002 break; 3003 } 3004 } 3005 3006 void 3007 ixgbe_configure_ivars(struct ix_softc *sc) 3008 { 3009 struct tx_ring *txr = sc->tx_rings; 3010 struct rx_ring *rxr = sc->rx_rings; 3011 int i; 3012 3013 for (i = 0; i < sc->num_rx_queues; i++, rxr++) 3014 ixgbe_set_ivar(sc, i, rxr->msix, 0); 3015 3016 for (i = 0; i < sc->num_tx_queues; i++, txr++) 3017 ixgbe_set_ivar(sc, i, txr->msix, 1); 3018 3019 /* For the Link interrupt */ 3020 ixgbe_set_ivar(sc, 1, sc->linkvec, -1); 3021 } 3022 3023 /********************************************************************** 3024 * 3025 * Update the board statistics counters. 3026 * 3027 **********************************************************************/ 3028 void 3029 ixgbe_update_stats_counters(struct ix_softc *sc) 3030 { 3031 struct ifnet *ifp = &sc->arpcom.ac_if; 3032 struct ixgbe_hw *hw = &sc->hw; 3033 uint32_t missed_rx = 0, bprc, lxon, lxoff, total; 3034 int i; 3035 3036 sc->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 3037 3038 for (i = 0; i < 8; i++) { 3039 int mp; 3040 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 3041 missed_rx += mp; 3042 sc->stats.mpc[i] += mp; 3043 if (hw->mac.type == ixgbe_mac_82598EB) 3044 sc->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3045 } 3046 3047 /* Hardware workaround, gprc counts missed packets */ 3048 sc->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 3049 sc->stats.gprc -= missed_rx; 3050 3051 sc->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3052 sc->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3053 sc->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3054 3055 /* 3056 * Workaround: mprc hardware is incorrectly counting 3057 * broadcasts, so for now we subtract those. 3058 */ 3059 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3060 sc->stats.bprc += bprc; 3061 sc->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3062 sc->stats.mprc -= bprc; 3063 3064 sc->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3065 sc->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3066 sc->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3067 sc->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3068 sc->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3069 sc->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3070 sc->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3071 sc->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 3072 3073 sc->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3074 sc->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3075 3076 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3077 sc->stats.lxontxc += lxon; 3078 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3079 sc->stats.lxofftxc += lxoff; 3080 total = lxon + lxoff; 3081 3082 sc->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 3083 sc->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3084 sc->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3085 sc->stats.gptc -= total; 3086 sc->stats.mptc -= total; 3087 sc->stats.ptc64 -= total; 3088 sc->stats.gotc -= total * ETHER_MIN_LEN; 3089 3090 sc->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3091 sc->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3092 sc->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3093 sc->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3094 sc->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3095 sc->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3096 sc->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3097 sc->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3098 sc->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3099 sc->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3100 3101 #if 0 3102 /* Fill out the OS statistics structure */ 3103 ifp->if_ipackets = sc->stats.gprc; 3104 ifp->if_opackets = sc->stats.gptc; 3105 ifp->if_ibytes = sc->stats.gorc; 3106 ifp->if_obytes = sc->stats.gotc; 3107 ifp->if_imcasts = sc->stats.mprc; 3108 #endif 3109 ifp->if_collisions = 0; 3110 ifp->if_oerrors = sc->watchdog_events; 3111 ifp->if_ierrors = missed_rx + sc->stats.crcerrs + sc->stats.rlec; 3112 } 3113 3114 #ifdef IX_DEBUG 3115 /********************************************************************** 3116 * 3117 * This routine is called only when ixgbe_display_debug_stats is enabled. 3118 * This routine provides a way to take a look at important statistics 3119 * maintained by the driver and hardware. 3120 * 3121 **********************************************************************/ 3122 void 3123 ixgbe_print_hw_stats(struct ix_softc * sc) 3124 { 3125 struct ifnet *ifp = &sc->arpcom.ac_if;; 3126 3127 printf("%s: mbuf alloc failed %lu, mbuf cluster failed %lu, " 3128 "missed pkts %llu, rx len errs %llu, crc errs %llu, " 3129 "dropped pkts %lu, watchdog timeouts %ld, " 3130 "XON rx %llu, XON tx %llu, XOFF rx %llu, XOFF tx %llu, " 3131 "total pkts rx %llu, good pkts rx %llu, good pkts tx %llu, " 3132 "tso tx %lu\n", 3133 ifp->if_xname, 3134 sc->mbuf_alloc_failed, 3135 sc->mbuf_cluster_failed, 3136 (long long)sc->stats.mpc[0], 3137 (long long)sc->stats.roc + (long long)sc->stats.ruc, 3138 (long long)sc->stats.crcerrs, 3139 sc->dropped_pkts, 3140 sc->watchdog_events, 3141 (long long)sc->stats.lxonrxc, 3142 (long long)sc->stats.lxontxc, 3143 (long long)sc->stats.lxoffrxc, 3144 (long long)sc->stats.lxofftxc, 3145 (long long)sc->stats.tpr, 3146 (long long)sc->stats.gprc, 3147 (long long)sc->stats.gptc, 3148 sc->tso_tx); 3149 } 3150 #endif 3151