1 /* 2 * Copyright (c) 2004 Joerg Sonnenberger <joerg@bec.de>. All rights reserved. 3 * 4 * Copyright (c) 2001-2008, Intel Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 * 34 * Copyright (c) 2005 The DragonFly Project. All rights reserved. 35 * 36 * This code is derived from software contributed to The DragonFly Project 37 * by Matthew Dillon <dillon@backplane.com> 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 43 * 1. Redistributions of source code must retain the above copyright 44 * notice, this list of conditions and the following disclaimer. 45 * 2. Redistributions in binary form must reproduce the above copyright 46 * notice, this list of conditions and the following disclaimer in 47 * the documentation and/or other materials provided with the 48 * distribution. 49 * 3. Neither the name of The DragonFly Project nor the names of its 50 * contributors may be used to endorse or promote products derived 51 * from this software without specific, prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 55 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 56 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 57 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 58 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 59 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 60 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 61 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 62 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 63 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 64 * SUCH DAMAGE. 65 */ 66 67 #include "opt_ifpoll.h" 68 #include "opt_emx.h" 69 70 #include <sys/param.h> 71 #include <sys/bus.h> 72 #include <sys/endian.h> 73 #include <sys/interrupt.h> 74 #include <sys/kernel.h> 75 #include <sys/ktr.h> 76 #include <sys/malloc.h> 77 #include <sys/mbuf.h> 78 #include <sys/proc.h> 79 #include <sys/rman.h> 80 #include <sys/serialize.h> 81 #include <sys/serialize2.h> 82 #include <sys/socket.h> 83 #include <sys/sockio.h> 84 #include <sys/sysctl.h> 85 #include <sys/systm.h> 86 87 #include <net/bpf.h> 88 #include <net/ethernet.h> 89 #include <net/if.h> 90 #include <net/if_arp.h> 91 #include <net/if_dl.h> 92 #include <net/if_media.h> 93 #include <net/ifq_var.h> 94 #include <net/toeplitz.h> 95 #include <net/toeplitz2.h> 96 #include <net/vlan/if_vlan_var.h> 97 #include <net/vlan/if_vlan_ether.h> 98 #include <net/if_poll.h> 99 100 #include <netinet/in_systm.h> 101 #include <netinet/in.h> 102 #include <netinet/ip.h> 103 #include <netinet/tcp.h> 104 #include <netinet/udp.h> 105 106 #include <bus/pci/pcivar.h> 107 #include <bus/pci/pcireg.h> 108 109 #include <dev/netif/ig_hal/e1000_api.h> 110 #include <dev/netif/ig_hal/e1000_82571.h> 111 #include <dev/netif/ig_hal/e1000_dragonfly.h> 112 #include <dev/netif/emx/if_emx.h> 113 114 #define DEBUG_HW 0 115 116 #ifdef EMX_RSS_DEBUG 117 #define EMX_RSS_DPRINTF(sc, lvl, fmt, ...) \ 118 do { \ 119 if (sc->rss_debug >= lvl) \ 120 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \ 121 } while (0) 122 #else /* !EMX_RSS_DEBUG */ 123 #define EMX_RSS_DPRINTF(sc, lvl, fmt, ...) ((void)0) 124 #endif /* EMX_RSS_DEBUG */ 125 126 #define EMX_NAME "Intel(R) PRO/1000 " 127 128 #define EMX_DEVICE(id) \ 129 { EMX_VENDOR_ID, E1000_DEV_ID_##id, EMX_NAME #id } 130 #define EMX_DEVICE_NULL { 0, 0, NULL } 131 132 static const struct emx_device { 133 uint16_t vid; 134 uint16_t did; 135 const char *desc; 136 } emx_devices[] = { 137 EMX_DEVICE(82571EB_COPPER), 138 EMX_DEVICE(82571EB_FIBER), 139 EMX_DEVICE(82571EB_SERDES), 140 EMX_DEVICE(82571EB_SERDES_DUAL), 141 EMX_DEVICE(82571EB_SERDES_QUAD), 142 EMX_DEVICE(82571EB_QUAD_COPPER), 143 EMX_DEVICE(82571EB_QUAD_COPPER_BP), 144 EMX_DEVICE(82571EB_QUAD_COPPER_LP), 145 EMX_DEVICE(82571EB_QUAD_FIBER), 146 EMX_DEVICE(82571PT_QUAD_COPPER), 147 148 EMX_DEVICE(82572EI_COPPER), 149 EMX_DEVICE(82572EI_FIBER), 150 EMX_DEVICE(82572EI_SERDES), 151 EMX_DEVICE(82572EI), 152 153 EMX_DEVICE(82573E), 154 EMX_DEVICE(82573E_IAMT), 155 EMX_DEVICE(82573L), 156 157 EMX_DEVICE(80003ES2LAN_COPPER_SPT), 158 EMX_DEVICE(80003ES2LAN_SERDES_SPT), 159 EMX_DEVICE(80003ES2LAN_COPPER_DPT), 160 EMX_DEVICE(80003ES2LAN_SERDES_DPT), 161 162 EMX_DEVICE(82574L), 163 EMX_DEVICE(82574LA), 164 165 EMX_DEVICE(PCH_LPT_I217_LM), 166 EMX_DEVICE(PCH_LPT_I217_V), 167 EMX_DEVICE(PCH_LPTLP_I218_LM), 168 EMX_DEVICE(PCH_LPTLP_I218_V), 169 EMX_DEVICE(PCH_I218_LM2), 170 EMX_DEVICE(PCH_I218_V2), 171 EMX_DEVICE(PCH_I218_LM3), 172 EMX_DEVICE(PCH_I218_V3), 173 174 /* required last entry */ 175 EMX_DEVICE_NULL 176 }; 177 178 static int emx_probe(device_t); 179 static int emx_attach(device_t); 180 static int emx_detach(device_t); 181 static int emx_shutdown(device_t); 182 static int emx_suspend(device_t); 183 static int emx_resume(device_t); 184 185 static void emx_init(void *); 186 static void emx_stop(struct emx_softc *); 187 static int emx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 188 static void emx_start(struct ifnet *, struct ifaltq_subque *); 189 #ifdef IFPOLL_ENABLE 190 static void emx_npoll(struct ifnet *, struct ifpoll_info *); 191 static void emx_npoll_status(struct ifnet *); 192 static void emx_npoll_tx(struct ifnet *, void *, int); 193 static void emx_npoll_rx(struct ifnet *, void *, int); 194 #endif 195 static void emx_watchdog(struct ifaltq_subque *); 196 static void emx_media_status(struct ifnet *, struct ifmediareq *); 197 static int emx_media_change(struct ifnet *); 198 static void emx_timer(void *); 199 static void emx_serialize(struct ifnet *, enum ifnet_serialize); 200 static void emx_deserialize(struct ifnet *, enum ifnet_serialize); 201 static int emx_tryserialize(struct ifnet *, enum ifnet_serialize); 202 #ifdef INVARIANTS 203 static void emx_serialize_assert(struct ifnet *, enum ifnet_serialize, 204 boolean_t); 205 #endif 206 207 static void emx_intr(void *); 208 static void emx_intr_mask(void *); 209 static void emx_intr_body(struct emx_softc *, boolean_t); 210 static void emx_rxeof(struct emx_rxdata *, int); 211 static void emx_txeof(struct emx_txdata *); 212 static void emx_tx_collect(struct emx_txdata *); 213 static void emx_tx_purge(struct emx_softc *); 214 static void emx_enable_intr(struct emx_softc *); 215 static void emx_disable_intr(struct emx_softc *); 216 217 static int emx_dma_alloc(struct emx_softc *); 218 static void emx_dma_free(struct emx_softc *); 219 static void emx_init_tx_ring(struct emx_txdata *); 220 static int emx_init_rx_ring(struct emx_rxdata *); 221 static void emx_free_tx_ring(struct emx_txdata *); 222 static void emx_free_rx_ring(struct emx_rxdata *); 223 static int emx_create_tx_ring(struct emx_txdata *); 224 static int emx_create_rx_ring(struct emx_rxdata *); 225 static void emx_destroy_tx_ring(struct emx_txdata *, int); 226 static void emx_destroy_rx_ring(struct emx_rxdata *, int); 227 static int emx_newbuf(struct emx_rxdata *, int, int); 228 static int emx_encap(struct emx_txdata *, struct mbuf **, int *, int *); 229 static int emx_txcsum(struct emx_txdata *, struct mbuf *, 230 uint32_t *, uint32_t *); 231 static int emx_tso_pullup(struct emx_txdata *, struct mbuf **); 232 static int emx_tso_setup(struct emx_txdata *, struct mbuf *, 233 uint32_t *, uint32_t *); 234 static int emx_get_txring_inuse(const struct emx_softc *, boolean_t); 235 236 static int emx_is_valid_eaddr(const uint8_t *); 237 static int emx_reset(struct emx_softc *); 238 static void emx_setup_ifp(struct emx_softc *); 239 static void emx_init_tx_unit(struct emx_softc *); 240 static void emx_init_rx_unit(struct emx_softc *); 241 static void emx_update_stats(struct emx_softc *); 242 static void emx_set_promisc(struct emx_softc *); 243 static void emx_disable_promisc(struct emx_softc *); 244 static void emx_set_multi(struct emx_softc *); 245 static void emx_update_link_status(struct emx_softc *); 246 static void emx_smartspeed(struct emx_softc *); 247 static void emx_set_itr(struct emx_softc *, uint32_t); 248 static void emx_disable_aspm(struct emx_softc *); 249 250 static void emx_print_debug_info(struct emx_softc *); 251 static void emx_print_nvm_info(struct emx_softc *); 252 static void emx_print_hw_stats(struct emx_softc *); 253 254 static int emx_sysctl_stats(SYSCTL_HANDLER_ARGS); 255 static int emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 256 static int emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS); 257 static int emx_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS); 258 static int emx_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS); 259 #ifdef IFPOLL_ENABLE 260 static int emx_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS); 261 static int emx_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS); 262 #endif 263 static void emx_add_sysctl(struct emx_softc *); 264 265 static void emx_serialize_skipmain(struct emx_softc *); 266 static void emx_deserialize_skipmain(struct emx_softc *); 267 268 /* Management and WOL Support */ 269 static void emx_get_mgmt(struct emx_softc *); 270 static void emx_rel_mgmt(struct emx_softc *); 271 static void emx_get_hw_control(struct emx_softc *); 272 static void emx_rel_hw_control(struct emx_softc *); 273 static void emx_enable_wol(device_t); 274 275 static device_method_t emx_methods[] = { 276 /* Device interface */ 277 DEVMETHOD(device_probe, emx_probe), 278 DEVMETHOD(device_attach, emx_attach), 279 DEVMETHOD(device_detach, emx_detach), 280 DEVMETHOD(device_shutdown, emx_shutdown), 281 DEVMETHOD(device_suspend, emx_suspend), 282 DEVMETHOD(device_resume, emx_resume), 283 DEVMETHOD_END 284 }; 285 286 static driver_t emx_driver = { 287 "emx", 288 emx_methods, 289 sizeof(struct emx_softc), 290 }; 291 292 static devclass_t emx_devclass; 293 294 DECLARE_DUMMY_MODULE(if_emx); 295 MODULE_DEPEND(emx, ig_hal, 1, 1, 1); 296 DRIVER_MODULE(if_emx, pci, emx_driver, emx_devclass, NULL, NULL); 297 298 /* 299 * Tunables 300 */ 301 static int emx_int_throttle_ceil = EMX_DEFAULT_ITR; 302 static int emx_rxd = EMX_DEFAULT_RXD; 303 static int emx_txd = EMX_DEFAULT_TXD; 304 static int emx_smart_pwr_down = 0; 305 static int emx_rxr = 0; 306 static int emx_txr = 1; 307 308 /* Controls whether promiscuous also shows bad packets */ 309 static int emx_debug_sbp = 0; 310 311 static int emx_82573_workaround = 1; 312 static int emx_msi_enable = 1; 313 314 static char emx_flowctrl[IFM_ETH_FC_STRLEN] = IFM_ETH_FC_RXPAUSE; 315 316 TUNABLE_INT("hw.emx.int_throttle_ceil", &emx_int_throttle_ceil); 317 TUNABLE_INT("hw.emx.rxd", &emx_rxd); 318 TUNABLE_INT("hw.emx.rxr", &emx_rxr); 319 TUNABLE_INT("hw.emx.txd", &emx_txd); 320 TUNABLE_INT("hw.emx.txr", &emx_txr); 321 TUNABLE_INT("hw.emx.smart_pwr_down", &emx_smart_pwr_down); 322 TUNABLE_INT("hw.emx.sbp", &emx_debug_sbp); 323 TUNABLE_INT("hw.emx.82573_workaround", &emx_82573_workaround); 324 TUNABLE_INT("hw.emx.msi.enable", &emx_msi_enable); 325 TUNABLE_STR("hw.emx.flow_ctrl", emx_flowctrl, sizeof(emx_flowctrl)); 326 327 /* Global used in WOL setup with multiport cards */ 328 static int emx_global_quad_port_a = 0; 329 330 /* Set this to one to display debug statistics */ 331 static int emx_display_debug_stats = 0; 332 333 #if !defined(KTR_IF_EMX) 334 #define KTR_IF_EMX KTR_ALL 335 #endif 336 KTR_INFO_MASTER(if_emx); 337 KTR_INFO(KTR_IF_EMX, if_emx, intr_beg, 0, "intr begin"); 338 KTR_INFO(KTR_IF_EMX, if_emx, intr_end, 1, "intr end"); 339 KTR_INFO(KTR_IF_EMX, if_emx, pkt_receive, 4, "rx packet"); 340 KTR_INFO(KTR_IF_EMX, if_emx, pkt_txqueue, 5, "tx packet"); 341 KTR_INFO(KTR_IF_EMX, if_emx, pkt_txclean, 6, "tx clean"); 342 #define logif(name) KTR_LOG(if_emx_ ## name) 343 344 static __inline void 345 emx_setup_rxdesc(emx_rxdesc_t *rxd, const struct emx_rxbuf *rxbuf) 346 { 347 rxd->rxd_bufaddr = htole64(rxbuf->paddr); 348 /* DD bit must be cleared */ 349 rxd->rxd_staterr = 0; 350 } 351 352 static __inline void 353 emx_rxcsum(uint32_t staterr, struct mbuf *mp) 354 { 355 /* Ignore Checksum bit is set */ 356 if (staterr & E1000_RXD_STAT_IXSM) 357 return; 358 359 if ((staterr & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == 360 E1000_RXD_STAT_IPCS) 361 mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID; 362 363 if ((staterr & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == 364 E1000_RXD_STAT_TCPCS) { 365 mp->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 366 CSUM_PSEUDO_HDR | 367 CSUM_FRAG_NOT_CHECKED; 368 mp->m_pkthdr.csum_data = htons(0xffff); 369 } 370 } 371 372 static __inline struct pktinfo * 373 emx_rssinfo(struct mbuf *m, struct pktinfo *pi, 374 uint32_t mrq, uint32_t hash, uint32_t staterr) 375 { 376 switch (mrq & EMX_RXDMRQ_RSSTYPE_MASK) { 377 case EMX_RXDMRQ_IPV4_TCP: 378 pi->pi_netisr = NETISR_IP; 379 pi->pi_flags = 0; 380 pi->pi_l3proto = IPPROTO_TCP; 381 break; 382 383 case EMX_RXDMRQ_IPV6_TCP: 384 pi->pi_netisr = NETISR_IPV6; 385 pi->pi_flags = 0; 386 pi->pi_l3proto = IPPROTO_TCP; 387 break; 388 389 case EMX_RXDMRQ_IPV4: 390 if (staterr & E1000_RXD_STAT_IXSM) 391 return NULL; 392 393 if ((staterr & 394 (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == 395 E1000_RXD_STAT_TCPCS) { 396 pi->pi_netisr = NETISR_IP; 397 pi->pi_flags = 0; 398 pi->pi_l3proto = IPPROTO_UDP; 399 break; 400 } 401 /* FALL THROUGH */ 402 default: 403 return NULL; 404 } 405 406 m->m_flags |= M_HASH; 407 m->m_pkthdr.hash = toeplitz_hash(hash); 408 return pi; 409 } 410 411 static int 412 emx_probe(device_t dev) 413 { 414 const struct emx_device *d; 415 uint16_t vid, did; 416 417 vid = pci_get_vendor(dev); 418 did = pci_get_device(dev); 419 420 for (d = emx_devices; d->desc != NULL; ++d) { 421 if (vid == d->vid && did == d->did) { 422 device_set_desc(dev, d->desc); 423 device_set_async_attach(dev, TRUE); 424 return 0; 425 } 426 } 427 return ENXIO; 428 } 429 430 static int 431 emx_attach(device_t dev) 432 { 433 struct emx_softc *sc = device_get_softc(dev); 434 int error = 0, i, throttle, msi_enable, tx_ring_max; 435 u_int intr_flags; 436 uint16_t eeprom_data, device_id, apme_mask; 437 driver_intr_t *intr_func; 438 char flowctrl[IFM_ETH_FC_STRLEN]; 439 #ifdef IFPOLL_ENABLE 440 int offset, offset_def; 441 #endif 442 443 /* 444 * Setup RX rings 445 */ 446 for (i = 0; i < EMX_NRX_RING; ++i) { 447 sc->rx_data[i].sc = sc; 448 sc->rx_data[i].idx = i; 449 } 450 451 /* 452 * Setup TX ring 453 */ 454 for (i = 0; i < EMX_NTX_RING; ++i) { 455 sc->tx_data[i].sc = sc; 456 sc->tx_data[i].idx = i; 457 } 458 459 /* 460 * Initialize serializers 461 */ 462 lwkt_serialize_init(&sc->main_serialize); 463 for (i = 0; i < EMX_NTX_RING; ++i) 464 lwkt_serialize_init(&sc->tx_data[i].tx_serialize); 465 for (i = 0; i < EMX_NRX_RING; ++i) 466 lwkt_serialize_init(&sc->rx_data[i].rx_serialize); 467 468 /* 469 * Initialize serializer array 470 */ 471 i = 0; 472 473 KKASSERT(i < EMX_NSERIALIZE); 474 sc->serializes[i++] = &sc->main_serialize; 475 476 KKASSERT(i < EMX_NSERIALIZE); 477 sc->serializes[i++] = &sc->tx_data[0].tx_serialize; 478 KKASSERT(i < EMX_NSERIALIZE); 479 sc->serializes[i++] = &sc->tx_data[1].tx_serialize; 480 481 KKASSERT(i < EMX_NSERIALIZE); 482 sc->serializes[i++] = &sc->rx_data[0].rx_serialize; 483 KKASSERT(i < EMX_NSERIALIZE); 484 sc->serializes[i++] = &sc->rx_data[1].rx_serialize; 485 486 KKASSERT(i == EMX_NSERIALIZE); 487 488 ifmedia_init(&sc->media, IFM_IMASK | IFM_ETH_FCMASK, 489 emx_media_change, emx_media_status); 490 callout_init_mp(&sc->timer); 491 492 sc->dev = sc->osdep.dev = dev; 493 494 /* 495 * Determine hardware and mac type 496 */ 497 sc->hw.vendor_id = pci_get_vendor(dev); 498 sc->hw.device_id = pci_get_device(dev); 499 sc->hw.revision_id = pci_get_revid(dev); 500 sc->hw.subsystem_vendor_id = pci_get_subvendor(dev); 501 sc->hw.subsystem_device_id = pci_get_subdevice(dev); 502 503 if (e1000_set_mac_type(&sc->hw)) 504 return ENXIO; 505 506 /* Enable bus mastering */ 507 pci_enable_busmaster(dev); 508 509 /* 510 * Allocate IO memory 511 */ 512 sc->memory_rid = EMX_BAR_MEM; 513 sc->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 514 &sc->memory_rid, RF_ACTIVE); 515 if (sc->memory == NULL) { 516 device_printf(dev, "Unable to allocate bus resource: memory\n"); 517 error = ENXIO; 518 goto fail; 519 } 520 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->memory); 521 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->memory); 522 523 /* XXX This is quite goofy, it is not actually used */ 524 sc->hw.hw_addr = (uint8_t *)&sc->osdep.mem_bus_space_handle; 525 526 /* 527 * Don't enable MSI-X on 82574, see: 528 * 82574 specification update errata #15 529 * 530 * Don't enable MSI on 82571/82572, see: 531 * 82571/82572 specification update errata #63 532 */ 533 msi_enable = emx_msi_enable; 534 if (msi_enable && 535 (sc->hw.mac.type == e1000_82571 || 536 sc->hw.mac.type == e1000_82572)) 537 msi_enable = 0; 538 539 /* 540 * Allocate interrupt 541 */ 542 sc->intr_type = pci_alloc_1intr(dev, msi_enable, 543 &sc->intr_rid, &intr_flags); 544 545 if (sc->intr_type == PCI_INTR_TYPE_LEGACY) { 546 int unshared; 547 548 unshared = device_getenv_int(dev, "irq.unshared", 0); 549 if (!unshared) { 550 sc->flags |= EMX_FLAG_SHARED_INTR; 551 if (bootverbose) 552 device_printf(dev, "IRQ shared\n"); 553 } else { 554 intr_flags &= ~RF_SHAREABLE; 555 if (bootverbose) 556 device_printf(dev, "IRQ unshared\n"); 557 } 558 } 559 560 sc->intr_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->intr_rid, 561 intr_flags); 562 if (sc->intr_res == NULL) { 563 device_printf(dev, "Unable to allocate bus resource: " 564 "interrupt\n"); 565 error = ENXIO; 566 goto fail; 567 } 568 569 /* Save PCI command register for Shared Code */ 570 sc->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 571 sc->hw.back = &sc->osdep; 572 573 /* 574 * For I217/I218, we need to map the flash memory and this 575 * must happen after the MAC is identified. 576 */ 577 if (sc->hw.mac.type == e1000_pch_lpt) { 578 sc->flash_rid = EMX_BAR_FLASH; 579 580 sc->flash = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 581 &sc->flash_rid, RF_ACTIVE); 582 if (sc->flash == NULL) { 583 device_printf(dev, "Mapping of Flash failed\n"); 584 error = ENXIO; 585 goto fail; 586 } 587 sc->osdep.flash_bus_space_tag = rman_get_bustag(sc->flash); 588 sc->osdep.flash_bus_space_handle = 589 rman_get_bushandle(sc->flash); 590 591 /* 592 * This is used in the shared code 593 * XXX this goof is actually not used. 594 */ 595 sc->hw.flash_address = (uint8_t *)sc->flash; 596 } 597 598 /* Do Shared Code initialization */ 599 if (e1000_setup_init_funcs(&sc->hw, TRUE)) { 600 device_printf(dev, "Setup of Shared code failed\n"); 601 error = ENXIO; 602 goto fail; 603 } 604 e1000_get_bus_info(&sc->hw); 605 606 sc->hw.mac.autoneg = EMX_DO_AUTO_NEG; 607 sc->hw.phy.autoneg_wait_to_complete = FALSE; 608 sc->hw.phy.autoneg_advertised = EMX_AUTONEG_ADV_DEFAULT; 609 610 /* 611 * Interrupt throttle rate 612 */ 613 throttle = device_getenv_int(dev, "int_throttle_ceil", 614 emx_int_throttle_ceil); 615 if (throttle == 0) { 616 sc->int_throttle_ceil = 0; 617 } else { 618 if (throttle < 0) 619 throttle = EMX_DEFAULT_ITR; 620 621 /* Recalculate the tunable value to get the exact frequency. */ 622 throttle = 1000000000 / 256 / throttle; 623 624 /* Upper 16bits of ITR is reserved and should be zero */ 625 if (throttle & 0xffff0000) 626 throttle = 1000000000 / 256 / EMX_DEFAULT_ITR; 627 628 sc->int_throttle_ceil = 1000000000 / 256 / throttle; 629 } 630 631 e1000_init_script_state_82541(&sc->hw, TRUE); 632 e1000_set_tbi_compatibility_82543(&sc->hw, TRUE); 633 634 /* Copper options */ 635 if (sc->hw.phy.media_type == e1000_media_type_copper) { 636 sc->hw.phy.mdix = EMX_AUTO_ALL_MODES; 637 sc->hw.phy.disable_polarity_correction = FALSE; 638 sc->hw.phy.ms_type = EMX_MASTER_SLAVE; 639 } 640 641 /* Set the frame limits assuming standard ethernet sized frames. */ 642 sc->hw.mac.max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN; 643 644 /* This controls when hardware reports transmit completion status. */ 645 sc->hw.mac.report_tx_early = 1; 646 647 /* Calculate # of RX rings */ 648 sc->rx_ring_cnt = device_getenv_int(dev, "rxr", emx_rxr); 649 sc->rx_ring_cnt = if_ring_count2(sc->rx_ring_cnt, EMX_NRX_RING); 650 651 /* 652 * Calculate # of TX rings 653 * 654 * XXX 655 * I217/I218 claims to have 2 TX queues 656 * 657 * NOTE: 658 * Don't enable multiple TX queues on 82574; it always gives 659 * watchdog timeout on TX queue0, when multiple TCP streams are 660 * received. It was originally suspected that the hardware TX 661 * checksum offloading caused this watchdog timeout, since only 662 * TCP ACKs are sent during TCP receiving tests. However, even 663 * if the hardware TX checksum offloading is disable, TX queue0 664 * still will give watchdog. 665 */ 666 tx_ring_max = 1; 667 if (sc->hw.mac.type == e1000_82571 || 668 sc->hw.mac.type == e1000_82572 || 669 sc->hw.mac.type == e1000_80003es2lan || 670 sc->hw.mac.type == e1000_pch_lpt || 671 sc->hw.mac.type == e1000_82574) 672 tx_ring_max = EMX_NTX_RING; 673 sc->tx_ring_cnt = device_getenv_int(dev, "txr", emx_txr); 674 sc->tx_ring_cnt = if_ring_count2(sc->tx_ring_cnt, tx_ring_max); 675 676 /* Allocate RX/TX rings' busdma(9) stuffs */ 677 error = emx_dma_alloc(sc); 678 if (error) 679 goto fail; 680 681 /* Allocate multicast array memory. */ 682 sc->mta = kmalloc(ETH_ADDR_LEN * EMX_MCAST_ADDR_MAX, 683 M_DEVBUF, M_WAITOK); 684 685 /* Indicate SOL/IDER usage */ 686 if (e1000_check_reset_block(&sc->hw)) { 687 device_printf(dev, 688 "PHY reset is blocked due to SOL/IDER session.\n"); 689 } 690 691 /* Disable EEE on I217/I218 */ 692 sc->hw.dev_spec.ich8lan.eee_disable = 1; 693 694 /* 695 * Start from a known state, this is important in reading the 696 * nvm and mac from that. 697 */ 698 e1000_reset_hw(&sc->hw); 699 700 /* Make sure we have a good EEPROM before we read from it */ 701 if (e1000_validate_nvm_checksum(&sc->hw) < 0) { 702 /* 703 * Some PCI-E parts fail the first check due to 704 * the link being in sleep state, call it again, 705 * if it fails a second time its a real issue. 706 */ 707 if (e1000_validate_nvm_checksum(&sc->hw) < 0) { 708 device_printf(dev, 709 "The EEPROM Checksum Is Not Valid\n"); 710 error = EIO; 711 goto fail; 712 } 713 } 714 715 /* Copy the permanent MAC address out of the EEPROM */ 716 if (e1000_read_mac_addr(&sc->hw) < 0) { 717 device_printf(dev, "EEPROM read error while reading MAC" 718 " address\n"); 719 error = EIO; 720 goto fail; 721 } 722 if (!emx_is_valid_eaddr(sc->hw.mac.addr)) { 723 device_printf(dev, "Invalid MAC address\n"); 724 error = EIO; 725 goto fail; 726 } 727 728 /* Disable ULP support */ 729 e1000_disable_ulp_lpt_lp(&sc->hw, TRUE); 730 731 /* Determine if we have to control management hardware */ 732 if (e1000_enable_mng_pass_thru(&sc->hw)) 733 sc->flags |= EMX_FLAG_HAS_MGMT; 734 735 /* 736 * Setup Wake-on-Lan 737 */ 738 apme_mask = EMX_EEPROM_APME; 739 eeprom_data = 0; 740 switch (sc->hw.mac.type) { 741 case e1000_82573: 742 sc->flags |= EMX_FLAG_HAS_AMT; 743 /* FALL THROUGH */ 744 745 case e1000_82571: 746 case e1000_82572: 747 case e1000_80003es2lan: 748 if (sc->hw.bus.func == 1) { 749 e1000_read_nvm(&sc->hw, 750 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 751 } else { 752 e1000_read_nvm(&sc->hw, 753 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 754 } 755 break; 756 757 default: 758 e1000_read_nvm(&sc->hw, 759 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 760 break; 761 } 762 if (eeprom_data & apme_mask) 763 sc->wol = E1000_WUFC_MAG | E1000_WUFC_MC; 764 765 /* 766 * We have the eeprom settings, now apply the special cases 767 * where the eeprom may be wrong or the board won't support 768 * wake on lan on a particular port 769 */ 770 device_id = pci_get_device(dev); 771 switch (device_id) { 772 case E1000_DEV_ID_82571EB_FIBER: 773 /* 774 * Wake events only supported on port A for dual fiber 775 * regardless of eeprom setting 776 */ 777 if (E1000_READ_REG(&sc->hw, E1000_STATUS) & 778 E1000_STATUS_FUNC_1) 779 sc->wol = 0; 780 break; 781 782 case E1000_DEV_ID_82571EB_QUAD_COPPER: 783 case E1000_DEV_ID_82571EB_QUAD_FIBER: 784 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 785 /* if quad port sc, disable WoL on all but port A */ 786 if (emx_global_quad_port_a != 0) 787 sc->wol = 0; 788 /* Reset for multiple quad port adapters */ 789 if (++emx_global_quad_port_a == 4) 790 emx_global_quad_port_a = 0; 791 break; 792 } 793 794 /* XXX disable wol */ 795 sc->wol = 0; 796 797 #ifdef IFPOLL_ENABLE 798 /* 799 * NPOLLING RX CPU offset 800 */ 801 if (sc->rx_ring_cnt == ncpus2) { 802 offset = 0; 803 } else { 804 offset_def = (sc->rx_ring_cnt * device_get_unit(dev)) % ncpus2; 805 offset = device_getenv_int(dev, "npoll.rxoff", offset_def); 806 if (offset >= ncpus2 || 807 offset % sc->rx_ring_cnt != 0) { 808 device_printf(dev, "invalid npoll.rxoff %d, use %d\n", 809 offset, offset_def); 810 offset = offset_def; 811 } 812 } 813 sc->rx_npoll_off = offset; 814 815 /* 816 * NPOLLING TX CPU offset 817 */ 818 if (sc->tx_ring_cnt == ncpus2) { 819 offset = 0; 820 } else { 821 offset_def = (sc->tx_ring_cnt * device_get_unit(dev)) % ncpus2; 822 offset = device_getenv_int(dev, "npoll.txoff", offset_def); 823 if (offset >= ncpus2 || 824 offset % sc->tx_ring_cnt != 0) { 825 device_printf(dev, "invalid npoll.txoff %d, use %d\n", 826 offset, offset_def); 827 offset = offset_def; 828 } 829 } 830 sc->tx_npoll_off = offset; 831 #endif 832 sc->tx_ring_inuse = emx_get_txring_inuse(sc, FALSE); 833 834 /* Setup flow control. */ 835 device_getenv_string(dev, "flow_ctrl", flowctrl, sizeof(flowctrl), 836 emx_flowctrl); 837 sc->ifm_flowctrl = ifmedia_str2ethfc(flowctrl); 838 839 /* Setup OS specific network interface */ 840 emx_setup_ifp(sc); 841 842 /* Add sysctl tree, must after em_setup_ifp() */ 843 emx_add_sysctl(sc); 844 845 /* Reset the hardware */ 846 error = emx_reset(sc); 847 if (error) { 848 /* 849 * Some 82573 parts fail the first reset, call it again, 850 * if it fails a second time its a real issue. 851 */ 852 error = emx_reset(sc); 853 if (error) { 854 device_printf(dev, "Unable to reset the hardware\n"); 855 ether_ifdetach(&sc->arpcom.ac_if); 856 goto fail; 857 } 858 } 859 860 /* Initialize statistics */ 861 emx_update_stats(sc); 862 863 sc->hw.mac.get_link_status = 1; 864 emx_update_link_status(sc); 865 866 /* Non-AMT based hardware can now take control from firmware */ 867 if ((sc->flags & (EMX_FLAG_HAS_MGMT | EMX_FLAG_HAS_AMT)) == 868 EMX_FLAG_HAS_MGMT) 869 emx_get_hw_control(sc); 870 871 /* 872 * Missing Interrupt Following ICR read: 873 * 874 * 82571/82572 specification update errata #76 875 * 82573 specification update errata #31 876 * 82574 specification update errata #12 877 */ 878 intr_func = emx_intr; 879 if ((sc->flags & EMX_FLAG_SHARED_INTR) && 880 (sc->hw.mac.type == e1000_82571 || 881 sc->hw.mac.type == e1000_82572 || 882 sc->hw.mac.type == e1000_82573 || 883 sc->hw.mac.type == e1000_82574)) 884 intr_func = emx_intr_mask; 885 886 error = bus_setup_intr(dev, sc->intr_res, INTR_MPSAFE, intr_func, sc, 887 &sc->intr_tag, &sc->main_serialize); 888 if (error) { 889 device_printf(dev, "Failed to register interrupt handler"); 890 ether_ifdetach(&sc->arpcom.ac_if); 891 goto fail; 892 } 893 return (0); 894 fail: 895 emx_detach(dev); 896 return (error); 897 } 898 899 static int 900 emx_detach(device_t dev) 901 { 902 struct emx_softc *sc = device_get_softc(dev); 903 904 if (device_is_attached(dev)) { 905 struct ifnet *ifp = &sc->arpcom.ac_if; 906 907 ifnet_serialize_all(ifp); 908 909 emx_stop(sc); 910 911 e1000_phy_hw_reset(&sc->hw); 912 913 emx_rel_mgmt(sc); 914 emx_rel_hw_control(sc); 915 916 if (sc->wol) { 917 E1000_WRITE_REG(&sc->hw, E1000_WUC, E1000_WUC_PME_EN); 918 E1000_WRITE_REG(&sc->hw, E1000_WUFC, sc->wol); 919 emx_enable_wol(dev); 920 } 921 922 bus_teardown_intr(dev, sc->intr_res, sc->intr_tag); 923 924 ifnet_deserialize_all(ifp); 925 926 ether_ifdetach(ifp); 927 } else if (sc->memory != NULL) { 928 emx_rel_hw_control(sc); 929 } 930 931 ifmedia_removeall(&sc->media); 932 bus_generic_detach(dev); 933 934 if (sc->intr_res != NULL) { 935 bus_release_resource(dev, SYS_RES_IRQ, sc->intr_rid, 936 sc->intr_res); 937 } 938 939 if (sc->intr_type == PCI_INTR_TYPE_MSI) 940 pci_release_msi(dev); 941 942 if (sc->memory != NULL) { 943 bus_release_resource(dev, SYS_RES_MEMORY, sc->memory_rid, 944 sc->memory); 945 } 946 947 if (sc->flash != NULL) { 948 bus_release_resource(dev, SYS_RES_MEMORY, sc->flash_rid, 949 sc->flash); 950 } 951 952 emx_dma_free(sc); 953 954 if (sc->mta != NULL) 955 kfree(sc->mta, M_DEVBUF); 956 957 return (0); 958 } 959 960 static int 961 emx_shutdown(device_t dev) 962 { 963 return emx_suspend(dev); 964 } 965 966 static int 967 emx_suspend(device_t dev) 968 { 969 struct emx_softc *sc = device_get_softc(dev); 970 struct ifnet *ifp = &sc->arpcom.ac_if; 971 972 ifnet_serialize_all(ifp); 973 974 emx_stop(sc); 975 976 emx_rel_mgmt(sc); 977 emx_rel_hw_control(sc); 978 979 if (sc->wol) { 980 E1000_WRITE_REG(&sc->hw, E1000_WUC, E1000_WUC_PME_EN); 981 E1000_WRITE_REG(&sc->hw, E1000_WUFC, sc->wol); 982 emx_enable_wol(dev); 983 } 984 985 ifnet_deserialize_all(ifp); 986 987 return bus_generic_suspend(dev); 988 } 989 990 static int 991 emx_resume(device_t dev) 992 { 993 struct emx_softc *sc = device_get_softc(dev); 994 struct ifnet *ifp = &sc->arpcom.ac_if; 995 int i; 996 997 ifnet_serialize_all(ifp); 998 999 emx_init(sc); 1000 emx_get_mgmt(sc); 1001 for (i = 0; i < sc->tx_ring_inuse; ++i) 1002 ifsq_devstart_sched(sc->tx_data[i].ifsq); 1003 1004 ifnet_deserialize_all(ifp); 1005 1006 return bus_generic_resume(dev); 1007 } 1008 1009 static void 1010 emx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 1011 { 1012 struct emx_softc *sc = ifp->if_softc; 1013 struct emx_txdata *tdata = ifsq_get_priv(ifsq); 1014 struct mbuf *m_head; 1015 int idx = -1, nsegs = 0; 1016 1017 KKASSERT(tdata->ifsq == ifsq); 1018 ASSERT_SERIALIZED(&tdata->tx_serialize); 1019 1020 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq)) 1021 return; 1022 1023 if (!sc->link_active || (tdata->tx_flags & EMX_TXFLAG_ENABLED) == 0) { 1024 ifsq_purge(ifsq); 1025 return; 1026 } 1027 1028 while (!ifsq_is_empty(ifsq)) { 1029 /* Now do we at least have a minimal? */ 1030 if (EMX_IS_OACTIVE(tdata)) { 1031 emx_tx_collect(tdata); 1032 if (EMX_IS_OACTIVE(tdata)) { 1033 ifsq_set_oactive(ifsq); 1034 break; 1035 } 1036 } 1037 1038 logif(pkt_txqueue); 1039 m_head = ifsq_dequeue(ifsq); 1040 if (m_head == NULL) 1041 break; 1042 1043 if (emx_encap(tdata, &m_head, &nsegs, &idx)) { 1044 IFNET_STAT_INC(ifp, oerrors, 1); 1045 emx_tx_collect(tdata); 1046 continue; 1047 } 1048 1049 /* 1050 * TX interrupt are aggressively aggregated, so increasing 1051 * opackets at TX interrupt time will make the opackets 1052 * statistics vastly inaccurate; we do the opackets increment 1053 * now. 1054 */ 1055 IFNET_STAT_INC(ifp, opackets, 1); 1056 1057 if (nsegs >= tdata->tx_wreg_nsegs) { 1058 E1000_WRITE_REG(&sc->hw, E1000_TDT(tdata->idx), idx); 1059 nsegs = 0; 1060 idx = -1; 1061 } 1062 1063 /* Send a copy of the frame to the BPF listener */ 1064 ETHER_BPF_MTAP(ifp, m_head); 1065 1066 /* Set timeout in case hardware has problems transmitting. */ 1067 tdata->tx_watchdog.wd_timer = EMX_TX_TIMEOUT; 1068 } 1069 if (idx >= 0) 1070 E1000_WRITE_REG(&sc->hw, E1000_TDT(tdata->idx), idx); 1071 } 1072 1073 static int 1074 emx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) 1075 { 1076 struct emx_softc *sc = ifp->if_softc; 1077 struct ifreq *ifr = (struct ifreq *)data; 1078 uint16_t eeprom_data = 0; 1079 int max_frame_size, mask, reinit; 1080 int error = 0; 1081 1082 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1083 1084 switch (command) { 1085 case SIOCSIFMTU: 1086 switch (sc->hw.mac.type) { 1087 case e1000_82573: 1088 /* 1089 * 82573 only supports jumbo frames 1090 * if ASPM is disabled. 1091 */ 1092 e1000_read_nvm(&sc->hw, NVM_INIT_3GIO_3, 1, 1093 &eeprom_data); 1094 if (eeprom_data & NVM_WORD1A_ASPM_MASK) { 1095 max_frame_size = ETHER_MAX_LEN; 1096 break; 1097 } 1098 /* FALL THROUGH */ 1099 1100 /* Limit Jumbo Frame size */ 1101 case e1000_82571: 1102 case e1000_82572: 1103 case e1000_82574: 1104 case e1000_pch_lpt: 1105 case e1000_80003es2lan: 1106 max_frame_size = 9234; 1107 break; 1108 1109 default: 1110 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1111 break; 1112 } 1113 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1114 ETHER_CRC_LEN) { 1115 error = EINVAL; 1116 break; 1117 } 1118 1119 ifp->if_mtu = ifr->ifr_mtu; 1120 sc->hw.mac.max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + 1121 ETHER_CRC_LEN; 1122 1123 if (ifp->if_flags & IFF_RUNNING) 1124 emx_init(sc); 1125 break; 1126 1127 case SIOCSIFFLAGS: 1128 if (ifp->if_flags & IFF_UP) { 1129 if ((ifp->if_flags & IFF_RUNNING)) { 1130 if ((ifp->if_flags ^ sc->if_flags) & 1131 (IFF_PROMISC | IFF_ALLMULTI)) { 1132 emx_disable_promisc(sc); 1133 emx_set_promisc(sc); 1134 } 1135 } else { 1136 emx_init(sc); 1137 } 1138 } else if (ifp->if_flags & IFF_RUNNING) { 1139 emx_stop(sc); 1140 } 1141 sc->if_flags = ifp->if_flags; 1142 break; 1143 1144 case SIOCADDMULTI: 1145 case SIOCDELMULTI: 1146 if (ifp->if_flags & IFF_RUNNING) { 1147 emx_disable_intr(sc); 1148 emx_set_multi(sc); 1149 #ifdef IFPOLL_ENABLE 1150 if (!(ifp->if_flags & IFF_NPOLLING)) 1151 #endif 1152 emx_enable_intr(sc); 1153 } 1154 break; 1155 1156 case SIOCSIFMEDIA: 1157 /* Check SOL/IDER usage */ 1158 if (e1000_check_reset_block(&sc->hw)) { 1159 device_printf(sc->dev, "Media change is" 1160 " blocked due to SOL/IDER session.\n"); 1161 break; 1162 } 1163 /* FALL THROUGH */ 1164 1165 case SIOCGIFMEDIA: 1166 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 1167 break; 1168 1169 case SIOCSIFCAP: 1170 reinit = 0; 1171 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1172 if (mask & IFCAP_RXCSUM) { 1173 ifp->if_capenable ^= IFCAP_RXCSUM; 1174 reinit = 1; 1175 } 1176 if (mask & IFCAP_VLAN_HWTAGGING) { 1177 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1178 reinit = 1; 1179 } 1180 if (mask & IFCAP_TXCSUM) { 1181 ifp->if_capenable ^= IFCAP_TXCSUM; 1182 if (ifp->if_capenable & IFCAP_TXCSUM) 1183 ifp->if_hwassist |= EMX_CSUM_FEATURES; 1184 else 1185 ifp->if_hwassist &= ~EMX_CSUM_FEATURES; 1186 } 1187 if (mask & IFCAP_TSO) { 1188 ifp->if_capenable ^= IFCAP_TSO; 1189 if (ifp->if_capenable & IFCAP_TSO) 1190 ifp->if_hwassist |= CSUM_TSO; 1191 else 1192 ifp->if_hwassist &= ~CSUM_TSO; 1193 } 1194 if (mask & IFCAP_RSS) 1195 ifp->if_capenable ^= IFCAP_RSS; 1196 if (reinit && (ifp->if_flags & IFF_RUNNING)) 1197 emx_init(sc); 1198 break; 1199 1200 default: 1201 error = ether_ioctl(ifp, command, data); 1202 break; 1203 } 1204 return (error); 1205 } 1206 1207 static void 1208 emx_watchdog(struct ifaltq_subque *ifsq) 1209 { 1210 struct emx_txdata *tdata = ifsq_get_priv(ifsq); 1211 struct ifnet *ifp = ifsq_get_ifp(ifsq); 1212 struct emx_softc *sc = ifp->if_softc; 1213 int i; 1214 1215 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1216 1217 /* 1218 * The timer is set to 5 every time start queues a packet. 1219 * Then txeof keeps resetting it as long as it cleans at 1220 * least one descriptor. 1221 * Finally, anytime all descriptors are clean the timer is 1222 * set to 0. 1223 */ 1224 1225 if (E1000_READ_REG(&sc->hw, E1000_TDT(tdata->idx)) == 1226 E1000_READ_REG(&sc->hw, E1000_TDH(tdata->idx))) { 1227 /* 1228 * If we reach here, all TX jobs are completed and 1229 * the TX engine should have been idled for some time. 1230 * We don't need to call ifsq_devstart_sched() here. 1231 */ 1232 ifsq_clr_oactive(ifsq); 1233 tdata->tx_watchdog.wd_timer = 0; 1234 return; 1235 } 1236 1237 /* 1238 * If we are in this routine because of pause frames, then 1239 * don't reset the hardware. 1240 */ 1241 if (E1000_READ_REG(&sc->hw, E1000_STATUS) & E1000_STATUS_TXOFF) { 1242 tdata->tx_watchdog.wd_timer = EMX_TX_TIMEOUT; 1243 return; 1244 } 1245 1246 if_printf(ifp, "TX %d watchdog timeout -- resetting\n", tdata->idx); 1247 1248 IFNET_STAT_INC(ifp, oerrors, 1); 1249 1250 emx_init(sc); 1251 for (i = 0; i < sc->tx_ring_inuse; ++i) 1252 ifsq_devstart_sched(sc->tx_data[i].ifsq); 1253 } 1254 1255 static void 1256 emx_init(void *xsc) 1257 { 1258 struct emx_softc *sc = xsc; 1259 struct ifnet *ifp = &sc->arpcom.ac_if; 1260 device_t dev = sc->dev; 1261 boolean_t polling; 1262 int i; 1263 1264 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1265 1266 emx_stop(sc); 1267 1268 /* Get the latest mac address, User can use a LAA */ 1269 bcopy(IF_LLADDR(ifp), sc->hw.mac.addr, ETHER_ADDR_LEN); 1270 1271 /* Put the address into the Receive Address Array */ 1272 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1273 1274 /* 1275 * With the 82571 sc, RAR[0] may be overwritten 1276 * when the other port is reset, we make a duplicate 1277 * in RAR[14] for that eventuality, this assures 1278 * the interface continues to function. 1279 */ 1280 if (sc->hw.mac.type == e1000_82571) { 1281 e1000_set_laa_state_82571(&sc->hw, TRUE); 1282 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 1283 E1000_RAR_ENTRIES - 1); 1284 } 1285 1286 /* Initialize the hardware */ 1287 if (emx_reset(sc)) { 1288 device_printf(dev, "Unable to reset the hardware\n"); 1289 /* XXX emx_stop()? */ 1290 return; 1291 } 1292 emx_update_link_status(sc); 1293 1294 /* Setup VLAN support, basic and offload if available */ 1295 E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN); 1296 1297 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1298 uint32_t ctrl; 1299 1300 ctrl = E1000_READ_REG(&sc->hw, E1000_CTRL); 1301 ctrl |= E1000_CTRL_VME; 1302 E1000_WRITE_REG(&sc->hw, E1000_CTRL, ctrl); 1303 } 1304 1305 /* Configure for OS presence */ 1306 emx_get_mgmt(sc); 1307 1308 polling = FALSE; 1309 #ifdef IFPOLL_ENABLE 1310 if (ifp->if_flags & IFF_NPOLLING) 1311 polling = TRUE; 1312 #endif 1313 sc->tx_ring_inuse = emx_get_txring_inuse(sc, polling); 1314 ifq_set_subq_mask(&ifp->if_snd, sc->tx_ring_inuse - 1); 1315 1316 /* Prepare transmit descriptors and buffers */ 1317 for (i = 0; i < sc->tx_ring_inuse; ++i) 1318 emx_init_tx_ring(&sc->tx_data[i]); 1319 emx_init_tx_unit(sc); 1320 1321 /* Setup Multicast table */ 1322 emx_set_multi(sc); 1323 1324 /* Prepare receive descriptors and buffers */ 1325 for (i = 0; i < sc->rx_ring_cnt; ++i) { 1326 if (emx_init_rx_ring(&sc->rx_data[i])) { 1327 device_printf(dev, 1328 "Could not setup receive structures\n"); 1329 emx_stop(sc); 1330 return; 1331 } 1332 } 1333 emx_init_rx_unit(sc); 1334 1335 /* Don't lose promiscuous settings */ 1336 emx_set_promisc(sc); 1337 1338 ifp->if_flags |= IFF_RUNNING; 1339 for (i = 0; i < sc->tx_ring_inuse; ++i) { 1340 ifsq_clr_oactive(sc->tx_data[i].ifsq); 1341 ifsq_watchdog_start(&sc->tx_data[i].tx_watchdog); 1342 } 1343 1344 callout_reset(&sc->timer, hz, emx_timer, sc); 1345 e1000_clear_hw_cntrs_base_generic(&sc->hw); 1346 1347 /* MSI/X configuration for 82574 */ 1348 if (sc->hw.mac.type == e1000_82574) { 1349 int tmp; 1350 1351 tmp = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 1352 tmp |= E1000_CTRL_EXT_PBA_CLR; 1353 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, tmp); 1354 /* 1355 * XXX MSIX 1356 * Set the IVAR - interrupt vector routing. 1357 * Each nibble represents a vector, high bit 1358 * is enable, other 3 bits are the MSIX table 1359 * entry, we map RXQ0 to 0, TXQ0 to 1, and 1360 * Link (other) to 2, hence the magic number. 1361 */ 1362 E1000_WRITE_REG(&sc->hw, E1000_IVAR, 0x800A0908); 1363 } 1364 1365 /* 1366 * Only enable interrupts if we are not polling, make sure 1367 * they are off otherwise. 1368 */ 1369 if (polling) 1370 emx_disable_intr(sc); 1371 else 1372 emx_enable_intr(sc); 1373 1374 /* AMT based hardware can now take control from firmware */ 1375 if ((sc->flags & (EMX_FLAG_HAS_MGMT | EMX_FLAG_HAS_AMT)) == 1376 (EMX_FLAG_HAS_MGMT | EMX_FLAG_HAS_AMT)) 1377 emx_get_hw_control(sc); 1378 } 1379 1380 static void 1381 emx_intr(void *xsc) 1382 { 1383 emx_intr_body(xsc, TRUE); 1384 } 1385 1386 static void 1387 emx_intr_body(struct emx_softc *sc, boolean_t chk_asserted) 1388 { 1389 struct ifnet *ifp = &sc->arpcom.ac_if; 1390 uint32_t reg_icr; 1391 1392 logif(intr_beg); 1393 ASSERT_SERIALIZED(&sc->main_serialize); 1394 1395 reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); 1396 1397 if (chk_asserted && (reg_icr & E1000_ICR_INT_ASSERTED) == 0) { 1398 logif(intr_end); 1399 return; 1400 } 1401 1402 /* 1403 * XXX: some laptops trigger several spurious interrupts 1404 * on emx(4) when in the resume cycle. The ICR register 1405 * reports all-ones value in this case. Processing such 1406 * interrupts would lead to a freeze. I don't know why. 1407 */ 1408 if (reg_icr == 0xffffffff) { 1409 logif(intr_end); 1410 return; 1411 } 1412 1413 if (ifp->if_flags & IFF_RUNNING) { 1414 if (reg_icr & 1415 (E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_RXO)) { 1416 int i; 1417 1418 for (i = 0; i < sc->rx_ring_cnt; ++i) { 1419 lwkt_serialize_enter( 1420 &sc->rx_data[i].rx_serialize); 1421 emx_rxeof(&sc->rx_data[i], -1); 1422 lwkt_serialize_exit( 1423 &sc->rx_data[i].rx_serialize); 1424 } 1425 } 1426 if (reg_icr & E1000_ICR_TXDW) { 1427 struct emx_txdata *tdata = &sc->tx_data[0]; 1428 1429 lwkt_serialize_enter(&tdata->tx_serialize); 1430 emx_txeof(tdata); 1431 if (!ifsq_is_empty(tdata->ifsq)) 1432 ifsq_devstart(tdata->ifsq); 1433 lwkt_serialize_exit(&tdata->tx_serialize); 1434 } 1435 } 1436 1437 /* Link status change */ 1438 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1439 emx_serialize_skipmain(sc); 1440 1441 callout_stop(&sc->timer); 1442 sc->hw.mac.get_link_status = 1; 1443 emx_update_link_status(sc); 1444 1445 /* Deal with TX cruft when link lost */ 1446 emx_tx_purge(sc); 1447 1448 callout_reset(&sc->timer, hz, emx_timer, sc); 1449 1450 emx_deserialize_skipmain(sc); 1451 } 1452 1453 if (reg_icr & E1000_ICR_RXO) 1454 sc->rx_overruns++; 1455 1456 logif(intr_end); 1457 } 1458 1459 static void 1460 emx_intr_mask(void *xsc) 1461 { 1462 struct emx_softc *sc = xsc; 1463 1464 E1000_WRITE_REG(&sc->hw, E1000_IMC, 0xffffffff); 1465 /* 1466 * NOTE: 1467 * ICR.INT_ASSERTED bit will never be set if IMS is 0, 1468 * so don't check it. 1469 */ 1470 emx_intr_body(sc, FALSE); 1471 E1000_WRITE_REG(&sc->hw, E1000_IMS, IMS_ENABLE_MASK); 1472 } 1473 1474 static void 1475 emx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1476 { 1477 struct emx_softc *sc = ifp->if_softc; 1478 1479 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1480 1481 emx_update_link_status(sc); 1482 1483 ifmr->ifm_status = IFM_AVALID; 1484 ifmr->ifm_active = IFM_ETHER; 1485 1486 if (!sc->link_active) { 1487 if (sc->hw.mac.autoneg) 1488 ifmr->ifm_active |= IFM_NONE; 1489 else 1490 ifmr->ifm_active |= sc->media.ifm_media; 1491 return; 1492 } 1493 1494 ifmr->ifm_status |= IFM_ACTIVE; 1495 if (sc->ifm_flowctrl & IFM_ETH_FORCEPAUSE) 1496 ifmr->ifm_active |= sc->ifm_flowctrl; 1497 1498 if (sc->hw.phy.media_type == e1000_media_type_fiber || 1499 sc->hw.phy.media_type == e1000_media_type_internal_serdes) { 1500 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 1501 } else { 1502 switch (sc->link_speed) { 1503 case 10: 1504 ifmr->ifm_active |= IFM_10_T; 1505 break; 1506 case 100: 1507 ifmr->ifm_active |= IFM_100_TX; 1508 break; 1509 1510 case 1000: 1511 ifmr->ifm_active |= IFM_1000_T; 1512 break; 1513 } 1514 if (sc->link_duplex == FULL_DUPLEX) 1515 ifmr->ifm_active |= IFM_FDX; 1516 else 1517 ifmr->ifm_active |= IFM_HDX; 1518 } 1519 if (ifmr->ifm_active & IFM_FDX) 1520 ifmr->ifm_active |= e1000_fc2ifmedia(sc->hw.fc.current_mode); 1521 } 1522 1523 static int 1524 emx_media_change(struct ifnet *ifp) 1525 { 1526 struct emx_softc *sc = ifp->if_softc; 1527 struct ifmedia *ifm = &sc->media; 1528 1529 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1530 1531 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1532 return (EINVAL); 1533 1534 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1535 case IFM_AUTO: 1536 sc->hw.mac.autoneg = EMX_DO_AUTO_NEG; 1537 sc->hw.phy.autoneg_advertised = EMX_AUTONEG_ADV_DEFAULT; 1538 break; 1539 1540 case IFM_1000_SX: 1541 case IFM_1000_T: 1542 sc->hw.mac.autoneg = EMX_DO_AUTO_NEG; 1543 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1544 break; 1545 1546 case IFM_100_TX: 1547 if (IFM_OPTIONS(ifm->ifm_media) & IFM_FDX) { 1548 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1549 } else { 1550 if (IFM_OPTIONS(ifm->ifm_media) & 1551 (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) { 1552 if (bootverbose) { 1553 if_printf(ifp, "Flow control is not " 1554 "allowed for half-duplex\n"); 1555 } 1556 return EINVAL; 1557 } 1558 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1559 } 1560 sc->hw.mac.autoneg = FALSE; 1561 sc->hw.phy.autoneg_advertised = 0; 1562 break; 1563 1564 case IFM_10_T: 1565 if (IFM_OPTIONS(ifm->ifm_media) & IFM_FDX) { 1566 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1567 } else { 1568 if (IFM_OPTIONS(ifm->ifm_media) & 1569 (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) { 1570 if (bootverbose) { 1571 if_printf(ifp, "Flow control is not " 1572 "allowed for half-duplex\n"); 1573 } 1574 return EINVAL; 1575 } 1576 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1577 } 1578 sc->hw.mac.autoneg = FALSE; 1579 sc->hw.phy.autoneg_advertised = 0; 1580 break; 1581 1582 default: 1583 if (bootverbose) { 1584 if_printf(ifp, "Unsupported media type %d\n", 1585 IFM_SUBTYPE(ifm->ifm_media)); 1586 } 1587 return EINVAL; 1588 } 1589 sc->ifm_flowctrl = ifm->ifm_media & IFM_ETH_FCMASK; 1590 1591 if (ifp->if_flags & IFF_RUNNING) 1592 emx_init(sc); 1593 1594 return (0); 1595 } 1596 1597 static int 1598 emx_encap(struct emx_txdata *tdata, struct mbuf **m_headp, 1599 int *segs_used, int *idx) 1600 { 1601 bus_dma_segment_t segs[EMX_MAX_SCATTER]; 1602 bus_dmamap_t map; 1603 struct emx_txbuf *tx_buffer, *tx_buffer_mapped; 1604 struct e1000_tx_desc *ctxd = NULL; 1605 struct mbuf *m_head = *m_headp; 1606 uint32_t txd_upper, txd_lower, cmd = 0; 1607 int maxsegs, nsegs, i, j, first, last = 0, error; 1608 1609 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1610 error = emx_tso_pullup(tdata, m_headp); 1611 if (error) 1612 return error; 1613 m_head = *m_headp; 1614 } 1615 1616 txd_upper = txd_lower = 0; 1617 1618 /* 1619 * Capture the first descriptor index, this descriptor 1620 * will have the index of the EOP which is the only one 1621 * that now gets a DONE bit writeback. 1622 */ 1623 first = tdata->next_avail_tx_desc; 1624 tx_buffer = &tdata->tx_buf[first]; 1625 tx_buffer_mapped = tx_buffer; 1626 map = tx_buffer->map; 1627 1628 maxsegs = tdata->num_tx_desc_avail - EMX_TX_RESERVED; 1629 KASSERT(maxsegs >= tdata->spare_tx_desc, ("not enough spare TX desc")); 1630 if (maxsegs > EMX_MAX_SCATTER) 1631 maxsegs = EMX_MAX_SCATTER; 1632 1633 error = bus_dmamap_load_mbuf_defrag(tdata->txtag, map, m_headp, 1634 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 1635 if (error) { 1636 m_freem(*m_headp); 1637 *m_headp = NULL; 1638 return error; 1639 } 1640 bus_dmamap_sync(tdata->txtag, map, BUS_DMASYNC_PREWRITE); 1641 1642 m_head = *m_headp; 1643 tdata->tx_nsegs += nsegs; 1644 *segs_used += nsegs; 1645 1646 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1647 /* TSO will consume one TX desc */ 1648 i = emx_tso_setup(tdata, m_head, &txd_upper, &txd_lower); 1649 tdata->tx_nsegs += i; 1650 *segs_used += i; 1651 } else if (m_head->m_pkthdr.csum_flags & EMX_CSUM_FEATURES) { 1652 /* TX csum offloading will consume one TX desc */ 1653 i = emx_txcsum(tdata, m_head, &txd_upper, &txd_lower); 1654 tdata->tx_nsegs += i; 1655 *segs_used += i; 1656 } 1657 1658 /* Handle VLAN tag */ 1659 if (m_head->m_flags & M_VLANTAG) { 1660 /* Set the vlan id. */ 1661 txd_upper |= (htole16(m_head->m_pkthdr.ether_vlantag) << 16); 1662 /* Tell hardware to add tag */ 1663 txd_lower |= htole32(E1000_TXD_CMD_VLE); 1664 } 1665 1666 i = tdata->next_avail_tx_desc; 1667 1668 /* Set up our transmit descriptors */ 1669 for (j = 0; j < nsegs; j++) { 1670 tx_buffer = &tdata->tx_buf[i]; 1671 ctxd = &tdata->tx_desc_base[i]; 1672 1673 ctxd->buffer_addr = htole64(segs[j].ds_addr); 1674 ctxd->lower.data = htole32(E1000_TXD_CMD_IFCS | 1675 txd_lower | segs[j].ds_len); 1676 ctxd->upper.data = htole32(txd_upper); 1677 1678 last = i; 1679 if (++i == tdata->num_tx_desc) 1680 i = 0; 1681 } 1682 1683 tdata->next_avail_tx_desc = i; 1684 1685 KKASSERT(tdata->num_tx_desc_avail > nsegs); 1686 tdata->num_tx_desc_avail -= nsegs; 1687 1688 tx_buffer->m_head = m_head; 1689 tx_buffer_mapped->map = tx_buffer->map; 1690 tx_buffer->map = map; 1691 1692 if (tdata->tx_nsegs >= tdata->tx_intr_nsegs) { 1693 tdata->tx_nsegs = 0; 1694 1695 /* 1696 * Report Status (RS) is turned on 1697 * every tx_intr_nsegs descriptors. 1698 */ 1699 cmd = E1000_TXD_CMD_RS; 1700 1701 /* 1702 * Keep track of the descriptor, which will 1703 * be written back by hardware. 1704 */ 1705 tdata->tx_dd[tdata->tx_dd_tail] = last; 1706 EMX_INC_TXDD_IDX(tdata->tx_dd_tail); 1707 KKASSERT(tdata->tx_dd_tail != tdata->tx_dd_head); 1708 } 1709 1710 /* 1711 * Last Descriptor of Packet needs End Of Packet (EOP) 1712 */ 1713 ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | cmd); 1714 1715 /* 1716 * Defer TDT updating, until enough descriptors are setup 1717 */ 1718 *idx = i; 1719 1720 #ifdef EMX_TSS_DEBUG 1721 tdata->tx_pkts++; 1722 #endif 1723 1724 return (0); 1725 } 1726 1727 static void 1728 emx_set_promisc(struct emx_softc *sc) 1729 { 1730 struct ifnet *ifp = &sc->arpcom.ac_if; 1731 uint32_t reg_rctl; 1732 1733 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 1734 1735 if (ifp->if_flags & IFF_PROMISC) { 1736 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 1737 /* Turn this on if you want to see bad packets */ 1738 if (emx_debug_sbp) 1739 reg_rctl |= E1000_RCTL_SBP; 1740 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 1741 } else if (ifp->if_flags & IFF_ALLMULTI) { 1742 reg_rctl |= E1000_RCTL_MPE; 1743 reg_rctl &= ~E1000_RCTL_UPE; 1744 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 1745 } 1746 } 1747 1748 static void 1749 emx_disable_promisc(struct emx_softc *sc) 1750 { 1751 uint32_t reg_rctl; 1752 1753 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 1754 1755 reg_rctl &= ~E1000_RCTL_UPE; 1756 reg_rctl &= ~E1000_RCTL_MPE; 1757 reg_rctl &= ~E1000_RCTL_SBP; 1758 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 1759 } 1760 1761 static void 1762 emx_set_multi(struct emx_softc *sc) 1763 { 1764 struct ifnet *ifp = &sc->arpcom.ac_if; 1765 struct ifmultiaddr *ifma; 1766 uint32_t reg_rctl = 0; 1767 uint8_t *mta; 1768 int mcnt = 0; 1769 1770 mta = sc->mta; 1771 bzero(mta, ETH_ADDR_LEN * EMX_MCAST_ADDR_MAX); 1772 1773 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1774 if (ifma->ifma_addr->sa_family != AF_LINK) 1775 continue; 1776 1777 if (mcnt == EMX_MCAST_ADDR_MAX) 1778 break; 1779 1780 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1781 &mta[mcnt * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 1782 mcnt++; 1783 } 1784 1785 if (mcnt >= EMX_MCAST_ADDR_MAX) { 1786 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 1787 reg_rctl |= E1000_RCTL_MPE; 1788 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 1789 } else { 1790 e1000_update_mc_addr_list(&sc->hw, mta, mcnt); 1791 } 1792 } 1793 1794 /* 1795 * This routine checks for link status and updates statistics. 1796 */ 1797 static void 1798 emx_timer(void *xsc) 1799 { 1800 struct emx_softc *sc = xsc; 1801 struct ifnet *ifp = &sc->arpcom.ac_if; 1802 1803 lwkt_serialize_enter(&sc->main_serialize); 1804 1805 emx_update_link_status(sc); 1806 emx_update_stats(sc); 1807 1808 /* Reset LAA into RAR[0] on 82571 */ 1809 if (e1000_get_laa_state_82571(&sc->hw) == TRUE) 1810 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1811 1812 if (emx_display_debug_stats && (ifp->if_flags & IFF_RUNNING)) 1813 emx_print_hw_stats(sc); 1814 1815 emx_smartspeed(sc); 1816 1817 callout_reset(&sc->timer, hz, emx_timer, sc); 1818 1819 lwkt_serialize_exit(&sc->main_serialize); 1820 } 1821 1822 static void 1823 emx_update_link_status(struct emx_softc *sc) 1824 { 1825 struct e1000_hw *hw = &sc->hw; 1826 struct ifnet *ifp = &sc->arpcom.ac_if; 1827 device_t dev = sc->dev; 1828 uint32_t link_check = 0; 1829 1830 /* Get the cached link value or read phy for real */ 1831 switch (hw->phy.media_type) { 1832 case e1000_media_type_copper: 1833 if (hw->mac.get_link_status) { 1834 /* Do the work to read phy */ 1835 e1000_check_for_link(hw); 1836 link_check = !hw->mac.get_link_status; 1837 if (link_check) /* ESB2 fix */ 1838 e1000_cfg_on_link_up(hw); 1839 } else { 1840 link_check = TRUE; 1841 } 1842 break; 1843 1844 case e1000_media_type_fiber: 1845 e1000_check_for_link(hw); 1846 link_check = E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU; 1847 break; 1848 1849 case e1000_media_type_internal_serdes: 1850 e1000_check_for_link(hw); 1851 link_check = sc->hw.mac.serdes_has_link; 1852 break; 1853 1854 case e1000_media_type_unknown: 1855 default: 1856 break; 1857 } 1858 1859 /* Now check for a transition */ 1860 if (link_check && sc->link_active == 0) { 1861 e1000_get_speed_and_duplex(hw, &sc->link_speed, 1862 &sc->link_duplex); 1863 1864 /* 1865 * Check if we should enable/disable SPEED_MODE bit on 1866 * 82571EB/82572EI 1867 */ 1868 if (sc->link_speed != SPEED_1000 && 1869 (hw->mac.type == e1000_82571 || 1870 hw->mac.type == e1000_82572)) { 1871 int tarc0; 1872 1873 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 1874 tarc0 &= ~EMX_TARC_SPEED_MODE; 1875 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 1876 } 1877 if (bootverbose) { 1878 char flowctrl[IFM_ETH_FC_STRLEN]; 1879 1880 e1000_fc2str(hw->fc.current_mode, flowctrl, 1881 sizeof(flowctrl)); 1882 device_printf(dev, "Link is up %d Mbps %s, " 1883 "Flow control: %s\n", 1884 sc->link_speed, 1885 (sc->link_duplex == FULL_DUPLEX) ? 1886 "Full Duplex" : "Half Duplex", 1887 flowctrl); 1888 } 1889 if (sc->ifm_flowctrl & IFM_ETH_FORCEPAUSE) 1890 e1000_force_flowctrl(hw, sc->ifm_flowctrl); 1891 sc->link_active = 1; 1892 sc->smartspeed = 0; 1893 ifp->if_baudrate = sc->link_speed * 1000000; 1894 ifp->if_link_state = LINK_STATE_UP; 1895 if_link_state_change(ifp); 1896 } else if (!link_check && sc->link_active == 1) { 1897 ifp->if_baudrate = sc->link_speed = 0; 1898 sc->link_duplex = 0; 1899 if (bootverbose) 1900 device_printf(dev, "Link is Down\n"); 1901 sc->link_active = 0; 1902 ifp->if_link_state = LINK_STATE_DOWN; 1903 if_link_state_change(ifp); 1904 } 1905 } 1906 1907 static void 1908 emx_stop(struct emx_softc *sc) 1909 { 1910 struct ifnet *ifp = &sc->arpcom.ac_if; 1911 int i; 1912 1913 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1914 1915 emx_disable_intr(sc); 1916 1917 callout_stop(&sc->timer); 1918 1919 ifp->if_flags &= ~IFF_RUNNING; 1920 for (i = 0; i < sc->tx_ring_cnt; ++i) { 1921 struct emx_txdata *tdata = &sc->tx_data[i]; 1922 1923 ifsq_clr_oactive(tdata->ifsq); 1924 ifsq_watchdog_stop(&tdata->tx_watchdog); 1925 tdata->tx_flags &= ~EMX_TXFLAG_ENABLED; 1926 } 1927 1928 /* 1929 * Disable multiple receive queues. 1930 * 1931 * NOTE: 1932 * We should disable multiple receive queues before 1933 * resetting the hardware. 1934 */ 1935 E1000_WRITE_REG(&sc->hw, E1000_MRQC, 0); 1936 1937 e1000_reset_hw(&sc->hw); 1938 E1000_WRITE_REG(&sc->hw, E1000_WUC, 0); 1939 1940 for (i = 0; i < sc->tx_ring_cnt; ++i) 1941 emx_free_tx_ring(&sc->tx_data[i]); 1942 for (i = 0; i < sc->rx_ring_cnt; ++i) 1943 emx_free_rx_ring(&sc->rx_data[i]); 1944 } 1945 1946 static int 1947 emx_reset(struct emx_softc *sc) 1948 { 1949 device_t dev = sc->dev; 1950 uint16_t rx_buffer_size; 1951 uint32_t pba; 1952 1953 /* Set up smart power down as default off on newer adapters. */ 1954 if (!emx_smart_pwr_down && 1955 (sc->hw.mac.type == e1000_82571 || 1956 sc->hw.mac.type == e1000_82572)) { 1957 uint16_t phy_tmp = 0; 1958 1959 /* Speed up time to link by disabling smart power down. */ 1960 e1000_read_phy_reg(&sc->hw, 1961 IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 1962 phy_tmp &= ~IGP02E1000_PM_SPD; 1963 e1000_write_phy_reg(&sc->hw, 1964 IGP02E1000_PHY_POWER_MGMT, phy_tmp); 1965 } 1966 1967 /* 1968 * Packet Buffer Allocation (PBA) 1969 * Writing PBA sets the receive portion of the buffer 1970 * the remainder is used for the transmit buffer. 1971 */ 1972 switch (sc->hw.mac.type) { 1973 /* Total Packet Buffer on these is 48K */ 1974 case e1000_82571: 1975 case e1000_82572: 1976 case e1000_80003es2lan: 1977 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1978 break; 1979 1980 case e1000_82573: /* 82573: Total Packet Buffer is 32K */ 1981 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 1982 break; 1983 1984 case e1000_82574: 1985 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */ 1986 break; 1987 1988 case e1000_pch_lpt: 1989 pba = E1000_PBA_26K; 1990 break; 1991 1992 default: 1993 /* Devices before 82547 had a Packet Buffer of 64K. */ 1994 if (sc->hw.mac.max_frame_size > 8192) 1995 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 1996 else 1997 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 1998 } 1999 E1000_WRITE_REG(&sc->hw, E1000_PBA, pba); 2000 2001 /* 2002 * These parameters control the automatic generation (Tx) and 2003 * response (Rx) to Ethernet PAUSE frames. 2004 * - High water mark should allow for at least two frames to be 2005 * received after sending an XOFF. 2006 * - Low water mark works best when it is very near the high water mark. 2007 * This allows the receiver to restart by sending XON when it has 2008 * drained a bit. Here we use an arbitary value of 1500 which will 2009 * restart after one full frame is pulled from the buffer. There 2010 * could be several smaller frames in the buffer and if so they will 2011 * not trigger the XON until their total number reduces the buffer 2012 * by 1500. 2013 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 2014 */ 2015 rx_buffer_size = (E1000_READ_REG(&sc->hw, E1000_PBA) & 0xffff) << 10; 2016 2017 sc->hw.fc.high_water = rx_buffer_size - 2018 roundup2(sc->hw.mac.max_frame_size, 1024); 2019 sc->hw.fc.low_water = sc->hw.fc.high_water - 1500; 2020 2021 sc->hw.fc.pause_time = EMX_FC_PAUSE_TIME; 2022 sc->hw.fc.send_xon = TRUE; 2023 sc->hw.fc.requested_mode = e1000_ifmedia2fc(sc->ifm_flowctrl); 2024 2025 /* 2026 * Device specific overrides/settings 2027 */ 2028 if (sc->hw.mac.type == e1000_pch_lpt) { 2029 sc->hw.fc.high_water = 0x5C20; 2030 sc->hw.fc.low_water = 0x5048; 2031 sc->hw.fc.pause_time = 0x0650; 2032 sc->hw.fc.refresh_time = 0x0400; 2033 /* Jumbos need adjusted PBA */ 2034 if (sc->arpcom.ac_if.if_mtu > ETHERMTU) 2035 E1000_WRITE_REG(&sc->hw, E1000_PBA, 12); 2036 else 2037 E1000_WRITE_REG(&sc->hw, E1000_PBA, 26); 2038 } else if (sc->hw.mac.type == e1000_80003es2lan) { 2039 sc->hw.fc.pause_time = 0xFFFF; 2040 } 2041 2042 /* Issue a global reset */ 2043 e1000_reset_hw(&sc->hw); 2044 E1000_WRITE_REG(&sc->hw, E1000_WUC, 0); 2045 emx_disable_aspm(sc); 2046 2047 if (e1000_init_hw(&sc->hw) < 0) { 2048 device_printf(dev, "Hardware Initialization Failed\n"); 2049 return (EIO); 2050 } 2051 2052 E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN); 2053 e1000_get_phy_info(&sc->hw); 2054 e1000_check_for_link(&sc->hw); 2055 2056 return (0); 2057 } 2058 2059 static void 2060 emx_setup_ifp(struct emx_softc *sc) 2061 { 2062 struct ifnet *ifp = &sc->arpcom.ac_if; 2063 int i; 2064 2065 if_initname(ifp, device_get_name(sc->dev), 2066 device_get_unit(sc->dev)); 2067 ifp->if_softc = sc; 2068 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2069 ifp->if_init = emx_init; 2070 ifp->if_ioctl = emx_ioctl; 2071 ifp->if_start = emx_start; 2072 #ifdef IFPOLL_ENABLE 2073 ifp->if_npoll = emx_npoll; 2074 #endif 2075 ifp->if_serialize = emx_serialize; 2076 ifp->if_deserialize = emx_deserialize; 2077 ifp->if_tryserialize = emx_tryserialize; 2078 #ifdef INVARIANTS 2079 ifp->if_serialize_assert = emx_serialize_assert; 2080 #endif 2081 2082 ifp->if_nmbclusters = sc->rx_ring_cnt * sc->rx_data[0].num_rx_desc; 2083 2084 ifq_set_maxlen(&ifp->if_snd, sc->tx_data[0].num_tx_desc - 1); 2085 ifq_set_ready(&ifp->if_snd); 2086 ifq_set_subq_cnt(&ifp->if_snd, sc->tx_ring_cnt); 2087 2088 ifp->if_mapsubq = ifq_mapsubq_mask; 2089 ifq_set_subq_mask(&ifp->if_snd, 0); 2090 2091 ether_ifattach(ifp, sc->hw.mac.addr, NULL); 2092 2093 ifp->if_capabilities = IFCAP_HWCSUM | 2094 IFCAP_VLAN_HWTAGGING | 2095 IFCAP_VLAN_MTU | 2096 IFCAP_TSO; 2097 if (sc->rx_ring_cnt > 1) 2098 ifp->if_capabilities |= IFCAP_RSS; 2099 ifp->if_capenable = ifp->if_capabilities; 2100 ifp->if_hwassist = EMX_CSUM_FEATURES | CSUM_TSO; 2101 2102 /* 2103 * Tell the upper layer(s) we support long frames. 2104 */ 2105 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2106 2107 for (i = 0; i < sc->tx_ring_cnt; ++i) { 2108 struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i); 2109 struct emx_txdata *tdata = &sc->tx_data[i]; 2110 2111 ifsq_set_cpuid(ifsq, rman_get_cpuid(sc->intr_res)); 2112 ifsq_set_priv(ifsq, tdata); 2113 ifsq_set_hw_serialize(ifsq, &tdata->tx_serialize); 2114 tdata->ifsq = ifsq; 2115 2116 ifsq_watchdog_init(&tdata->tx_watchdog, ifsq, emx_watchdog); 2117 } 2118 2119 /* 2120 * Specify the media types supported by this sc and register 2121 * callbacks to update media and link information 2122 */ 2123 if (sc->hw.phy.media_type == e1000_media_type_fiber || 2124 sc->hw.phy.media_type == e1000_media_type_internal_serdes) { 2125 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 2126 0, NULL); 2127 } else { 2128 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 2129 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 2130 0, NULL); 2131 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 2132 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 2133 0, NULL); 2134 if (sc->hw.phy.type != e1000_phy_ife) { 2135 ifmedia_add(&sc->media, 2136 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2137 } 2138 } 2139 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2140 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO | sc->ifm_flowctrl); 2141 } 2142 2143 /* 2144 * Workaround for SmartSpeed on 82541 and 82547 controllers 2145 */ 2146 static void 2147 emx_smartspeed(struct emx_softc *sc) 2148 { 2149 uint16_t phy_tmp; 2150 2151 if (sc->link_active || sc->hw.phy.type != e1000_phy_igp || 2152 sc->hw.mac.autoneg == 0 || 2153 (sc->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) 2154 return; 2155 2156 if (sc->smartspeed == 0) { 2157 /* 2158 * If Master/Slave config fault is asserted twice, 2159 * we assume back-to-back 2160 */ 2161 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 2162 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) 2163 return; 2164 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 2165 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { 2166 e1000_read_phy_reg(&sc->hw, 2167 PHY_1000T_CTRL, &phy_tmp); 2168 if (phy_tmp & CR_1000T_MS_ENABLE) { 2169 phy_tmp &= ~CR_1000T_MS_ENABLE; 2170 e1000_write_phy_reg(&sc->hw, 2171 PHY_1000T_CTRL, phy_tmp); 2172 sc->smartspeed++; 2173 if (sc->hw.mac.autoneg && 2174 !e1000_phy_setup_autoneg(&sc->hw) && 2175 !e1000_read_phy_reg(&sc->hw, 2176 PHY_CONTROL, &phy_tmp)) { 2177 phy_tmp |= MII_CR_AUTO_NEG_EN | 2178 MII_CR_RESTART_AUTO_NEG; 2179 e1000_write_phy_reg(&sc->hw, 2180 PHY_CONTROL, phy_tmp); 2181 } 2182 } 2183 } 2184 return; 2185 } else if (sc->smartspeed == EMX_SMARTSPEED_DOWNSHIFT) { 2186 /* If still no link, perhaps using 2/3 pair cable */ 2187 e1000_read_phy_reg(&sc->hw, PHY_1000T_CTRL, &phy_tmp); 2188 phy_tmp |= CR_1000T_MS_ENABLE; 2189 e1000_write_phy_reg(&sc->hw, PHY_1000T_CTRL, phy_tmp); 2190 if (sc->hw.mac.autoneg && 2191 !e1000_phy_setup_autoneg(&sc->hw) && 2192 !e1000_read_phy_reg(&sc->hw, PHY_CONTROL, &phy_tmp)) { 2193 phy_tmp |= MII_CR_AUTO_NEG_EN | MII_CR_RESTART_AUTO_NEG; 2194 e1000_write_phy_reg(&sc->hw, PHY_CONTROL, phy_tmp); 2195 } 2196 } 2197 2198 /* Restart process after EMX_SMARTSPEED_MAX iterations */ 2199 if (sc->smartspeed++ == EMX_SMARTSPEED_MAX) 2200 sc->smartspeed = 0; 2201 } 2202 2203 static int 2204 emx_create_tx_ring(struct emx_txdata *tdata) 2205 { 2206 device_t dev = tdata->sc->dev; 2207 struct emx_txbuf *tx_buffer; 2208 int error, i, tsize, ntxd; 2209 2210 /* 2211 * Validate number of transmit descriptors. It must not exceed 2212 * hardware maximum, and must be multiple of E1000_DBA_ALIGN. 2213 */ 2214 ntxd = device_getenv_int(dev, "txd", emx_txd); 2215 if ((ntxd * sizeof(struct e1000_tx_desc)) % EMX_DBA_ALIGN != 0 || 2216 ntxd > EMX_MAX_TXD || ntxd < EMX_MIN_TXD) { 2217 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 2218 EMX_DEFAULT_TXD, ntxd); 2219 tdata->num_tx_desc = EMX_DEFAULT_TXD; 2220 } else { 2221 tdata->num_tx_desc = ntxd; 2222 } 2223 2224 /* 2225 * Allocate Transmit Descriptor ring 2226 */ 2227 tsize = roundup2(tdata->num_tx_desc * sizeof(struct e1000_tx_desc), 2228 EMX_DBA_ALIGN); 2229 tdata->tx_desc_base = bus_dmamem_coherent_any(tdata->sc->parent_dtag, 2230 EMX_DBA_ALIGN, tsize, BUS_DMA_WAITOK, 2231 &tdata->tx_desc_dtag, &tdata->tx_desc_dmap, 2232 &tdata->tx_desc_paddr); 2233 if (tdata->tx_desc_base == NULL) { 2234 device_printf(dev, "Unable to allocate tx_desc memory\n"); 2235 return ENOMEM; 2236 } 2237 2238 tsize = __VM_CACHELINE_ALIGN( 2239 sizeof(struct emx_txbuf) * tdata->num_tx_desc); 2240 tdata->tx_buf = kmalloc_cachealign(tsize, M_DEVBUF, M_WAITOK | M_ZERO); 2241 2242 /* 2243 * Create DMA tags for tx buffers 2244 */ 2245 error = bus_dma_tag_create(tdata->sc->parent_dtag, /* parent */ 2246 1, 0, /* alignment, bounds */ 2247 BUS_SPACE_MAXADDR, /* lowaddr */ 2248 BUS_SPACE_MAXADDR, /* highaddr */ 2249 NULL, NULL, /* filter, filterarg */ 2250 EMX_TSO_SIZE, /* maxsize */ 2251 EMX_MAX_SCATTER, /* nsegments */ 2252 EMX_MAX_SEGSIZE, /* maxsegsize */ 2253 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | 2254 BUS_DMA_ONEBPAGE, /* flags */ 2255 &tdata->txtag); 2256 if (error) { 2257 device_printf(dev, "Unable to allocate TX DMA tag\n"); 2258 kfree(tdata->tx_buf, M_DEVBUF); 2259 tdata->tx_buf = NULL; 2260 return error; 2261 } 2262 2263 /* 2264 * Create DMA maps for tx buffers 2265 */ 2266 for (i = 0; i < tdata->num_tx_desc; i++) { 2267 tx_buffer = &tdata->tx_buf[i]; 2268 2269 error = bus_dmamap_create(tdata->txtag, 2270 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 2271 &tx_buffer->map); 2272 if (error) { 2273 device_printf(dev, "Unable to create TX DMA map\n"); 2274 emx_destroy_tx_ring(tdata, i); 2275 return error; 2276 } 2277 } 2278 2279 /* 2280 * Setup TX parameters 2281 */ 2282 tdata->spare_tx_desc = EMX_TX_SPARE; 2283 tdata->tx_wreg_nsegs = EMX_DEFAULT_TXWREG; 2284 2285 /* 2286 * Keep following relationship between spare_tx_desc, oact_tx_desc 2287 * and tx_intr_nsegs: 2288 * (spare_tx_desc + EMX_TX_RESERVED) <= 2289 * oact_tx_desc <= EMX_TX_OACTIVE_MAX <= tx_intr_nsegs 2290 */ 2291 tdata->oact_tx_desc = tdata->num_tx_desc / 8; 2292 if (tdata->oact_tx_desc > EMX_TX_OACTIVE_MAX) 2293 tdata->oact_tx_desc = EMX_TX_OACTIVE_MAX; 2294 if (tdata->oact_tx_desc < tdata->spare_tx_desc + EMX_TX_RESERVED) 2295 tdata->oact_tx_desc = tdata->spare_tx_desc + EMX_TX_RESERVED; 2296 2297 tdata->tx_intr_nsegs = tdata->num_tx_desc / 16; 2298 if (tdata->tx_intr_nsegs < tdata->oact_tx_desc) 2299 tdata->tx_intr_nsegs = tdata->oact_tx_desc; 2300 2301 /* 2302 * Pullup extra 4bytes into the first data segment for TSO, see: 2303 * 82571/82572 specification update errata #7 2304 * 2305 * Same applies to I217 (and maybe I218). 2306 * 2307 * NOTE: 2308 * 4bytes instead of 2bytes, which are mentioned in the errata, 2309 * are pulled; mainly to keep rest of the data properly aligned. 2310 */ 2311 if (tdata->sc->hw.mac.type == e1000_82571 || 2312 tdata->sc->hw.mac.type == e1000_82572 || 2313 tdata->sc->hw.mac.type == e1000_pch_lpt) 2314 tdata->tx_flags |= EMX_TXFLAG_TSO_PULLEX; 2315 2316 return (0); 2317 } 2318 2319 static void 2320 emx_init_tx_ring(struct emx_txdata *tdata) 2321 { 2322 /* Clear the old ring contents */ 2323 bzero(tdata->tx_desc_base, 2324 sizeof(struct e1000_tx_desc) * tdata->num_tx_desc); 2325 2326 /* Reset state */ 2327 tdata->next_avail_tx_desc = 0; 2328 tdata->next_tx_to_clean = 0; 2329 tdata->num_tx_desc_avail = tdata->num_tx_desc; 2330 2331 tdata->tx_flags |= EMX_TXFLAG_ENABLED; 2332 if (tdata->sc->tx_ring_inuse > 1) { 2333 tdata->tx_flags |= EMX_TXFLAG_FORCECTX; 2334 if (bootverbose) { 2335 if_printf(&tdata->sc->arpcom.ac_if, 2336 "TX %d force ctx setup\n", tdata->idx); 2337 } 2338 } 2339 } 2340 2341 static void 2342 emx_init_tx_unit(struct emx_softc *sc) 2343 { 2344 uint32_t tctl, tarc, tipg = 0, txdctl; 2345 int i; 2346 2347 for (i = 0; i < sc->tx_ring_inuse; ++i) { 2348 struct emx_txdata *tdata = &sc->tx_data[i]; 2349 uint64_t bus_addr; 2350 2351 /* Setup the Base and Length of the Tx Descriptor Ring */ 2352 bus_addr = tdata->tx_desc_paddr; 2353 E1000_WRITE_REG(&sc->hw, E1000_TDLEN(i), 2354 tdata->num_tx_desc * sizeof(struct e1000_tx_desc)); 2355 E1000_WRITE_REG(&sc->hw, E1000_TDBAH(i), 2356 (uint32_t)(bus_addr >> 32)); 2357 E1000_WRITE_REG(&sc->hw, E1000_TDBAL(i), 2358 (uint32_t)bus_addr); 2359 /* Setup the HW Tx Head and Tail descriptor pointers */ 2360 E1000_WRITE_REG(&sc->hw, E1000_TDT(i), 0); 2361 E1000_WRITE_REG(&sc->hw, E1000_TDH(i), 0); 2362 } 2363 2364 /* Set the default values for the Tx Inter Packet Gap timer */ 2365 switch (sc->hw.mac.type) { 2366 case e1000_80003es2lan: 2367 tipg = DEFAULT_82543_TIPG_IPGR1; 2368 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 2369 E1000_TIPG_IPGR2_SHIFT; 2370 break; 2371 2372 default: 2373 if (sc->hw.phy.media_type == e1000_media_type_fiber || 2374 sc->hw.phy.media_type == e1000_media_type_internal_serdes) 2375 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 2376 else 2377 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 2378 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 2379 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 2380 break; 2381 } 2382 2383 E1000_WRITE_REG(&sc->hw, E1000_TIPG, tipg); 2384 2385 /* NOTE: 0 is not allowed for TIDV */ 2386 E1000_WRITE_REG(&sc->hw, E1000_TIDV, 1); 2387 E1000_WRITE_REG(&sc->hw, E1000_TADV, 0); 2388 2389 /* 2390 * Errata workaround (obtained from Linux). This is necessary 2391 * to make multiple TX queues work on 82574. 2392 * XXX can't find it in any published errata though. 2393 */ 2394 txdctl = E1000_READ_REG(&sc->hw, E1000_TXDCTL(0)); 2395 E1000_WRITE_REG(&sc->hw, E1000_TXDCTL(1), txdctl); 2396 2397 if (sc->hw.mac.type == e1000_82571 || 2398 sc->hw.mac.type == e1000_82572) { 2399 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(0)); 2400 tarc |= EMX_TARC_SPEED_MODE; 2401 E1000_WRITE_REG(&sc->hw, E1000_TARC(0), tarc); 2402 } else if (sc->hw.mac.type == e1000_80003es2lan) { 2403 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(0)); 2404 tarc |= 1; 2405 E1000_WRITE_REG(&sc->hw, E1000_TARC(0), tarc); 2406 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(1)); 2407 tarc |= 1; 2408 E1000_WRITE_REG(&sc->hw, E1000_TARC(1), tarc); 2409 } 2410 2411 /* Program the Transmit Control Register */ 2412 tctl = E1000_READ_REG(&sc->hw, E1000_TCTL); 2413 tctl &= ~E1000_TCTL_CT; 2414 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 2415 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2416 tctl |= E1000_TCTL_MULR; 2417 2418 /* This write will effectively turn on the transmit unit. */ 2419 E1000_WRITE_REG(&sc->hw, E1000_TCTL, tctl); 2420 2421 if (sc->hw.mac.type == e1000_82571 || 2422 sc->hw.mac.type == e1000_82572 || 2423 sc->hw.mac.type == e1000_80003es2lan) { 2424 /* Bit 28 of TARC1 must be cleared when MULR is enabled */ 2425 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(1)); 2426 tarc &= ~(1 << 28); 2427 E1000_WRITE_REG(&sc->hw, E1000_TARC(1), tarc); 2428 } 2429 2430 if (sc->tx_ring_inuse > 1) { 2431 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(0)); 2432 tarc &= ~EMX_TARC_COUNT_MASK; 2433 tarc |= 1; 2434 E1000_WRITE_REG(&sc->hw, E1000_TARC(0), tarc); 2435 2436 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(1)); 2437 tarc &= ~EMX_TARC_COUNT_MASK; 2438 tarc |= 1; 2439 E1000_WRITE_REG(&sc->hw, E1000_TARC(1), tarc); 2440 } 2441 } 2442 2443 static void 2444 emx_destroy_tx_ring(struct emx_txdata *tdata, int ndesc) 2445 { 2446 struct emx_txbuf *tx_buffer; 2447 int i; 2448 2449 /* Free Transmit Descriptor ring */ 2450 if (tdata->tx_desc_base) { 2451 bus_dmamap_unload(tdata->tx_desc_dtag, tdata->tx_desc_dmap); 2452 bus_dmamem_free(tdata->tx_desc_dtag, tdata->tx_desc_base, 2453 tdata->tx_desc_dmap); 2454 bus_dma_tag_destroy(tdata->tx_desc_dtag); 2455 2456 tdata->tx_desc_base = NULL; 2457 } 2458 2459 if (tdata->tx_buf == NULL) 2460 return; 2461 2462 for (i = 0; i < ndesc; i++) { 2463 tx_buffer = &tdata->tx_buf[i]; 2464 2465 KKASSERT(tx_buffer->m_head == NULL); 2466 bus_dmamap_destroy(tdata->txtag, tx_buffer->map); 2467 } 2468 bus_dma_tag_destroy(tdata->txtag); 2469 2470 kfree(tdata->tx_buf, M_DEVBUF); 2471 tdata->tx_buf = NULL; 2472 } 2473 2474 /* 2475 * The offload context needs to be set when we transfer the first 2476 * packet of a particular protocol (TCP/UDP). This routine has been 2477 * enhanced to deal with inserted VLAN headers. 2478 * 2479 * If the new packet's ether header length, ip header length and 2480 * csum offloading type are same as the previous packet, we should 2481 * avoid allocating a new csum context descriptor; mainly to take 2482 * advantage of the pipeline effect of the TX data read request. 2483 * 2484 * This function returns number of TX descrptors allocated for 2485 * csum context. 2486 */ 2487 static int 2488 emx_txcsum(struct emx_txdata *tdata, struct mbuf *mp, 2489 uint32_t *txd_upper, uint32_t *txd_lower) 2490 { 2491 struct e1000_context_desc *TXD; 2492 int curr_txd, ehdrlen, csum_flags; 2493 uint32_t cmd, hdr_len, ip_hlen; 2494 2495 csum_flags = mp->m_pkthdr.csum_flags & EMX_CSUM_FEATURES; 2496 ip_hlen = mp->m_pkthdr.csum_iphlen; 2497 ehdrlen = mp->m_pkthdr.csum_lhlen; 2498 2499 if ((tdata->tx_flags & EMX_TXFLAG_FORCECTX) == 0 && 2500 tdata->csum_lhlen == ehdrlen && tdata->csum_iphlen == ip_hlen && 2501 tdata->csum_flags == csum_flags) { 2502 /* 2503 * Same csum offload context as the previous packets; 2504 * just return. 2505 */ 2506 *txd_upper = tdata->csum_txd_upper; 2507 *txd_lower = tdata->csum_txd_lower; 2508 return 0; 2509 } 2510 2511 /* 2512 * Setup a new csum offload context. 2513 */ 2514 2515 curr_txd = tdata->next_avail_tx_desc; 2516 TXD = (struct e1000_context_desc *)&tdata->tx_desc_base[curr_txd]; 2517 2518 cmd = 0; 2519 2520 /* Setup of IP header checksum. */ 2521 if (csum_flags & CSUM_IP) { 2522 /* 2523 * Start offset for header checksum calculation. 2524 * End offset for header checksum calculation. 2525 * Offset of place to put the checksum. 2526 */ 2527 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 2528 TXD->lower_setup.ip_fields.ipcse = 2529 htole16(ehdrlen + ip_hlen - 1); 2530 TXD->lower_setup.ip_fields.ipcso = 2531 ehdrlen + offsetof(struct ip, ip_sum); 2532 cmd |= E1000_TXD_CMD_IP; 2533 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 2534 } 2535 hdr_len = ehdrlen + ip_hlen; 2536 2537 if (csum_flags & CSUM_TCP) { 2538 /* 2539 * Start offset for payload checksum calculation. 2540 * End offset for payload checksum calculation. 2541 * Offset of place to put the checksum. 2542 */ 2543 TXD->upper_setup.tcp_fields.tucss = hdr_len; 2544 TXD->upper_setup.tcp_fields.tucse = htole16(0); 2545 TXD->upper_setup.tcp_fields.tucso = 2546 hdr_len + offsetof(struct tcphdr, th_sum); 2547 cmd |= E1000_TXD_CMD_TCP; 2548 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 2549 } else if (csum_flags & CSUM_UDP) { 2550 /* 2551 * Start offset for header checksum calculation. 2552 * End offset for header checksum calculation. 2553 * Offset of place to put the checksum. 2554 */ 2555 TXD->upper_setup.tcp_fields.tucss = hdr_len; 2556 TXD->upper_setup.tcp_fields.tucse = htole16(0); 2557 TXD->upper_setup.tcp_fields.tucso = 2558 hdr_len + offsetof(struct udphdr, uh_sum); 2559 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 2560 } 2561 2562 *txd_lower = E1000_TXD_CMD_DEXT | /* Extended descr type */ 2563 E1000_TXD_DTYP_D; /* Data descr */ 2564 2565 /* Save the information for this csum offloading context */ 2566 tdata->csum_lhlen = ehdrlen; 2567 tdata->csum_iphlen = ip_hlen; 2568 tdata->csum_flags = csum_flags; 2569 tdata->csum_txd_upper = *txd_upper; 2570 tdata->csum_txd_lower = *txd_lower; 2571 2572 TXD->tcp_seg_setup.data = htole32(0); 2573 TXD->cmd_and_length = 2574 htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); 2575 2576 if (++curr_txd == tdata->num_tx_desc) 2577 curr_txd = 0; 2578 2579 KKASSERT(tdata->num_tx_desc_avail > 0); 2580 tdata->num_tx_desc_avail--; 2581 2582 tdata->next_avail_tx_desc = curr_txd; 2583 return 1; 2584 } 2585 2586 static void 2587 emx_txeof(struct emx_txdata *tdata) 2588 { 2589 struct emx_txbuf *tx_buffer; 2590 int first, num_avail; 2591 2592 if (tdata->tx_dd_head == tdata->tx_dd_tail) 2593 return; 2594 2595 if (tdata->num_tx_desc_avail == tdata->num_tx_desc) 2596 return; 2597 2598 num_avail = tdata->num_tx_desc_avail; 2599 first = tdata->next_tx_to_clean; 2600 2601 while (tdata->tx_dd_head != tdata->tx_dd_tail) { 2602 int dd_idx = tdata->tx_dd[tdata->tx_dd_head]; 2603 struct e1000_tx_desc *tx_desc; 2604 2605 tx_desc = &tdata->tx_desc_base[dd_idx]; 2606 if (tx_desc->upper.fields.status & E1000_TXD_STAT_DD) { 2607 EMX_INC_TXDD_IDX(tdata->tx_dd_head); 2608 2609 if (++dd_idx == tdata->num_tx_desc) 2610 dd_idx = 0; 2611 2612 while (first != dd_idx) { 2613 logif(pkt_txclean); 2614 2615 num_avail++; 2616 2617 tx_buffer = &tdata->tx_buf[first]; 2618 if (tx_buffer->m_head) { 2619 bus_dmamap_unload(tdata->txtag, 2620 tx_buffer->map); 2621 m_freem(tx_buffer->m_head); 2622 tx_buffer->m_head = NULL; 2623 } 2624 2625 if (++first == tdata->num_tx_desc) 2626 first = 0; 2627 } 2628 } else { 2629 break; 2630 } 2631 } 2632 tdata->next_tx_to_clean = first; 2633 tdata->num_tx_desc_avail = num_avail; 2634 2635 if (tdata->tx_dd_head == tdata->tx_dd_tail) { 2636 tdata->tx_dd_head = 0; 2637 tdata->tx_dd_tail = 0; 2638 } 2639 2640 if (!EMX_IS_OACTIVE(tdata)) { 2641 ifsq_clr_oactive(tdata->ifsq); 2642 2643 /* All clean, turn off the timer */ 2644 if (tdata->num_tx_desc_avail == tdata->num_tx_desc) 2645 tdata->tx_watchdog.wd_timer = 0; 2646 } 2647 } 2648 2649 static void 2650 emx_tx_collect(struct emx_txdata *tdata) 2651 { 2652 struct emx_txbuf *tx_buffer; 2653 int tdh, first, num_avail, dd_idx = -1; 2654 2655 if (tdata->num_tx_desc_avail == tdata->num_tx_desc) 2656 return; 2657 2658 tdh = E1000_READ_REG(&tdata->sc->hw, E1000_TDH(tdata->idx)); 2659 if (tdh == tdata->next_tx_to_clean) 2660 return; 2661 2662 if (tdata->tx_dd_head != tdata->tx_dd_tail) 2663 dd_idx = tdata->tx_dd[tdata->tx_dd_head]; 2664 2665 num_avail = tdata->num_tx_desc_avail; 2666 first = tdata->next_tx_to_clean; 2667 2668 while (first != tdh) { 2669 logif(pkt_txclean); 2670 2671 num_avail++; 2672 2673 tx_buffer = &tdata->tx_buf[first]; 2674 if (tx_buffer->m_head) { 2675 bus_dmamap_unload(tdata->txtag, 2676 tx_buffer->map); 2677 m_freem(tx_buffer->m_head); 2678 tx_buffer->m_head = NULL; 2679 } 2680 2681 if (first == dd_idx) { 2682 EMX_INC_TXDD_IDX(tdata->tx_dd_head); 2683 if (tdata->tx_dd_head == tdata->tx_dd_tail) { 2684 tdata->tx_dd_head = 0; 2685 tdata->tx_dd_tail = 0; 2686 dd_idx = -1; 2687 } else { 2688 dd_idx = tdata->tx_dd[tdata->tx_dd_head]; 2689 } 2690 } 2691 2692 if (++first == tdata->num_tx_desc) 2693 first = 0; 2694 } 2695 tdata->next_tx_to_clean = first; 2696 tdata->num_tx_desc_avail = num_avail; 2697 2698 if (!EMX_IS_OACTIVE(tdata)) { 2699 ifsq_clr_oactive(tdata->ifsq); 2700 2701 /* All clean, turn off the timer */ 2702 if (tdata->num_tx_desc_avail == tdata->num_tx_desc) 2703 tdata->tx_watchdog.wd_timer = 0; 2704 } 2705 } 2706 2707 /* 2708 * When Link is lost sometimes there is work still in the TX ring 2709 * which will result in a watchdog, rather than allow that do an 2710 * attempted cleanup and then reinit here. Note that this has been 2711 * seens mostly with fiber adapters. 2712 */ 2713 static void 2714 emx_tx_purge(struct emx_softc *sc) 2715 { 2716 int i; 2717 2718 if (sc->link_active) 2719 return; 2720 2721 for (i = 0; i < sc->tx_ring_inuse; ++i) { 2722 struct emx_txdata *tdata = &sc->tx_data[i]; 2723 2724 if (tdata->tx_watchdog.wd_timer) { 2725 emx_tx_collect(tdata); 2726 if (tdata->tx_watchdog.wd_timer) { 2727 if_printf(&sc->arpcom.ac_if, 2728 "Link lost, TX pending, reinit\n"); 2729 emx_init(sc); 2730 return; 2731 } 2732 } 2733 } 2734 } 2735 2736 static int 2737 emx_newbuf(struct emx_rxdata *rdata, int i, int init) 2738 { 2739 struct mbuf *m; 2740 bus_dma_segment_t seg; 2741 bus_dmamap_t map; 2742 struct emx_rxbuf *rx_buffer; 2743 int error, nseg; 2744 2745 m = m_getcl(init ? M_WAITOK : M_NOWAIT, MT_DATA, M_PKTHDR); 2746 if (m == NULL) { 2747 if (init) { 2748 if_printf(&rdata->sc->arpcom.ac_if, 2749 "Unable to allocate RX mbuf\n"); 2750 } 2751 return (ENOBUFS); 2752 } 2753 m->m_len = m->m_pkthdr.len = MCLBYTES; 2754 2755 if (rdata->sc->hw.mac.max_frame_size <= MCLBYTES - ETHER_ALIGN) 2756 m_adj(m, ETHER_ALIGN); 2757 2758 error = bus_dmamap_load_mbuf_segment(rdata->rxtag, 2759 rdata->rx_sparemap, m, 2760 &seg, 1, &nseg, BUS_DMA_NOWAIT); 2761 if (error) { 2762 m_freem(m); 2763 if (init) { 2764 if_printf(&rdata->sc->arpcom.ac_if, 2765 "Unable to load RX mbuf\n"); 2766 } 2767 return (error); 2768 } 2769 2770 rx_buffer = &rdata->rx_buf[i]; 2771 if (rx_buffer->m_head != NULL) 2772 bus_dmamap_unload(rdata->rxtag, rx_buffer->map); 2773 2774 map = rx_buffer->map; 2775 rx_buffer->map = rdata->rx_sparemap; 2776 rdata->rx_sparemap = map; 2777 2778 rx_buffer->m_head = m; 2779 rx_buffer->paddr = seg.ds_addr; 2780 2781 emx_setup_rxdesc(&rdata->rx_desc[i], rx_buffer); 2782 return (0); 2783 } 2784 2785 static int 2786 emx_create_rx_ring(struct emx_rxdata *rdata) 2787 { 2788 device_t dev = rdata->sc->dev; 2789 struct emx_rxbuf *rx_buffer; 2790 int i, error, rsize, nrxd; 2791 2792 /* 2793 * Validate number of receive descriptors. It must not exceed 2794 * hardware maximum, and must be multiple of E1000_DBA_ALIGN. 2795 */ 2796 nrxd = device_getenv_int(dev, "rxd", emx_rxd); 2797 if ((nrxd * sizeof(emx_rxdesc_t)) % EMX_DBA_ALIGN != 0 || 2798 nrxd > EMX_MAX_RXD || nrxd < EMX_MIN_RXD) { 2799 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 2800 EMX_DEFAULT_RXD, nrxd); 2801 rdata->num_rx_desc = EMX_DEFAULT_RXD; 2802 } else { 2803 rdata->num_rx_desc = nrxd; 2804 } 2805 2806 /* 2807 * Allocate Receive Descriptor ring 2808 */ 2809 rsize = roundup2(rdata->num_rx_desc * sizeof(emx_rxdesc_t), 2810 EMX_DBA_ALIGN); 2811 rdata->rx_desc = bus_dmamem_coherent_any(rdata->sc->parent_dtag, 2812 EMX_DBA_ALIGN, rsize, BUS_DMA_WAITOK, 2813 &rdata->rx_desc_dtag, &rdata->rx_desc_dmap, 2814 &rdata->rx_desc_paddr); 2815 if (rdata->rx_desc == NULL) { 2816 device_printf(dev, "Unable to allocate rx_desc memory\n"); 2817 return ENOMEM; 2818 } 2819 2820 rsize = __VM_CACHELINE_ALIGN( 2821 sizeof(struct emx_rxbuf) * rdata->num_rx_desc); 2822 rdata->rx_buf = kmalloc_cachealign(rsize, M_DEVBUF, M_WAITOK | M_ZERO); 2823 2824 /* 2825 * Create DMA tag for rx buffers 2826 */ 2827 error = bus_dma_tag_create(rdata->sc->parent_dtag, /* parent */ 2828 1, 0, /* alignment, bounds */ 2829 BUS_SPACE_MAXADDR, /* lowaddr */ 2830 BUS_SPACE_MAXADDR, /* highaddr */ 2831 NULL, NULL, /* filter, filterarg */ 2832 MCLBYTES, /* maxsize */ 2833 1, /* nsegments */ 2834 MCLBYTES, /* maxsegsize */ 2835 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, /* flags */ 2836 &rdata->rxtag); 2837 if (error) { 2838 device_printf(dev, "Unable to allocate RX DMA tag\n"); 2839 kfree(rdata->rx_buf, M_DEVBUF); 2840 rdata->rx_buf = NULL; 2841 return error; 2842 } 2843 2844 /* 2845 * Create spare DMA map for rx buffers 2846 */ 2847 error = bus_dmamap_create(rdata->rxtag, BUS_DMA_WAITOK, 2848 &rdata->rx_sparemap); 2849 if (error) { 2850 device_printf(dev, "Unable to create spare RX DMA map\n"); 2851 bus_dma_tag_destroy(rdata->rxtag); 2852 kfree(rdata->rx_buf, M_DEVBUF); 2853 rdata->rx_buf = NULL; 2854 return error; 2855 } 2856 2857 /* 2858 * Create DMA maps for rx buffers 2859 */ 2860 for (i = 0; i < rdata->num_rx_desc; i++) { 2861 rx_buffer = &rdata->rx_buf[i]; 2862 2863 error = bus_dmamap_create(rdata->rxtag, BUS_DMA_WAITOK, 2864 &rx_buffer->map); 2865 if (error) { 2866 device_printf(dev, "Unable to create RX DMA map\n"); 2867 emx_destroy_rx_ring(rdata, i); 2868 return error; 2869 } 2870 } 2871 return (0); 2872 } 2873 2874 static void 2875 emx_free_rx_ring(struct emx_rxdata *rdata) 2876 { 2877 int i; 2878 2879 for (i = 0; i < rdata->num_rx_desc; i++) { 2880 struct emx_rxbuf *rx_buffer = &rdata->rx_buf[i]; 2881 2882 if (rx_buffer->m_head != NULL) { 2883 bus_dmamap_unload(rdata->rxtag, rx_buffer->map); 2884 m_freem(rx_buffer->m_head); 2885 rx_buffer->m_head = NULL; 2886 } 2887 } 2888 2889 if (rdata->fmp != NULL) 2890 m_freem(rdata->fmp); 2891 rdata->fmp = NULL; 2892 rdata->lmp = NULL; 2893 } 2894 2895 static void 2896 emx_free_tx_ring(struct emx_txdata *tdata) 2897 { 2898 int i; 2899 2900 for (i = 0; i < tdata->num_tx_desc; i++) { 2901 struct emx_txbuf *tx_buffer = &tdata->tx_buf[i]; 2902 2903 if (tx_buffer->m_head != NULL) { 2904 bus_dmamap_unload(tdata->txtag, tx_buffer->map); 2905 m_freem(tx_buffer->m_head); 2906 tx_buffer->m_head = NULL; 2907 } 2908 } 2909 2910 tdata->tx_flags &= ~EMX_TXFLAG_FORCECTX; 2911 2912 tdata->csum_flags = 0; 2913 tdata->csum_lhlen = 0; 2914 tdata->csum_iphlen = 0; 2915 tdata->csum_thlen = 0; 2916 tdata->csum_mss = 0; 2917 tdata->csum_pktlen = 0; 2918 2919 tdata->tx_dd_head = 0; 2920 tdata->tx_dd_tail = 0; 2921 tdata->tx_nsegs = 0; 2922 } 2923 2924 static int 2925 emx_init_rx_ring(struct emx_rxdata *rdata) 2926 { 2927 int i, error; 2928 2929 /* Reset descriptor ring */ 2930 bzero(rdata->rx_desc, sizeof(emx_rxdesc_t) * rdata->num_rx_desc); 2931 2932 /* Allocate new ones. */ 2933 for (i = 0; i < rdata->num_rx_desc; i++) { 2934 error = emx_newbuf(rdata, i, 1); 2935 if (error) 2936 return (error); 2937 } 2938 2939 /* Setup our descriptor pointers */ 2940 rdata->next_rx_desc_to_check = 0; 2941 2942 return (0); 2943 } 2944 2945 static void 2946 emx_init_rx_unit(struct emx_softc *sc) 2947 { 2948 struct ifnet *ifp = &sc->arpcom.ac_if; 2949 uint64_t bus_addr; 2950 uint32_t rctl, itr, rfctl; 2951 int i; 2952 2953 /* 2954 * Make sure receives are disabled while setting 2955 * up the descriptor ring 2956 */ 2957 rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 2958 E1000_WRITE_REG(&sc->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 2959 2960 /* 2961 * Set the interrupt throttling rate. Value is calculated 2962 * as ITR = 1 / (INT_THROTTLE_CEIL * 256ns) 2963 */ 2964 if (sc->int_throttle_ceil) 2965 itr = 1000000000 / 256 / sc->int_throttle_ceil; 2966 else 2967 itr = 0; 2968 emx_set_itr(sc, itr); 2969 2970 /* Use extended RX descriptor */ 2971 rfctl = E1000_RFCTL_EXTEN; 2972 2973 /* Disable accelerated ackknowledge */ 2974 if (sc->hw.mac.type == e1000_82574) 2975 rfctl |= E1000_RFCTL_ACK_DIS; 2976 2977 E1000_WRITE_REG(&sc->hw, E1000_RFCTL, rfctl); 2978 2979 /* 2980 * Receive Checksum Offload for TCP and UDP 2981 * 2982 * Checksum offloading is also enabled if multiple receive 2983 * queue is to be supported, since we need it to figure out 2984 * packet type. 2985 */ 2986 if ((ifp->if_capenable & IFCAP_RXCSUM) || 2987 sc->rx_ring_cnt > 1) { 2988 uint32_t rxcsum; 2989 2990 rxcsum = E1000_READ_REG(&sc->hw, E1000_RXCSUM); 2991 2992 /* 2993 * NOTE: 2994 * PCSD must be enabled to enable multiple 2995 * receive queues. 2996 */ 2997 rxcsum |= E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL | 2998 E1000_RXCSUM_PCSD; 2999 E1000_WRITE_REG(&sc->hw, E1000_RXCSUM, rxcsum); 3000 } 3001 3002 /* 3003 * Configure multiple receive queue (RSS) 3004 */ 3005 if (sc->rx_ring_cnt > 1) { 3006 uint8_t key[EMX_NRSSRK * EMX_RSSRK_SIZE]; 3007 uint32_t reta; 3008 3009 KASSERT(sc->rx_ring_cnt == EMX_NRX_RING, 3010 ("invalid number of RX ring (%d)", sc->rx_ring_cnt)); 3011 3012 /* 3013 * NOTE: 3014 * When we reach here, RSS has already been disabled 3015 * in emx_stop(), so we could safely configure RSS key 3016 * and redirect table. 3017 */ 3018 3019 /* 3020 * Configure RSS key 3021 */ 3022 toeplitz_get_key(key, sizeof(key)); 3023 for (i = 0; i < EMX_NRSSRK; ++i) { 3024 uint32_t rssrk; 3025 3026 rssrk = EMX_RSSRK_VAL(key, i); 3027 EMX_RSS_DPRINTF(sc, 1, "rssrk%d 0x%08x\n", i, rssrk); 3028 3029 E1000_WRITE_REG(&sc->hw, E1000_RSSRK(i), rssrk); 3030 } 3031 3032 /* 3033 * Configure RSS redirect table in following fashion: 3034 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] 3035 */ 3036 reta = 0; 3037 for (i = 0; i < EMX_RETA_SIZE; ++i) { 3038 uint32_t q; 3039 3040 q = (i % sc->rx_ring_cnt) << EMX_RETA_RINGIDX_SHIFT; 3041 reta |= q << (8 * i); 3042 } 3043 EMX_RSS_DPRINTF(sc, 1, "reta 0x%08x\n", reta); 3044 3045 for (i = 0; i < EMX_NRETA; ++i) 3046 E1000_WRITE_REG(&sc->hw, E1000_RETA(i), reta); 3047 3048 /* 3049 * Enable multiple receive queues. 3050 * Enable IPv4 RSS standard hash functions. 3051 * Disable RSS interrupt. 3052 */ 3053 E1000_WRITE_REG(&sc->hw, E1000_MRQC, 3054 E1000_MRQC_ENABLE_RSS_2Q | 3055 E1000_MRQC_RSS_FIELD_IPV4_TCP | 3056 E1000_MRQC_RSS_FIELD_IPV4); 3057 } 3058 3059 /* 3060 * XXX TEMPORARY WORKAROUND: on some systems with 82573 3061 * long latencies are observed, like Lenovo X60. This 3062 * change eliminates the problem, but since having positive 3063 * values in RDTR is a known source of problems on other 3064 * platforms another solution is being sought. 3065 */ 3066 if (emx_82573_workaround && sc->hw.mac.type == e1000_82573) { 3067 E1000_WRITE_REG(&sc->hw, E1000_RADV, EMX_RADV_82573); 3068 E1000_WRITE_REG(&sc->hw, E1000_RDTR, EMX_RDTR_82573); 3069 } 3070 3071 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3072 struct emx_rxdata *rdata = &sc->rx_data[i]; 3073 3074 /* 3075 * Setup the Base and Length of the Rx Descriptor Ring 3076 */ 3077 bus_addr = rdata->rx_desc_paddr; 3078 E1000_WRITE_REG(&sc->hw, E1000_RDLEN(i), 3079 rdata->num_rx_desc * sizeof(emx_rxdesc_t)); 3080 E1000_WRITE_REG(&sc->hw, E1000_RDBAH(i), 3081 (uint32_t)(bus_addr >> 32)); 3082 E1000_WRITE_REG(&sc->hw, E1000_RDBAL(i), 3083 (uint32_t)bus_addr); 3084 3085 /* 3086 * Setup the HW Rx Head and Tail Descriptor Pointers 3087 */ 3088 E1000_WRITE_REG(&sc->hw, E1000_RDH(i), 0); 3089 E1000_WRITE_REG(&sc->hw, E1000_RDT(i), 3090 sc->rx_data[i].num_rx_desc - 1); 3091 } 3092 3093 if (sc->hw.mac.type >= e1000_pch2lan) { 3094 if (ifp->if_mtu > ETHERMTU) 3095 e1000_lv_jumbo_workaround_ich8lan(&sc->hw, TRUE); 3096 else 3097 e1000_lv_jumbo_workaround_ich8lan(&sc->hw, FALSE); 3098 } 3099 3100 /* Setup the Receive Control Register */ 3101 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 3102 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 3103 E1000_RCTL_RDMTS_HALF | E1000_RCTL_SECRC | 3104 (sc->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 3105 3106 /* Make sure VLAN Filters are off */ 3107 rctl &= ~E1000_RCTL_VFE; 3108 3109 /* Don't store bad paket */ 3110 rctl &= ~E1000_RCTL_SBP; 3111 3112 /* MCLBYTES */ 3113 rctl |= E1000_RCTL_SZ_2048; 3114 3115 if (ifp->if_mtu > ETHERMTU) 3116 rctl |= E1000_RCTL_LPE; 3117 else 3118 rctl &= ~E1000_RCTL_LPE; 3119 3120 /* Enable Receives */ 3121 E1000_WRITE_REG(&sc->hw, E1000_RCTL, rctl); 3122 } 3123 3124 static void 3125 emx_destroy_rx_ring(struct emx_rxdata *rdata, int ndesc) 3126 { 3127 struct emx_rxbuf *rx_buffer; 3128 int i; 3129 3130 /* Free Receive Descriptor ring */ 3131 if (rdata->rx_desc) { 3132 bus_dmamap_unload(rdata->rx_desc_dtag, rdata->rx_desc_dmap); 3133 bus_dmamem_free(rdata->rx_desc_dtag, rdata->rx_desc, 3134 rdata->rx_desc_dmap); 3135 bus_dma_tag_destroy(rdata->rx_desc_dtag); 3136 3137 rdata->rx_desc = NULL; 3138 } 3139 3140 if (rdata->rx_buf == NULL) 3141 return; 3142 3143 for (i = 0; i < ndesc; i++) { 3144 rx_buffer = &rdata->rx_buf[i]; 3145 3146 KKASSERT(rx_buffer->m_head == NULL); 3147 bus_dmamap_destroy(rdata->rxtag, rx_buffer->map); 3148 } 3149 bus_dmamap_destroy(rdata->rxtag, rdata->rx_sparemap); 3150 bus_dma_tag_destroy(rdata->rxtag); 3151 3152 kfree(rdata->rx_buf, M_DEVBUF); 3153 rdata->rx_buf = NULL; 3154 } 3155 3156 static void 3157 emx_rxeof(struct emx_rxdata *rdata, int count) 3158 { 3159 struct ifnet *ifp = &rdata->sc->arpcom.ac_if; 3160 uint32_t staterr; 3161 emx_rxdesc_t *current_desc; 3162 struct mbuf *mp; 3163 int i, cpuid = mycpuid; 3164 3165 i = rdata->next_rx_desc_to_check; 3166 current_desc = &rdata->rx_desc[i]; 3167 staterr = le32toh(current_desc->rxd_staterr); 3168 3169 if (!(staterr & E1000_RXD_STAT_DD)) 3170 return; 3171 3172 while ((staterr & E1000_RXD_STAT_DD) && count != 0) { 3173 struct pktinfo *pi = NULL, pi0; 3174 struct emx_rxbuf *rx_buf = &rdata->rx_buf[i]; 3175 struct mbuf *m = NULL; 3176 int eop, len; 3177 3178 logif(pkt_receive); 3179 3180 mp = rx_buf->m_head; 3181 3182 /* 3183 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT 3184 * needs to access the last received byte in the mbuf. 3185 */ 3186 bus_dmamap_sync(rdata->rxtag, rx_buf->map, 3187 BUS_DMASYNC_POSTREAD); 3188 3189 len = le16toh(current_desc->rxd_length); 3190 if (staterr & E1000_RXD_STAT_EOP) { 3191 count--; 3192 eop = 1; 3193 } else { 3194 eop = 0; 3195 } 3196 3197 if (!(staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK)) { 3198 uint16_t vlan = 0; 3199 uint32_t mrq, rss_hash; 3200 3201 /* 3202 * Save several necessary information, 3203 * before emx_newbuf() destroy it. 3204 */ 3205 if ((staterr & E1000_RXD_STAT_VP) && eop) 3206 vlan = le16toh(current_desc->rxd_vlan); 3207 3208 mrq = le32toh(current_desc->rxd_mrq); 3209 rss_hash = le32toh(current_desc->rxd_rss); 3210 3211 EMX_RSS_DPRINTF(rdata->sc, 10, 3212 "ring%d, mrq 0x%08x, rss_hash 0x%08x\n", 3213 rdata->idx, mrq, rss_hash); 3214 3215 if (emx_newbuf(rdata, i, 0) != 0) { 3216 IFNET_STAT_INC(ifp, iqdrops, 1); 3217 goto discard; 3218 } 3219 3220 /* Assign correct length to the current fragment */ 3221 mp->m_len = len; 3222 3223 if (rdata->fmp == NULL) { 3224 mp->m_pkthdr.len = len; 3225 rdata->fmp = mp; /* Store the first mbuf */ 3226 rdata->lmp = mp; 3227 } else { 3228 /* 3229 * Chain mbuf's together 3230 */ 3231 rdata->lmp->m_next = mp; 3232 rdata->lmp = rdata->lmp->m_next; 3233 rdata->fmp->m_pkthdr.len += len; 3234 } 3235 3236 if (eop) { 3237 rdata->fmp->m_pkthdr.rcvif = ifp; 3238 IFNET_STAT_INC(ifp, ipackets, 1); 3239 3240 if (ifp->if_capenable & IFCAP_RXCSUM) 3241 emx_rxcsum(staterr, rdata->fmp); 3242 3243 if (staterr & E1000_RXD_STAT_VP) { 3244 rdata->fmp->m_pkthdr.ether_vlantag = 3245 vlan; 3246 rdata->fmp->m_flags |= M_VLANTAG; 3247 } 3248 m = rdata->fmp; 3249 rdata->fmp = NULL; 3250 rdata->lmp = NULL; 3251 3252 if (ifp->if_capenable & IFCAP_RSS) { 3253 pi = emx_rssinfo(m, &pi0, mrq, 3254 rss_hash, staterr); 3255 } 3256 #ifdef EMX_RSS_DEBUG 3257 rdata->rx_pkts++; 3258 #endif 3259 } 3260 } else { 3261 IFNET_STAT_INC(ifp, ierrors, 1); 3262 discard: 3263 emx_setup_rxdesc(current_desc, rx_buf); 3264 if (rdata->fmp != NULL) { 3265 m_freem(rdata->fmp); 3266 rdata->fmp = NULL; 3267 rdata->lmp = NULL; 3268 } 3269 m = NULL; 3270 } 3271 3272 if (m != NULL) 3273 ifp->if_input(ifp, m, pi, cpuid); 3274 3275 /* Advance our pointers to the next descriptor. */ 3276 if (++i == rdata->num_rx_desc) 3277 i = 0; 3278 3279 current_desc = &rdata->rx_desc[i]; 3280 staterr = le32toh(current_desc->rxd_staterr); 3281 } 3282 rdata->next_rx_desc_to_check = i; 3283 3284 /* Advance the E1000's Receive Queue "Tail Pointer". */ 3285 if (--i < 0) 3286 i = rdata->num_rx_desc - 1; 3287 E1000_WRITE_REG(&rdata->sc->hw, E1000_RDT(rdata->idx), i); 3288 } 3289 3290 static void 3291 emx_enable_intr(struct emx_softc *sc) 3292 { 3293 uint32_t ims_mask = IMS_ENABLE_MASK; 3294 3295 lwkt_serialize_handler_enable(&sc->main_serialize); 3296 3297 #if 0 3298 if (sc->hw.mac.type == e1000_82574) { 3299 E1000_WRITE_REG(hw, EMX_EIAC, EM_MSIX_MASK); 3300 ims_mask |= EM_MSIX_MASK; 3301 } 3302 #endif 3303 E1000_WRITE_REG(&sc->hw, E1000_IMS, ims_mask); 3304 } 3305 3306 static void 3307 emx_disable_intr(struct emx_softc *sc) 3308 { 3309 if (sc->hw.mac.type == e1000_82574) 3310 E1000_WRITE_REG(&sc->hw, EMX_EIAC, 0); 3311 E1000_WRITE_REG(&sc->hw, E1000_IMC, 0xffffffff); 3312 3313 lwkt_serialize_handler_disable(&sc->main_serialize); 3314 } 3315 3316 /* 3317 * Bit of a misnomer, what this really means is 3318 * to enable OS management of the system... aka 3319 * to disable special hardware management features 3320 */ 3321 static void 3322 emx_get_mgmt(struct emx_softc *sc) 3323 { 3324 /* A shared code workaround */ 3325 if (sc->flags & EMX_FLAG_HAS_MGMT) { 3326 int manc2h = E1000_READ_REG(&sc->hw, E1000_MANC2H); 3327 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 3328 3329 /* disable hardware interception of ARP */ 3330 manc &= ~(E1000_MANC_ARP_EN); 3331 3332 /* enable receiving management packets to the host */ 3333 manc |= E1000_MANC_EN_MNG2HOST; 3334 #define E1000_MNG2HOST_PORT_623 (1 << 5) 3335 #define E1000_MNG2HOST_PORT_664 (1 << 6) 3336 manc2h |= E1000_MNG2HOST_PORT_623; 3337 manc2h |= E1000_MNG2HOST_PORT_664; 3338 E1000_WRITE_REG(&sc->hw, E1000_MANC2H, manc2h); 3339 3340 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 3341 } 3342 } 3343 3344 /* 3345 * Give control back to hardware management 3346 * controller if there is one. 3347 */ 3348 static void 3349 emx_rel_mgmt(struct emx_softc *sc) 3350 { 3351 if (sc->flags & EMX_FLAG_HAS_MGMT) { 3352 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 3353 3354 /* re-enable hardware interception of ARP */ 3355 manc |= E1000_MANC_ARP_EN; 3356 manc &= ~E1000_MANC_EN_MNG2HOST; 3357 3358 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 3359 } 3360 } 3361 3362 /* 3363 * emx_get_hw_control() sets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3364 * For ASF and Pass Through versions of f/w this means that 3365 * the driver is loaded. For AMT version (only with 82573) 3366 * of the f/w this means that the network i/f is open. 3367 */ 3368 static void 3369 emx_get_hw_control(struct emx_softc *sc) 3370 { 3371 /* Let firmware know the driver has taken over */ 3372 if (sc->hw.mac.type == e1000_82573) { 3373 uint32_t swsm; 3374 3375 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 3376 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 3377 swsm | E1000_SWSM_DRV_LOAD); 3378 } else { 3379 uint32_t ctrl_ext; 3380 3381 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 3382 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 3383 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 3384 } 3385 sc->flags |= EMX_FLAG_HW_CTRL; 3386 } 3387 3388 /* 3389 * emx_rel_hw_control() resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3390 * For ASF and Pass Through versions of f/w this means that the 3391 * driver is no longer loaded. For AMT version (only with 82573) 3392 * of the f/w this means that the network i/f is closed. 3393 */ 3394 static void 3395 emx_rel_hw_control(struct emx_softc *sc) 3396 { 3397 if ((sc->flags & EMX_FLAG_HW_CTRL) == 0) 3398 return; 3399 sc->flags &= ~EMX_FLAG_HW_CTRL; 3400 3401 /* Let firmware taken over control of h/w */ 3402 if (sc->hw.mac.type == e1000_82573) { 3403 uint32_t swsm; 3404 3405 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 3406 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 3407 swsm & ~E1000_SWSM_DRV_LOAD); 3408 } else { 3409 uint32_t ctrl_ext; 3410 3411 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 3412 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 3413 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 3414 } 3415 } 3416 3417 static int 3418 emx_is_valid_eaddr(const uint8_t *addr) 3419 { 3420 char zero_addr[ETHER_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 }; 3421 3422 if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3423 return (FALSE); 3424 3425 return (TRUE); 3426 } 3427 3428 /* 3429 * Enable PCI Wake On Lan capability 3430 */ 3431 void 3432 emx_enable_wol(device_t dev) 3433 { 3434 uint16_t cap, status; 3435 uint8_t id; 3436 3437 /* First find the capabilities pointer*/ 3438 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 3439 3440 /* Read the PM Capabilities */ 3441 id = pci_read_config(dev, cap, 1); 3442 if (id != PCIY_PMG) /* Something wrong */ 3443 return; 3444 3445 /* 3446 * OK, we have the power capabilities, 3447 * so now get the status register 3448 */ 3449 cap += PCIR_POWER_STATUS; 3450 status = pci_read_config(dev, cap, 2); 3451 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 3452 pci_write_config(dev, cap, status, 2); 3453 } 3454 3455 static void 3456 emx_update_stats(struct emx_softc *sc) 3457 { 3458 struct ifnet *ifp = &sc->arpcom.ac_if; 3459 3460 if (sc->hw.phy.media_type == e1000_media_type_copper || 3461 (E1000_READ_REG(&sc->hw, E1000_STATUS) & E1000_STATUS_LU)) { 3462 sc->stats.symerrs += E1000_READ_REG(&sc->hw, E1000_SYMERRS); 3463 sc->stats.sec += E1000_READ_REG(&sc->hw, E1000_SEC); 3464 } 3465 sc->stats.crcerrs += E1000_READ_REG(&sc->hw, E1000_CRCERRS); 3466 sc->stats.mpc += E1000_READ_REG(&sc->hw, E1000_MPC); 3467 sc->stats.scc += E1000_READ_REG(&sc->hw, E1000_SCC); 3468 sc->stats.ecol += E1000_READ_REG(&sc->hw, E1000_ECOL); 3469 3470 sc->stats.mcc += E1000_READ_REG(&sc->hw, E1000_MCC); 3471 sc->stats.latecol += E1000_READ_REG(&sc->hw, E1000_LATECOL); 3472 sc->stats.colc += E1000_READ_REG(&sc->hw, E1000_COLC); 3473 sc->stats.dc += E1000_READ_REG(&sc->hw, E1000_DC); 3474 sc->stats.rlec += E1000_READ_REG(&sc->hw, E1000_RLEC); 3475 sc->stats.xonrxc += E1000_READ_REG(&sc->hw, E1000_XONRXC); 3476 sc->stats.xontxc += E1000_READ_REG(&sc->hw, E1000_XONTXC); 3477 sc->stats.xoffrxc += E1000_READ_REG(&sc->hw, E1000_XOFFRXC); 3478 sc->stats.xofftxc += E1000_READ_REG(&sc->hw, E1000_XOFFTXC); 3479 sc->stats.fcruc += E1000_READ_REG(&sc->hw, E1000_FCRUC); 3480 sc->stats.prc64 += E1000_READ_REG(&sc->hw, E1000_PRC64); 3481 sc->stats.prc127 += E1000_READ_REG(&sc->hw, E1000_PRC127); 3482 sc->stats.prc255 += E1000_READ_REG(&sc->hw, E1000_PRC255); 3483 sc->stats.prc511 += E1000_READ_REG(&sc->hw, E1000_PRC511); 3484 sc->stats.prc1023 += E1000_READ_REG(&sc->hw, E1000_PRC1023); 3485 sc->stats.prc1522 += E1000_READ_REG(&sc->hw, E1000_PRC1522); 3486 sc->stats.gprc += E1000_READ_REG(&sc->hw, E1000_GPRC); 3487 sc->stats.bprc += E1000_READ_REG(&sc->hw, E1000_BPRC); 3488 sc->stats.mprc += E1000_READ_REG(&sc->hw, E1000_MPRC); 3489 sc->stats.gptc += E1000_READ_REG(&sc->hw, E1000_GPTC); 3490 3491 /* For the 64-bit byte counters the low dword must be read first. */ 3492 /* Both registers clear on the read of the high dword */ 3493 3494 sc->stats.gorc += E1000_READ_REG(&sc->hw, E1000_GORCH); 3495 sc->stats.gotc += E1000_READ_REG(&sc->hw, E1000_GOTCH); 3496 3497 sc->stats.rnbc += E1000_READ_REG(&sc->hw, E1000_RNBC); 3498 sc->stats.ruc += E1000_READ_REG(&sc->hw, E1000_RUC); 3499 sc->stats.rfc += E1000_READ_REG(&sc->hw, E1000_RFC); 3500 sc->stats.roc += E1000_READ_REG(&sc->hw, E1000_ROC); 3501 sc->stats.rjc += E1000_READ_REG(&sc->hw, E1000_RJC); 3502 3503 sc->stats.tor += E1000_READ_REG(&sc->hw, E1000_TORH); 3504 sc->stats.tot += E1000_READ_REG(&sc->hw, E1000_TOTH); 3505 3506 sc->stats.tpr += E1000_READ_REG(&sc->hw, E1000_TPR); 3507 sc->stats.tpt += E1000_READ_REG(&sc->hw, E1000_TPT); 3508 sc->stats.ptc64 += E1000_READ_REG(&sc->hw, E1000_PTC64); 3509 sc->stats.ptc127 += E1000_READ_REG(&sc->hw, E1000_PTC127); 3510 sc->stats.ptc255 += E1000_READ_REG(&sc->hw, E1000_PTC255); 3511 sc->stats.ptc511 += E1000_READ_REG(&sc->hw, E1000_PTC511); 3512 sc->stats.ptc1023 += E1000_READ_REG(&sc->hw, E1000_PTC1023); 3513 sc->stats.ptc1522 += E1000_READ_REG(&sc->hw, E1000_PTC1522); 3514 sc->stats.mptc += E1000_READ_REG(&sc->hw, E1000_MPTC); 3515 sc->stats.bptc += E1000_READ_REG(&sc->hw, E1000_BPTC); 3516 3517 sc->stats.algnerrc += E1000_READ_REG(&sc->hw, E1000_ALGNERRC); 3518 sc->stats.rxerrc += E1000_READ_REG(&sc->hw, E1000_RXERRC); 3519 sc->stats.tncrs += E1000_READ_REG(&sc->hw, E1000_TNCRS); 3520 sc->stats.cexterr += E1000_READ_REG(&sc->hw, E1000_CEXTERR); 3521 sc->stats.tsctc += E1000_READ_REG(&sc->hw, E1000_TSCTC); 3522 sc->stats.tsctfc += E1000_READ_REG(&sc->hw, E1000_TSCTFC); 3523 3524 IFNET_STAT_SET(ifp, collisions, sc->stats.colc); 3525 3526 /* Rx Errors */ 3527 IFNET_STAT_SET(ifp, ierrors, 3528 sc->stats.rxerrc + sc->stats.crcerrs + sc->stats.algnerrc + 3529 sc->stats.ruc + sc->stats.roc + sc->stats.mpc + sc->stats.cexterr); 3530 3531 /* Tx Errors */ 3532 IFNET_STAT_SET(ifp, oerrors, sc->stats.ecol + sc->stats.latecol); 3533 } 3534 3535 static void 3536 emx_print_debug_info(struct emx_softc *sc) 3537 { 3538 device_t dev = sc->dev; 3539 uint8_t *hw_addr = sc->hw.hw_addr; 3540 int i; 3541 3542 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 3543 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 3544 E1000_READ_REG(&sc->hw, E1000_CTRL), 3545 E1000_READ_REG(&sc->hw, E1000_RCTL)); 3546 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 3547 ((E1000_READ_REG(&sc->hw, E1000_PBA) & 0xffff0000) >> 16),\ 3548 (E1000_READ_REG(&sc->hw, E1000_PBA) & 0xffff) ); 3549 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 3550 sc->hw.fc.high_water, sc->hw.fc.low_water); 3551 device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n", 3552 E1000_READ_REG(&sc->hw, E1000_TIDV), 3553 E1000_READ_REG(&sc->hw, E1000_TADV)); 3554 device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n", 3555 E1000_READ_REG(&sc->hw, E1000_RDTR), 3556 E1000_READ_REG(&sc->hw, E1000_RADV)); 3557 3558 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3559 device_printf(dev, "hw %d tdh = %d, hw tdt = %d\n", i, 3560 E1000_READ_REG(&sc->hw, E1000_TDH(i)), 3561 E1000_READ_REG(&sc->hw, E1000_TDT(i))); 3562 } 3563 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3564 device_printf(dev, "hw %d rdh = %d, hw rdt = %d\n", i, 3565 E1000_READ_REG(&sc->hw, E1000_RDH(i)), 3566 E1000_READ_REG(&sc->hw, E1000_RDT(i))); 3567 } 3568 3569 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3570 device_printf(dev, "TX %d Tx descriptors avail = %d\n", i, 3571 sc->tx_data[i].num_tx_desc_avail); 3572 device_printf(dev, "TX %d TSO segments = %lu\n", i, 3573 sc->tx_data[i].tso_segments); 3574 device_printf(dev, "TX %d TSO ctx reused = %lu\n", i, 3575 sc->tx_data[i].tso_ctx_reused); 3576 } 3577 } 3578 3579 static void 3580 emx_print_hw_stats(struct emx_softc *sc) 3581 { 3582 device_t dev = sc->dev; 3583 3584 device_printf(dev, "Excessive collisions = %lld\n", 3585 (long long)sc->stats.ecol); 3586 #if (DEBUG_HW > 0) /* Dont output these errors normally */ 3587 device_printf(dev, "Symbol errors = %lld\n", 3588 (long long)sc->stats.symerrs); 3589 #endif 3590 device_printf(dev, "Sequence errors = %lld\n", 3591 (long long)sc->stats.sec); 3592 device_printf(dev, "Defer count = %lld\n", 3593 (long long)sc->stats.dc); 3594 device_printf(dev, "Missed Packets = %lld\n", 3595 (long long)sc->stats.mpc); 3596 device_printf(dev, "Receive No Buffers = %lld\n", 3597 (long long)sc->stats.rnbc); 3598 /* RLEC is inaccurate on some hardware, calculate our own. */ 3599 device_printf(dev, "Receive Length Errors = %lld\n", 3600 ((long long)sc->stats.roc + (long long)sc->stats.ruc)); 3601 device_printf(dev, "Receive errors = %lld\n", 3602 (long long)sc->stats.rxerrc); 3603 device_printf(dev, "Crc errors = %lld\n", 3604 (long long)sc->stats.crcerrs); 3605 device_printf(dev, "Alignment errors = %lld\n", 3606 (long long)sc->stats.algnerrc); 3607 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 3608 (long long)sc->stats.cexterr); 3609 device_printf(dev, "RX overruns = %ld\n", sc->rx_overruns); 3610 device_printf(dev, "XON Rcvd = %lld\n", 3611 (long long)sc->stats.xonrxc); 3612 device_printf(dev, "XON Xmtd = %lld\n", 3613 (long long)sc->stats.xontxc); 3614 device_printf(dev, "XOFF Rcvd = %lld\n", 3615 (long long)sc->stats.xoffrxc); 3616 device_printf(dev, "XOFF Xmtd = %lld\n", 3617 (long long)sc->stats.xofftxc); 3618 device_printf(dev, "Good Packets Rcvd = %lld\n", 3619 (long long)sc->stats.gprc); 3620 device_printf(dev, "Good Packets Xmtd = %lld\n", 3621 (long long)sc->stats.gptc); 3622 } 3623 3624 static void 3625 emx_print_nvm_info(struct emx_softc *sc) 3626 { 3627 uint16_t eeprom_data; 3628 int i, j, row = 0; 3629 3630 /* Its a bit crude, but it gets the job done */ 3631 kprintf("\nInterface EEPROM Dump:\n"); 3632 kprintf("Offset\n0x0000 "); 3633 for (i = 0, j = 0; i < 32; i++, j++) { 3634 if (j == 8) { /* Make the offset block */ 3635 j = 0; ++row; 3636 kprintf("\n0x00%x0 ",row); 3637 } 3638 e1000_read_nvm(&sc->hw, i, 1, &eeprom_data); 3639 kprintf("%04x ", eeprom_data); 3640 } 3641 kprintf("\n"); 3642 } 3643 3644 static int 3645 emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 3646 { 3647 struct emx_softc *sc; 3648 struct ifnet *ifp; 3649 int error, result; 3650 3651 result = -1; 3652 error = sysctl_handle_int(oidp, &result, 0, req); 3653 if (error || !req->newptr) 3654 return (error); 3655 3656 sc = (struct emx_softc *)arg1; 3657 ifp = &sc->arpcom.ac_if; 3658 3659 ifnet_serialize_all(ifp); 3660 3661 if (result == 1) 3662 emx_print_debug_info(sc); 3663 3664 /* 3665 * This value will cause a hex dump of the 3666 * first 32 16-bit words of the EEPROM to 3667 * the screen. 3668 */ 3669 if (result == 2) 3670 emx_print_nvm_info(sc); 3671 3672 ifnet_deserialize_all(ifp); 3673 3674 return (error); 3675 } 3676 3677 static int 3678 emx_sysctl_stats(SYSCTL_HANDLER_ARGS) 3679 { 3680 int error, result; 3681 3682 result = -1; 3683 error = sysctl_handle_int(oidp, &result, 0, req); 3684 if (error || !req->newptr) 3685 return (error); 3686 3687 if (result == 1) { 3688 struct emx_softc *sc = (struct emx_softc *)arg1; 3689 struct ifnet *ifp = &sc->arpcom.ac_if; 3690 3691 ifnet_serialize_all(ifp); 3692 emx_print_hw_stats(sc); 3693 ifnet_deserialize_all(ifp); 3694 } 3695 return (error); 3696 } 3697 3698 static void 3699 emx_add_sysctl(struct emx_softc *sc) 3700 { 3701 struct sysctl_ctx_list *ctx; 3702 struct sysctl_oid *tree; 3703 #if defined(EMX_RSS_DEBUG) || defined(EMX_TSS_DEBUG) 3704 char pkt_desc[32]; 3705 int i; 3706 #endif 3707 3708 ctx = device_get_sysctl_ctx(sc->dev); 3709 tree = device_get_sysctl_tree(sc->dev); 3710 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 3711 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 3712 emx_sysctl_debug_info, "I", "Debug Information"); 3713 3714 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 3715 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 3716 emx_sysctl_stats, "I", "Statistics"); 3717 3718 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 3719 OID_AUTO, "rxd", CTLFLAG_RD, &sc->rx_data[0].num_rx_desc, 0, 3720 "# of RX descs"); 3721 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 3722 OID_AUTO, "txd", CTLFLAG_RD, &sc->tx_data[0].num_tx_desc, 0, 3723 "# of TX descs"); 3724 3725 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 3726 OID_AUTO, "int_throttle_ceil", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 3727 emx_sysctl_int_throttle, "I", "interrupt throttling rate"); 3728 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 3729 OID_AUTO, "tx_intr_nsegs", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 3730 emx_sysctl_tx_intr_nsegs, "I", "# segments per TX interrupt"); 3731 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 3732 OID_AUTO, "tx_wreg_nsegs", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 3733 emx_sysctl_tx_wreg_nsegs, "I", 3734 "# segments sent before write to hardware register"); 3735 3736 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 3737 OID_AUTO, "rx_ring_cnt", CTLFLAG_RD, &sc->rx_ring_cnt, 0, 3738 "# of RX rings"); 3739 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 3740 OID_AUTO, "tx_ring_cnt", CTLFLAG_RD, &sc->tx_ring_cnt, 0, 3741 "# of TX rings"); 3742 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 3743 OID_AUTO, "tx_ring_inuse", CTLFLAG_RD, &sc->tx_ring_inuse, 0, 3744 "# of TX rings used"); 3745 3746 #ifdef IFPOLL_ENABLE 3747 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 3748 OID_AUTO, "npoll_rxoff", CTLTYPE_INT|CTLFLAG_RW, 3749 sc, 0, emx_sysctl_npoll_rxoff, "I", 3750 "NPOLLING RX cpu offset"); 3751 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 3752 OID_AUTO, "npoll_txoff", CTLTYPE_INT|CTLFLAG_RW, 3753 sc, 0, emx_sysctl_npoll_txoff, "I", 3754 "NPOLLING TX cpu offset"); 3755 #endif 3756 3757 #ifdef EMX_RSS_DEBUG 3758 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 3759 OID_AUTO, "rss_debug", CTLFLAG_RW, &sc->rss_debug, 3760 0, "RSS debug level"); 3761 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3762 ksnprintf(pkt_desc, sizeof(pkt_desc), "rx%d_pkt", i); 3763 SYSCTL_ADD_ULONG(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, 3764 pkt_desc, CTLFLAG_RW, &sc->rx_data[i].rx_pkts, 3765 "RXed packets"); 3766 } 3767 #endif 3768 #ifdef EMX_TSS_DEBUG 3769 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3770 ksnprintf(pkt_desc, sizeof(pkt_desc), "tx%d_pkt", i); 3771 SYSCTL_ADD_ULONG(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, 3772 pkt_desc, CTLFLAG_RW, &sc->tx_data[i].tx_pkts, 3773 "TXed packets"); 3774 } 3775 #endif 3776 } 3777 3778 static int 3779 emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS) 3780 { 3781 struct emx_softc *sc = (void *)arg1; 3782 struct ifnet *ifp = &sc->arpcom.ac_if; 3783 int error, throttle; 3784 3785 throttle = sc->int_throttle_ceil; 3786 error = sysctl_handle_int(oidp, &throttle, 0, req); 3787 if (error || req->newptr == NULL) 3788 return error; 3789 if (throttle < 0 || throttle > 1000000000 / 256) 3790 return EINVAL; 3791 3792 if (throttle) { 3793 /* 3794 * Set the interrupt throttling rate in 256ns increments, 3795 * recalculate sysctl value assignment to get exact frequency. 3796 */ 3797 throttle = 1000000000 / 256 / throttle; 3798 3799 /* Upper 16bits of ITR is reserved and should be zero */ 3800 if (throttle & 0xffff0000) 3801 return EINVAL; 3802 } 3803 3804 ifnet_serialize_all(ifp); 3805 3806 if (throttle) 3807 sc->int_throttle_ceil = 1000000000 / 256 / throttle; 3808 else 3809 sc->int_throttle_ceil = 0; 3810 3811 if (ifp->if_flags & IFF_RUNNING) 3812 emx_set_itr(sc, throttle); 3813 3814 ifnet_deserialize_all(ifp); 3815 3816 if (bootverbose) { 3817 if_printf(ifp, "Interrupt moderation set to %d/sec\n", 3818 sc->int_throttle_ceil); 3819 } 3820 return 0; 3821 } 3822 3823 static int 3824 emx_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS) 3825 { 3826 struct emx_softc *sc = (void *)arg1; 3827 struct ifnet *ifp = &sc->arpcom.ac_if; 3828 struct emx_txdata *tdata = &sc->tx_data[0]; 3829 int error, segs; 3830 3831 segs = tdata->tx_intr_nsegs; 3832 error = sysctl_handle_int(oidp, &segs, 0, req); 3833 if (error || req->newptr == NULL) 3834 return error; 3835 if (segs <= 0) 3836 return EINVAL; 3837 3838 ifnet_serialize_all(ifp); 3839 3840 /* 3841 * Don't allow tx_intr_nsegs to become: 3842 * o Less the oact_tx_desc 3843 * o Too large that no TX desc will cause TX interrupt to 3844 * be generated (OACTIVE will never recover) 3845 * o Too small that will cause tx_dd[] overflow 3846 */ 3847 if (segs < tdata->oact_tx_desc || 3848 segs >= tdata->num_tx_desc - tdata->oact_tx_desc || 3849 segs < tdata->num_tx_desc / EMX_TXDD_SAFE) { 3850 error = EINVAL; 3851 } else { 3852 int i; 3853 3854 error = 0; 3855 for (i = 0; i < sc->tx_ring_cnt; ++i) 3856 sc->tx_data[i].tx_intr_nsegs = segs; 3857 } 3858 3859 ifnet_deserialize_all(ifp); 3860 3861 return error; 3862 } 3863 3864 static int 3865 emx_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS) 3866 { 3867 struct emx_softc *sc = (void *)arg1; 3868 struct ifnet *ifp = &sc->arpcom.ac_if; 3869 int error, nsegs, i; 3870 3871 nsegs = sc->tx_data[0].tx_wreg_nsegs; 3872 error = sysctl_handle_int(oidp, &nsegs, 0, req); 3873 if (error || req->newptr == NULL) 3874 return error; 3875 3876 ifnet_serialize_all(ifp); 3877 for (i = 0; i < sc->tx_ring_cnt; ++i) 3878 sc->tx_data[i].tx_wreg_nsegs =nsegs; 3879 ifnet_deserialize_all(ifp); 3880 3881 return 0; 3882 } 3883 3884 #ifdef IFPOLL_ENABLE 3885 3886 static int 3887 emx_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS) 3888 { 3889 struct emx_softc *sc = (void *)arg1; 3890 struct ifnet *ifp = &sc->arpcom.ac_if; 3891 int error, off; 3892 3893 off = sc->rx_npoll_off; 3894 error = sysctl_handle_int(oidp, &off, 0, req); 3895 if (error || req->newptr == NULL) 3896 return error; 3897 if (off < 0) 3898 return EINVAL; 3899 3900 ifnet_serialize_all(ifp); 3901 if (off >= ncpus2 || off % sc->rx_ring_cnt != 0) { 3902 error = EINVAL; 3903 } else { 3904 error = 0; 3905 sc->rx_npoll_off = off; 3906 } 3907 ifnet_deserialize_all(ifp); 3908 3909 return error; 3910 } 3911 3912 static int 3913 emx_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS) 3914 { 3915 struct emx_softc *sc = (void *)arg1; 3916 struct ifnet *ifp = &sc->arpcom.ac_if; 3917 int error, off; 3918 3919 off = sc->tx_npoll_off; 3920 error = sysctl_handle_int(oidp, &off, 0, req); 3921 if (error || req->newptr == NULL) 3922 return error; 3923 if (off < 0) 3924 return EINVAL; 3925 3926 ifnet_serialize_all(ifp); 3927 if (off >= ncpus2 || off % sc->tx_ring_cnt != 0) { 3928 error = EINVAL; 3929 } else { 3930 error = 0; 3931 sc->tx_npoll_off = off; 3932 } 3933 ifnet_deserialize_all(ifp); 3934 3935 return error; 3936 } 3937 3938 #endif /* IFPOLL_ENABLE */ 3939 3940 static int 3941 emx_dma_alloc(struct emx_softc *sc) 3942 { 3943 int error, i; 3944 3945 /* 3946 * Create top level busdma tag 3947 */ 3948 error = bus_dma_tag_create(NULL, 1, 0, 3949 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 3950 NULL, NULL, 3951 BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 3952 0, &sc->parent_dtag); 3953 if (error) { 3954 device_printf(sc->dev, "could not create top level DMA tag\n"); 3955 return error; 3956 } 3957 3958 /* 3959 * Allocate transmit descriptors ring and buffers 3960 */ 3961 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3962 error = emx_create_tx_ring(&sc->tx_data[i]); 3963 if (error) { 3964 device_printf(sc->dev, 3965 "Could not setup transmit structures\n"); 3966 return error; 3967 } 3968 } 3969 3970 /* 3971 * Allocate receive descriptors ring and buffers 3972 */ 3973 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3974 error = emx_create_rx_ring(&sc->rx_data[i]); 3975 if (error) { 3976 device_printf(sc->dev, 3977 "Could not setup receive structures\n"); 3978 return error; 3979 } 3980 } 3981 return 0; 3982 } 3983 3984 static void 3985 emx_dma_free(struct emx_softc *sc) 3986 { 3987 int i; 3988 3989 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3990 emx_destroy_tx_ring(&sc->tx_data[i], 3991 sc->tx_data[i].num_tx_desc); 3992 } 3993 3994 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3995 emx_destroy_rx_ring(&sc->rx_data[i], 3996 sc->rx_data[i].num_rx_desc); 3997 } 3998 3999 /* Free top level busdma tag */ 4000 if (sc->parent_dtag != NULL) 4001 bus_dma_tag_destroy(sc->parent_dtag); 4002 } 4003 4004 static void 4005 emx_serialize(struct ifnet *ifp, enum ifnet_serialize slz) 4006 { 4007 struct emx_softc *sc = ifp->if_softc; 4008 4009 ifnet_serialize_array_enter(sc->serializes, EMX_NSERIALIZE, slz); 4010 } 4011 4012 static void 4013 emx_deserialize(struct ifnet *ifp, enum ifnet_serialize slz) 4014 { 4015 struct emx_softc *sc = ifp->if_softc; 4016 4017 ifnet_serialize_array_exit(sc->serializes, EMX_NSERIALIZE, slz); 4018 } 4019 4020 static int 4021 emx_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz) 4022 { 4023 struct emx_softc *sc = ifp->if_softc; 4024 4025 return ifnet_serialize_array_try(sc->serializes, EMX_NSERIALIZE, slz); 4026 } 4027 4028 static void 4029 emx_serialize_skipmain(struct emx_softc *sc) 4030 { 4031 lwkt_serialize_array_enter(sc->serializes, EMX_NSERIALIZE, 1); 4032 } 4033 4034 static void 4035 emx_deserialize_skipmain(struct emx_softc *sc) 4036 { 4037 lwkt_serialize_array_exit(sc->serializes, EMX_NSERIALIZE, 1); 4038 } 4039 4040 #ifdef INVARIANTS 4041 4042 static void 4043 emx_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz, 4044 boolean_t serialized) 4045 { 4046 struct emx_softc *sc = ifp->if_softc; 4047 4048 ifnet_serialize_array_assert(sc->serializes, EMX_NSERIALIZE, 4049 slz, serialized); 4050 } 4051 4052 #endif /* INVARIANTS */ 4053 4054 #ifdef IFPOLL_ENABLE 4055 4056 static void 4057 emx_npoll_status(struct ifnet *ifp) 4058 { 4059 struct emx_softc *sc = ifp->if_softc; 4060 uint32_t reg_icr; 4061 4062 ASSERT_SERIALIZED(&sc->main_serialize); 4063 4064 reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); 4065 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 4066 callout_stop(&sc->timer); 4067 sc->hw.mac.get_link_status = 1; 4068 emx_update_link_status(sc); 4069 callout_reset(&sc->timer, hz, emx_timer, sc); 4070 } 4071 } 4072 4073 static void 4074 emx_npoll_tx(struct ifnet *ifp, void *arg, int cycle __unused) 4075 { 4076 struct emx_txdata *tdata = arg; 4077 4078 ASSERT_SERIALIZED(&tdata->tx_serialize); 4079 4080 emx_txeof(tdata); 4081 if (!ifsq_is_empty(tdata->ifsq)) 4082 ifsq_devstart(tdata->ifsq); 4083 } 4084 4085 static void 4086 emx_npoll_rx(struct ifnet *ifp __unused, void *arg, int cycle) 4087 { 4088 struct emx_rxdata *rdata = arg; 4089 4090 ASSERT_SERIALIZED(&rdata->rx_serialize); 4091 4092 emx_rxeof(rdata, cycle); 4093 } 4094 4095 static void 4096 emx_npoll(struct ifnet *ifp, struct ifpoll_info *info) 4097 { 4098 struct emx_softc *sc = ifp->if_softc; 4099 int i, txr_cnt; 4100 4101 ASSERT_IFNET_SERIALIZED_ALL(ifp); 4102 4103 if (info) { 4104 int off; 4105 4106 info->ifpi_status.status_func = emx_npoll_status; 4107 info->ifpi_status.serializer = &sc->main_serialize; 4108 4109 txr_cnt = emx_get_txring_inuse(sc, TRUE); 4110 off = sc->tx_npoll_off; 4111 for (i = 0; i < txr_cnt; ++i) { 4112 struct emx_txdata *tdata = &sc->tx_data[i]; 4113 int idx = i + off; 4114 4115 KKASSERT(idx < ncpus2); 4116 info->ifpi_tx[idx].poll_func = emx_npoll_tx; 4117 info->ifpi_tx[idx].arg = tdata; 4118 info->ifpi_tx[idx].serializer = &tdata->tx_serialize; 4119 ifsq_set_cpuid(tdata->ifsq, idx); 4120 } 4121 4122 off = sc->rx_npoll_off; 4123 for (i = 0; i < sc->rx_ring_cnt; ++i) { 4124 struct emx_rxdata *rdata = &sc->rx_data[i]; 4125 int idx = i + off; 4126 4127 KKASSERT(idx < ncpus2); 4128 info->ifpi_rx[idx].poll_func = emx_npoll_rx; 4129 info->ifpi_rx[idx].arg = rdata; 4130 info->ifpi_rx[idx].serializer = &rdata->rx_serialize; 4131 } 4132 4133 if (ifp->if_flags & IFF_RUNNING) { 4134 if (txr_cnt == sc->tx_ring_inuse) 4135 emx_disable_intr(sc); 4136 else 4137 emx_init(sc); 4138 } 4139 } else { 4140 for (i = 0; i < sc->tx_ring_cnt; ++i) { 4141 struct emx_txdata *tdata = &sc->tx_data[i]; 4142 4143 ifsq_set_cpuid(tdata->ifsq, 4144 rman_get_cpuid(sc->intr_res)); 4145 } 4146 4147 if (ifp->if_flags & IFF_RUNNING) { 4148 txr_cnt = emx_get_txring_inuse(sc, FALSE); 4149 if (txr_cnt == sc->tx_ring_inuse) 4150 emx_enable_intr(sc); 4151 else 4152 emx_init(sc); 4153 } 4154 } 4155 } 4156 4157 #endif /* IFPOLL_ENABLE */ 4158 4159 static void 4160 emx_set_itr(struct emx_softc *sc, uint32_t itr) 4161 { 4162 E1000_WRITE_REG(&sc->hw, E1000_ITR, itr); 4163 if (sc->hw.mac.type == e1000_82574) { 4164 int i; 4165 4166 /* 4167 * When using MSIX interrupts we need to 4168 * throttle using the EITR register 4169 */ 4170 for (i = 0; i < 4; ++i) 4171 E1000_WRITE_REG(&sc->hw, E1000_EITR_82574(i), itr); 4172 } 4173 } 4174 4175 /* 4176 * Disable the L0s, 82574L Errata #20 4177 */ 4178 static void 4179 emx_disable_aspm(struct emx_softc *sc) 4180 { 4181 uint16_t link_cap, link_ctrl, disable; 4182 uint8_t pcie_ptr, reg; 4183 device_t dev = sc->dev; 4184 4185 switch (sc->hw.mac.type) { 4186 case e1000_82571: 4187 case e1000_82572: 4188 case e1000_82573: 4189 /* 4190 * 82573 specification update 4191 * errata #8 disable L0s 4192 * errata #41 disable L1 4193 * 4194 * 82571/82572 specification update 4195 # errata #13 disable L1 4196 * errata #68 disable L0s 4197 */ 4198 disable = PCIEM_LNKCTL_ASPM_L0S | PCIEM_LNKCTL_ASPM_L1; 4199 break; 4200 4201 case e1000_82574: 4202 /* 4203 * 82574 specification update errata #20 4204 * 4205 * There is no need to disable L1 4206 */ 4207 disable = PCIEM_LNKCTL_ASPM_L0S; 4208 break; 4209 4210 default: 4211 return; 4212 } 4213 4214 pcie_ptr = pci_get_pciecap_ptr(dev); 4215 if (pcie_ptr == 0) 4216 return; 4217 4218 link_cap = pci_read_config(dev, pcie_ptr + PCIER_LINKCAP, 2); 4219 if ((link_cap & PCIEM_LNKCAP_ASPM_MASK) == 0) 4220 return; 4221 4222 if (bootverbose) 4223 if_printf(&sc->arpcom.ac_if, "disable ASPM %#02x\n", disable); 4224 4225 reg = pcie_ptr + PCIER_LINKCTRL; 4226 link_ctrl = pci_read_config(dev, reg, 2); 4227 link_ctrl &= ~disable; 4228 pci_write_config(dev, reg, link_ctrl, 2); 4229 } 4230 4231 static int 4232 emx_tso_pullup(struct emx_txdata *tdata, struct mbuf **mp) 4233 { 4234 int iphlen, hoff, thoff, ex = 0; 4235 struct mbuf *m; 4236 struct ip *ip; 4237 4238 m = *mp; 4239 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable")); 4240 4241 iphlen = m->m_pkthdr.csum_iphlen; 4242 thoff = m->m_pkthdr.csum_thlen; 4243 hoff = m->m_pkthdr.csum_lhlen; 4244 4245 KASSERT(iphlen > 0, ("invalid ip hlen")); 4246 KASSERT(thoff > 0, ("invalid tcp hlen")); 4247 KASSERT(hoff > 0, ("invalid ether hlen")); 4248 4249 if (tdata->tx_flags & EMX_TXFLAG_TSO_PULLEX) 4250 ex = 4; 4251 4252 if (m->m_len < hoff + iphlen + thoff + ex) { 4253 m = m_pullup(m, hoff + iphlen + thoff + ex); 4254 if (m == NULL) { 4255 *mp = NULL; 4256 return ENOBUFS; 4257 } 4258 *mp = m; 4259 } 4260 ip = mtodoff(m, struct ip *, hoff); 4261 ip->ip_len = 0; 4262 4263 return 0; 4264 } 4265 4266 static int 4267 emx_tso_setup(struct emx_txdata *tdata, struct mbuf *mp, 4268 uint32_t *txd_upper, uint32_t *txd_lower) 4269 { 4270 struct e1000_context_desc *TXD; 4271 int hoff, iphlen, thoff, hlen; 4272 int mss, pktlen, curr_txd; 4273 4274 #ifdef EMX_TSO_DEBUG 4275 tdata->tso_segments++; 4276 #endif 4277 4278 iphlen = mp->m_pkthdr.csum_iphlen; 4279 thoff = mp->m_pkthdr.csum_thlen; 4280 hoff = mp->m_pkthdr.csum_lhlen; 4281 mss = mp->m_pkthdr.tso_segsz; 4282 pktlen = mp->m_pkthdr.len; 4283 4284 if ((tdata->tx_flags & EMX_TXFLAG_FORCECTX) == 0 && 4285 tdata->csum_flags == CSUM_TSO && 4286 tdata->csum_iphlen == iphlen && 4287 tdata->csum_lhlen == hoff && 4288 tdata->csum_thlen == thoff && 4289 tdata->csum_mss == mss && 4290 tdata->csum_pktlen == pktlen) { 4291 *txd_upper = tdata->csum_txd_upper; 4292 *txd_lower = tdata->csum_txd_lower; 4293 #ifdef EMX_TSO_DEBUG 4294 tdata->tso_ctx_reused++; 4295 #endif 4296 return 0; 4297 } 4298 hlen = hoff + iphlen + thoff; 4299 4300 /* 4301 * Setup a new TSO context. 4302 */ 4303 4304 curr_txd = tdata->next_avail_tx_desc; 4305 TXD = (struct e1000_context_desc *)&tdata->tx_desc_base[curr_txd]; 4306 4307 *txd_lower = E1000_TXD_CMD_DEXT | /* Extended descr type */ 4308 E1000_TXD_DTYP_D | /* Data descr type */ 4309 E1000_TXD_CMD_TSE; /* Do TSE on this packet */ 4310 4311 /* IP and/or TCP header checksum calculation and insertion. */ 4312 *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; 4313 4314 /* 4315 * Start offset for header checksum calculation. 4316 * End offset for header checksum calculation. 4317 * Offset of place put the checksum. 4318 */ 4319 TXD->lower_setup.ip_fields.ipcss = hoff; 4320 TXD->lower_setup.ip_fields.ipcse = htole16(hoff + iphlen - 1); 4321 TXD->lower_setup.ip_fields.ipcso = hoff + offsetof(struct ip, ip_sum); 4322 4323 /* 4324 * Start offset for payload checksum calculation. 4325 * End offset for payload checksum calculation. 4326 * Offset of place to put the checksum. 4327 */ 4328 TXD->upper_setup.tcp_fields.tucss = hoff + iphlen; 4329 TXD->upper_setup.tcp_fields.tucse = 0; 4330 TXD->upper_setup.tcp_fields.tucso = 4331 hoff + iphlen + offsetof(struct tcphdr, th_sum); 4332 4333 /* 4334 * Payload size per packet w/o any headers. 4335 * Length of all headers up to payload. 4336 */ 4337 TXD->tcp_seg_setup.fields.mss = htole16(mss); 4338 TXD->tcp_seg_setup.fields.hdr_len = hlen; 4339 TXD->cmd_and_length = htole32(E1000_TXD_CMD_IFCS | 4340 E1000_TXD_CMD_DEXT | /* Extended descr */ 4341 E1000_TXD_CMD_TSE | /* TSE context */ 4342 E1000_TXD_CMD_IP | /* Do IP csum */ 4343 E1000_TXD_CMD_TCP | /* Do TCP checksum */ 4344 (pktlen - hlen)); /* Total len */ 4345 4346 /* Save the information for this TSO context */ 4347 tdata->csum_flags = CSUM_TSO; 4348 tdata->csum_lhlen = hoff; 4349 tdata->csum_iphlen = iphlen; 4350 tdata->csum_thlen = thoff; 4351 tdata->csum_mss = mss; 4352 tdata->csum_pktlen = pktlen; 4353 tdata->csum_txd_upper = *txd_upper; 4354 tdata->csum_txd_lower = *txd_lower; 4355 4356 if (++curr_txd == tdata->num_tx_desc) 4357 curr_txd = 0; 4358 4359 KKASSERT(tdata->num_tx_desc_avail > 0); 4360 tdata->num_tx_desc_avail--; 4361 4362 tdata->next_avail_tx_desc = curr_txd; 4363 return 1; 4364 } 4365 4366 static int 4367 emx_get_txring_inuse(const struct emx_softc *sc, boolean_t polling) 4368 { 4369 if (polling) 4370 return sc->tx_ring_cnt; 4371 else 4372 return 1; 4373 } 4374