1 /* 2 * Copyright (c) 2004 Joerg Sonnenberger <joerg@bec.de>. All rights reserved. 3 * 4 * Copyright (c) 2001-2008, Intel Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 * 34 * Copyright (c) 2005 The DragonFly Project. All rights reserved. 35 * 36 * This code is derived from software contributed to The DragonFly Project 37 * by Matthew Dillon <dillon@backplane.com> 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 43 * 1. Redistributions of source code must retain the above copyright 44 * notice, this list of conditions and the following disclaimer. 45 * 2. Redistributions in binary form must reproduce the above copyright 46 * notice, this list of conditions and the following disclaimer in 47 * the documentation and/or other materials provided with the 48 * distribution. 49 * 3. Neither the name of The DragonFly Project nor the names of its 50 * contributors may be used to endorse or promote products derived 51 * from this software without specific, prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 55 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 56 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 57 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 58 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 59 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 60 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 61 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 62 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 63 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 64 * SUCH DAMAGE. 65 */ 66 67 #include "opt_ifpoll.h" 68 #include "opt_emx.h" 69 70 #include <sys/param.h> 71 #include <sys/bus.h> 72 #include <sys/endian.h> 73 #include <sys/interrupt.h> 74 #include <sys/kernel.h> 75 #include <sys/ktr.h> 76 #include <sys/malloc.h> 77 #include <sys/mbuf.h> 78 #include <sys/proc.h> 79 #include <sys/rman.h> 80 #include <sys/serialize.h> 81 #include <sys/serialize2.h> 82 #include <sys/socket.h> 83 #include <sys/sockio.h> 84 #include <sys/sysctl.h> 85 #include <sys/systm.h> 86 87 #include <net/bpf.h> 88 #include <net/ethernet.h> 89 #include <net/if.h> 90 #include <net/if_arp.h> 91 #include <net/if_dl.h> 92 #include <net/if_media.h> 93 #include <net/ifq_var.h> 94 #include <net/toeplitz.h> 95 #include <net/toeplitz2.h> 96 #include <net/vlan/if_vlan_var.h> 97 #include <net/vlan/if_vlan_ether.h> 98 #include <net/if_poll.h> 99 100 #include <netinet/in_systm.h> 101 #include <netinet/in.h> 102 #include <netinet/ip.h> 103 #include <netinet/tcp.h> 104 #include <netinet/udp.h> 105 106 #include <bus/pci/pcivar.h> 107 #include <bus/pci/pcireg.h> 108 109 #include <dev/netif/ig_hal/e1000_api.h> 110 #include <dev/netif/ig_hal/e1000_82571.h> 111 #include <dev/netif/emx/if_emx.h> 112 113 #define DEBUG_HW 0 114 115 #ifdef EMX_RSS_DEBUG 116 #define EMX_RSS_DPRINTF(sc, lvl, fmt, ...) \ 117 do { \ 118 if (sc->rss_debug >= lvl) \ 119 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \ 120 } while (0) 121 #else /* !EMX_RSS_DEBUG */ 122 #define EMX_RSS_DPRINTF(sc, lvl, fmt, ...) ((void)0) 123 #endif /* EMX_RSS_DEBUG */ 124 125 #define EMX_NAME "Intel(R) PRO/1000 " 126 127 #define EMX_DEVICE(id) \ 128 { EMX_VENDOR_ID, E1000_DEV_ID_##id, EMX_NAME #id } 129 #define EMX_DEVICE_NULL { 0, 0, NULL } 130 131 static const struct emx_device { 132 uint16_t vid; 133 uint16_t did; 134 const char *desc; 135 } emx_devices[] = { 136 EMX_DEVICE(82571EB_COPPER), 137 EMX_DEVICE(82571EB_FIBER), 138 EMX_DEVICE(82571EB_SERDES), 139 EMX_DEVICE(82571EB_SERDES_DUAL), 140 EMX_DEVICE(82571EB_SERDES_QUAD), 141 EMX_DEVICE(82571EB_QUAD_COPPER), 142 EMX_DEVICE(82571EB_QUAD_COPPER_BP), 143 EMX_DEVICE(82571EB_QUAD_COPPER_LP), 144 EMX_DEVICE(82571EB_QUAD_FIBER), 145 EMX_DEVICE(82571PT_QUAD_COPPER), 146 147 EMX_DEVICE(82572EI_COPPER), 148 EMX_DEVICE(82572EI_FIBER), 149 EMX_DEVICE(82572EI_SERDES), 150 EMX_DEVICE(82572EI), 151 152 EMX_DEVICE(82573E), 153 EMX_DEVICE(82573E_IAMT), 154 EMX_DEVICE(82573L), 155 156 EMX_DEVICE(80003ES2LAN_COPPER_SPT), 157 EMX_DEVICE(80003ES2LAN_SERDES_SPT), 158 EMX_DEVICE(80003ES2LAN_COPPER_DPT), 159 EMX_DEVICE(80003ES2LAN_SERDES_DPT), 160 161 EMX_DEVICE(82574L), 162 EMX_DEVICE(82574LA), 163 164 EMX_DEVICE(PCH_LPT_I217_LM), 165 EMX_DEVICE(PCH_LPT_I217_V), 166 EMX_DEVICE(PCH_LPTLP_I218_LM), 167 EMX_DEVICE(PCH_LPTLP_I218_V), 168 169 /* required last entry */ 170 EMX_DEVICE_NULL 171 }; 172 173 static int emx_probe(device_t); 174 static int emx_attach(device_t); 175 static int emx_detach(device_t); 176 static int emx_shutdown(device_t); 177 static int emx_suspend(device_t); 178 static int emx_resume(device_t); 179 180 static void emx_init(void *); 181 static void emx_stop(struct emx_softc *); 182 static int emx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 183 static void emx_start(struct ifnet *, struct ifaltq_subque *); 184 #ifdef IFPOLL_ENABLE 185 static void emx_npoll(struct ifnet *, struct ifpoll_info *); 186 static void emx_npoll_status(struct ifnet *); 187 static void emx_npoll_tx(struct ifnet *, void *, int); 188 static void emx_npoll_rx(struct ifnet *, void *, int); 189 #endif 190 static void emx_watchdog(struct ifaltq_subque *); 191 static void emx_media_status(struct ifnet *, struct ifmediareq *); 192 static int emx_media_change(struct ifnet *); 193 static void emx_timer(void *); 194 static void emx_serialize(struct ifnet *, enum ifnet_serialize); 195 static void emx_deserialize(struct ifnet *, enum ifnet_serialize); 196 static int emx_tryserialize(struct ifnet *, enum ifnet_serialize); 197 #ifdef INVARIANTS 198 static void emx_serialize_assert(struct ifnet *, enum ifnet_serialize, 199 boolean_t); 200 #endif 201 202 static void emx_intr(void *); 203 static void emx_intr_mask(void *); 204 static void emx_intr_body(struct emx_softc *, boolean_t); 205 static void emx_rxeof(struct emx_rxdata *, int); 206 static void emx_txeof(struct emx_txdata *); 207 static void emx_tx_collect(struct emx_txdata *); 208 static void emx_tx_purge(struct emx_softc *); 209 static void emx_enable_intr(struct emx_softc *); 210 static void emx_disable_intr(struct emx_softc *); 211 212 static int emx_dma_alloc(struct emx_softc *); 213 static void emx_dma_free(struct emx_softc *); 214 static void emx_init_tx_ring(struct emx_txdata *); 215 static int emx_init_rx_ring(struct emx_rxdata *); 216 static void emx_free_tx_ring(struct emx_txdata *); 217 static void emx_free_rx_ring(struct emx_rxdata *); 218 static int emx_create_tx_ring(struct emx_txdata *); 219 static int emx_create_rx_ring(struct emx_rxdata *); 220 static void emx_destroy_tx_ring(struct emx_txdata *, int); 221 static void emx_destroy_rx_ring(struct emx_rxdata *, int); 222 static int emx_newbuf(struct emx_rxdata *, int, int); 223 static int emx_encap(struct emx_txdata *, struct mbuf **, int *, int *); 224 static int emx_txcsum(struct emx_txdata *, struct mbuf *, 225 uint32_t *, uint32_t *); 226 static int emx_tso_pullup(struct emx_txdata *, struct mbuf **); 227 static int emx_tso_setup(struct emx_txdata *, struct mbuf *, 228 uint32_t *, uint32_t *); 229 static int emx_get_txring_inuse(const struct emx_softc *, boolean_t); 230 231 static int emx_is_valid_eaddr(const uint8_t *); 232 static int emx_reset(struct emx_softc *); 233 static void emx_setup_ifp(struct emx_softc *); 234 static void emx_init_tx_unit(struct emx_softc *); 235 static void emx_init_rx_unit(struct emx_softc *); 236 static void emx_update_stats(struct emx_softc *); 237 static void emx_set_promisc(struct emx_softc *); 238 static void emx_disable_promisc(struct emx_softc *); 239 static void emx_set_multi(struct emx_softc *); 240 static void emx_update_link_status(struct emx_softc *); 241 static void emx_smartspeed(struct emx_softc *); 242 static void emx_set_itr(struct emx_softc *, uint32_t); 243 static void emx_disable_aspm(struct emx_softc *); 244 245 static void emx_print_debug_info(struct emx_softc *); 246 static void emx_print_nvm_info(struct emx_softc *); 247 static void emx_print_hw_stats(struct emx_softc *); 248 249 static int emx_sysctl_stats(SYSCTL_HANDLER_ARGS); 250 static int emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 251 static int emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS); 252 static int emx_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS); 253 static int emx_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS); 254 #ifdef IFPOLL_ENABLE 255 static int emx_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS); 256 static int emx_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS); 257 #endif 258 static void emx_add_sysctl(struct emx_softc *); 259 260 static void emx_serialize_skipmain(struct emx_softc *); 261 static void emx_deserialize_skipmain(struct emx_softc *); 262 263 /* Management and WOL Support */ 264 static void emx_get_mgmt(struct emx_softc *); 265 static void emx_rel_mgmt(struct emx_softc *); 266 static void emx_get_hw_control(struct emx_softc *); 267 static void emx_rel_hw_control(struct emx_softc *); 268 static void emx_enable_wol(device_t); 269 270 static device_method_t emx_methods[] = { 271 /* Device interface */ 272 DEVMETHOD(device_probe, emx_probe), 273 DEVMETHOD(device_attach, emx_attach), 274 DEVMETHOD(device_detach, emx_detach), 275 DEVMETHOD(device_shutdown, emx_shutdown), 276 DEVMETHOD(device_suspend, emx_suspend), 277 DEVMETHOD(device_resume, emx_resume), 278 DEVMETHOD_END 279 }; 280 281 static driver_t emx_driver = { 282 "emx", 283 emx_methods, 284 sizeof(struct emx_softc), 285 }; 286 287 static devclass_t emx_devclass; 288 289 DECLARE_DUMMY_MODULE(if_emx); 290 MODULE_DEPEND(emx, ig_hal, 1, 1, 1); 291 DRIVER_MODULE(if_emx, pci, emx_driver, emx_devclass, NULL, NULL); 292 293 /* 294 * Tunables 295 */ 296 static int emx_int_throttle_ceil = EMX_DEFAULT_ITR; 297 static int emx_rxd = EMX_DEFAULT_RXD; 298 static int emx_txd = EMX_DEFAULT_TXD; 299 static int emx_smart_pwr_down = 0; 300 static int emx_rxr = 0; 301 static int emx_txr = 1; 302 303 /* Controls whether promiscuous also shows bad packets */ 304 static int emx_debug_sbp = 0; 305 306 static int emx_82573_workaround = 1; 307 static int emx_msi_enable = 1; 308 309 TUNABLE_INT("hw.emx.int_throttle_ceil", &emx_int_throttle_ceil); 310 TUNABLE_INT("hw.emx.rxd", &emx_rxd); 311 TUNABLE_INT("hw.emx.rxr", &emx_rxr); 312 TUNABLE_INT("hw.emx.txd", &emx_txd); 313 TUNABLE_INT("hw.emx.txr", &emx_txr); 314 TUNABLE_INT("hw.emx.smart_pwr_down", &emx_smart_pwr_down); 315 TUNABLE_INT("hw.emx.sbp", &emx_debug_sbp); 316 TUNABLE_INT("hw.emx.82573_workaround", &emx_82573_workaround); 317 TUNABLE_INT("hw.emx.msi.enable", &emx_msi_enable); 318 319 /* Global used in WOL setup with multiport cards */ 320 static int emx_global_quad_port_a = 0; 321 322 /* Set this to one to display debug statistics */ 323 static int emx_display_debug_stats = 0; 324 325 #if !defined(KTR_IF_EMX) 326 #define KTR_IF_EMX KTR_ALL 327 #endif 328 KTR_INFO_MASTER(if_emx); 329 KTR_INFO(KTR_IF_EMX, if_emx, intr_beg, 0, "intr begin"); 330 KTR_INFO(KTR_IF_EMX, if_emx, intr_end, 1, "intr end"); 331 KTR_INFO(KTR_IF_EMX, if_emx, pkt_receive, 4, "rx packet"); 332 KTR_INFO(KTR_IF_EMX, if_emx, pkt_txqueue, 5, "tx packet"); 333 KTR_INFO(KTR_IF_EMX, if_emx, pkt_txclean, 6, "tx clean"); 334 #define logif(name) KTR_LOG(if_emx_ ## name) 335 336 static __inline void 337 emx_setup_rxdesc(emx_rxdesc_t *rxd, const struct emx_rxbuf *rxbuf) 338 { 339 rxd->rxd_bufaddr = htole64(rxbuf->paddr); 340 /* DD bit must be cleared */ 341 rxd->rxd_staterr = 0; 342 } 343 344 static __inline void 345 emx_rxcsum(uint32_t staterr, struct mbuf *mp) 346 { 347 /* Ignore Checksum bit is set */ 348 if (staterr & E1000_RXD_STAT_IXSM) 349 return; 350 351 if ((staterr & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == 352 E1000_RXD_STAT_IPCS) 353 mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID; 354 355 if ((staterr & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == 356 E1000_RXD_STAT_TCPCS) { 357 mp->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 358 CSUM_PSEUDO_HDR | 359 CSUM_FRAG_NOT_CHECKED; 360 mp->m_pkthdr.csum_data = htons(0xffff); 361 } 362 } 363 364 static __inline struct pktinfo * 365 emx_rssinfo(struct mbuf *m, struct pktinfo *pi, 366 uint32_t mrq, uint32_t hash, uint32_t staterr) 367 { 368 switch (mrq & EMX_RXDMRQ_RSSTYPE_MASK) { 369 case EMX_RXDMRQ_IPV4_TCP: 370 pi->pi_netisr = NETISR_IP; 371 pi->pi_flags = 0; 372 pi->pi_l3proto = IPPROTO_TCP; 373 break; 374 375 case EMX_RXDMRQ_IPV6_TCP: 376 pi->pi_netisr = NETISR_IPV6; 377 pi->pi_flags = 0; 378 pi->pi_l3proto = IPPROTO_TCP; 379 break; 380 381 case EMX_RXDMRQ_IPV4: 382 if (staterr & E1000_RXD_STAT_IXSM) 383 return NULL; 384 385 if ((staterr & 386 (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == 387 E1000_RXD_STAT_TCPCS) { 388 pi->pi_netisr = NETISR_IP; 389 pi->pi_flags = 0; 390 pi->pi_l3proto = IPPROTO_UDP; 391 break; 392 } 393 /* FALL THROUGH */ 394 default: 395 return NULL; 396 } 397 398 m->m_flags |= M_HASH; 399 m->m_pkthdr.hash = toeplitz_hash(hash); 400 return pi; 401 } 402 403 static int 404 emx_probe(device_t dev) 405 { 406 const struct emx_device *d; 407 uint16_t vid, did; 408 409 vid = pci_get_vendor(dev); 410 did = pci_get_device(dev); 411 412 for (d = emx_devices; d->desc != NULL; ++d) { 413 if (vid == d->vid && did == d->did) { 414 device_set_desc(dev, d->desc); 415 device_set_async_attach(dev, TRUE); 416 return 0; 417 } 418 } 419 return ENXIO; 420 } 421 422 static int 423 emx_attach(device_t dev) 424 { 425 struct emx_softc *sc = device_get_softc(dev); 426 int error = 0, i, throttle, msi_enable, tx_ring_max; 427 u_int intr_flags; 428 uint16_t eeprom_data, device_id, apme_mask; 429 driver_intr_t *intr_func; 430 #ifdef IFPOLL_ENABLE 431 int offset, offset_def; 432 #endif 433 434 /* 435 * Setup RX rings 436 */ 437 for (i = 0; i < EMX_NRX_RING; ++i) { 438 sc->rx_data[i].sc = sc; 439 sc->rx_data[i].idx = i; 440 } 441 442 /* 443 * Setup TX ring 444 */ 445 for (i = 0; i < EMX_NTX_RING; ++i) { 446 sc->tx_data[i].sc = sc; 447 sc->tx_data[i].idx = i; 448 } 449 450 /* 451 * Initialize serializers 452 */ 453 lwkt_serialize_init(&sc->main_serialize); 454 for (i = 0; i < EMX_NTX_RING; ++i) 455 lwkt_serialize_init(&sc->tx_data[i].tx_serialize); 456 for (i = 0; i < EMX_NRX_RING; ++i) 457 lwkt_serialize_init(&sc->rx_data[i].rx_serialize); 458 459 /* 460 * Initialize serializer array 461 */ 462 i = 0; 463 464 KKASSERT(i < EMX_NSERIALIZE); 465 sc->serializes[i++] = &sc->main_serialize; 466 467 KKASSERT(i < EMX_NSERIALIZE); 468 sc->serializes[i++] = &sc->tx_data[0].tx_serialize; 469 KKASSERT(i < EMX_NSERIALIZE); 470 sc->serializes[i++] = &sc->tx_data[1].tx_serialize; 471 472 KKASSERT(i < EMX_NSERIALIZE); 473 sc->serializes[i++] = &sc->rx_data[0].rx_serialize; 474 KKASSERT(i < EMX_NSERIALIZE); 475 sc->serializes[i++] = &sc->rx_data[1].rx_serialize; 476 477 KKASSERT(i == EMX_NSERIALIZE); 478 479 ifmedia_init(&sc->media, IFM_IMASK, emx_media_change, emx_media_status); 480 callout_init_mp(&sc->timer); 481 482 sc->dev = sc->osdep.dev = dev; 483 484 /* 485 * Determine hardware and mac type 486 */ 487 sc->hw.vendor_id = pci_get_vendor(dev); 488 sc->hw.device_id = pci_get_device(dev); 489 sc->hw.revision_id = pci_get_revid(dev); 490 sc->hw.subsystem_vendor_id = pci_get_subvendor(dev); 491 sc->hw.subsystem_device_id = pci_get_subdevice(dev); 492 493 if (e1000_set_mac_type(&sc->hw)) 494 return ENXIO; 495 496 /* Enable bus mastering */ 497 pci_enable_busmaster(dev); 498 499 /* 500 * Allocate IO memory 501 */ 502 sc->memory_rid = EMX_BAR_MEM; 503 sc->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 504 &sc->memory_rid, RF_ACTIVE); 505 if (sc->memory == NULL) { 506 device_printf(dev, "Unable to allocate bus resource: memory\n"); 507 error = ENXIO; 508 goto fail; 509 } 510 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->memory); 511 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->memory); 512 513 /* XXX This is quite goofy, it is not actually used */ 514 sc->hw.hw_addr = (uint8_t *)&sc->osdep.mem_bus_space_handle; 515 516 /* 517 * Don't enable MSI-X on 82574, see: 518 * 82574 specification update errata #15 519 * 520 * Don't enable MSI on 82571/82572, see: 521 * 82571/82572 specification update errata #63 522 */ 523 msi_enable = emx_msi_enable; 524 if (msi_enable && 525 (sc->hw.mac.type == e1000_82571 || 526 sc->hw.mac.type == e1000_82572)) 527 msi_enable = 0; 528 529 /* 530 * Allocate interrupt 531 */ 532 sc->intr_type = pci_alloc_1intr(dev, msi_enable, 533 &sc->intr_rid, &intr_flags); 534 535 if (sc->intr_type == PCI_INTR_TYPE_LEGACY) { 536 int unshared; 537 538 unshared = device_getenv_int(dev, "irq.unshared", 0); 539 if (!unshared) { 540 sc->flags |= EMX_FLAG_SHARED_INTR; 541 if (bootverbose) 542 device_printf(dev, "IRQ shared\n"); 543 } else { 544 intr_flags &= ~RF_SHAREABLE; 545 if (bootverbose) 546 device_printf(dev, "IRQ unshared\n"); 547 } 548 } 549 550 sc->intr_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->intr_rid, 551 intr_flags); 552 if (sc->intr_res == NULL) { 553 device_printf(dev, "Unable to allocate bus resource: " 554 "interrupt\n"); 555 error = ENXIO; 556 goto fail; 557 } 558 559 /* Save PCI command register for Shared Code */ 560 sc->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 561 sc->hw.back = &sc->osdep; 562 563 /* 564 * For I217/I218, we need to map the flash memory and this 565 * must happen after the MAC is identified. 566 */ 567 if (sc->hw.mac.type == e1000_pch_lpt) { 568 sc->flash_rid = EMX_BAR_FLASH; 569 570 sc->flash = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 571 &sc->flash_rid, RF_ACTIVE); 572 if (sc->flash == NULL) { 573 device_printf(dev, "Mapping of Flash failed\n"); 574 error = ENXIO; 575 goto fail; 576 } 577 sc->osdep.flash_bus_space_tag = rman_get_bustag(sc->flash); 578 sc->osdep.flash_bus_space_handle = 579 rman_get_bushandle(sc->flash); 580 581 /* 582 * This is used in the shared code 583 * XXX this goof is actually not used. 584 */ 585 sc->hw.flash_address = (uint8_t *)sc->flash; 586 } 587 588 /* Do Shared Code initialization */ 589 if (e1000_setup_init_funcs(&sc->hw, TRUE)) { 590 device_printf(dev, "Setup of Shared code failed\n"); 591 error = ENXIO; 592 goto fail; 593 } 594 e1000_get_bus_info(&sc->hw); 595 596 sc->hw.mac.autoneg = EMX_DO_AUTO_NEG; 597 sc->hw.phy.autoneg_wait_to_complete = FALSE; 598 sc->hw.phy.autoneg_advertised = EMX_AUTONEG_ADV_DEFAULT; 599 600 /* 601 * Interrupt throttle rate 602 */ 603 throttle = device_getenv_int(dev, "int_throttle_ceil", 604 emx_int_throttle_ceil); 605 if (throttle == 0) { 606 sc->int_throttle_ceil = 0; 607 } else { 608 if (throttle < 0) 609 throttle = EMX_DEFAULT_ITR; 610 611 /* Recalculate the tunable value to get the exact frequency. */ 612 throttle = 1000000000 / 256 / throttle; 613 614 /* Upper 16bits of ITR is reserved and should be zero */ 615 if (throttle & 0xffff0000) 616 throttle = 1000000000 / 256 / EMX_DEFAULT_ITR; 617 618 sc->int_throttle_ceil = 1000000000 / 256 / throttle; 619 } 620 621 e1000_init_script_state_82541(&sc->hw, TRUE); 622 e1000_set_tbi_compatibility_82543(&sc->hw, TRUE); 623 624 /* Copper options */ 625 if (sc->hw.phy.media_type == e1000_media_type_copper) { 626 sc->hw.phy.mdix = EMX_AUTO_ALL_MODES; 627 sc->hw.phy.disable_polarity_correction = FALSE; 628 sc->hw.phy.ms_type = EMX_MASTER_SLAVE; 629 } 630 631 /* Set the frame limits assuming standard ethernet sized frames. */ 632 sc->hw.mac.max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN; 633 634 /* This controls when hardware reports transmit completion status. */ 635 sc->hw.mac.report_tx_early = 1; 636 637 /* Calculate # of RX rings */ 638 sc->rx_ring_cnt = device_getenv_int(dev, "rxr", emx_rxr); 639 sc->rx_ring_cnt = if_ring_count2(sc->rx_ring_cnt, EMX_NRX_RING); 640 641 /* 642 * Calculate # of TX rings 643 * 644 * XXX 645 * I217/I218 claims to have 2 TX queues 646 * 647 * NOTE: 648 * Don't enable multiple TX queues on 82574; it always gives 649 * watchdog timeout on TX queue0, when multiple TCP streams are 650 * received. It was originally suspected that the hardware TX 651 * checksum offloading caused this watchdog timeout, since only 652 * TCP ACKs are sent during TCP receiving tests. However, even 653 * if the hardware TX checksum offloading is disable, TX queue0 654 * still will give watchdog. 655 */ 656 tx_ring_max = 1; 657 if (sc->hw.mac.type == e1000_82571 || 658 sc->hw.mac.type == e1000_82572 || 659 sc->hw.mac.type == e1000_80003es2lan) 660 tx_ring_max = EMX_NTX_RING; 661 sc->tx_ring_cnt = device_getenv_int(dev, "txr", emx_txr); 662 sc->tx_ring_cnt = if_ring_count2(sc->tx_ring_cnt, tx_ring_max); 663 664 /* Allocate RX/TX rings' busdma(9) stuffs */ 665 error = emx_dma_alloc(sc); 666 if (error) 667 goto fail; 668 669 /* Allocate multicast array memory. */ 670 sc->mta = kmalloc(ETH_ADDR_LEN * EMX_MCAST_ADDR_MAX, 671 M_DEVBUF, M_WAITOK); 672 673 /* Indicate SOL/IDER usage */ 674 if (e1000_check_reset_block(&sc->hw)) { 675 device_printf(dev, 676 "PHY reset is blocked due to SOL/IDER session.\n"); 677 } 678 679 /* Disable EEE on I217/I218 */ 680 sc->hw.dev_spec.ich8lan.eee_disable = 1; 681 682 /* 683 * Start from a known state, this is important in reading the 684 * nvm and mac from that. 685 */ 686 e1000_reset_hw(&sc->hw); 687 688 /* Make sure we have a good EEPROM before we read from it */ 689 if (e1000_validate_nvm_checksum(&sc->hw) < 0) { 690 /* 691 * Some PCI-E parts fail the first check due to 692 * the link being in sleep state, call it again, 693 * if it fails a second time its a real issue. 694 */ 695 if (e1000_validate_nvm_checksum(&sc->hw) < 0) { 696 device_printf(dev, 697 "The EEPROM Checksum Is Not Valid\n"); 698 error = EIO; 699 goto fail; 700 } 701 } 702 703 /* Copy the permanent MAC address out of the EEPROM */ 704 if (e1000_read_mac_addr(&sc->hw) < 0) { 705 device_printf(dev, "EEPROM read error while reading MAC" 706 " address\n"); 707 error = EIO; 708 goto fail; 709 } 710 if (!emx_is_valid_eaddr(sc->hw.mac.addr)) { 711 device_printf(dev, "Invalid MAC address\n"); 712 error = EIO; 713 goto fail; 714 } 715 716 /* Determine if we have to control management hardware */ 717 if (e1000_enable_mng_pass_thru(&sc->hw)) 718 sc->flags |= EMX_FLAG_HAS_MGMT; 719 720 /* 721 * Setup Wake-on-Lan 722 */ 723 apme_mask = EMX_EEPROM_APME; 724 eeprom_data = 0; 725 switch (sc->hw.mac.type) { 726 case e1000_82573: 727 sc->flags |= EMX_FLAG_HAS_AMT; 728 /* FALL THROUGH */ 729 730 case e1000_82571: 731 case e1000_82572: 732 case e1000_80003es2lan: 733 if (sc->hw.bus.func == 1) { 734 e1000_read_nvm(&sc->hw, 735 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 736 } else { 737 e1000_read_nvm(&sc->hw, 738 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 739 } 740 break; 741 742 default: 743 e1000_read_nvm(&sc->hw, 744 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 745 break; 746 } 747 if (eeprom_data & apme_mask) 748 sc->wol = E1000_WUFC_MAG | E1000_WUFC_MC; 749 750 /* 751 * We have the eeprom settings, now apply the special cases 752 * where the eeprom may be wrong or the board won't support 753 * wake on lan on a particular port 754 */ 755 device_id = pci_get_device(dev); 756 switch (device_id) { 757 case E1000_DEV_ID_82571EB_FIBER: 758 /* 759 * Wake events only supported on port A for dual fiber 760 * regardless of eeprom setting 761 */ 762 if (E1000_READ_REG(&sc->hw, E1000_STATUS) & 763 E1000_STATUS_FUNC_1) 764 sc->wol = 0; 765 break; 766 767 case E1000_DEV_ID_82571EB_QUAD_COPPER: 768 case E1000_DEV_ID_82571EB_QUAD_FIBER: 769 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 770 /* if quad port sc, disable WoL on all but port A */ 771 if (emx_global_quad_port_a != 0) 772 sc->wol = 0; 773 /* Reset for multiple quad port adapters */ 774 if (++emx_global_quad_port_a == 4) 775 emx_global_quad_port_a = 0; 776 break; 777 } 778 779 /* XXX disable wol */ 780 sc->wol = 0; 781 782 #ifdef IFPOLL_ENABLE 783 /* 784 * NPOLLING RX CPU offset 785 */ 786 if (sc->rx_ring_cnt == ncpus2) { 787 offset = 0; 788 } else { 789 offset_def = (sc->rx_ring_cnt * device_get_unit(dev)) % ncpus2; 790 offset = device_getenv_int(dev, "npoll.rxoff", offset_def); 791 if (offset >= ncpus2 || 792 offset % sc->rx_ring_cnt != 0) { 793 device_printf(dev, "invalid npoll.rxoff %d, use %d\n", 794 offset, offset_def); 795 offset = offset_def; 796 } 797 } 798 sc->rx_npoll_off = offset; 799 800 /* 801 * NPOLLING TX CPU offset 802 */ 803 if (sc->tx_ring_cnt == ncpus2) { 804 offset = 0; 805 } else { 806 offset_def = (sc->tx_ring_cnt * device_get_unit(dev)) % ncpus2; 807 offset = device_getenv_int(dev, "npoll.txoff", offset_def); 808 if (offset >= ncpus2 || 809 offset % sc->tx_ring_cnt != 0) { 810 device_printf(dev, "invalid npoll.txoff %d, use %d\n", 811 offset, offset_def); 812 offset = offset_def; 813 } 814 } 815 sc->tx_npoll_off = offset; 816 #endif 817 sc->tx_ring_inuse = emx_get_txring_inuse(sc, FALSE); 818 819 /* Setup OS specific network interface */ 820 emx_setup_ifp(sc); 821 822 /* Add sysctl tree, must after em_setup_ifp() */ 823 emx_add_sysctl(sc); 824 825 /* Reset the hardware */ 826 error = emx_reset(sc); 827 if (error) { 828 /* 829 * Some 82573 parts fail the first reset, call it again, 830 * if it fails a second time its a real issue. 831 */ 832 error = emx_reset(sc); 833 if (error) { 834 device_printf(dev, "Unable to reset the hardware\n"); 835 ether_ifdetach(&sc->arpcom.ac_if); 836 goto fail; 837 } 838 } 839 840 /* Initialize statistics */ 841 emx_update_stats(sc); 842 843 sc->hw.mac.get_link_status = 1; 844 emx_update_link_status(sc); 845 846 /* Non-AMT based hardware can now take control from firmware */ 847 if ((sc->flags & (EMX_FLAG_HAS_MGMT | EMX_FLAG_HAS_AMT)) == 848 EMX_FLAG_HAS_MGMT) 849 emx_get_hw_control(sc); 850 851 /* 852 * Missing Interrupt Following ICR read: 853 * 854 * 82571/82572 specification update errata #76 855 * 82573 specification update errata #31 856 * 82574 specification update errata #12 857 */ 858 intr_func = emx_intr; 859 if ((sc->flags & EMX_FLAG_SHARED_INTR) && 860 (sc->hw.mac.type == e1000_82571 || 861 sc->hw.mac.type == e1000_82572 || 862 sc->hw.mac.type == e1000_82573 || 863 sc->hw.mac.type == e1000_82574)) 864 intr_func = emx_intr_mask; 865 866 error = bus_setup_intr(dev, sc->intr_res, INTR_MPSAFE, intr_func, sc, 867 &sc->intr_tag, &sc->main_serialize); 868 if (error) { 869 device_printf(dev, "Failed to register interrupt handler"); 870 ether_ifdetach(&sc->arpcom.ac_if); 871 goto fail; 872 } 873 return (0); 874 fail: 875 emx_detach(dev); 876 return (error); 877 } 878 879 static int 880 emx_detach(device_t dev) 881 { 882 struct emx_softc *sc = device_get_softc(dev); 883 884 if (device_is_attached(dev)) { 885 struct ifnet *ifp = &sc->arpcom.ac_if; 886 887 ifnet_serialize_all(ifp); 888 889 emx_stop(sc); 890 891 e1000_phy_hw_reset(&sc->hw); 892 893 emx_rel_mgmt(sc); 894 emx_rel_hw_control(sc); 895 896 if (sc->wol) { 897 E1000_WRITE_REG(&sc->hw, E1000_WUC, E1000_WUC_PME_EN); 898 E1000_WRITE_REG(&sc->hw, E1000_WUFC, sc->wol); 899 emx_enable_wol(dev); 900 } 901 902 bus_teardown_intr(dev, sc->intr_res, sc->intr_tag); 903 904 ifnet_deserialize_all(ifp); 905 906 ether_ifdetach(ifp); 907 } else if (sc->memory != NULL) { 908 emx_rel_hw_control(sc); 909 } 910 911 ifmedia_removeall(&sc->media); 912 bus_generic_detach(dev); 913 914 if (sc->intr_res != NULL) { 915 bus_release_resource(dev, SYS_RES_IRQ, sc->intr_rid, 916 sc->intr_res); 917 } 918 919 if (sc->intr_type == PCI_INTR_TYPE_MSI) 920 pci_release_msi(dev); 921 922 if (sc->memory != NULL) { 923 bus_release_resource(dev, SYS_RES_MEMORY, sc->memory_rid, 924 sc->memory); 925 } 926 927 if (sc->flash != NULL) { 928 bus_release_resource(dev, SYS_RES_MEMORY, sc->flash_rid, 929 sc->flash); 930 } 931 932 emx_dma_free(sc); 933 934 /* Free sysctl tree */ 935 if (sc->sysctl_tree != NULL) 936 sysctl_ctx_free(&sc->sysctl_ctx); 937 938 if (sc->mta != NULL) 939 kfree(sc->mta, M_DEVBUF); 940 941 return (0); 942 } 943 944 static int 945 emx_shutdown(device_t dev) 946 { 947 return emx_suspend(dev); 948 } 949 950 static int 951 emx_suspend(device_t dev) 952 { 953 struct emx_softc *sc = device_get_softc(dev); 954 struct ifnet *ifp = &sc->arpcom.ac_if; 955 956 ifnet_serialize_all(ifp); 957 958 emx_stop(sc); 959 960 emx_rel_mgmt(sc); 961 emx_rel_hw_control(sc); 962 963 if (sc->wol) { 964 E1000_WRITE_REG(&sc->hw, E1000_WUC, E1000_WUC_PME_EN); 965 E1000_WRITE_REG(&sc->hw, E1000_WUFC, sc->wol); 966 emx_enable_wol(dev); 967 } 968 969 ifnet_deserialize_all(ifp); 970 971 return bus_generic_suspend(dev); 972 } 973 974 static int 975 emx_resume(device_t dev) 976 { 977 struct emx_softc *sc = device_get_softc(dev); 978 struct ifnet *ifp = &sc->arpcom.ac_if; 979 int i; 980 981 ifnet_serialize_all(ifp); 982 983 emx_init(sc); 984 emx_get_mgmt(sc); 985 for (i = 0; i < sc->tx_ring_inuse; ++i) 986 ifsq_devstart_sched(sc->tx_data[i].ifsq); 987 988 ifnet_deserialize_all(ifp); 989 990 return bus_generic_resume(dev); 991 } 992 993 static void 994 emx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 995 { 996 struct emx_softc *sc = ifp->if_softc; 997 struct emx_txdata *tdata = ifsq_get_priv(ifsq); 998 struct mbuf *m_head; 999 int idx = -1, nsegs = 0; 1000 1001 KKASSERT(tdata->ifsq == ifsq); 1002 ASSERT_SERIALIZED(&tdata->tx_serialize); 1003 1004 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq)) 1005 return; 1006 1007 if (!sc->link_active || (tdata->tx_flags & EMX_TXFLAG_ENABLED) == 0) { 1008 ifsq_purge(ifsq); 1009 return; 1010 } 1011 1012 while (!ifsq_is_empty(ifsq)) { 1013 /* Now do we at least have a minimal? */ 1014 if (EMX_IS_OACTIVE(tdata)) { 1015 emx_tx_collect(tdata); 1016 if (EMX_IS_OACTIVE(tdata)) { 1017 ifsq_set_oactive(ifsq); 1018 break; 1019 } 1020 } 1021 1022 logif(pkt_txqueue); 1023 m_head = ifsq_dequeue(ifsq); 1024 if (m_head == NULL) 1025 break; 1026 1027 if (emx_encap(tdata, &m_head, &nsegs, &idx)) { 1028 IFNET_STAT_INC(ifp, oerrors, 1); 1029 emx_tx_collect(tdata); 1030 continue; 1031 } 1032 1033 if (nsegs >= tdata->tx_wreg_nsegs) { 1034 E1000_WRITE_REG(&sc->hw, E1000_TDT(tdata->idx), idx); 1035 nsegs = 0; 1036 idx = -1; 1037 } 1038 1039 /* Send a copy of the frame to the BPF listener */ 1040 ETHER_BPF_MTAP(ifp, m_head); 1041 1042 /* Set timeout in case hardware has problems transmitting. */ 1043 tdata->tx_watchdog.wd_timer = EMX_TX_TIMEOUT; 1044 } 1045 if (idx >= 0) 1046 E1000_WRITE_REG(&sc->hw, E1000_TDT(tdata->idx), idx); 1047 } 1048 1049 static int 1050 emx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) 1051 { 1052 struct emx_softc *sc = ifp->if_softc; 1053 struct ifreq *ifr = (struct ifreq *)data; 1054 uint16_t eeprom_data = 0; 1055 int max_frame_size, mask, reinit; 1056 int error = 0; 1057 1058 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1059 1060 switch (command) { 1061 case SIOCSIFMTU: 1062 switch (sc->hw.mac.type) { 1063 case e1000_82573: 1064 /* 1065 * 82573 only supports jumbo frames 1066 * if ASPM is disabled. 1067 */ 1068 e1000_read_nvm(&sc->hw, NVM_INIT_3GIO_3, 1, 1069 &eeprom_data); 1070 if (eeprom_data & NVM_WORD1A_ASPM_MASK) { 1071 max_frame_size = ETHER_MAX_LEN; 1072 break; 1073 } 1074 /* FALL THROUGH */ 1075 1076 /* Limit Jumbo Frame size */ 1077 case e1000_82571: 1078 case e1000_82572: 1079 case e1000_82574: 1080 case e1000_pch_lpt: 1081 case e1000_80003es2lan: 1082 max_frame_size = 9234; 1083 break; 1084 1085 default: 1086 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1087 break; 1088 } 1089 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1090 ETHER_CRC_LEN) { 1091 error = EINVAL; 1092 break; 1093 } 1094 1095 ifp->if_mtu = ifr->ifr_mtu; 1096 sc->hw.mac.max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + 1097 ETHER_CRC_LEN; 1098 1099 if (ifp->if_flags & IFF_RUNNING) 1100 emx_init(sc); 1101 break; 1102 1103 case SIOCSIFFLAGS: 1104 if (ifp->if_flags & IFF_UP) { 1105 if ((ifp->if_flags & IFF_RUNNING)) { 1106 if ((ifp->if_flags ^ sc->if_flags) & 1107 (IFF_PROMISC | IFF_ALLMULTI)) { 1108 emx_disable_promisc(sc); 1109 emx_set_promisc(sc); 1110 } 1111 } else { 1112 emx_init(sc); 1113 } 1114 } else if (ifp->if_flags & IFF_RUNNING) { 1115 emx_stop(sc); 1116 } 1117 sc->if_flags = ifp->if_flags; 1118 break; 1119 1120 case SIOCADDMULTI: 1121 case SIOCDELMULTI: 1122 if (ifp->if_flags & IFF_RUNNING) { 1123 emx_disable_intr(sc); 1124 emx_set_multi(sc); 1125 #ifdef IFPOLL_ENABLE 1126 if (!(ifp->if_flags & IFF_NPOLLING)) 1127 #endif 1128 emx_enable_intr(sc); 1129 } 1130 break; 1131 1132 case SIOCSIFMEDIA: 1133 /* Check SOL/IDER usage */ 1134 if (e1000_check_reset_block(&sc->hw)) { 1135 device_printf(sc->dev, "Media change is" 1136 " blocked due to SOL/IDER session.\n"); 1137 break; 1138 } 1139 /* FALL THROUGH */ 1140 1141 case SIOCGIFMEDIA: 1142 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 1143 break; 1144 1145 case SIOCSIFCAP: 1146 reinit = 0; 1147 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1148 if (mask & IFCAP_RXCSUM) { 1149 ifp->if_capenable ^= IFCAP_RXCSUM; 1150 reinit = 1; 1151 } 1152 if (mask & IFCAP_VLAN_HWTAGGING) { 1153 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1154 reinit = 1; 1155 } 1156 if (mask & IFCAP_TXCSUM) { 1157 ifp->if_capenable ^= IFCAP_TXCSUM; 1158 if (ifp->if_capenable & IFCAP_TXCSUM) 1159 ifp->if_hwassist |= EMX_CSUM_FEATURES; 1160 else 1161 ifp->if_hwassist &= ~EMX_CSUM_FEATURES; 1162 } 1163 if (mask & IFCAP_TSO) { 1164 ifp->if_capenable ^= IFCAP_TSO; 1165 if (ifp->if_capenable & IFCAP_TSO) 1166 ifp->if_hwassist |= CSUM_TSO; 1167 else 1168 ifp->if_hwassist &= ~CSUM_TSO; 1169 } 1170 if (mask & IFCAP_RSS) 1171 ifp->if_capenable ^= IFCAP_RSS; 1172 if (reinit && (ifp->if_flags & IFF_RUNNING)) 1173 emx_init(sc); 1174 break; 1175 1176 default: 1177 error = ether_ioctl(ifp, command, data); 1178 break; 1179 } 1180 return (error); 1181 } 1182 1183 static void 1184 emx_watchdog(struct ifaltq_subque *ifsq) 1185 { 1186 struct emx_txdata *tdata = ifsq_get_priv(ifsq); 1187 struct ifnet *ifp = ifsq_get_ifp(ifsq); 1188 struct emx_softc *sc = ifp->if_softc; 1189 int i; 1190 1191 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1192 1193 /* 1194 * The timer is set to 5 every time start queues a packet. 1195 * Then txeof keeps resetting it as long as it cleans at 1196 * least one descriptor. 1197 * Finally, anytime all descriptors are clean the timer is 1198 * set to 0. 1199 */ 1200 1201 if (E1000_READ_REG(&sc->hw, E1000_TDT(tdata->idx)) == 1202 E1000_READ_REG(&sc->hw, E1000_TDH(tdata->idx))) { 1203 /* 1204 * If we reach here, all TX jobs are completed and 1205 * the TX engine should have been idled for some time. 1206 * We don't need to call ifsq_devstart_sched() here. 1207 */ 1208 ifsq_clr_oactive(ifsq); 1209 tdata->tx_watchdog.wd_timer = 0; 1210 return; 1211 } 1212 1213 /* 1214 * If we are in this routine because of pause frames, then 1215 * don't reset the hardware. 1216 */ 1217 if (E1000_READ_REG(&sc->hw, E1000_STATUS) & E1000_STATUS_TXOFF) { 1218 tdata->tx_watchdog.wd_timer = EMX_TX_TIMEOUT; 1219 return; 1220 } 1221 1222 if_printf(ifp, "TX %d watchdog timeout -- resetting\n", tdata->idx); 1223 1224 IFNET_STAT_INC(ifp, oerrors, 1); 1225 1226 emx_init(sc); 1227 for (i = 0; i < sc->tx_ring_inuse; ++i) 1228 ifsq_devstart_sched(sc->tx_data[i].ifsq); 1229 } 1230 1231 static void 1232 emx_init(void *xsc) 1233 { 1234 struct emx_softc *sc = xsc; 1235 struct ifnet *ifp = &sc->arpcom.ac_if; 1236 device_t dev = sc->dev; 1237 boolean_t polling; 1238 int i; 1239 1240 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1241 1242 emx_stop(sc); 1243 1244 /* Get the latest mac address, User can use a LAA */ 1245 bcopy(IF_LLADDR(ifp), sc->hw.mac.addr, ETHER_ADDR_LEN); 1246 1247 /* Put the address into the Receive Address Array */ 1248 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1249 1250 /* 1251 * With the 82571 sc, RAR[0] may be overwritten 1252 * when the other port is reset, we make a duplicate 1253 * in RAR[14] for that eventuality, this assures 1254 * the interface continues to function. 1255 */ 1256 if (sc->hw.mac.type == e1000_82571) { 1257 e1000_set_laa_state_82571(&sc->hw, TRUE); 1258 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 1259 E1000_RAR_ENTRIES - 1); 1260 } 1261 1262 /* Initialize the hardware */ 1263 if (emx_reset(sc)) { 1264 device_printf(dev, "Unable to reset the hardware\n"); 1265 /* XXX emx_stop()? */ 1266 return; 1267 } 1268 emx_update_link_status(sc); 1269 1270 /* Setup VLAN support, basic and offload if available */ 1271 E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN); 1272 1273 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1274 uint32_t ctrl; 1275 1276 ctrl = E1000_READ_REG(&sc->hw, E1000_CTRL); 1277 ctrl |= E1000_CTRL_VME; 1278 E1000_WRITE_REG(&sc->hw, E1000_CTRL, ctrl); 1279 } 1280 1281 /* Configure for OS presence */ 1282 emx_get_mgmt(sc); 1283 1284 polling = FALSE; 1285 #ifdef IFPOLL_ENABLE 1286 if (ifp->if_flags & IFF_NPOLLING) 1287 polling = TRUE; 1288 #endif 1289 sc->tx_ring_inuse = emx_get_txring_inuse(sc, polling); 1290 ifq_set_subq_mask(&ifp->if_snd, sc->tx_ring_inuse - 1); 1291 1292 /* Prepare transmit descriptors and buffers */ 1293 for (i = 0; i < sc->tx_ring_inuse; ++i) 1294 emx_init_tx_ring(&sc->tx_data[i]); 1295 emx_init_tx_unit(sc); 1296 1297 /* Setup Multicast table */ 1298 emx_set_multi(sc); 1299 1300 /* Prepare receive descriptors and buffers */ 1301 for (i = 0; i < sc->rx_ring_cnt; ++i) { 1302 if (emx_init_rx_ring(&sc->rx_data[i])) { 1303 device_printf(dev, 1304 "Could not setup receive structures\n"); 1305 emx_stop(sc); 1306 return; 1307 } 1308 } 1309 emx_init_rx_unit(sc); 1310 1311 /* Don't lose promiscuous settings */ 1312 emx_set_promisc(sc); 1313 1314 ifp->if_flags |= IFF_RUNNING; 1315 for (i = 0; i < sc->tx_ring_inuse; ++i) { 1316 ifsq_clr_oactive(sc->tx_data[i].ifsq); 1317 ifsq_watchdog_start(&sc->tx_data[i].tx_watchdog); 1318 } 1319 1320 callout_reset(&sc->timer, hz, emx_timer, sc); 1321 e1000_clear_hw_cntrs_base_generic(&sc->hw); 1322 1323 /* MSI/X configuration for 82574 */ 1324 if (sc->hw.mac.type == e1000_82574) { 1325 int tmp; 1326 1327 tmp = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 1328 tmp |= E1000_CTRL_EXT_PBA_CLR; 1329 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, tmp); 1330 /* 1331 * XXX MSIX 1332 * Set the IVAR - interrupt vector routing. 1333 * Each nibble represents a vector, high bit 1334 * is enable, other 3 bits are the MSIX table 1335 * entry, we map RXQ0 to 0, TXQ0 to 1, and 1336 * Link (other) to 2, hence the magic number. 1337 */ 1338 E1000_WRITE_REG(&sc->hw, E1000_IVAR, 0x800A0908); 1339 } 1340 1341 /* 1342 * Only enable interrupts if we are not polling, make sure 1343 * they are off otherwise. 1344 */ 1345 if (polling) 1346 emx_disable_intr(sc); 1347 else 1348 emx_enable_intr(sc); 1349 1350 /* AMT based hardware can now take control from firmware */ 1351 if ((sc->flags & (EMX_FLAG_HAS_MGMT | EMX_FLAG_HAS_AMT)) == 1352 (EMX_FLAG_HAS_MGMT | EMX_FLAG_HAS_AMT)) 1353 emx_get_hw_control(sc); 1354 } 1355 1356 static void 1357 emx_intr(void *xsc) 1358 { 1359 emx_intr_body(xsc, TRUE); 1360 } 1361 1362 static void 1363 emx_intr_body(struct emx_softc *sc, boolean_t chk_asserted) 1364 { 1365 struct ifnet *ifp = &sc->arpcom.ac_if; 1366 uint32_t reg_icr; 1367 1368 logif(intr_beg); 1369 ASSERT_SERIALIZED(&sc->main_serialize); 1370 1371 reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); 1372 1373 if (chk_asserted && (reg_icr & E1000_ICR_INT_ASSERTED) == 0) { 1374 logif(intr_end); 1375 return; 1376 } 1377 1378 /* 1379 * XXX: some laptops trigger several spurious interrupts 1380 * on emx(4) when in the resume cycle. The ICR register 1381 * reports all-ones value in this case. Processing such 1382 * interrupts would lead to a freeze. I don't know why. 1383 */ 1384 if (reg_icr == 0xffffffff) { 1385 logif(intr_end); 1386 return; 1387 } 1388 1389 if (ifp->if_flags & IFF_RUNNING) { 1390 if (reg_icr & 1391 (E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_RXO)) { 1392 int i; 1393 1394 for (i = 0; i < sc->rx_ring_cnt; ++i) { 1395 lwkt_serialize_enter( 1396 &sc->rx_data[i].rx_serialize); 1397 emx_rxeof(&sc->rx_data[i], -1); 1398 lwkt_serialize_exit( 1399 &sc->rx_data[i].rx_serialize); 1400 } 1401 } 1402 if (reg_icr & E1000_ICR_TXDW) { 1403 struct emx_txdata *tdata = &sc->tx_data[0]; 1404 1405 lwkt_serialize_enter(&tdata->tx_serialize); 1406 emx_txeof(tdata); 1407 if (!ifsq_is_empty(tdata->ifsq)) 1408 ifsq_devstart(tdata->ifsq); 1409 lwkt_serialize_exit(&tdata->tx_serialize); 1410 } 1411 } 1412 1413 /* Link status change */ 1414 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1415 emx_serialize_skipmain(sc); 1416 1417 callout_stop(&sc->timer); 1418 sc->hw.mac.get_link_status = 1; 1419 emx_update_link_status(sc); 1420 1421 /* Deal with TX cruft when link lost */ 1422 emx_tx_purge(sc); 1423 1424 callout_reset(&sc->timer, hz, emx_timer, sc); 1425 1426 emx_deserialize_skipmain(sc); 1427 } 1428 1429 if (reg_icr & E1000_ICR_RXO) 1430 sc->rx_overruns++; 1431 1432 logif(intr_end); 1433 } 1434 1435 static void 1436 emx_intr_mask(void *xsc) 1437 { 1438 struct emx_softc *sc = xsc; 1439 1440 E1000_WRITE_REG(&sc->hw, E1000_IMC, 0xffffffff); 1441 /* 1442 * NOTE: 1443 * ICR.INT_ASSERTED bit will never be set if IMS is 0, 1444 * so don't check it. 1445 */ 1446 emx_intr_body(sc, FALSE); 1447 E1000_WRITE_REG(&sc->hw, E1000_IMS, IMS_ENABLE_MASK); 1448 } 1449 1450 static void 1451 emx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1452 { 1453 struct emx_softc *sc = ifp->if_softc; 1454 1455 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1456 1457 emx_update_link_status(sc); 1458 1459 ifmr->ifm_status = IFM_AVALID; 1460 ifmr->ifm_active = IFM_ETHER; 1461 1462 if (!sc->link_active) 1463 return; 1464 1465 ifmr->ifm_status |= IFM_ACTIVE; 1466 1467 if (sc->hw.phy.media_type == e1000_media_type_fiber || 1468 sc->hw.phy.media_type == e1000_media_type_internal_serdes) { 1469 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 1470 } else { 1471 switch (sc->link_speed) { 1472 case 10: 1473 ifmr->ifm_active |= IFM_10_T; 1474 break; 1475 case 100: 1476 ifmr->ifm_active |= IFM_100_TX; 1477 break; 1478 1479 case 1000: 1480 ifmr->ifm_active |= IFM_1000_T; 1481 break; 1482 } 1483 if (sc->link_duplex == FULL_DUPLEX) 1484 ifmr->ifm_active |= IFM_FDX; 1485 else 1486 ifmr->ifm_active |= IFM_HDX; 1487 } 1488 } 1489 1490 static int 1491 emx_media_change(struct ifnet *ifp) 1492 { 1493 struct emx_softc *sc = ifp->if_softc; 1494 struct ifmedia *ifm = &sc->media; 1495 1496 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1497 1498 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1499 return (EINVAL); 1500 1501 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1502 case IFM_AUTO: 1503 sc->hw.mac.autoneg = EMX_DO_AUTO_NEG; 1504 sc->hw.phy.autoneg_advertised = EMX_AUTONEG_ADV_DEFAULT; 1505 break; 1506 1507 case IFM_1000_LX: 1508 case IFM_1000_SX: 1509 case IFM_1000_T: 1510 sc->hw.mac.autoneg = EMX_DO_AUTO_NEG; 1511 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1512 break; 1513 1514 case IFM_100_TX: 1515 sc->hw.mac.autoneg = FALSE; 1516 sc->hw.phy.autoneg_advertised = 0; 1517 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1518 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1519 else 1520 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1521 break; 1522 1523 case IFM_10_T: 1524 sc->hw.mac.autoneg = FALSE; 1525 sc->hw.phy.autoneg_advertised = 0; 1526 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1527 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1528 else 1529 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1530 break; 1531 1532 default: 1533 if_printf(ifp, "Unsupported media type\n"); 1534 break; 1535 } 1536 1537 emx_init(sc); 1538 1539 return (0); 1540 } 1541 1542 static int 1543 emx_encap(struct emx_txdata *tdata, struct mbuf **m_headp, 1544 int *segs_used, int *idx) 1545 { 1546 bus_dma_segment_t segs[EMX_MAX_SCATTER]; 1547 bus_dmamap_t map; 1548 struct emx_txbuf *tx_buffer, *tx_buffer_mapped; 1549 struct e1000_tx_desc *ctxd = NULL; 1550 struct mbuf *m_head = *m_headp; 1551 uint32_t txd_upper, txd_lower, cmd = 0; 1552 int maxsegs, nsegs, i, j, first, last = 0, error; 1553 1554 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1555 error = emx_tso_pullup(tdata, m_headp); 1556 if (error) 1557 return error; 1558 m_head = *m_headp; 1559 } 1560 1561 txd_upper = txd_lower = 0; 1562 1563 /* 1564 * Capture the first descriptor index, this descriptor 1565 * will have the index of the EOP which is the only one 1566 * that now gets a DONE bit writeback. 1567 */ 1568 first = tdata->next_avail_tx_desc; 1569 tx_buffer = &tdata->tx_buf[first]; 1570 tx_buffer_mapped = tx_buffer; 1571 map = tx_buffer->map; 1572 1573 maxsegs = tdata->num_tx_desc_avail - EMX_TX_RESERVED; 1574 KASSERT(maxsegs >= tdata->spare_tx_desc, ("not enough spare TX desc")); 1575 if (maxsegs > EMX_MAX_SCATTER) 1576 maxsegs = EMX_MAX_SCATTER; 1577 1578 error = bus_dmamap_load_mbuf_defrag(tdata->txtag, map, m_headp, 1579 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 1580 if (error) { 1581 m_freem(*m_headp); 1582 *m_headp = NULL; 1583 return error; 1584 } 1585 bus_dmamap_sync(tdata->txtag, map, BUS_DMASYNC_PREWRITE); 1586 1587 m_head = *m_headp; 1588 tdata->tx_nsegs += nsegs; 1589 *segs_used += nsegs; 1590 1591 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1592 /* TSO will consume one TX desc */ 1593 i = emx_tso_setup(tdata, m_head, &txd_upper, &txd_lower); 1594 tdata->tx_nsegs += i; 1595 *segs_used += i; 1596 } else if (m_head->m_pkthdr.csum_flags & EMX_CSUM_FEATURES) { 1597 /* TX csum offloading will consume one TX desc */ 1598 i = emx_txcsum(tdata, m_head, &txd_upper, &txd_lower); 1599 tdata->tx_nsegs += i; 1600 *segs_used += i; 1601 } 1602 1603 /* Handle VLAN tag */ 1604 if (m_head->m_flags & M_VLANTAG) { 1605 /* Set the vlan id. */ 1606 txd_upper |= (htole16(m_head->m_pkthdr.ether_vlantag) << 16); 1607 /* Tell hardware to add tag */ 1608 txd_lower |= htole32(E1000_TXD_CMD_VLE); 1609 } 1610 1611 i = tdata->next_avail_tx_desc; 1612 1613 /* Set up our transmit descriptors */ 1614 for (j = 0; j < nsegs; j++) { 1615 tx_buffer = &tdata->tx_buf[i]; 1616 ctxd = &tdata->tx_desc_base[i]; 1617 1618 ctxd->buffer_addr = htole64(segs[j].ds_addr); 1619 ctxd->lower.data = htole32(E1000_TXD_CMD_IFCS | 1620 txd_lower | segs[j].ds_len); 1621 ctxd->upper.data = htole32(txd_upper); 1622 1623 last = i; 1624 if (++i == tdata->num_tx_desc) 1625 i = 0; 1626 } 1627 1628 tdata->next_avail_tx_desc = i; 1629 1630 KKASSERT(tdata->num_tx_desc_avail > nsegs); 1631 tdata->num_tx_desc_avail -= nsegs; 1632 1633 tx_buffer->m_head = m_head; 1634 tx_buffer_mapped->map = tx_buffer->map; 1635 tx_buffer->map = map; 1636 1637 if (tdata->tx_nsegs >= tdata->tx_intr_nsegs) { 1638 tdata->tx_nsegs = 0; 1639 1640 /* 1641 * Report Status (RS) is turned on 1642 * every tx_intr_nsegs descriptors. 1643 */ 1644 cmd = E1000_TXD_CMD_RS; 1645 1646 /* 1647 * Keep track of the descriptor, which will 1648 * be written back by hardware. 1649 */ 1650 tdata->tx_dd[tdata->tx_dd_tail] = last; 1651 EMX_INC_TXDD_IDX(tdata->tx_dd_tail); 1652 KKASSERT(tdata->tx_dd_tail != tdata->tx_dd_head); 1653 } 1654 1655 /* 1656 * Last Descriptor of Packet needs End Of Packet (EOP) 1657 */ 1658 ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | cmd); 1659 1660 /* 1661 * Defer TDT updating, until enough descriptors are setup 1662 */ 1663 *idx = i; 1664 1665 #ifdef EMX_TSS_DEBUG 1666 tdata->tx_pkts++; 1667 #endif 1668 1669 return (0); 1670 } 1671 1672 static void 1673 emx_set_promisc(struct emx_softc *sc) 1674 { 1675 struct ifnet *ifp = &sc->arpcom.ac_if; 1676 uint32_t reg_rctl; 1677 1678 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 1679 1680 if (ifp->if_flags & IFF_PROMISC) { 1681 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 1682 /* Turn this on if you want to see bad packets */ 1683 if (emx_debug_sbp) 1684 reg_rctl |= E1000_RCTL_SBP; 1685 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 1686 } else if (ifp->if_flags & IFF_ALLMULTI) { 1687 reg_rctl |= E1000_RCTL_MPE; 1688 reg_rctl &= ~E1000_RCTL_UPE; 1689 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 1690 } 1691 } 1692 1693 static void 1694 emx_disable_promisc(struct emx_softc *sc) 1695 { 1696 uint32_t reg_rctl; 1697 1698 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 1699 1700 reg_rctl &= ~E1000_RCTL_UPE; 1701 reg_rctl &= ~E1000_RCTL_MPE; 1702 reg_rctl &= ~E1000_RCTL_SBP; 1703 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 1704 } 1705 1706 static void 1707 emx_set_multi(struct emx_softc *sc) 1708 { 1709 struct ifnet *ifp = &sc->arpcom.ac_if; 1710 struct ifmultiaddr *ifma; 1711 uint32_t reg_rctl = 0; 1712 uint8_t *mta; 1713 int mcnt = 0; 1714 1715 mta = sc->mta; 1716 bzero(mta, ETH_ADDR_LEN * EMX_MCAST_ADDR_MAX); 1717 1718 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1719 if (ifma->ifma_addr->sa_family != AF_LINK) 1720 continue; 1721 1722 if (mcnt == EMX_MCAST_ADDR_MAX) 1723 break; 1724 1725 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1726 &mta[mcnt * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 1727 mcnt++; 1728 } 1729 1730 if (mcnt >= EMX_MCAST_ADDR_MAX) { 1731 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 1732 reg_rctl |= E1000_RCTL_MPE; 1733 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 1734 } else { 1735 e1000_update_mc_addr_list(&sc->hw, mta, mcnt); 1736 } 1737 } 1738 1739 /* 1740 * This routine checks for link status and updates statistics. 1741 */ 1742 static void 1743 emx_timer(void *xsc) 1744 { 1745 struct emx_softc *sc = xsc; 1746 struct ifnet *ifp = &sc->arpcom.ac_if; 1747 1748 lwkt_serialize_enter(&sc->main_serialize); 1749 1750 emx_update_link_status(sc); 1751 emx_update_stats(sc); 1752 1753 /* Reset LAA into RAR[0] on 82571 */ 1754 if (e1000_get_laa_state_82571(&sc->hw) == TRUE) 1755 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1756 1757 if (emx_display_debug_stats && (ifp->if_flags & IFF_RUNNING)) 1758 emx_print_hw_stats(sc); 1759 1760 emx_smartspeed(sc); 1761 1762 callout_reset(&sc->timer, hz, emx_timer, sc); 1763 1764 lwkt_serialize_exit(&sc->main_serialize); 1765 } 1766 1767 static void 1768 emx_update_link_status(struct emx_softc *sc) 1769 { 1770 struct e1000_hw *hw = &sc->hw; 1771 struct ifnet *ifp = &sc->arpcom.ac_if; 1772 device_t dev = sc->dev; 1773 uint32_t link_check = 0; 1774 1775 /* Get the cached link value or read phy for real */ 1776 switch (hw->phy.media_type) { 1777 case e1000_media_type_copper: 1778 if (hw->mac.get_link_status) { 1779 /* Do the work to read phy */ 1780 e1000_check_for_link(hw); 1781 link_check = !hw->mac.get_link_status; 1782 if (link_check) /* ESB2 fix */ 1783 e1000_cfg_on_link_up(hw); 1784 } else { 1785 link_check = TRUE; 1786 } 1787 break; 1788 1789 case e1000_media_type_fiber: 1790 e1000_check_for_link(hw); 1791 link_check = E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU; 1792 break; 1793 1794 case e1000_media_type_internal_serdes: 1795 e1000_check_for_link(hw); 1796 link_check = sc->hw.mac.serdes_has_link; 1797 break; 1798 1799 case e1000_media_type_unknown: 1800 default: 1801 break; 1802 } 1803 1804 /* Now check for a transition */ 1805 if (link_check && sc->link_active == 0) { 1806 e1000_get_speed_and_duplex(hw, &sc->link_speed, 1807 &sc->link_duplex); 1808 1809 /* 1810 * Check if we should enable/disable SPEED_MODE bit on 1811 * 82571EB/82572EI 1812 */ 1813 if (sc->link_speed != SPEED_1000 && 1814 (hw->mac.type == e1000_82571 || 1815 hw->mac.type == e1000_82572)) { 1816 int tarc0; 1817 1818 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 1819 tarc0 &= ~EMX_TARC_SPEED_MODE; 1820 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 1821 } 1822 if (bootverbose) { 1823 device_printf(dev, "Link is up %d Mbps %s\n", 1824 sc->link_speed, 1825 ((sc->link_duplex == FULL_DUPLEX) ? 1826 "Full Duplex" : "Half Duplex")); 1827 } 1828 sc->link_active = 1; 1829 sc->smartspeed = 0; 1830 ifp->if_baudrate = sc->link_speed * 1000000; 1831 ifp->if_link_state = LINK_STATE_UP; 1832 if_link_state_change(ifp); 1833 } else if (!link_check && sc->link_active == 1) { 1834 ifp->if_baudrate = sc->link_speed = 0; 1835 sc->link_duplex = 0; 1836 if (bootverbose) 1837 device_printf(dev, "Link is Down\n"); 1838 sc->link_active = 0; 1839 ifp->if_link_state = LINK_STATE_DOWN; 1840 if_link_state_change(ifp); 1841 } 1842 } 1843 1844 static void 1845 emx_stop(struct emx_softc *sc) 1846 { 1847 struct ifnet *ifp = &sc->arpcom.ac_if; 1848 int i; 1849 1850 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1851 1852 emx_disable_intr(sc); 1853 1854 callout_stop(&sc->timer); 1855 1856 ifp->if_flags &= ~IFF_RUNNING; 1857 for (i = 0; i < sc->tx_ring_cnt; ++i) { 1858 struct emx_txdata *tdata = &sc->tx_data[i]; 1859 1860 ifsq_clr_oactive(tdata->ifsq); 1861 ifsq_watchdog_stop(&tdata->tx_watchdog); 1862 tdata->tx_flags &= ~EMX_TXFLAG_ENABLED; 1863 } 1864 1865 /* 1866 * Disable multiple receive queues. 1867 * 1868 * NOTE: 1869 * We should disable multiple receive queues before 1870 * resetting the hardware. 1871 */ 1872 E1000_WRITE_REG(&sc->hw, E1000_MRQC, 0); 1873 1874 e1000_reset_hw(&sc->hw); 1875 E1000_WRITE_REG(&sc->hw, E1000_WUC, 0); 1876 1877 for (i = 0; i < sc->tx_ring_cnt; ++i) 1878 emx_free_tx_ring(&sc->tx_data[i]); 1879 for (i = 0; i < sc->rx_ring_cnt; ++i) 1880 emx_free_rx_ring(&sc->rx_data[i]); 1881 } 1882 1883 static int 1884 emx_reset(struct emx_softc *sc) 1885 { 1886 device_t dev = sc->dev; 1887 uint16_t rx_buffer_size; 1888 uint32_t pba; 1889 1890 /* Set up smart power down as default off on newer adapters. */ 1891 if (!emx_smart_pwr_down && 1892 (sc->hw.mac.type == e1000_82571 || 1893 sc->hw.mac.type == e1000_82572)) { 1894 uint16_t phy_tmp = 0; 1895 1896 /* Speed up time to link by disabling smart power down. */ 1897 e1000_read_phy_reg(&sc->hw, 1898 IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 1899 phy_tmp &= ~IGP02E1000_PM_SPD; 1900 e1000_write_phy_reg(&sc->hw, 1901 IGP02E1000_PHY_POWER_MGMT, phy_tmp); 1902 } 1903 1904 /* 1905 * Packet Buffer Allocation (PBA) 1906 * Writing PBA sets the receive portion of the buffer 1907 * the remainder is used for the transmit buffer. 1908 */ 1909 switch (sc->hw.mac.type) { 1910 /* Total Packet Buffer on these is 48K */ 1911 case e1000_82571: 1912 case e1000_82572: 1913 case e1000_80003es2lan: 1914 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1915 break; 1916 1917 case e1000_82573: /* 82573: Total Packet Buffer is 32K */ 1918 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 1919 break; 1920 1921 case e1000_82574: 1922 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */ 1923 break; 1924 1925 case e1000_pch_lpt: 1926 pba = E1000_PBA_26K; 1927 break; 1928 1929 default: 1930 /* Devices before 82547 had a Packet Buffer of 64K. */ 1931 if (sc->hw.mac.max_frame_size > 8192) 1932 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 1933 else 1934 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 1935 } 1936 E1000_WRITE_REG(&sc->hw, E1000_PBA, pba); 1937 1938 /* 1939 * These parameters control the automatic generation (Tx) and 1940 * response (Rx) to Ethernet PAUSE frames. 1941 * - High water mark should allow for at least two frames to be 1942 * received after sending an XOFF. 1943 * - Low water mark works best when it is very near the high water mark. 1944 * This allows the receiver to restart by sending XON when it has 1945 * drained a bit. Here we use an arbitary value of 1500 which will 1946 * restart after one full frame is pulled from the buffer. There 1947 * could be several smaller frames in the buffer and if so they will 1948 * not trigger the XON until their total number reduces the buffer 1949 * by 1500. 1950 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 1951 */ 1952 rx_buffer_size = (E1000_READ_REG(&sc->hw, E1000_PBA) & 0xffff) << 10; 1953 1954 sc->hw.fc.high_water = rx_buffer_size - 1955 roundup2(sc->hw.mac.max_frame_size, 1024); 1956 sc->hw.fc.low_water = sc->hw.fc.high_water - 1500; 1957 1958 sc->hw.fc.pause_time = EMX_FC_PAUSE_TIME; 1959 sc->hw.fc.send_xon = TRUE; 1960 sc->hw.fc.requested_mode = e1000_fc_full; 1961 1962 /* 1963 * Device specific overrides/settings 1964 */ 1965 if (sc->hw.mac.type == e1000_pch_lpt) { 1966 sc->hw.fc.high_water = 0x5C20; 1967 sc->hw.fc.low_water = 0x5048; 1968 sc->hw.fc.pause_time = 0x0650; 1969 sc->hw.fc.refresh_time = 0x0400; 1970 /* Jumbos need adjusted PBA */ 1971 if (sc->arpcom.ac_if.if_mtu > ETHERMTU) 1972 E1000_WRITE_REG(&sc->hw, E1000_PBA, 12); 1973 else 1974 E1000_WRITE_REG(&sc->hw, E1000_PBA, 26); 1975 } else if (sc->hw.mac.type == e1000_80003es2lan) { 1976 sc->hw.fc.pause_time = 0xFFFF; 1977 } 1978 1979 /* Issue a global reset */ 1980 e1000_reset_hw(&sc->hw); 1981 E1000_WRITE_REG(&sc->hw, E1000_WUC, 0); 1982 emx_disable_aspm(sc); 1983 1984 if (e1000_init_hw(&sc->hw) < 0) { 1985 device_printf(dev, "Hardware Initialization Failed\n"); 1986 return (EIO); 1987 } 1988 1989 E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN); 1990 e1000_get_phy_info(&sc->hw); 1991 e1000_check_for_link(&sc->hw); 1992 1993 return (0); 1994 } 1995 1996 static void 1997 emx_setup_ifp(struct emx_softc *sc) 1998 { 1999 struct ifnet *ifp = &sc->arpcom.ac_if; 2000 int i; 2001 2002 if_initname(ifp, device_get_name(sc->dev), 2003 device_get_unit(sc->dev)); 2004 ifp->if_softc = sc; 2005 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2006 ifp->if_init = emx_init; 2007 ifp->if_ioctl = emx_ioctl; 2008 ifp->if_start = emx_start; 2009 #ifdef IFPOLL_ENABLE 2010 ifp->if_npoll = emx_npoll; 2011 #endif 2012 ifp->if_serialize = emx_serialize; 2013 ifp->if_deserialize = emx_deserialize; 2014 ifp->if_tryserialize = emx_tryserialize; 2015 #ifdef INVARIANTS 2016 ifp->if_serialize_assert = emx_serialize_assert; 2017 #endif 2018 2019 ifq_set_maxlen(&ifp->if_snd, sc->tx_data[0].num_tx_desc - 1); 2020 ifq_set_ready(&ifp->if_snd); 2021 ifq_set_subq_cnt(&ifp->if_snd, sc->tx_ring_cnt); 2022 2023 ifp->if_mapsubq = ifq_mapsubq_mask; 2024 ifq_set_subq_mask(&ifp->if_snd, 0); 2025 2026 ether_ifattach(ifp, sc->hw.mac.addr, NULL); 2027 2028 ifp->if_capabilities = IFCAP_HWCSUM | 2029 IFCAP_VLAN_HWTAGGING | 2030 IFCAP_VLAN_MTU | 2031 IFCAP_TSO; 2032 if (sc->rx_ring_cnt > 1) 2033 ifp->if_capabilities |= IFCAP_RSS; 2034 ifp->if_capenable = ifp->if_capabilities; 2035 ifp->if_hwassist = EMX_CSUM_FEATURES | CSUM_TSO; 2036 2037 /* 2038 * Tell the upper layer(s) we support long frames. 2039 */ 2040 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2041 2042 for (i = 0; i < sc->tx_ring_cnt; ++i) { 2043 struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i); 2044 struct emx_txdata *tdata = &sc->tx_data[i]; 2045 2046 ifsq_set_cpuid(ifsq, rman_get_cpuid(sc->intr_res)); 2047 ifsq_set_priv(ifsq, tdata); 2048 ifsq_set_hw_serialize(ifsq, &tdata->tx_serialize); 2049 tdata->ifsq = ifsq; 2050 2051 ifsq_watchdog_init(&tdata->tx_watchdog, ifsq, emx_watchdog); 2052 } 2053 2054 /* 2055 * Specify the media types supported by this sc and register 2056 * callbacks to update media and link information 2057 */ 2058 if (sc->hw.phy.media_type == e1000_media_type_fiber || 2059 sc->hw.phy.media_type == e1000_media_type_internal_serdes) { 2060 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 2061 0, NULL); 2062 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 2063 } else { 2064 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 2065 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 2066 0, NULL); 2067 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 2068 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 2069 0, NULL); 2070 if (sc->hw.phy.type != e1000_phy_ife) { 2071 ifmedia_add(&sc->media, 2072 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2073 ifmedia_add(&sc->media, 2074 IFM_ETHER | IFM_1000_T, 0, NULL); 2075 } 2076 } 2077 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2078 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 2079 } 2080 2081 /* 2082 * Workaround for SmartSpeed on 82541 and 82547 controllers 2083 */ 2084 static void 2085 emx_smartspeed(struct emx_softc *sc) 2086 { 2087 uint16_t phy_tmp; 2088 2089 if (sc->link_active || sc->hw.phy.type != e1000_phy_igp || 2090 sc->hw.mac.autoneg == 0 || 2091 (sc->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) 2092 return; 2093 2094 if (sc->smartspeed == 0) { 2095 /* 2096 * If Master/Slave config fault is asserted twice, 2097 * we assume back-to-back 2098 */ 2099 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 2100 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) 2101 return; 2102 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 2103 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { 2104 e1000_read_phy_reg(&sc->hw, 2105 PHY_1000T_CTRL, &phy_tmp); 2106 if (phy_tmp & CR_1000T_MS_ENABLE) { 2107 phy_tmp &= ~CR_1000T_MS_ENABLE; 2108 e1000_write_phy_reg(&sc->hw, 2109 PHY_1000T_CTRL, phy_tmp); 2110 sc->smartspeed++; 2111 if (sc->hw.mac.autoneg && 2112 !e1000_phy_setup_autoneg(&sc->hw) && 2113 !e1000_read_phy_reg(&sc->hw, 2114 PHY_CONTROL, &phy_tmp)) { 2115 phy_tmp |= MII_CR_AUTO_NEG_EN | 2116 MII_CR_RESTART_AUTO_NEG; 2117 e1000_write_phy_reg(&sc->hw, 2118 PHY_CONTROL, phy_tmp); 2119 } 2120 } 2121 } 2122 return; 2123 } else if (sc->smartspeed == EMX_SMARTSPEED_DOWNSHIFT) { 2124 /* If still no link, perhaps using 2/3 pair cable */ 2125 e1000_read_phy_reg(&sc->hw, PHY_1000T_CTRL, &phy_tmp); 2126 phy_tmp |= CR_1000T_MS_ENABLE; 2127 e1000_write_phy_reg(&sc->hw, PHY_1000T_CTRL, phy_tmp); 2128 if (sc->hw.mac.autoneg && 2129 !e1000_phy_setup_autoneg(&sc->hw) && 2130 !e1000_read_phy_reg(&sc->hw, PHY_CONTROL, &phy_tmp)) { 2131 phy_tmp |= MII_CR_AUTO_NEG_EN | MII_CR_RESTART_AUTO_NEG; 2132 e1000_write_phy_reg(&sc->hw, PHY_CONTROL, phy_tmp); 2133 } 2134 } 2135 2136 /* Restart process after EMX_SMARTSPEED_MAX iterations */ 2137 if (sc->smartspeed++ == EMX_SMARTSPEED_MAX) 2138 sc->smartspeed = 0; 2139 } 2140 2141 static int 2142 emx_create_tx_ring(struct emx_txdata *tdata) 2143 { 2144 device_t dev = tdata->sc->dev; 2145 struct emx_txbuf *tx_buffer; 2146 int error, i, tsize, ntxd; 2147 2148 /* 2149 * Validate number of transmit descriptors. It must not exceed 2150 * hardware maximum, and must be multiple of E1000_DBA_ALIGN. 2151 */ 2152 ntxd = device_getenv_int(dev, "txd", emx_txd); 2153 if ((ntxd * sizeof(struct e1000_tx_desc)) % EMX_DBA_ALIGN != 0 || 2154 ntxd > EMX_MAX_TXD || ntxd < EMX_MIN_TXD) { 2155 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 2156 EMX_DEFAULT_TXD, ntxd); 2157 tdata->num_tx_desc = EMX_DEFAULT_TXD; 2158 } else { 2159 tdata->num_tx_desc = ntxd; 2160 } 2161 2162 /* 2163 * Allocate Transmit Descriptor ring 2164 */ 2165 tsize = roundup2(tdata->num_tx_desc * sizeof(struct e1000_tx_desc), 2166 EMX_DBA_ALIGN); 2167 tdata->tx_desc_base = bus_dmamem_coherent_any(tdata->sc->parent_dtag, 2168 EMX_DBA_ALIGN, tsize, BUS_DMA_WAITOK, 2169 &tdata->tx_desc_dtag, &tdata->tx_desc_dmap, 2170 &tdata->tx_desc_paddr); 2171 if (tdata->tx_desc_base == NULL) { 2172 device_printf(dev, "Unable to allocate tx_desc memory\n"); 2173 return ENOMEM; 2174 } 2175 2176 tsize = __VM_CACHELINE_ALIGN( 2177 sizeof(struct emx_txbuf) * tdata->num_tx_desc); 2178 tdata->tx_buf = kmalloc_cachealign(tsize, M_DEVBUF, M_WAITOK | M_ZERO); 2179 2180 /* 2181 * Create DMA tags for tx buffers 2182 */ 2183 error = bus_dma_tag_create(tdata->sc->parent_dtag, /* parent */ 2184 1, 0, /* alignment, bounds */ 2185 BUS_SPACE_MAXADDR, /* lowaddr */ 2186 BUS_SPACE_MAXADDR, /* highaddr */ 2187 NULL, NULL, /* filter, filterarg */ 2188 EMX_TSO_SIZE, /* maxsize */ 2189 EMX_MAX_SCATTER, /* nsegments */ 2190 EMX_MAX_SEGSIZE, /* maxsegsize */ 2191 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | 2192 BUS_DMA_ONEBPAGE, /* flags */ 2193 &tdata->txtag); 2194 if (error) { 2195 device_printf(dev, "Unable to allocate TX DMA tag\n"); 2196 kfree(tdata->tx_buf, M_DEVBUF); 2197 tdata->tx_buf = NULL; 2198 return error; 2199 } 2200 2201 /* 2202 * Create DMA maps for tx buffers 2203 */ 2204 for (i = 0; i < tdata->num_tx_desc; i++) { 2205 tx_buffer = &tdata->tx_buf[i]; 2206 2207 error = bus_dmamap_create(tdata->txtag, 2208 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 2209 &tx_buffer->map); 2210 if (error) { 2211 device_printf(dev, "Unable to create TX DMA map\n"); 2212 emx_destroy_tx_ring(tdata, i); 2213 return error; 2214 } 2215 } 2216 2217 /* 2218 * Setup TX parameters 2219 */ 2220 tdata->spare_tx_desc = EMX_TX_SPARE; 2221 tdata->tx_wreg_nsegs = EMX_DEFAULT_TXWREG; 2222 2223 /* 2224 * Keep following relationship between spare_tx_desc, oact_tx_desc 2225 * and tx_intr_nsegs: 2226 * (spare_tx_desc + EMX_TX_RESERVED) <= 2227 * oact_tx_desc <= EMX_TX_OACTIVE_MAX <= tx_intr_nsegs 2228 */ 2229 tdata->oact_tx_desc = tdata->num_tx_desc / 8; 2230 if (tdata->oact_tx_desc > EMX_TX_OACTIVE_MAX) 2231 tdata->oact_tx_desc = EMX_TX_OACTIVE_MAX; 2232 if (tdata->oact_tx_desc < tdata->spare_tx_desc + EMX_TX_RESERVED) 2233 tdata->oact_tx_desc = tdata->spare_tx_desc + EMX_TX_RESERVED; 2234 2235 tdata->tx_intr_nsegs = tdata->num_tx_desc / 16; 2236 if (tdata->tx_intr_nsegs < tdata->oact_tx_desc) 2237 tdata->tx_intr_nsegs = tdata->oact_tx_desc; 2238 2239 /* 2240 * Pullup extra 4bytes into the first data segment, see: 2241 * 82571/82572 specification update errata #7 2242 * 2243 * NOTE: 2244 * 4bytes instead of 2bytes, which are mentioned in the errata, 2245 * are pulled; mainly to keep rest of the data properly aligned. 2246 */ 2247 if (tdata->sc->hw.mac.type == e1000_82571 || 2248 tdata->sc->hw.mac.type == e1000_82572) 2249 tdata->tx_flags |= EMX_TXFLAG_TSO_PULLEX; 2250 2251 return (0); 2252 } 2253 2254 static void 2255 emx_init_tx_ring(struct emx_txdata *tdata) 2256 { 2257 /* Clear the old ring contents */ 2258 bzero(tdata->tx_desc_base, 2259 sizeof(struct e1000_tx_desc) * tdata->num_tx_desc); 2260 2261 /* Reset state */ 2262 tdata->next_avail_tx_desc = 0; 2263 tdata->next_tx_to_clean = 0; 2264 tdata->num_tx_desc_avail = tdata->num_tx_desc; 2265 2266 tdata->tx_flags |= EMX_TXFLAG_ENABLED; 2267 if (tdata->sc->tx_ring_inuse > 1) { 2268 tdata->tx_flags |= EMX_TXFLAG_FORCECTX; 2269 if (bootverbose) { 2270 if_printf(&tdata->sc->arpcom.ac_if, 2271 "TX %d force ctx setup\n", tdata->idx); 2272 } 2273 } 2274 } 2275 2276 static void 2277 emx_init_tx_unit(struct emx_softc *sc) 2278 { 2279 uint32_t tctl, tarc, tipg = 0; 2280 int i; 2281 2282 for (i = 0; i < sc->tx_ring_inuse; ++i) { 2283 struct emx_txdata *tdata = &sc->tx_data[i]; 2284 uint64_t bus_addr; 2285 2286 /* Setup the Base and Length of the Tx Descriptor Ring */ 2287 bus_addr = tdata->tx_desc_paddr; 2288 E1000_WRITE_REG(&sc->hw, E1000_TDLEN(i), 2289 tdata->num_tx_desc * sizeof(struct e1000_tx_desc)); 2290 E1000_WRITE_REG(&sc->hw, E1000_TDBAH(i), 2291 (uint32_t)(bus_addr >> 32)); 2292 E1000_WRITE_REG(&sc->hw, E1000_TDBAL(i), 2293 (uint32_t)bus_addr); 2294 /* Setup the HW Tx Head and Tail descriptor pointers */ 2295 E1000_WRITE_REG(&sc->hw, E1000_TDT(i), 0); 2296 E1000_WRITE_REG(&sc->hw, E1000_TDH(i), 0); 2297 } 2298 2299 /* Set the default values for the Tx Inter Packet Gap timer */ 2300 switch (sc->hw.mac.type) { 2301 case e1000_80003es2lan: 2302 tipg = DEFAULT_82543_TIPG_IPGR1; 2303 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 2304 E1000_TIPG_IPGR2_SHIFT; 2305 break; 2306 2307 default: 2308 if (sc->hw.phy.media_type == e1000_media_type_fiber || 2309 sc->hw.phy.media_type == e1000_media_type_internal_serdes) 2310 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 2311 else 2312 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 2313 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 2314 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 2315 break; 2316 } 2317 2318 E1000_WRITE_REG(&sc->hw, E1000_TIPG, tipg); 2319 2320 /* NOTE: 0 is not allowed for TIDV */ 2321 E1000_WRITE_REG(&sc->hw, E1000_TIDV, 1); 2322 E1000_WRITE_REG(&sc->hw, E1000_TADV, 0); 2323 2324 if (sc->hw.mac.type == e1000_82571 || 2325 sc->hw.mac.type == e1000_82572) { 2326 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(0)); 2327 tarc |= EMX_TARC_SPEED_MODE; 2328 E1000_WRITE_REG(&sc->hw, E1000_TARC(0), tarc); 2329 } else if (sc->hw.mac.type == e1000_80003es2lan) { 2330 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(0)); 2331 tarc |= 1; 2332 E1000_WRITE_REG(&sc->hw, E1000_TARC(0), tarc); 2333 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(1)); 2334 tarc |= 1; 2335 E1000_WRITE_REG(&sc->hw, E1000_TARC(1), tarc); 2336 } 2337 2338 /* Program the Transmit Control Register */ 2339 tctl = E1000_READ_REG(&sc->hw, E1000_TCTL); 2340 tctl &= ~E1000_TCTL_CT; 2341 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 2342 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2343 tctl |= E1000_TCTL_MULR; 2344 2345 /* This write will effectively turn on the transmit unit. */ 2346 E1000_WRITE_REG(&sc->hw, E1000_TCTL, tctl); 2347 2348 if (sc->hw.mac.type == e1000_82571 || 2349 sc->hw.mac.type == e1000_82572 || 2350 sc->hw.mac.type == e1000_80003es2lan) { 2351 /* Bit 28 of TARC1 must be cleared when MULR is enabled */ 2352 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(1)); 2353 tarc &= ~(1 << 28); 2354 E1000_WRITE_REG(&sc->hw, E1000_TARC(1), tarc); 2355 } 2356 2357 if (sc->tx_ring_inuse > 1) { 2358 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(0)); 2359 tarc &= ~EMX_TARC_COUNT_MASK; 2360 tarc |= 1; 2361 E1000_WRITE_REG(&sc->hw, E1000_TARC(0), tarc); 2362 2363 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(1)); 2364 tarc &= ~EMX_TARC_COUNT_MASK; 2365 tarc |= 1; 2366 E1000_WRITE_REG(&sc->hw, E1000_TARC(1), tarc); 2367 } 2368 } 2369 2370 static void 2371 emx_destroy_tx_ring(struct emx_txdata *tdata, int ndesc) 2372 { 2373 struct emx_txbuf *tx_buffer; 2374 int i; 2375 2376 /* Free Transmit Descriptor ring */ 2377 if (tdata->tx_desc_base) { 2378 bus_dmamap_unload(tdata->tx_desc_dtag, tdata->tx_desc_dmap); 2379 bus_dmamem_free(tdata->tx_desc_dtag, tdata->tx_desc_base, 2380 tdata->tx_desc_dmap); 2381 bus_dma_tag_destroy(tdata->tx_desc_dtag); 2382 2383 tdata->tx_desc_base = NULL; 2384 } 2385 2386 if (tdata->tx_buf == NULL) 2387 return; 2388 2389 for (i = 0; i < ndesc; i++) { 2390 tx_buffer = &tdata->tx_buf[i]; 2391 2392 KKASSERT(tx_buffer->m_head == NULL); 2393 bus_dmamap_destroy(tdata->txtag, tx_buffer->map); 2394 } 2395 bus_dma_tag_destroy(tdata->txtag); 2396 2397 kfree(tdata->tx_buf, M_DEVBUF); 2398 tdata->tx_buf = NULL; 2399 } 2400 2401 /* 2402 * The offload context needs to be set when we transfer the first 2403 * packet of a particular protocol (TCP/UDP). This routine has been 2404 * enhanced to deal with inserted VLAN headers. 2405 * 2406 * If the new packet's ether header length, ip header length and 2407 * csum offloading type are same as the previous packet, we should 2408 * avoid allocating a new csum context descriptor; mainly to take 2409 * advantage of the pipeline effect of the TX data read request. 2410 * 2411 * This function returns number of TX descrptors allocated for 2412 * csum context. 2413 */ 2414 static int 2415 emx_txcsum(struct emx_txdata *tdata, struct mbuf *mp, 2416 uint32_t *txd_upper, uint32_t *txd_lower) 2417 { 2418 struct e1000_context_desc *TXD; 2419 int curr_txd, ehdrlen, csum_flags; 2420 uint32_t cmd, hdr_len, ip_hlen; 2421 2422 csum_flags = mp->m_pkthdr.csum_flags & EMX_CSUM_FEATURES; 2423 ip_hlen = mp->m_pkthdr.csum_iphlen; 2424 ehdrlen = mp->m_pkthdr.csum_lhlen; 2425 2426 if ((tdata->tx_flags & EMX_TXFLAG_FORCECTX) == 0 && 2427 tdata->csum_lhlen == ehdrlen && tdata->csum_iphlen == ip_hlen && 2428 tdata->csum_flags == csum_flags) { 2429 /* 2430 * Same csum offload context as the previous packets; 2431 * just return. 2432 */ 2433 *txd_upper = tdata->csum_txd_upper; 2434 *txd_lower = tdata->csum_txd_lower; 2435 return 0; 2436 } 2437 2438 /* 2439 * Setup a new csum offload context. 2440 */ 2441 2442 curr_txd = tdata->next_avail_tx_desc; 2443 TXD = (struct e1000_context_desc *)&tdata->tx_desc_base[curr_txd]; 2444 2445 cmd = 0; 2446 2447 /* Setup of IP header checksum. */ 2448 if (csum_flags & CSUM_IP) { 2449 /* 2450 * Start offset for header checksum calculation. 2451 * End offset for header checksum calculation. 2452 * Offset of place to put the checksum. 2453 */ 2454 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 2455 TXD->lower_setup.ip_fields.ipcse = 2456 htole16(ehdrlen + ip_hlen - 1); 2457 TXD->lower_setup.ip_fields.ipcso = 2458 ehdrlen + offsetof(struct ip, ip_sum); 2459 cmd |= E1000_TXD_CMD_IP; 2460 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 2461 } 2462 hdr_len = ehdrlen + ip_hlen; 2463 2464 if (csum_flags & CSUM_TCP) { 2465 /* 2466 * Start offset for payload checksum calculation. 2467 * End offset for payload checksum calculation. 2468 * Offset of place to put the checksum. 2469 */ 2470 TXD->upper_setup.tcp_fields.tucss = hdr_len; 2471 TXD->upper_setup.tcp_fields.tucse = htole16(0); 2472 TXD->upper_setup.tcp_fields.tucso = 2473 hdr_len + offsetof(struct tcphdr, th_sum); 2474 cmd |= E1000_TXD_CMD_TCP; 2475 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 2476 } else if (csum_flags & CSUM_UDP) { 2477 /* 2478 * Start offset for header checksum calculation. 2479 * End offset for header checksum calculation. 2480 * Offset of place to put the checksum. 2481 */ 2482 TXD->upper_setup.tcp_fields.tucss = hdr_len; 2483 TXD->upper_setup.tcp_fields.tucse = htole16(0); 2484 TXD->upper_setup.tcp_fields.tucso = 2485 hdr_len + offsetof(struct udphdr, uh_sum); 2486 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 2487 } 2488 2489 *txd_lower = E1000_TXD_CMD_DEXT | /* Extended descr type */ 2490 E1000_TXD_DTYP_D; /* Data descr */ 2491 2492 /* Save the information for this csum offloading context */ 2493 tdata->csum_lhlen = ehdrlen; 2494 tdata->csum_iphlen = ip_hlen; 2495 tdata->csum_flags = csum_flags; 2496 tdata->csum_txd_upper = *txd_upper; 2497 tdata->csum_txd_lower = *txd_lower; 2498 2499 TXD->tcp_seg_setup.data = htole32(0); 2500 TXD->cmd_and_length = 2501 htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); 2502 2503 if (++curr_txd == tdata->num_tx_desc) 2504 curr_txd = 0; 2505 2506 KKASSERT(tdata->num_tx_desc_avail > 0); 2507 tdata->num_tx_desc_avail--; 2508 2509 tdata->next_avail_tx_desc = curr_txd; 2510 return 1; 2511 } 2512 2513 static void 2514 emx_txeof(struct emx_txdata *tdata) 2515 { 2516 struct ifnet *ifp = &tdata->sc->arpcom.ac_if; 2517 struct emx_txbuf *tx_buffer; 2518 int first, num_avail; 2519 2520 if (tdata->tx_dd_head == tdata->tx_dd_tail) 2521 return; 2522 2523 if (tdata->num_tx_desc_avail == tdata->num_tx_desc) 2524 return; 2525 2526 num_avail = tdata->num_tx_desc_avail; 2527 first = tdata->next_tx_to_clean; 2528 2529 while (tdata->tx_dd_head != tdata->tx_dd_tail) { 2530 int dd_idx = tdata->tx_dd[tdata->tx_dd_head]; 2531 struct e1000_tx_desc *tx_desc; 2532 2533 tx_desc = &tdata->tx_desc_base[dd_idx]; 2534 if (tx_desc->upper.fields.status & E1000_TXD_STAT_DD) { 2535 EMX_INC_TXDD_IDX(tdata->tx_dd_head); 2536 2537 if (++dd_idx == tdata->num_tx_desc) 2538 dd_idx = 0; 2539 2540 while (first != dd_idx) { 2541 logif(pkt_txclean); 2542 2543 num_avail++; 2544 2545 tx_buffer = &tdata->tx_buf[first]; 2546 if (tx_buffer->m_head) { 2547 IFNET_STAT_INC(ifp, opackets, 1); 2548 bus_dmamap_unload(tdata->txtag, 2549 tx_buffer->map); 2550 m_freem(tx_buffer->m_head); 2551 tx_buffer->m_head = NULL; 2552 } 2553 2554 if (++first == tdata->num_tx_desc) 2555 first = 0; 2556 } 2557 } else { 2558 break; 2559 } 2560 } 2561 tdata->next_tx_to_clean = first; 2562 tdata->num_tx_desc_avail = num_avail; 2563 2564 if (tdata->tx_dd_head == tdata->tx_dd_tail) { 2565 tdata->tx_dd_head = 0; 2566 tdata->tx_dd_tail = 0; 2567 } 2568 2569 if (!EMX_IS_OACTIVE(tdata)) { 2570 ifsq_clr_oactive(tdata->ifsq); 2571 2572 /* All clean, turn off the timer */ 2573 if (tdata->num_tx_desc_avail == tdata->num_tx_desc) 2574 tdata->tx_watchdog.wd_timer = 0; 2575 } 2576 } 2577 2578 static void 2579 emx_tx_collect(struct emx_txdata *tdata) 2580 { 2581 struct ifnet *ifp = &tdata->sc->arpcom.ac_if; 2582 struct emx_txbuf *tx_buffer; 2583 int tdh, first, num_avail, dd_idx = -1; 2584 2585 if (tdata->num_tx_desc_avail == tdata->num_tx_desc) 2586 return; 2587 2588 tdh = E1000_READ_REG(&tdata->sc->hw, E1000_TDH(tdata->idx)); 2589 if (tdh == tdata->next_tx_to_clean) 2590 return; 2591 2592 if (tdata->tx_dd_head != tdata->tx_dd_tail) 2593 dd_idx = tdata->tx_dd[tdata->tx_dd_head]; 2594 2595 num_avail = tdata->num_tx_desc_avail; 2596 first = tdata->next_tx_to_clean; 2597 2598 while (first != tdh) { 2599 logif(pkt_txclean); 2600 2601 num_avail++; 2602 2603 tx_buffer = &tdata->tx_buf[first]; 2604 if (tx_buffer->m_head) { 2605 IFNET_STAT_INC(ifp, opackets, 1); 2606 bus_dmamap_unload(tdata->txtag, 2607 tx_buffer->map); 2608 m_freem(tx_buffer->m_head); 2609 tx_buffer->m_head = NULL; 2610 } 2611 2612 if (first == dd_idx) { 2613 EMX_INC_TXDD_IDX(tdata->tx_dd_head); 2614 if (tdata->tx_dd_head == tdata->tx_dd_tail) { 2615 tdata->tx_dd_head = 0; 2616 tdata->tx_dd_tail = 0; 2617 dd_idx = -1; 2618 } else { 2619 dd_idx = tdata->tx_dd[tdata->tx_dd_head]; 2620 } 2621 } 2622 2623 if (++first == tdata->num_tx_desc) 2624 first = 0; 2625 } 2626 tdata->next_tx_to_clean = first; 2627 tdata->num_tx_desc_avail = num_avail; 2628 2629 if (!EMX_IS_OACTIVE(tdata)) { 2630 ifsq_clr_oactive(tdata->ifsq); 2631 2632 /* All clean, turn off the timer */ 2633 if (tdata->num_tx_desc_avail == tdata->num_tx_desc) 2634 tdata->tx_watchdog.wd_timer = 0; 2635 } 2636 } 2637 2638 /* 2639 * When Link is lost sometimes there is work still in the TX ring 2640 * which will result in a watchdog, rather than allow that do an 2641 * attempted cleanup and then reinit here. Note that this has been 2642 * seens mostly with fiber adapters. 2643 */ 2644 static void 2645 emx_tx_purge(struct emx_softc *sc) 2646 { 2647 int i; 2648 2649 if (sc->link_active) 2650 return; 2651 2652 for (i = 0; i < sc->tx_ring_inuse; ++i) { 2653 struct emx_txdata *tdata = &sc->tx_data[i]; 2654 2655 if (tdata->tx_watchdog.wd_timer) { 2656 emx_tx_collect(tdata); 2657 if (tdata->tx_watchdog.wd_timer) { 2658 if_printf(&sc->arpcom.ac_if, 2659 "Link lost, TX pending, reinit\n"); 2660 emx_init(sc); 2661 return; 2662 } 2663 } 2664 } 2665 } 2666 2667 static int 2668 emx_newbuf(struct emx_rxdata *rdata, int i, int init) 2669 { 2670 struct mbuf *m; 2671 bus_dma_segment_t seg; 2672 bus_dmamap_t map; 2673 struct emx_rxbuf *rx_buffer; 2674 int error, nseg; 2675 2676 m = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR); 2677 if (m == NULL) { 2678 if (init) { 2679 if_printf(&rdata->sc->arpcom.ac_if, 2680 "Unable to allocate RX mbuf\n"); 2681 } 2682 return (ENOBUFS); 2683 } 2684 m->m_len = m->m_pkthdr.len = MCLBYTES; 2685 2686 if (rdata->sc->hw.mac.max_frame_size <= MCLBYTES - ETHER_ALIGN) 2687 m_adj(m, ETHER_ALIGN); 2688 2689 error = bus_dmamap_load_mbuf_segment(rdata->rxtag, 2690 rdata->rx_sparemap, m, 2691 &seg, 1, &nseg, BUS_DMA_NOWAIT); 2692 if (error) { 2693 m_freem(m); 2694 if (init) { 2695 if_printf(&rdata->sc->arpcom.ac_if, 2696 "Unable to load RX mbuf\n"); 2697 } 2698 return (error); 2699 } 2700 2701 rx_buffer = &rdata->rx_buf[i]; 2702 if (rx_buffer->m_head != NULL) 2703 bus_dmamap_unload(rdata->rxtag, rx_buffer->map); 2704 2705 map = rx_buffer->map; 2706 rx_buffer->map = rdata->rx_sparemap; 2707 rdata->rx_sparemap = map; 2708 2709 rx_buffer->m_head = m; 2710 rx_buffer->paddr = seg.ds_addr; 2711 2712 emx_setup_rxdesc(&rdata->rx_desc[i], rx_buffer); 2713 return (0); 2714 } 2715 2716 static int 2717 emx_create_rx_ring(struct emx_rxdata *rdata) 2718 { 2719 device_t dev = rdata->sc->dev; 2720 struct emx_rxbuf *rx_buffer; 2721 int i, error, rsize, nrxd; 2722 2723 /* 2724 * Validate number of receive descriptors. It must not exceed 2725 * hardware maximum, and must be multiple of E1000_DBA_ALIGN. 2726 */ 2727 nrxd = device_getenv_int(dev, "rxd", emx_rxd); 2728 if ((nrxd * sizeof(emx_rxdesc_t)) % EMX_DBA_ALIGN != 0 || 2729 nrxd > EMX_MAX_RXD || nrxd < EMX_MIN_RXD) { 2730 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 2731 EMX_DEFAULT_RXD, nrxd); 2732 rdata->num_rx_desc = EMX_DEFAULT_RXD; 2733 } else { 2734 rdata->num_rx_desc = nrxd; 2735 } 2736 2737 /* 2738 * Allocate Receive Descriptor ring 2739 */ 2740 rsize = roundup2(rdata->num_rx_desc * sizeof(emx_rxdesc_t), 2741 EMX_DBA_ALIGN); 2742 rdata->rx_desc = bus_dmamem_coherent_any(rdata->sc->parent_dtag, 2743 EMX_DBA_ALIGN, rsize, BUS_DMA_WAITOK, 2744 &rdata->rx_desc_dtag, &rdata->rx_desc_dmap, 2745 &rdata->rx_desc_paddr); 2746 if (rdata->rx_desc == NULL) { 2747 device_printf(dev, "Unable to allocate rx_desc memory\n"); 2748 return ENOMEM; 2749 } 2750 2751 rsize = __VM_CACHELINE_ALIGN( 2752 sizeof(struct emx_rxbuf) * rdata->num_rx_desc); 2753 rdata->rx_buf = kmalloc_cachealign(rsize, M_DEVBUF, M_WAITOK | M_ZERO); 2754 2755 /* 2756 * Create DMA tag for rx buffers 2757 */ 2758 error = bus_dma_tag_create(rdata->sc->parent_dtag, /* parent */ 2759 1, 0, /* alignment, bounds */ 2760 BUS_SPACE_MAXADDR, /* lowaddr */ 2761 BUS_SPACE_MAXADDR, /* highaddr */ 2762 NULL, NULL, /* filter, filterarg */ 2763 MCLBYTES, /* maxsize */ 2764 1, /* nsegments */ 2765 MCLBYTES, /* maxsegsize */ 2766 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, /* flags */ 2767 &rdata->rxtag); 2768 if (error) { 2769 device_printf(dev, "Unable to allocate RX DMA tag\n"); 2770 kfree(rdata->rx_buf, M_DEVBUF); 2771 rdata->rx_buf = NULL; 2772 return error; 2773 } 2774 2775 /* 2776 * Create spare DMA map for rx buffers 2777 */ 2778 error = bus_dmamap_create(rdata->rxtag, BUS_DMA_WAITOK, 2779 &rdata->rx_sparemap); 2780 if (error) { 2781 device_printf(dev, "Unable to create spare RX DMA map\n"); 2782 bus_dma_tag_destroy(rdata->rxtag); 2783 kfree(rdata->rx_buf, M_DEVBUF); 2784 rdata->rx_buf = NULL; 2785 return error; 2786 } 2787 2788 /* 2789 * Create DMA maps for rx buffers 2790 */ 2791 for (i = 0; i < rdata->num_rx_desc; i++) { 2792 rx_buffer = &rdata->rx_buf[i]; 2793 2794 error = bus_dmamap_create(rdata->rxtag, BUS_DMA_WAITOK, 2795 &rx_buffer->map); 2796 if (error) { 2797 device_printf(dev, "Unable to create RX DMA map\n"); 2798 emx_destroy_rx_ring(rdata, i); 2799 return error; 2800 } 2801 } 2802 return (0); 2803 } 2804 2805 static void 2806 emx_free_rx_ring(struct emx_rxdata *rdata) 2807 { 2808 int i; 2809 2810 for (i = 0; i < rdata->num_rx_desc; i++) { 2811 struct emx_rxbuf *rx_buffer = &rdata->rx_buf[i]; 2812 2813 if (rx_buffer->m_head != NULL) { 2814 bus_dmamap_unload(rdata->rxtag, rx_buffer->map); 2815 m_freem(rx_buffer->m_head); 2816 rx_buffer->m_head = NULL; 2817 } 2818 } 2819 2820 if (rdata->fmp != NULL) 2821 m_freem(rdata->fmp); 2822 rdata->fmp = NULL; 2823 rdata->lmp = NULL; 2824 } 2825 2826 static void 2827 emx_free_tx_ring(struct emx_txdata *tdata) 2828 { 2829 int i; 2830 2831 for (i = 0; i < tdata->num_tx_desc; i++) { 2832 struct emx_txbuf *tx_buffer = &tdata->tx_buf[i]; 2833 2834 if (tx_buffer->m_head != NULL) { 2835 bus_dmamap_unload(tdata->txtag, tx_buffer->map); 2836 m_freem(tx_buffer->m_head); 2837 tx_buffer->m_head = NULL; 2838 } 2839 } 2840 2841 tdata->tx_flags &= ~EMX_TXFLAG_FORCECTX; 2842 2843 tdata->csum_flags = 0; 2844 tdata->csum_lhlen = 0; 2845 tdata->csum_iphlen = 0; 2846 tdata->csum_thlen = 0; 2847 tdata->csum_mss = 0; 2848 tdata->csum_pktlen = 0; 2849 2850 tdata->tx_dd_head = 0; 2851 tdata->tx_dd_tail = 0; 2852 tdata->tx_nsegs = 0; 2853 } 2854 2855 static int 2856 emx_init_rx_ring(struct emx_rxdata *rdata) 2857 { 2858 int i, error; 2859 2860 /* Reset descriptor ring */ 2861 bzero(rdata->rx_desc, sizeof(emx_rxdesc_t) * rdata->num_rx_desc); 2862 2863 /* Allocate new ones. */ 2864 for (i = 0; i < rdata->num_rx_desc; i++) { 2865 error = emx_newbuf(rdata, i, 1); 2866 if (error) 2867 return (error); 2868 } 2869 2870 /* Setup our descriptor pointers */ 2871 rdata->next_rx_desc_to_check = 0; 2872 2873 return (0); 2874 } 2875 2876 static void 2877 emx_init_rx_unit(struct emx_softc *sc) 2878 { 2879 struct ifnet *ifp = &sc->arpcom.ac_if; 2880 uint64_t bus_addr; 2881 uint32_t rctl, itr, rfctl; 2882 int i; 2883 2884 /* 2885 * Make sure receives are disabled while setting 2886 * up the descriptor ring 2887 */ 2888 rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 2889 E1000_WRITE_REG(&sc->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 2890 2891 /* 2892 * Set the interrupt throttling rate. Value is calculated 2893 * as ITR = 1 / (INT_THROTTLE_CEIL * 256ns) 2894 */ 2895 if (sc->int_throttle_ceil) 2896 itr = 1000000000 / 256 / sc->int_throttle_ceil; 2897 else 2898 itr = 0; 2899 emx_set_itr(sc, itr); 2900 2901 /* Use extended RX descriptor */ 2902 rfctl = E1000_RFCTL_EXTEN; 2903 2904 /* Disable accelerated ackknowledge */ 2905 if (sc->hw.mac.type == e1000_82574) 2906 rfctl |= E1000_RFCTL_ACK_DIS; 2907 2908 E1000_WRITE_REG(&sc->hw, E1000_RFCTL, rfctl); 2909 2910 /* 2911 * Receive Checksum Offload for TCP and UDP 2912 * 2913 * Checksum offloading is also enabled if multiple receive 2914 * queue is to be supported, since we need it to figure out 2915 * packet type. 2916 */ 2917 if ((ifp->if_capenable & IFCAP_RXCSUM) || 2918 sc->rx_ring_cnt > 1) { 2919 uint32_t rxcsum; 2920 2921 rxcsum = E1000_READ_REG(&sc->hw, E1000_RXCSUM); 2922 2923 /* 2924 * NOTE: 2925 * PCSD must be enabled to enable multiple 2926 * receive queues. 2927 */ 2928 rxcsum |= E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL | 2929 E1000_RXCSUM_PCSD; 2930 E1000_WRITE_REG(&sc->hw, E1000_RXCSUM, rxcsum); 2931 } 2932 2933 /* 2934 * Configure multiple receive queue (RSS) 2935 */ 2936 if (sc->rx_ring_cnt > 1) { 2937 uint8_t key[EMX_NRSSRK * EMX_RSSRK_SIZE]; 2938 uint32_t reta; 2939 2940 KASSERT(sc->rx_ring_cnt == EMX_NRX_RING, 2941 ("invalid number of RX ring (%d)", sc->rx_ring_cnt)); 2942 2943 /* 2944 * NOTE: 2945 * When we reach here, RSS has already been disabled 2946 * in emx_stop(), so we could safely configure RSS key 2947 * and redirect table. 2948 */ 2949 2950 /* 2951 * Configure RSS key 2952 */ 2953 toeplitz_get_key(key, sizeof(key)); 2954 for (i = 0; i < EMX_NRSSRK; ++i) { 2955 uint32_t rssrk; 2956 2957 rssrk = EMX_RSSRK_VAL(key, i); 2958 EMX_RSS_DPRINTF(sc, 1, "rssrk%d 0x%08x\n", i, rssrk); 2959 2960 E1000_WRITE_REG(&sc->hw, E1000_RSSRK(i), rssrk); 2961 } 2962 2963 /* 2964 * Configure RSS redirect table in following fashion: 2965 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] 2966 */ 2967 reta = 0; 2968 for (i = 0; i < EMX_RETA_SIZE; ++i) { 2969 uint32_t q; 2970 2971 q = (i % sc->rx_ring_cnt) << EMX_RETA_RINGIDX_SHIFT; 2972 reta |= q << (8 * i); 2973 } 2974 EMX_RSS_DPRINTF(sc, 1, "reta 0x%08x\n", reta); 2975 2976 for (i = 0; i < EMX_NRETA; ++i) 2977 E1000_WRITE_REG(&sc->hw, E1000_RETA(i), reta); 2978 2979 /* 2980 * Enable multiple receive queues. 2981 * Enable IPv4 RSS standard hash functions. 2982 * Disable RSS interrupt. 2983 */ 2984 E1000_WRITE_REG(&sc->hw, E1000_MRQC, 2985 E1000_MRQC_ENABLE_RSS_2Q | 2986 E1000_MRQC_RSS_FIELD_IPV4_TCP | 2987 E1000_MRQC_RSS_FIELD_IPV4); 2988 } 2989 2990 /* 2991 * XXX TEMPORARY WORKAROUND: on some systems with 82573 2992 * long latencies are observed, like Lenovo X60. This 2993 * change eliminates the problem, but since having positive 2994 * values in RDTR is a known source of problems on other 2995 * platforms another solution is being sought. 2996 */ 2997 if (emx_82573_workaround && sc->hw.mac.type == e1000_82573) { 2998 E1000_WRITE_REG(&sc->hw, E1000_RADV, EMX_RADV_82573); 2999 E1000_WRITE_REG(&sc->hw, E1000_RDTR, EMX_RDTR_82573); 3000 } 3001 3002 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3003 struct emx_rxdata *rdata = &sc->rx_data[i]; 3004 3005 /* 3006 * Setup the Base and Length of the Rx Descriptor Ring 3007 */ 3008 bus_addr = rdata->rx_desc_paddr; 3009 E1000_WRITE_REG(&sc->hw, E1000_RDLEN(i), 3010 rdata->num_rx_desc * sizeof(emx_rxdesc_t)); 3011 E1000_WRITE_REG(&sc->hw, E1000_RDBAH(i), 3012 (uint32_t)(bus_addr >> 32)); 3013 E1000_WRITE_REG(&sc->hw, E1000_RDBAL(i), 3014 (uint32_t)bus_addr); 3015 3016 /* 3017 * Setup the HW Rx Head and Tail Descriptor Pointers 3018 */ 3019 E1000_WRITE_REG(&sc->hw, E1000_RDH(i), 0); 3020 E1000_WRITE_REG(&sc->hw, E1000_RDT(i), 3021 sc->rx_data[i].num_rx_desc - 1); 3022 } 3023 3024 if (sc->hw.mac.type >= e1000_pch2lan) { 3025 if (ifp->if_mtu > ETHERMTU) 3026 e1000_lv_jumbo_workaround_ich8lan(&sc->hw, TRUE); 3027 else 3028 e1000_lv_jumbo_workaround_ich8lan(&sc->hw, FALSE); 3029 } 3030 3031 /* Setup the Receive Control Register */ 3032 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 3033 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 3034 E1000_RCTL_RDMTS_HALF | E1000_RCTL_SECRC | 3035 (sc->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 3036 3037 /* Make sure VLAN Filters are off */ 3038 rctl &= ~E1000_RCTL_VFE; 3039 3040 /* Don't store bad paket */ 3041 rctl &= ~E1000_RCTL_SBP; 3042 3043 /* MCLBYTES */ 3044 rctl |= E1000_RCTL_SZ_2048; 3045 3046 if (ifp->if_mtu > ETHERMTU) 3047 rctl |= E1000_RCTL_LPE; 3048 else 3049 rctl &= ~E1000_RCTL_LPE; 3050 3051 /* Enable Receives */ 3052 E1000_WRITE_REG(&sc->hw, E1000_RCTL, rctl); 3053 } 3054 3055 static void 3056 emx_destroy_rx_ring(struct emx_rxdata *rdata, int ndesc) 3057 { 3058 struct emx_rxbuf *rx_buffer; 3059 int i; 3060 3061 /* Free Receive Descriptor ring */ 3062 if (rdata->rx_desc) { 3063 bus_dmamap_unload(rdata->rx_desc_dtag, rdata->rx_desc_dmap); 3064 bus_dmamem_free(rdata->rx_desc_dtag, rdata->rx_desc, 3065 rdata->rx_desc_dmap); 3066 bus_dma_tag_destroy(rdata->rx_desc_dtag); 3067 3068 rdata->rx_desc = NULL; 3069 } 3070 3071 if (rdata->rx_buf == NULL) 3072 return; 3073 3074 for (i = 0; i < ndesc; i++) { 3075 rx_buffer = &rdata->rx_buf[i]; 3076 3077 KKASSERT(rx_buffer->m_head == NULL); 3078 bus_dmamap_destroy(rdata->rxtag, rx_buffer->map); 3079 } 3080 bus_dmamap_destroy(rdata->rxtag, rdata->rx_sparemap); 3081 bus_dma_tag_destroy(rdata->rxtag); 3082 3083 kfree(rdata->rx_buf, M_DEVBUF); 3084 rdata->rx_buf = NULL; 3085 } 3086 3087 static void 3088 emx_rxeof(struct emx_rxdata *rdata, int count) 3089 { 3090 struct ifnet *ifp = &rdata->sc->arpcom.ac_if; 3091 uint32_t staterr; 3092 emx_rxdesc_t *current_desc; 3093 struct mbuf *mp; 3094 int i, cpuid = mycpuid; 3095 3096 i = rdata->next_rx_desc_to_check; 3097 current_desc = &rdata->rx_desc[i]; 3098 staterr = le32toh(current_desc->rxd_staterr); 3099 3100 if (!(staterr & E1000_RXD_STAT_DD)) 3101 return; 3102 3103 while ((staterr & E1000_RXD_STAT_DD) && count != 0) { 3104 struct pktinfo *pi = NULL, pi0; 3105 struct emx_rxbuf *rx_buf = &rdata->rx_buf[i]; 3106 struct mbuf *m = NULL; 3107 int eop, len; 3108 3109 logif(pkt_receive); 3110 3111 mp = rx_buf->m_head; 3112 3113 /* 3114 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT 3115 * needs to access the last received byte in the mbuf. 3116 */ 3117 bus_dmamap_sync(rdata->rxtag, rx_buf->map, 3118 BUS_DMASYNC_POSTREAD); 3119 3120 len = le16toh(current_desc->rxd_length); 3121 if (staterr & E1000_RXD_STAT_EOP) { 3122 count--; 3123 eop = 1; 3124 } else { 3125 eop = 0; 3126 } 3127 3128 if (!(staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK)) { 3129 uint16_t vlan = 0; 3130 uint32_t mrq, rss_hash; 3131 3132 /* 3133 * Save several necessary information, 3134 * before emx_newbuf() destroy it. 3135 */ 3136 if ((staterr & E1000_RXD_STAT_VP) && eop) 3137 vlan = le16toh(current_desc->rxd_vlan); 3138 3139 mrq = le32toh(current_desc->rxd_mrq); 3140 rss_hash = le32toh(current_desc->rxd_rss); 3141 3142 EMX_RSS_DPRINTF(rdata->sc, 10, 3143 "ring%d, mrq 0x%08x, rss_hash 0x%08x\n", 3144 rdata->idx, mrq, rss_hash); 3145 3146 if (emx_newbuf(rdata, i, 0) != 0) { 3147 IFNET_STAT_INC(ifp, iqdrops, 1); 3148 goto discard; 3149 } 3150 3151 /* Assign correct length to the current fragment */ 3152 mp->m_len = len; 3153 3154 if (rdata->fmp == NULL) { 3155 mp->m_pkthdr.len = len; 3156 rdata->fmp = mp; /* Store the first mbuf */ 3157 rdata->lmp = mp; 3158 } else { 3159 /* 3160 * Chain mbuf's together 3161 */ 3162 rdata->lmp->m_next = mp; 3163 rdata->lmp = rdata->lmp->m_next; 3164 rdata->fmp->m_pkthdr.len += len; 3165 } 3166 3167 if (eop) { 3168 rdata->fmp->m_pkthdr.rcvif = ifp; 3169 IFNET_STAT_INC(ifp, ipackets, 1); 3170 3171 if (ifp->if_capenable & IFCAP_RXCSUM) 3172 emx_rxcsum(staterr, rdata->fmp); 3173 3174 if (staterr & E1000_RXD_STAT_VP) { 3175 rdata->fmp->m_pkthdr.ether_vlantag = 3176 vlan; 3177 rdata->fmp->m_flags |= M_VLANTAG; 3178 } 3179 m = rdata->fmp; 3180 rdata->fmp = NULL; 3181 rdata->lmp = NULL; 3182 3183 if (ifp->if_capenable & IFCAP_RSS) { 3184 pi = emx_rssinfo(m, &pi0, mrq, 3185 rss_hash, staterr); 3186 } 3187 #ifdef EMX_RSS_DEBUG 3188 rdata->rx_pkts++; 3189 #endif 3190 } 3191 } else { 3192 IFNET_STAT_INC(ifp, ierrors, 1); 3193 discard: 3194 emx_setup_rxdesc(current_desc, rx_buf); 3195 if (rdata->fmp != NULL) { 3196 m_freem(rdata->fmp); 3197 rdata->fmp = NULL; 3198 rdata->lmp = NULL; 3199 } 3200 m = NULL; 3201 } 3202 3203 if (m != NULL) 3204 ether_input_pkt(ifp, m, pi, cpuid); 3205 3206 /* Advance our pointers to the next descriptor. */ 3207 if (++i == rdata->num_rx_desc) 3208 i = 0; 3209 3210 current_desc = &rdata->rx_desc[i]; 3211 staterr = le32toh(current_desc->rxd_staterr); 3212 } 3213 rdata->next_rx_desc_to_check = i; 3214 3215 /* Advance the E1000's Receive Queue "Tail Pointer". */ 3216 if (--i < 0) 3217 i = rdata->num_rx_desc - 1; 3218 E1000_WRITE_REG(&rdata->sc->hw, E1000_RDT(rdata->idx), i); 3219 } 3220 3221 static void 3222 emx_enable_intr(struct emx_softc *sc) 3223 { 3224 uint32_t ims_mask = IMS_ENABLE_MASK; 3225 3226 lwkt_serialize_handler_enable(&sc->main_serialize); 3227 3228 #if 0 3229 if (sc->hw.mac.type == e1000_82574) { 3230 E1000_WRITE_REG(hw, EMX_EIAC, EM_MSIX_MASK); 3231 ims_mask |= EM_MSIX_MASK; 3232 } 3233 #endif 3234 E1000_WRITE_REG(&sc->hw, E1000_IMS, ims_mask); 3235 } 3236 3237 static void 3238 emx_disable_intr(struct emx_softc *sc) 3239 { 3240 if (sc->hw.mac.type == e1000_82574) 3241 E1000_WRITE_REG(&sc->hw, EMX_EIAC, 0); 3242 E1000_WRITE_REG(&sc->hw, E1000_IMC, 0xffffffff); 3243 3244 lwkt_serialize_handler_disable(&sc->main_serialize); 3245 } 3246 3247 /* 3248 * Bit of a misnomer, what this really means is 3249 * to enable OS management of the system... aka 3250 * to disable special hardware management features 3251 */ 3252 static void 3253 emx_get_mgmt(struct emx_softc *sc) 3254 { 3255 /* A shared code workaround */ 3256 if (sc->flags & EMX_FLAG_HAS_MGMT) { 3257 int manc2h = E1000_READ_REG(&sc->hw, E1000_MANC2H); 3258 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 3259 3260 /* disable hardware interception of ARP */ 3261 manc &= ~(E1000_MANC_ARP_EN); 3262 3263 /* enable receiving management packets to the host */ 3264 manc |= E1000_MANC_EN_MNG2HOST; 3265 #define E1000_MNG2HOST_PORT_623 (1 << 5) 3266 #define E1000_MNG2HOST_PORT_664 (1 << 6) 3267 manc2h |= E1000_MNG2HOST_PORT_623; 3268 manc2h |= E1000_MNG2HOST_PORT_664; 3269 E1000_WRITE_REG(&sc->hw, E1000_MANC2H, manc2h); 3270 3271 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 3272 } 3273 } 3274 3275 /* 3276 * Give control back to hardware management 3277 * controller if there is one. 3278 */ 3279 static void 3280 emx_rel_mgmt(struct emx_softc *sc) 3281 { 3282 if (sc->flags & EMX_FLAG_HAS_MGMT) { 3283 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 3284 3285 /* re-enable hardware interception of ARP */ 3286 manc |= E1000_MANC_ARP_EN; 3287 manc &= ~E1000_MANC_EN_MNG2HOST; 3288 3289 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 3290 } 3291 } 3292 3293 /* 3294 * emx_get_hw_control() sets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3295 * For ASF and Pass Through versions of f/w this means that 3296 * the driver is loaded. For AMT version (only with 82573) 3297 * of the f/w this means that the network i/f is open. 3298 */ 3299 static void 3300 emx_get_hw_control(struct emx_softc *sc) 3301 { 3302 /* Let firmware know the driver has taken over */ 3303 if (sc->hw.mac.type == e1000_82573) { 3304 uint32_t swsm; 3305 3306 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 3307 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 3308 swsm | E1000_SWSM_DRV_LOAD); 3309 } else { 3310 uint32_t ctrl_ext; 3311 3312 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 3313 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 3314 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 3315 } 3316 sc->flags |= EMX_FLAG_HW_CTRL; 3317 } 3318 3319 /* 3320 * emx_rel_hw_control() resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3321 * For ASF and Pass Through versions of f/w this means that the 3322 * driver is no longer loaded. For AMT version (only with 82573) 3323 * of the f/w this means that the network i/f is closed. 3324 */ 3325 static void 3326 emx_rel_hw_control(struct emx_softc *sc) 3327 { 3328 if ((sc->flags & EMX_FLAG_HW_CTRL) == 0) 3329 return; 3330 sc->flags &= ~EMX_FLAG_HW_CTRL; 3331 3332 /* Let firmware taken over control of h/w */ 3333 if (sc->hw.mac.type == e1000_82573) { 3334 uint32_t swsm; 3335 3336 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 3337 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 3338 swsm & ~E1000_SWSM_DRV_LOAD); 3339 } else { 3340 uint32_t ctrl_ext; 3341 3342 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 3343 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 3344 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 3345 } 3346 } 3347 3348 static int 3349 emx_is_valid_eaddr(const uint8_t *addr) 3350 { 3351 char zero_addr[ETHER_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 }; 3352 3353 if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3354 return (FALSE); 3355 3356 return (TRUE); 3357 } 3358 3359 /* 3360 * Enable PCI Wake On Lan capability 3361 */ 3362 void 3363 emx_enable_wol(device_t dev) 3364 { 3365 uint16_t cap, status; 3366 uint8_t id; 3367 3368 /* First find the capabilities pointer*/ 3369 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 3370 3371 /* Read the PM Capabilities */ 3372 id = pci_read_config(dev, cap, 1); 3373 if (id != PCIY_PMG) /* Something wrong */ 3374 return; 3375 3376 /* 3377 * OK, we have the power capabilities, 3378 * so now get the status register 3379 */ 3380 cap += PCIR_POWER_STATUS; 3381 status = pci_read_config(dev, cap, 2); 3382 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 3383 pci_write_config(dev, cap, status, 2); 3384 } 3385 3386 static void 3387 emx_update_stats(struct emx_softc *sc) 3388 { 3389 struct ifnet *ifp = &sc->arpcom.ac_if; 3390 3391 if (sc->hw.phy.media_type == e1000_media_type_copper || 3392 (E1000_READ_REG(&sc->hw, E1000_STATUS) & E1000_STATUS_LU)) { 3393 sc->stats.symerrs += E1000_READ_REG(&sc->hw, E1000_SYMERRS); 3394 sc->stats.sec += E1000_READ_REG(&sc->hw, E1000_SEC); 3395 } 3396 sc->stats.crcerrs += E1000_READ_REG(&sc->hw, E1000_CRCERRS); 3397 sc->stats.mpc += E1000_READ_REG(&sc->hw, E1000_MPC); 3398 sc->stats.scc += E1000_READ_REG(&sc->hw, E1000_SCC); 3399 sc->stats.ecol += E1000_READ_REG(&sc->hw, E1000_ECOL); 3400 3401 sc->stats.mcc += E1000_READ_REG(&sc->hw, E1000_MCC); 3402 sc->stats.latecol += E1000_READ_REG(&sc->hw, E1000_LATECOL); 3403 sc->stats.colc += E1000_READ_REG(&sc->hw, E1000_COLC); 3404 sc->stats.dc += E1000_READ_REG(&sc->hw, E1000_DC); 3405 sc->stats.rlec += E1000_READ_REG(&sc->hw, E1000_RLEC); 3406 sc->stats.xonrxc += E1000_READ_REG(&sc->hw, E1000_XONRXC); 3407 sc->stats.xontxc += E1000_READ_REG(&sc->hw, E1000_XONTXC); 3408 sc->stats.xoffrxc += E1000_READ_REG(&sc->hw, E1000_XOFFRXC); 3409 sc->stats.xofftxc += E1000_READ_REG(&sc->hw, E1000_XOFFTXC); 3410 sc->stats.fcruc += E1000_READ_REG(&sc->hw, E1000_FCRUC); 3411 sc->stats.prc64 += E1000_READ_REG(&sc->hw, E1000_PRC64); 3412 sc->stats.prc127 += E1000_READ_REG(&sc->hw, E1000_PRC127); 3413 sc->stats.prc255 += E1000_READ_REG(&sc->hw, E1000_PRC255); 3414 sc->stats.prc511 += E1000_READ_REG(&sc->hw, E1000_PRC511); 3415 sc->stats.prc1023 += E1000_READ_REG(&sc->hw, E1000_PRC1023); 3416 sc->stats.prc1522 += E1000_READ_REG(&sc->hw, E1000_PRC1522); 3417 sc->stats.gprc += E1000_READ_REG(&sc->hw, E1000_GPRC); 3418 sc->stats.bprc += E1000_READ_REG(&sc->hw, E1000_BPRC); 3419 sc->stats.mprc += E1000_READ_REG(&sc->hw, E1000_MPRC); 3420 sc->stats.gptc += E1000_READ_REG(&sc->hw, E1000_GPTC); 3421 3422 /* For the 64-bit byte counters the low dword must be read first. */ 3423 /* Both registers clear on the read of the high dword */ 3424 3425 sc->stats.gorc += E1000_READ_REG(&sc->hw, E1000_GORCH); 3426 sc->stats.gotc += E1000_READ_REG(&sc->hw, E1000_GOTCH); 3427 3428 sc->stats.rnbc += E1000_READ_REG(&sc->hw, E1000_RNBC); 3429 sc->stats.ruc += E1000_READ_REG(&sc->hw, E1000_RUC); 3430 sc->stats.rfc += E1000_READ_REG(&sc->hw, E1000_RFC); 3431 sc->stats.roc += E1000_READ_REG(&sc->hw, E1000_ROC); 3432 sc->stats.rjc += E1000_READ_REG(&sc->hw, E1000_RJC); 3433 3434 sc->stats.tor += E1000_READ_REG(&sc->hw, E1000_TORH); 3435 sc->stats.tot += E1000_READ_REG(&sc->hw, E1000_TOTH); 3436 3437 sc->stats.tpr += E1000_READ_REG(&sc->hw, E1000_TPR); 3438 sc->stats.tpt += E1000_READ_REG(&sc->hw, E1000_TPT); 3439 sc->stats.ptc64 += E1000_READ_REG(&sc->hw, E1000_PTC64); 3440 sc->stats.ptc127 += E1000_READ_REG(&sc->hw, E1000_PTC127); 3441 sc->stats.ptc255 += E1000_READ_REG(&sc->hw, E1000_PTC255); 3442 sc->stats.ptc511 += E1000_READ_REG(&sc->hw, E1000_PTC511); 3443 sc->stats.ptc1023 += E1000_READ_REG(&sc->hw, E1000_PTC1023); 3444 sc->stats.ptc1522 += E1000_READ_REG(&sc->hw, E1000_PTC1522); 3445 sc->stats.mptc += E1000_READ_REG(&sc->hw, E1000_MPTC); 3446 sc->stats.bptc += E1000_READ_REG(&sc->hw, E1000_BPTC); 3447 3448 sc->stats.algnerrc += E1000_READ_REG(&sc->hw, E1000_ALGNERRC); 3449 sc->stats.rxerrc += E1000_READ_REG(&sc->hw, E1000_RXERRC); 3450 sc->stats.tncrs += E1000_READ_REG(&sc->hw, E1000_TNCRS); 3451 sc->stats.cexterr += E1000_READ_REG(&sc->hw, E1000_CEXTERR); 3452 sc->stats.tsctc += E1000_READ_REG(&sc->hw, E1000_TSCTC); 3453 sc->stats.tsctfc += E1000_READ_REG(&sc->hw, E1000_TSCTFC); 3454 3455 IFNET_STAT_SET(ifp, collisions, sc->stats.colc); 3456 3457 /* Rx Errors */ 3458 IFNET_STAT_SET(ifp, ierrors, 3459 sc->stats.rxerrc + sc->stats.crcerrs + sc->stats.algnerrc + 3460 sc->stats.ruc + sc->stats.roc + sc->stats.mpc + sc->stats.cexterr); 3461 3462 /* Tx Errors */ 3463 IFNET_STAT_SET(ifp, oerrors, sc->stats.ecol + sc->stats.latecol); 3464 } 3465 3466 static void 3467 emx_print_debug_info(struct emx_softc *sc) 3468 { 3469 device_t dev = sc->dev; 3470 uint8_t *hw_addr = sc->hw.hw_addr; 3471 int i; 3472 3473 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 3474 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 3475 E1000_READ_REG(&sc->hw, E1000_CTRL), 3476 E1000_READ_REG(&sc->hw, E1000_RCTL)); 3477 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 3478 ((E1000_READ_REG(&sc->hw, E1000_PBA) & 0xffff0000) >> 16),\ 3479 (E1000_READ_REG(&sc->hw, E1000_PBA) & 0xffff) ); 3480 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 3481 sc->hw.fc.high_water, sc->hw.fc.low_water); 3482 device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n", 3483 E1000_READ_REG(&sc->hw, E1000_TIDV), 3484 E1000_READ_REG(&sc->hw, E1000_TADV)); 3485 device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n", 3486 E1000_READ_REG(&sc->hw, E1000_RDTR), 3487 E1000_READ_REG(&sc->hw, E1000_RADV)); 3488 3489 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3490 device_printf(dev, "hw %d tdh = %d, hw tdt = %d\n", i, 3491 E1000_READ_REG(&sc->hw, E1000_TDH(i)), 3492 E1000_READ_REG(&sc->hw, E1000_TDT(i))); 3493 } 3494 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3495 device_printf(dev, "hw %d rdh = %d, hw rdt = %d\n", i, 3496 E1000_READ_REG(&sc->hw, E1000_RDH(i)), 3497 E1000_READ_REG(&sc->hw, E1000_RDT(i))); 3498 } 3499 3500 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3501 device_printf(dev, "TX %d Tx descriptors avail = %d\n", i, 3502 sc->tx_data[i].num_tx_desc_avail); 3503 device_printf(dev, "TX %d TSO segments = %lu\n", i, 3504 sc->tx_data[i].tso_segments); 3505 device_printf(dev, "TX %d TSO ctx reused = %lu\n", i, 3506 sc->tx_data[i].tso_ctx_reused); 3507 } 3508 } 3509 3510 static void 3511 emx_print_hw_stats(struct emx_softc *sc) 3512 { 3513 device_t dev = sc->dev; 3514 3515 device_printf(dev, "Excessive collisions = %lld\n", 3516 (long long)sc->stats.ecol); 3517 #if (DEBUG_HW > 0) /* Dont output these errors normally */ 3518 device_printf(dev, "Symbol errors = %lld\n", 3519 (long long)sc->stats.symerrs); 3520 #endif 3521 device_printf(dev, "Sequence errors = %lld\n", 3522 (long long)sc->stats.sec); 3523 device_printf(dev, "Defer count = %lld\n", 3524 (long long)sc->stats.dc); 3525 device_printf(dev, "Missed Packets = %lld\n", 3526 (long long)sc->stats.mpc); 3527 device_printf(dev, "Receive No Buffers = %lld\n", 3528 (long long)sc->stats.rnbc); 3529 /* RLEC is inaccurate on some hardware, calculate our own. */ 3530 device_printf(dev, "Receive Length Errors = %lld\n", 3531 ((long long)sc->stats.roc + (long long)sc->stats.ruc)); 3532 device_printf(dev, "Receive errors = %lld\n", 3533 (long long)sc->stats.rxerrc); 3534 device_printf(dev, "Crc errors = %lld\n", 3535 (long long)sc->stats.crcerrs); 3536 device_printf(dev, "Alignment errors = %lld\n", 3537 (long long)sc->stats.algnerrc); 3538 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 3539 (long long)sc->stats.cexterr); 3540 device_printf(dev, "RX overruns = %ld\n", sc->rx_overruns); 3541 device_printf(dev, "XON Rcvd = %lld\n", 3542 (long long)sc->stats.xonrxc); 3543 device_printf(dev, "XON Xmtd = %lld\n", 3544 (long long)sc->stats.xontxc); 3545 device_printf(dev, "XOFF Rcvd = %lld\n", 3546 (long long)sc->stats.xoffrxc); 3547 device_printf(dev, "XOFF Xmtd = %lld\n", 3548 (long long)sc->stats.xofftxc); 3549 device_printf(dev, "Good Packets Rcvd = %lld\n", 3550 (long long)sc->stats.gprc); 3551 device_printf(dev, "Good Packets Xmtd = %lld\n", 3552 (long long)sc->stats.gptc); 3553 } 3554 3555 static void 3556 emx_print_nvm_info(struct emx_softc *sc) 3557 { 3558 uint16_t eeprom_data; 3559 int i, j, row = 0; 3560 3561 /* Its a bit crude, but it gets the job done */ 3562 kprintf("\nInterface EEPROM Dump:\n"); 3563 kprintf("Offset\n0x0000 "); 3564 for (i = 0, j = 0; i < 32; i++, j++) { 3565 if (j == 8) { /* Make the offset block */ 3566 j = 0; ++row; 3567 kprintf("\n0x00%x0 ",row); 3568 } 3569 e1000_read_nvm(&sc->hw, i, 1, &eeprom_data); 3570 kprintf("%04x ", eeprom_data); 3571 } 3572 kprintf("\n"); 3573 } 3574 3575 static int 3576 emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 3577 { 3578 struct emx_softc *sc; 3579 struct ifnet *ifp; 3580 int error, result; 3581 3582 result = -1; 3583 error = sysctl_handle_int(oidp, &result, 0, req); 3584 if (error || !req->newptr) 3585 return (error); 3586 3587 sc = (struct emx_softc *)arg1; 3588 ifp = &sc->arpcom.ac_if; 3589 3590 ifnet_serialize_all(ifp); 3591 3592 if (result == 1) 3593 emx_print_debug_info(sc); 3594 3595 /* 3596 * This value will cause a hex dump of the 3597 * first 32 16-bit words of the EEPROM to 3598 * the screen. 3599 */ 3600 if (result == 2) 3601 emx_print_nvm_info(sc); 3602 3603 ifnet_deserialize_all(ifp); 3604 3605 return (error); 3606 } 3607 3608 static int 3609 emx_sysctl_stats(SYSCTL_HANDLER_ARGS) 3610 { 3611 int error, result; 3612 3613 result = -1; 3614 error = sysctl_handle_int(oidp, &result, 0, req); 3615 if (error || !req->newptr) 3616 return (error); 3617 3618 if (result == 1) { 3619 struct emx_softc *sc = (struct emx_softc *)arg1; 3620 struct ifnet *ifp = &sc->arpcom.ac_if; 3621 3622 ifnet_serialize_all(ifp); 3623 emx_print_hw_stats(sc); 3624 ifnet_deserialize_all(ifp); 3625 } 3626 return (error); 3627 } 3628 3629 static void 3630 emx_add_sysctl(struct emx_softc *sc) 3631 { 3632 #if defined(EMX_RSS_DEBUG) || defined(EMX_TSS_DEBUG) 3633 char pkt_desc[32]; 3634 int i; 3635 #endif 3636 3637 sysctl_ctx_init(&sc->sysctl_ctx); 3638 sc->sysctl_tree = SYSCTL_ADD_NODE(&sc->sysctl_ctx, 3639 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO, 3640 device_get_nameunit(sc->dev), 3641 CTLFLAG_RD, 0, ""); 3642 if (sc->sysctl_tree == NULL) { 3643 device_printf(sc->dev, "can't add sysctl node\n"); 3644 return; 3645 } 3646 3647 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3648 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 3649 emx_sysctl_debug_info, "I", "Debug Information"); 3650 3651 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3652 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 3653 emx_sysctl_stats, "I", "Statistics"); 3654 3655 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3656 OID_AUTO, "rxd", CTLFLAG_RD, &sc->rx_data[0].num_rx_desc, 0, 3657 "# of RX descs"); 3658 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3659 OID_AUTO, "txd", CTLFLAG_RD, &sc->tx_data[0].num_tx_desc, 0, 3660 "# of TX descs"); 3661 3662 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3663 OID_AUTO, "int_throttle_ceil", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 3664 emx_sysctl_int_throttle, "I", "interrupt throttling rate"); 3665 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3666 OID_AUTO, "tx_intr_nsegs", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 3667 emx_sysctl_tx_intr_nsegs, "I", "# segments per TX interrupt"); 3668 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3669 OID_AUTO, "tx_wreg_nsegs", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 3670 emx_sysctl_tx_wreg_nsegs, "I", 3671 "# segments sent before write to hardware register"); 3672 3673 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3674 OID_AUTO, "rx_ring_cnt", CTLFLAG_RD, &sc->rx_ring_cnt, 0, 3675 "# of RX rings"); 3676 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3677 OID_AUTO, "tx_ring_cnt", CTLFLAG_RD, &sc->tx_ring_cnt, 0, 3678 "# of TX rings"); 3679 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3680 OID_AUTO, "tx_ring_inuse", CTLFLAG_RD, &sc->tx_ring_inuse, 0, 3681 "# of TX rings used"); 3682 3683 #ifdef IFPOLL_ENABLE 3684 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3685 OID_AUTO, "npoll_rxoff", CTLTYPE_INT|CTLFLAG_RW, 3686 sc, 0, emx_sysctl_npoll_rxoff, "I", 3687 "NPOLLING RX cpu offset"); 3688 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3689 OID_AUTO, "npoll_txoff", CTLTYPE_INT|CTLFLAG_RW, 3690 sc, 0, emx_sysctl_npoll_txoff, "I", 3691 "NPOLLING TX cpu offset"); 3692 #endif 3693 3694 #ifdef EMX_RSS_DEBUG 3695 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3696 OID_AUTO, "rss_debug", CTLFLAG_RW, &sc->rss_debug, 3697 0, "RSS debug level"); 3698 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3699 ksnprintf(pkt_desc, sizeof(pkt_desc), "rx%d_pkt", i); 3700 SYSCTL_ADD_ULONG(&sc->sysctl_ctx, 3701 SYSCTL_CHILDREN(sc->sysctl_tree), OID_AUTO, 3702 pkt_desc, CTLFLAG_RW, &sc->rx_data[i].rx_pkts, 3703 "RXed packets"); 3704 } 3705 #endif 3706 #ifdef EMX_TSS_DEBUG 3707 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3708 ksnprintf(pkt_desc, sizeof(pkt_desc), "tx%d_pkt", i); 3709 SYSCTL_ADD_ULONG(&sc->sysctl_ctx, 3710 SYSCTL_CHILDREN(sc->sysctl_tree), OID_AUTO, 3711 pkt_desc, CTLFLAG_RW, &sc->tx_data[i].tx_pkts, 3712 "TXed packets"); 3713 } 3714 #endif 3715 } 3716 3717 static int 3718 emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS) 3719 { 3720 struct emx_softc *sc = (void *)arg1; 3721 struct ifnet *ifp = &sc->arpcom.ac_if; 3722 int error, throttle; 3723 3724 throttle = sc->int_throttle_ceil; 3725 error = sysctl_handle_int(oidp, &throttle, 0, req); 3726 if (error || req->newptr == NULL) 3727 return error; 3728 if (throttle < 0 || throttle > 1000000000 / 256) 3729 return EINVAL; 3730 3731 if (throttle) { 3732 /* 3733 * Set the interrupt throttling rate in 256ns increments, 3734 * recalculate sysctl value assignment to get exact frequency. 3735 */ 3736 throttle = 1000000000 / 256 / throttle; 3737 3738 /* Upper 16bits of ITR is reserved and should be zero */ 3739 if (throttle & 0xffff0000) 3740 return EINVAL; 3741 } 3742 3743 ifnet_serialize_all(ifp); 3744 3745 if (throttle) 3746 sc->int_throttle_ceil = 1000000000 / 256 / throttle; 3747 else 3748 sc->int_throttle_ceil = 0; 3749 3750 if (ifp->if_flags & IFF_RUNNING) 3751 emx_set_itr(sc, throttle); 3752 3753 ifnet_deserialize_all(ifp); 3754 3755 if (bootverbose) { 3756 if_printf(ifp, "Interrupt moderation set to %d/sec\n", 3757 sc->int_throttle_ceil); 3758 } 3759 return 0; 3760 } 3761 3762 static int 3763 emx_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS) 3764 { 3765 struct emx_softc *sc = (void *)arg1; 3766 struct ifnet *ifp = &sc->arpcom.ac_if; 3767 struct emx_txdata *tdata = &sc->tx_data[0]; 3768 int error, segs; 3769 3770 segs = tdata->tx_intr_nsegs; 3771 error = sysctl_handle_int(oidp, &segs, 0, req); 3772 if (error || req->newptr == NULL) 3773 return error; 3774 if (segs <= 0) 3775 return EINVAL; 3776 3777 ifnet_serialize_all(ifp); 3778 3779 /* 3780 * Don't allow tx_intr_nsegs to become: 3781 * o Less the oact_tx_desc 3782 * o Too large that no TX desc will cause TX interrupt to 3783 * be generated (OACTIVE will never recover) 3784 * o Too small that will cause tx_dd[] overflow 3785 */ 3786 if (segs < tdata->oact_tx_desc || 3787 segs >= tdata->num_tx_desc - tdata->oact_tx_desc || 3788 segs < tdata->num_tx_desc / EMX_TXDD_SAFE) { 3789 error = EINVAL; 3790 } else { 3791 int i; 3792 3793 error = 0; 3794 for (i = 0; i < sc->tx_ring_cnt; ++i) 3795 sc->tx_data[i].tx_intr_nsegs = segs; 3796 } 3797 3798 ifnet_deserialize_all(ifp); 3799 3800 return error; 3801 } 3802 3803 static int 3804 emx_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS) 3805 { 3806 struct emx_softc *sc = (void *)arg1; 3807 struct ifnet *ifp = &sc->arpcom.ac_if; 3808 int error, nsegs, i; 3809 3810 nsegs = sc->tx_data[0].tx_wreg_nsegs; 3811 error = sysctl_handle_int(oidp, &nsegs, 0, req); 3812 if (error || req->newptr == NULL) 3813 return error; 3814 3815 ifnet_serialize_all(ifp); 3816 for (i = 0; i < sc->tx_ring_cnt; ++i) 3817 sc->tx_data[i].tx_wreg_nsegs =nsegs; 3818 ifnet_deserialize_all(ifp); 3819 3820 return 0; 3821 } 3822 3823 #ifdef IFPOLL_ENABLE 3824 3825 static int 3826 emx_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS) 3827 { 3828 struct emx_softc *sc = (void *)arg1; 3829 struct ifnet *ifp = &sc->arpcom.ac_if; 3830 int error, off; 3831 3832 off = sc->rx_npoll_off; 3833 error = sysctl_handle_int(oidp, &off, 0, req); 3834 if (error || req->newptr == NULL) 3835 return error; 3836 if (off < 0) 3837 return EINVAL; 3838 3839 ifnet_serialize_all(ifp); 3840 if (off >= ncpus2 || off % sc->rx_ring_cnt != 0) { 3841 error = EINVAL; 3842 } else { 3843 error = 0; 3844 sc->rx_npoll_off = off; 3845 } 3846 ifnet_deserialize_all(ifp); 3847 3848 return error; 3849 } 3850 3851 static int 3852 emx_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS) 3853 { 3854 struct emx_softc *sc = (void *)arg1; 3855 struct ifnet *ifp = &sc->arpcom.ac_if; 3856 int error, off; 3857 3858 off = sc->tx_npoll_off; 3859 error = sysctl_handle_int(oidp, &off, 0, req); 3860 if (error || req->newptr == NULL) 3861 return error; 3862 if (off < 0) 3863 return EINVAL; 3864 3865 ifnet_serialize_all(ifp); 3866 if (off >= ncpus2 || off % sc->tx_ring_cnt != 0) { 3867 error = EINVAL; 3868 } else { 3869 error = 0; 3870 sc->tx_npoll_off = off; 3871 } 3872 ifnet_deserialize_all(ifp); 3873 3874 return error; 3875 } 3876 3877 #endif /* IFPOLL_ENABLE */ 3878 3879 static int 3880 emx_dma_alloc(struct emx_softc *sc) 3881 { 3882 int error, i; 3883 3884 /* 3885 * Create top level busdma tag 3886 */ 3887 error = bus_dma_tag_create(NULL, 1, 0, 3888 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 3889 NULL, NULL, 3890 BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 3891 0, &sc->parent_dtag); 3892 if (error) { 3893 device_printf(sc->dev, "could not create top level DMA tag\n"); 3894 return error; 3895 } 3896 3897 /* 3898 * Allocate transmit descriptors ring and buffers 3899 */ 3900 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3901 error = emx_create_tx_ring(&sc->tx_data[i]); 3902 if (error) { 3903 device_printf(sc->dev, 3904 "Could not setup transmit structures\n"); 3905 return error; 3906 } 3907 } 3908 3909 /* 3910 * Allocate receive descriptors ring and buffers 3911 */ 3912 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3913 error = emx_create_rx_ring(&sc->rx_data[i]); 3914 if (error) { 3915 device_printf(sc->dev, 3916 "Could not setup receive structures\n"); 3917 return error; 3918 } 3919 } 3920 return 0; 3921 } 3922 3923 static void 3924 emx_dma_free(struct emx_softc *sc) 3925 { 3926 int i; 3927 3928 for (i = 0; i < sc->tx_ring_cnt; ++i) { 3929 emx_destroy_tx_ring(&sc->tx_data[i], 3930 sc->tx_data[i].num_tx_desc); 3931 } 3932 3933 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3934 emx_destroy_rx_ring(&sc->rx_data[i], 3935 sc->rx_data[i].num_rx_desc); 3936 } 3937 3938 /* Free top level busdma tag */ 3939 if (sc->parent_dtag != NULL) 3940 bus_dma_tag_destroy(sc->parent_dtag); 3941 } 3942 3943 static void 3944 emx_serialize(struct ifnet *ifp, enum ifnet_serialize slz) 3945 { 3946 struct emx_softc *sc = ifp->if_softc; 3947 3948 ifnet_serialize_array_enter(sc->serializes, EMX_NSERIALIZE, slz); 3949 } 3950 3951 static void 3952 emx_deserialize(struct ifnet *ifp, enum ifnet_serialize slz) 3953 { 3954 struct emx_softc *sc = ifp->if_softc; 3955 3956 ifnet_serialize_array_exit(sc->serializes, EMX_NSERIALIZE, slz); 3957 } 3958 3959 static int 3960 emx_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz) 3961 { 3962 struct emx_softc *sc = ifp->if_softc; 3963 3964 return ifnet_serialize_array_try(sc->serializes, EMX_NSERIALIZE, slz); 3965 } 3966 3967 static void 3968 emx_serialize_skipmain(struct emx_softc *sc) 3969 { 3970 lwkt_serialize_array_enter(sc->serializes, EMX_NSERIALIZE, 1); 3971 } 3972 3973 static void 3974 emx_deserialize_skipmain(struct emx_softc *sc) 3975 { 3976 lwkt_serialize_array_exit(sc->serializes, EMX_NSERIALIZE, 1); 3977 } 3978 3979 #ifdef INVARIANTS 3980 3981 static void 3982 emx_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz, 3983 boolean_t serialized) 3984 { 3985 struct emx_softc *sc = ifp->if_softc; 3986 3987 ifnet_serialize_array_assert(sc->serializes, EMX_NSERIALIZE, 3988 slz, serialized); 3989 } 3990 3991 #endif /* INVARIANTS */ 3992 3993 #ifdef IFPOLL_ENABLE 3994 3995 static void 3996 emx_npoll_status(struct ifnet *ifp) 3997 { 3998 struct emx_softc *sc = ifp->if_softc; 3999 uint32_t reg_icr; 4000 4001 ASSERT_SERIALIZED(&sc->main_serialize); 4002 4003 reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); 4004 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 4005 callout_stop(&sc->timer); 4006 sc->hw.mac.get_link_status = 1; 4007 emx_update_link_status(sc); 4008 callout_reset(&sc->timer, hz, emx_timer, sc); 4009 } 4010 } 4011 4012 static void 4013 emx_npoll_tx(struct ifnet *ifp, void *arg, int cycle __unused) 4014 { 4015 struct emx_txdata *tdata = arg; 4016 4017 ASSERT_SERIALIZED(&tdata->tx_serialize); 4018 4019 emx_txeof(tdata); 4020 if (!ifsq_is_empty(tdata->ifsq)) 4021 ifsq_devstart(tdata->ifsq); 4022 } 4023 4024 static void 4025 emx_npoll_rx(struct ifnet *ifp __unused, void *arg, int cycle) 4026 { 4027 struct emx_rxdata *rdata = arg; 4028 4029 ASSERT_SERIALIZED(&rdata->rx_serialize); 4030 4031 emx_rxeof(rdata, cycle); 4032 } 4033 4034 static void 4035 emx_npoll(struct ifnet *ifp, struct ifpoll_info *info) 4036 { 4037 struct emx_softc *sc = ifp->if_softc; 4038 int i, txr_cnt; 4039 4040 ASSERT_IFNET_SERIALIZED_ALL(ifp); 4041 4042 if (info) { 4043 int off; 4044 4045 info->ifpi_status.status_func = emx_npoll_status; 4046 info->ifpi_status.serializer = &sc->main_serialize; 4047 4048 txr_cnt = emx_get_txring_inuse(sc, TRUE); 4049 off = sc->tx_npoll_off; 4050 for (i = 0; i < txr_cnt; ++i) { 4051 struct emx_txdata *tdata = &sc->tx_data[i]; 4052 int idx = i + off; 4053 4054 KKASSERT(idx < ncpus2); 4055 info->ifpi_tx[idx].poll_func = emx_npoll_tx; 4056 info->ifpi_tx[idx].arg = tdata; 4057 info->ifpi_tx[idx].serializer = &tdata->tx_serialize; 4058 ifsq_set_cpuid(tdata->ifsq, idx); 4059 } 4060 4061 off = sc->rx_npoll_off; 4062 for (i = 0; i < sc->rx_ring_cnt; ++i) { 4063 struct emx_rxdata *rdata = &sc->rx_data[i]; 4064 int idx = i + off; 4065 4066 KKASSERT(idx < ncpus2); 4067 info->ifpi_rx[idx].poll_func = emx_npoll_rx; 4068 info->ifpi_rx[idx].arg = rdata; 4069 info->ifpi_rx[idx].serializer = &rdata->rx_serialize; 4070 } 4071 4072 if (ifp->if_flags & IFF_RUNNING) { 4073 if (txr_cnt == sc->tx_ring_inuse) 4074 emx_disable_intr(sc); 4075 else 4076 emx_init(sc); 4077 } 4078 } else { 4079 for (i = 0; i < sc->tx_ring_cnt; ++i) { 4080 struct emx_txdata *tdata = &sc->tx_data[i]; 4081 4082 ifsq_set_cpuid(tdata->ifsq, 4083 rman_get_cpuid(sc->intr_res)); 4084 } 4085 4086 if (ifp->if_flags & IFF_RUNNING) { 4087 txr_cnt = emx_get_txring_inuse(sc, FALSE); 4088 if (txr_cnt == sc->tx_ring_inuse) 4089 emx_enable_intr(sc); 4090 else 4091 emx_init(sc); 4092 } 4093 } 4094 } 4095 4096 #endif /* IFPOLL_ENABLE */ 4097 4098 static void 4099 emx_set_itr(struct emx_softc *sc, uint32_t itr) 4100 { 4101 E1000_WRITE_REG(&sc->hw, E1000_ITR, itr); 4102 if (sc->hw.mac.type == e1000_82574) { 4103 int i; 4104 4105 /* 4106 * When using MSIX interrupts we need to 4107 * throttle using the EITR register 4108 */ 4109 for (i = 0; i < 4; ++i) 4110 E1000_WRITE_REG(&sc->hw, E1000_EITR_82574(i), itr); 4111 } 4112 } 4113 4114 /* 4115 * Disable the L0s, 82574L Errata #20 4116 */ 4117 static void 4118 emx_disable_aspm(struct emx_softc *sc) 4119 { 4120 uint16_t link_cap, link_ctrl, disable; 4121 uint8_t pcie_ptr, reg; 4122 device_t dev = sc->dev; 4123 4124 switch (sc->hw.mac.type) { 4125 case e1000_82571: 4126 case e1000_82572: 4127 case e1000_82573: 4128 /* 4129 * 82573 specification update 4130 * errata #8 disable L0s 4131 * errata #41 disable L1 4132 * 4133 * 82571/82572 specification update 4134 # errata #13 disable L1 4135 * errata #68 disable L0s 4136 */ 4137 disable = PCIEM_LNKCTL_ASPM_L0S | PCIEM_LNKCTL_ASPM_L1; 4138 break; 4139 4140 case e1000_82574: 4141 /* 4142 * 82574 specification update errata #20 4143 * 4144 * There is no need to disable L1 4145 */ 4146 disable = PCIEM_LNKCTL_ASPM_L0S; 4147 break; 4148 4149 default: 4150 return; 4151 } 4152 4153 pcie_ptr = pci_get_pciecap_ptr(dev); 4154 if (pcie_ptr == 0) 4155 return; 4156 4157 link_cap = pci_read_config(dev, pcie_ptr + PCIER_LINKCAP, 2); 4158 if ((link_cap & PCIEM_LNKCAP_ASPM_MASK) == 0) 4159 return; 4160 4161 if (bootverbose) 4162 if_printf(&sc->arpcom.ac_if, "disable ASPM %#02x\n", disable); 4163 4164 reg = pcie_ptr + PCIER_LINKCTRL; 4165 link_ctrl = pci_read_config(dev, reg, 2); 4166 link_ctrl &= ~disable; 4167 pci_write_config(dev, reg, link_ctrl, 2); 4168 } 4169 4170 static int 4171 emx_tso_pullup(struct emx_txdata *tdata, struct mbuf **mp) 4172 { 4173 int iphlen, hoff, thoff, ex = 0; 4174 struct mbuf *m; 4175 struct ip *ip; 4176 4177 m = *mp; 4178 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable")); 4179 4180 iphlen = m->m_pkthdr.csum_iphlen; 4181 thoff = m->m_pkthdr.csum_thlen; 4182 hoff = m->m_pkthdr.csum_lhlen; 4183 4184 KASSERT(iphlen > 0, ("invalid ip hlen")); 4185 KASSERT(thoff > 0, ("invalid tcp hlen")); 4186 KASSERT(hoff > 0, ("invalid ether hlen")); 4187 4188 if (tdata->tx_flags & EMX_TXFLAG_TSO_PULLEX) 4189 ex = 4; 4190 4191 if (m->m_len < hoff + iphlen + thoff + ex) { 4192 m = m_pullup(m, hoff + iphlen + thoff + ex); 4193 if (m == NULL) { 4194 *mp = NULL; 4195 return ENOBUFS; 4196 } 4197 *mp = m; 4198 } 4199 ip = mtodoff(m, struct ip *, hoff); 4200 ip->ip_len = 0; 4201 4202 return 0; 4203 } 4204 4205 static int 4206 emx_tso_setup(struct emx_txdata *tdata, struct mbuf *mp, 4207 uint32_t *txd_upper, uint32_t *txd_lower) 4208 { 4209 struct e1000_context_desc *TXD; 4210 int hoff, iphlen, thoff, hlen; 4211 int mss, pktlen, curr_txd; 4212 4213 #ifdef EMX_TSO_DEBUG 4214 tdata->tso_segments++; 4215 #endif 4216 4217 iphlen = mp->m_pkthdr.csum_iphlen; 4218 thoff = mp->m_pkthdr.csum_thlen; 4219 hoff = mp->m_pkthdr.csum_lhlen; 4220 mss = mp->m_pkthdr.tso_segsz; 4221 pktlen = mp->m_pkthdr.len; 4222 4223 if ((tdata->tx_flags & EMX_TXFLAG_FORCECTX) == 0 && 4224 tdata->csum_flags == CSUM_TSO && 4225 tdata->csum_iphlen == iphlen && 4226 tdata->csum_lhlen == hoff && 4227 tdata->csum_thlen == thoff && 4228 tdata->csum_mss == mss && 4229 tdata->csum_pktlen == pktlen) { 4230 *txd_upper = tdata->csum_txd_upper; 4231 *txd_lower = tdata->csum_txd_lower; 4232 #ifdef EMX_TSO_DEBUG 4233 tdata->tso_ctx_reused++; 4234 #endif 4235 return 0; 4236 } 4237 hlen = hoff + iphlen + thoff; 4238 4239 /* 4240 * Setup a new TSO context. 4241 */ 4242 4243 curr_txd = tdata->next_avail_tx_desc; 4244 TXD = (struct e1000_context_desc *)&tdata->tx_desc_base[curr_txd]; 4245 4246 *txd_lower = E1000_TXD_CMD_DEXT | /* Extended descr type */ 4247 E1000_TXD_DTYP_D | /* Data descr type */ 4248 E1000_TXD_CMD_TSE; /* Do TSE on this packet */ 4249 4250 /* IP and/or TCP header checksum calculation and insertion. */ 4251 *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; 4252 4253 /* 4254 * Start offset for header checksum calculation. 4255 * End offset for header checksum calculation. 4256 * Offset of place put the checksum. 4257 */ 4258 TXD->lower_setup.ip_fields.ipcss = hoff; 4259 TXD->lower_setup.ip_fields.ipcse = htole16(hoff + iphlen - 1); 4260 TXD->lower_setup.ip_fields.ipcso = hoff + offsetof(struct ip, ip_sum); 4261 4262 /* 4263 * Start offset for payload checksum calculation. 4264 * End offset for payload checksum calculation. 4265 * Offset of place to put the checksum. 4266 */ 4267 TXD->upper_setup.tcp_fields.tucss = hoff + iphlen; 4268 TXD->upper_setup.tcp_fields.tucse = 0; 4269 TXD->upper_setup.tcp_fields.tucso = 4270 hoff + iphlen + offsetof(struct tcphdr, th_sum); 4271 4272 /* 4273 * Payload size per packet w/o any headers. 4274 * Length of all headers up to payload. 4275 */ 4276 TXD->tcp_seg_setup.fields.mss = htole16(mss); 4277 TXD->tcp_seg_setup.fields.hdr_len = hlen; 4278 TXD->cmd_and_length = htole32(E1000_TXD_CMD_IFCS | 4279 E1000_TXD_CMD_DEXT | /* Extended descr */ 4280 E1000_TXD_CMD_TSE | /* TSE context */ 4281 E1000_TXD_CMD_IP | /* Do IP csum */ 4282 E1000_TXD_CMD_TCP | /* Do TCP checksum */ 4283 (pktlen - hlen)); /* Total len */ 4284 4285 /* Save the information for this TSO context */ 4286 tdata->csum_flags = CSUM_TSO; 4287 tdata->csum_lhlen = hoff; 4288 tdata->csum_iphlen = iphlen; 4289 tdata->csum_thlen = thoff; 4290 tdata->csum_mss = mss; 4291 tdata->csum_pktlen = pktlen; 4292 tdata->csum_txd_upper = *txd_upper; 4293 tdata->csum_txd_lower = *txd_lower; 4294 4295 if (++curr_txd == tdata->num_tx_desc) 4296 curr_txd = 0; 4297 4298 KKASSERT(tdata->num_tx_desc_avail > 0); 4299 tdata->num_tx_desc_avail--; 4300 4301 tdata->next_avail_tx_desc = curr_txd; 4302 return 1; 4303 } 4304 4305 static int 4306 emx_get_txring_inuse(const struct emx_softc *sc, boolean_t polling) 4307 { 4308 if (polling) 4309 return sc->tx_ring_cnt; 4310 else 4311 return 1; 4312 } 4313