1 /* 2 * Copyright (c) 2004 Joerg Sonnenberger <joerg@bec.de>. All rights reserved. 3 * 4 * Copyright (c) 2001-2008, Intel Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 * 34 * Copyright (c) 2005 The DragonFly Project. All rights reserved. 35 * 36 * This code is derived from software contributed to The DragonFly Project 37 * by Matthew Dillon <dillon@backplane.com> 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 43 * 1. Redistributions of source code must retain the above copyright 44 * notice, this list of conditions and the following disclaimer. 45 * 2. Redistributions in binary form must reproduce the above copyright 46 * notice, this list of conditions and the following disclaimer in 47 * the documentation and/or other materials provided with the 48 * distribution. 49 * 3. Neither the name of The DragonFly Project nor the names of its 50 * contributors may be used to endorse or promote products derived 51 * from this software without specific, prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 55 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 56 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 57 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 58 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 59 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 60 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 61 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 62 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 63 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 64 * SUCH DAMAGE. 65 */ 66 67 #include "opt_ifpoll.h" 68 #include "opt_rss.h" 69 #include "opt_emx.h" 70 71 #include <sys/param.h> 72 #include <sys/bus.h> 73 #include <sys/endian.h> 74 #include <sys/interrupt.h> 75 #include <sys/kernel.h> 76 #include <sys/ktr.h> 77 #include <sys/malloc.h> 78 #include <sys/mbuf.h> 79 #include <sys/proc.h> 80 #include <sys/rman.h> 81 #include <sys/serialize.h> 82 #include <sys/serialize2.h> 83 #include <sys/socket.h> 84 #include <sys/sockio.h> 85 #include <sys/sysctl.h> 86 #include <sys/systm.h> 87 88 #include <net/bpf.h> 89 #include <net/ethernet.h> 90 #include <net/if.h> 91 #include <net/if_arp.h> 92 #include <net/if_dl.h> 93 #include <net/if_media.h> 94 #include <net/ifq_var.h> 95 #include <net/toeplitz.h> 96 #include <net/toeplitz2.h> 97 #include <net/vlan/if_vlan_var.h> 98 #include <net/vlan/if_vlan_ether.h> 99 #include <net/if_poll.h> 100 101 #include <netinet/in_systm.h> 102 #include <netinet/in.h> 103 #include <netinet/ip.h> 104 #include <netinet/tcp.h> 105 #include <netinet/udp.h> 106 107 #include <bus/pci/pcivar.h> 108 #include <bus/pci/pcireg.h> 109 110 #include <dev/netif/ig_hal/e1000_api.h> 111 #include <dev/netif/ig_hal/e1000_82571.h> 112 #include <dev/netif/emx/if_emx.h> 113 114 #ifdef EMX_RSS_DEBUG 115 #define EMX_RSS_DPRINTF(sc, lvl, fmt, ...) \ 116 do { \ 117 if (sc->rss_debug >= lvl) \ 118 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \ 119 } while (0) 120 #else /* !EMX_RSS_DEBUG */ 121 #define EMX_RSS_DPRINTF(sc, lvl, fmt, ...) ((void)0) 122 #endif /* EMX_RSS_DEBUG */ 123 124 #define EMX_NAME "Intel(R) PRO/1000 " 125 126 #define EMX_DEVICE(id) \ 127 { EMX_VENDOR_ID, E1000_DEV_ID_##id, EMX_NAME #id } 128 #define EMX_DEVICE_NULL { 0, 0, NULL } 129 130 static const struct emx_device { 131 uint16_t vid; 132 uint16_t did; 133 const char *desc; 134 } emx_devices[] = { 135 EMX_DEVICE(82571EB_COPPER), 136 EMX_DEVICE(82571EB_FIBER), 137 EMX_DEVICE(82571EB_SERDES), 138 EMX_DEVICE(82571EB_SERDES_DUAL), 139 EMX_DEVICE(82571EB_SERDES_QUAD), 140 EMX_DEVICE(82571EB_QUAD_COPPER), 141 EMX_DEVICE(82571EB_QUAD_COPPER_BP), 142 EMX_DEVICE(82571EB_QUAD_COPPER_LP), 143 EMX_DEVICE(82571EB_QUAD_FIBER), 144 EMX_DEVICE(82571PT_QUAD_COPPER), 145 146 EMX_DEVICE(82572EI_COPPER), 147 EMX_DEVICE(82572EI_FIBER), 148 EMX_DEVICE(82572EI_SERDES), 149 EMX_DEVICE(82572EI), 150 151 EMX_DEVICE(82573E), 152 EMX_DEVICE(82573E_IAMT), 153 EMX_DEVICE(82573L), 154 155 EMX_DEVICE(80003ES2LAN_COPPER_SPT), 156 EMX_DEVICE(80003ES2LAN_SERDES_SPT), 157 EMX_DEVICE(80003ES2LAN_COPPER_DPT), 158 EMX_DEVICE(80003ES2LAN_SERDES_DPT), 159 160 EMX_DEVICE(82574L), 161 EMX_DEVICE(82574LA), 162 163 /* required last entry */ 164 EMX_DEVICE_NULL 165 }; 166 167 static int emx_probe(device_t); 168 static int emx_attach(device_t); 169 static int emx_detach(device_t); 170 static int emx_shutdown(device_t); 171 static int emx_suspend(device_t); 172 static int emx_resume(device_t); 173 174 static void emx_init(void *); 175 static void emx_stop(struct emx_softc *); 176 static int emx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 177 static void emx_start(struct ifnet *); 178 #ifdef IFPOLL_ENABLE 179 static void emx_qpoll(struct ifnet *, struct ifpoll_info *); 180 #endif 181 static void emx_watchdog(struct ifnet *); 182 static void emx_media_status(struct ifnet *, struct ifmediareq *); 183 static int emx_media_change(struct ifnet *); 184 static void emx_timer(void *); 185 static void emx_serialize(struct ifnet *, enum ifnet_serialize); 186 static void emx_deserialize(struct ifnet *, enum ifnet_serialize); 187 static int emx_tryserialize(struct ifnet *, enum ifnet_serialize); 188 #ifdef INVARIANTS 189 static void emx_serialize_assert(struct ifnet *, enum ifnet_serialize, 190 boolean_t); 191 #endif 192 193 static void emx_intr(void *); 194 static void emx_rxeof(struct emx_softc *, int, int); 195 static void emx_txeof(struct emx_softc *); 196 static void emx_tx_collect(struct emx_softc *); 197 static void emx_tx_purge(struct emx_softc *); 198 static void emx_enable_intr(struct emx_softc *); 199 static void emx_disable_intr(struct emx_softc *); 200 201 static int emx_dma_alloc(struct emx_softc *); 202 static void emx_dma_free(struct emx_softc *); 203 static void emx_init_tx_ring(struct emx_softc *); 204 static int emx_init_rx_ring(struct emx_softc *, struct emx_rxdata *); 205 static void emx_free_rx_ring(struct emx_softc *, struct emx_rxdata *); 206 static int emx_create_tx_ring(struct emx_softc *); 207 static int emx_create_rx_ring(struct emx_softc *, struct emx_rxdata *); 208 static void emx_destroy_tx_ring(struct emx_softc *, int); 209 static void emx_destroy_rx_ring(struct emx_softc *, 210 struct emx_rxdata *, int); 211 static int emx_newbuf(struct emx_softc *, struct emx_rxdata *, int, int); 212 static int emx_encap(struct emx_softc *, struct mbuf **); 213 static int emx_txcsum_pullup(struct emx_softc *, struct mbuf **); 214 static int emx_txcsum(struct emx_softc *, struct mbuf *, 215 uint32_t *, uint32_t *); 216 217 static int emx_is_valid_eaddr(const uint8_t *); 218 static int emx_reset(struct emx_softc *); 219 static void emx_setup_ifp(struct emx_softc *); 220 static void emx_init_tx_unit(struct emx_softc *); 221 static void emx_init_rx_unit(struct emx_softc *); 222 static void emx_update_stats(struct emx_softc *); 223 static void emx_set_promisc(struct emx_softc *); 224 static void emx_disable_promisc(struct emx_softc *); 225 static void emx_set_multi(struct emx_softc *); 226 static void emx_update_link_status(struct emx_softc *); 227 static void emx_smartspeed(struct emx_softc *); 228 static void emx_set_itr(struct emx_softc *, uint32_t); 229 230 static void emx_print_debug_info(struct emx_softc *); 231 static void emx_print_nvm_info(struct emx_softc *); 232 static void emx_print_hw_stats(struct emx_softc *); 233 234 static int emx_sysctl_stats(SYSCTL_HANDLER_ARGS); 235 static int emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 236 static int emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS); 237 static int emx_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS); 238 static void emx_add_sysctl(struct emx_softc *); 239 240 static void emx_serialize_skipmain(struct emx_softc *); 241 static void emx_deserialize_skipmain(struct emx_softc *); 242 243 /* Management and WOL Support */ 244 static void emx_get_mgmt(struct emx_softc *); 245 static void emx_rel_mgmt(struct emx_softc *); 246 static void emx_get_hw_control(struct emx_softc *); 247 static void emx_rel_hw_control(struct emx_softc *); 248 static void emx_enable_wol(device_t); 249 250 static device_method_t emx_methods[] = { 251 /* Device interface */ 252 DEVMETHOD(device_probe, emx_probe), 253 DEVMETHOD(device_attach, emx_attach), 254 DEVMETHOD(device_detach, emx_detach), 255 DEVMETHOD(device_shutdown, emx_shutdown), 256 DEVMETHOD(device_suspend, emx_suspend), 257 DEVMETHOD(device_resume, emx_resume), 258 { 0, 0 } 259 }; 260 261 static driver_t emx_driver = { 262 "emx", 263 emx_methods, 264 sizeof(struct emx_softc), 265 }; 266 267 static devclass_t emx_devclass; 268 269 DECLARE_DUMMY_MODULE(if_emx); 270 MODULE_DEPEND(emx, ig_hal, 1, 1, 1); 271 DRIVER_MODULE(if_emx, pci, emx_driver, emx_devclass, NULL, NULL); 272 273 /* 274 * Tunables 275 */ 276 static int emx_int_throttle_ceil = EMX_DEFAULT_ITR; 277 static int emx_rxd = EMX_DEFAULT_RXD; 278 static int emx_txd = EMX_DEFAULT_TXD; 279 static int emx_smart_pwr_down = FALSE; 280 281 /* Controls whether promiscuous also shows bad packets */ 282 static int emx_debug_sbp = FALSE; 283 284 static int emx_82573_workaround = TRUE; 285 286 TUNABLE_INT("hw.emx.int_throttle_ceil", &emx_int_throttle_ceil); 287 TUNABLE_INT("hw.emx.rxd", &emx_rxd); 288 TUNABLE_INT("hw.emx.txd", &emx_txd); 289 TUNABLE_INT("hw.emx.smart_pwr_down", &emx_smart_pwr_down); 290 TUNABLE_INT("hw.emx.sbp", &emx_debug_sbp); 291 TUNABLE_INT("hw.emx.82573_workaround", &emx_82573_workaround); 292 293 /* Global used in WOL setup with multiport cards */ 294 static int emx_global_quad_port_a = 0; 295 296 /* Set this to one to display debug statistics */ 297 static int emx_display_debug_stats = 0; 298 299 #if !defined(KTR_IF_EMX) 300 #define KTR_IF_EMX KTR_ALL 301 #endif 302 KTR_INFO_MASTER(if_emx); 303 KTR_INFO(KTR_IF_EMX, if_emx, intr_beg, 0, "intr begin", 0); 304 KTR_INFO(KTR_IF_EMX, if_emx, intr_end, 1, "intr end", 0); 305 KTR_INFO(KTR_IF_EMX, if_emx, pkt_receive, 4, "rx packet", 0); 306 KTR_INFO(KTR_IF_EMX, if_emx, pkt_txqueue, 5, "tx packet", 0); 307 KTR_INFO(KTR_IF_EMX, if_emx, pkt_txclean, 6, "tx clean", 0); 308 #define logif(name) KTR_LOG(if_emx_ ## name) 309 310 static __inline void 311 emx_setup_rxdesc(emx_rxdesc_t *rxd, const struct emx_rxbuf *rxbuf) 312 { 313 rxd->rxd_bufaddr = htole64(rxbuf->paddr); 314 /* DD bit must be cleared */ 315 rxd->rxd_staterr = 0; 316 } 317 318 static __inline void 319 emx_rxcsum(uint32_t staterr, struct mbuf *mp) 320 { 321 /* Ignore Checksum bit is set */ 322 if (staterr & E1000_RXD_STAT_IXSM) 323 return; 324 325 if ((staterr & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == 326 E1000_RXD_STAT_IPCS) 327 mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID; 328 329 if ((staterr & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == 330 E1000_RXD_STAT_TCPCS) { 331 mp->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 332 CSUM_PSEUDO_HDR | 333 CSUM_FRAG_NOT_CHECKED; 334 mp->m_pkthdr.csum_data = htons(0xffff); 335 } 336 } 337 338 static __inline struct pktinfo * 339 emx_rssinfo(struct mbuf *m, struct pktinfo *pi, 340 uint32_t mrq, uint32_t hash, uint32_t staterr) 341 { 342 switch (mrq & EMX_RXDMRQ_RSSTYPE_MASK) { 343 case EMX_RXDMRQ_IPV4_TCP: 344 pi->pi_netisr = NETISR_IP; 345 pi->pi_flags = 0; 346 pi->pi_l3proto = IPPROTO_TCP; 347 break; 348 349 case EMX_RXDMRQ_IPV6_TCP: 350 pi->pi_netisr = NETISR_IPV6; 351 pi->pi_flags = 0; 352 pi->pi_l3proto = IPPROTO_TCP; 353 break; 354 355 case EMX_RXDMRQ_IPV4: 356 if (staterr & E1000_RXD_STAT_IXSM) 357 return NULL; 358 359 if ((staterr & 360 (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == 361 E1000_RXD_STAT_TCPCS) { 362 pi->pi_netisr = NETISR_IP; 363 pi->pi_flags = 0; 364 pi->pi_l3proto = IPPROTO_UDP; 365 break; 366 } 367 /* FALL THROUGH */ 368 default: 369 return NULL; 370 } 371 372 m->m_flags |= M_HASH; 373 m->m_pkthdr.hash = toeplitz_hash(hash); 374 return pi; 375 } 376 377 static int 378 emx_probe(device_t dev) 379 { 380 const struct emx_device *d; 381 uint16_t vid, did; 382 383 vid = pci_get_vendor(dev); 384 did = pci_get_device(dev); 385 386 for (d = emx_devices; d->desc != NULL; ++d) { 387 if (vid == d->vid && did == d->did) { 388 device_set_desc(dev, d->desc); 389 device_set_async_attach(dev, TRUE); 390 return 0; 391 } 392 } 393 return ENXIO; 394 } 395 396 static int 397 emx_attach(device_t dev) 398 { 399 struct emx_softc *sc = device_get_softc(dev); 400 struct ifnet *ifp = &sc->arpcom.ac_if; 401 int error = 0, i; 402 uint16_t eeprom_data, device_id, apme_mask; 403 404 lwkt_serialize_init(&sc->main_serialize); 405 lwkt_serialize_init(&sc->tx_serialize); 406 for (i = 0; i < EMX_NRX_RING; ++i) 407 lwkt_serialize_init(&sc->rx_data[i].rx_serialize); 408 409 i = 0; 410 sc->serializes[i++] = &sc->main_serialize; 411 sc->serializes[i++] = &sc->tx_serialize; 412 sc->serializes[i++] = &sc->rx_data[0].rx_serialize; 413 sc->serializes[i++] = &sc->rx_data[1].rx_serialize; 414 KKASSERT(i == EMX_NSERIALIZE); 415 416 callout_init_mp(&sc->timer); 417 418 sc->dev = sc->osdep.dev = dev; 419 420 /* 421 * Determine hardware and mac type 422 */ 423 sc->hw.vendor_id = pci_get_vendor(dev); 424 sc->hw.device_id = pci_get_device(dev); 425 sc->hw.revision_id = pci_get_revid(dev); 426 sc->hw.subsystem_vendor_id = pci_get_subvendor(dev); 427 sc->hw.subsystem_device_id = pci_get_subdevice(dev); 428 429 if (e1000_set_mac_type(&sc->hw)) 430 return ENXIO; 431 432 /* Enable bus mastering */ 433 pci_enable_busmaster(dev); 434 435 /* 436 * Allocate IO memory 437 */ 438 sc->memory_rid = EMX_BAR_MEM; 439 sc->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 440 &sc->memory_rid, RF_ACTIVE); 441 if (sc->memory == NULL) { 442 device_printf(dev, "Unable to allocate bus resource: memory\n"); 443 error = ENXIO; 444 goto fail; 445 } 446 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->memory); 447 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->memory); 448 449 /* XXX This is quite goofy, it is not actually used */ 450 sc->hw.hw_addr = (uint8_t *)&sc->osdep.mem_bus_space_handle; 451 452 /* 453 * Allocate interrupt 454 */ 455 sc->intr_rid = 0; 456 sc->intr_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->intr_rid, 457 RF_SHAREABLE | RF_ACTIVE); 458 if (sc->intr_res == NULL) { 459 device_printf(dev, "Unable to allocate bus resource: " 460 "interrupt\n"); 461 error = ENXIO; 462 goto fail; 463 } 464 465 /* Save PCI command register for Shared Code */ 466 sc->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 467 sc->hw.back = &sc->osdep; 468 469 /* Do Shared Code initialization */ 470 if (e1000_setup_init_funcs(&sc->hw, TRUE)) { 471 device_printf(dev, "Setup of Shared code failed\n"); 472 error = ENXIO; 473 goto fail; 474 } 475 e1000_get_bus_info(&sc->hw); 476 477 sc->hw.mac.autoneg = EMX_DO_AUTO_NEG; 478 sc->hw.phy.autoneg_wait_to_complete = FALSE; 479 sc->hw.phy.autoneg_advertised = EMX_AUTONEG_ADV_DEFAULT; 480 481 /* 482 * Interrupt throttle rate 483 */ 484 if (emx_int_throttle_ceil == 0) { 485 sc->int_throttle_ceil = 0; 486 } else { 487 int throttle = emx_int_throttle_ceil; 488 489 if (throttle < 0) 490 throttle = EMX_DEFAULT_ITR; 491 492 /* Recalculate the tunable value to get the exact frequency. */ 493 throttle = 1000000000 / 256 / throttle; 494 495 /* Upper 16bits of ITR is reserved and should be zero */ 496 if (throttle & 0xffff0000) 497 throttle = 1000000000 / 256 / EMX_DEFAULT_ITR; 498 499 sc->int_throttle_ceil = 1000000000 / 256 / throttle; 500 } 501 502 e1000_init_script_state_82541(&sc->hw, TRUE); 503 e1000_set_tbi_compatibility_82543(&sc->hw, TRUE); 504 505 /* Copper options */ 506 if (sc->hw.phy.media_type == e1000_media_type_copper) { 507 sc->hw.phy.mdix = EMX_AUTO_ALL_MODES; 508 sc->hw.phy.disable_polarity_correction = FALSE; 509 sc->hw.phy.ms_type = EMX_MASTER_SLAVE; 510 } 511 512 /* Set the frame limits assuming standard ethernet sized frames. */ 513 sc->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN; 514 sc->min_frame_size = ETHER_MIN_LEN; 515 516 /* This controls when hardware reports transmit completion status. */ 517 sc->hw.mac.report_tx_early = 1; 518 519 /* Calculate # of RX rings */ 520 if (ncpus > 1) 521 sc->rx_ring_cnt = EMX_NRX_RING; 522 else 523 sc->rx_ring_cnt = 1; 524 sc->rx_ring_inuse = sc->rx_ring_cnt; 525 526 /* Allocate RX/TX rings' busdma(9) stuffs */ 527 error = emx_dma_alloc(sc); 528 if (error) 529 goto fail; 530 531 /* Allocate multicast array memory. */ 532 sc->mta = kmalloc(ETH_ADDR_LEN * EMX_MCAST_ADDR_MAX, 533 M_DEVBUF, M_WAITOK); 534 535 /* Indicate SOL/IDER usage */ 536 if (e1000_check_reset_block(&sc->hw)) { 537 device_printf(dev, 538 "PHY reset is blocked due to SOL/IDER session.\n"); 539 } 540 541 /* 542 * Start from a known state, this is important in reading the 543 * nvm and mac from that. 544 */ 545 e1000_reset_hw(&sc->hw); 546 547 /* Make sure we have a good EEPROM before we read from it */ 548 if (e1000_validate_nvm_checksum(&sc->hw) < 0) { 549 /* 550 * Some PCI-E parts fail the first check due to 551 * the link being in sleep state, call it again, 552 * if it fails a second time its a real issue. 553 */ 554 if (e1000_validate_nvm_checksum(&sc->hw) < 0) { 555 device_printf(dev, 556 "The EEPROM Checksum Is Not Valid\n"); 557 error = EIO; 558 goto fail; 559 } 560 } 561 562 /* Copy the permanent MAC address out of the EEPROM */ 563 if (e1000_read_mac_addr(&sc->hw) < 0) { 564 device_printf(dev, "EEPROM read error while reading MAC" 565 " address\n"); 566 error = EIO; 567 goto fail; 568 } 569 if (!emx_is_valid_eaddr(sc->hw.mac.addr)) { 570 device_printf(dev, "Invalid MAC address\n"); 571 error = EIO; 572 goto fail; 573 } 574 575 /* Determine if we have to control management hardware */ 576 sc->has_manage = e1000_enable_mng_pass_thru(&sc->hw); 577 578 /* 579 * Setup Wake-on-Lan 580 */ 581 apme_mask = EMX_EEPROM_APME; 582 eeprom_data = 0; 583 switch (sc->hw.mac.type) { 584 case e1000_82573: 585 sc->has_amt = 1; 586 /* FALL THROUGH */ 587 588 case e1000_82571: 589 case e1000_82572: 590 case e1000_80003es2lan: 591 if (sc->hw.bus.func == 1) { 592 e1000_read_nvm(&sc->hw, 593 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 594 } else { 595 e1000_read_nvm(&sc->hw, 596 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 597 } 598 break; 599 600 default: 601 e1000_read_nvm(&sc->hw, 602 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 603 break; 604 } 605 if (eeprom_data & apme_mask) 606 sc->wol = E1000_WUFC_MAG | E1000_WUFC_MC; 607 608 /* 609 * We have the eeprom settings, now apply the special cases 610 * where the eeprom may be wrong or the board won't support 611 * wake on lan on a particular port 612 */ 613 device_id = pci_get_device(dev); 614 switch (device_id) { 615 case E1000_DEV_ID_82571EB_FIBER: 616 /* 617 * Wake events only supported on port A for dual fiber 618 * regardless of eeprom setting 619 */ 620 if (E1000_READ_REG(&sc->hw, E1000_STATUS) & 621 E1000_STATUS_FUNC_1) 622 sc->wol = 0; 623 break; 624 625 case E1000_DEV_ID_82571EB_QUAD_COPPER: 626 case E1000_DEV_ID_82571EB_QUAD_FIBER: 627 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 628 /* if quad port sc, disable WoL on all but port A */ 629 if (emx_global_quad_port_a != 0) 630 sc->wol = 0; 631 /* Reset for multiple quad port adapters */ 632 if (++emx_global_quad_port_a == 4) 633 emx_global_quad_port_a = 0; 634 break; 635 } 636 637 /* XXX disable wol */ 638 sc->wol = 0; 639 640 /* Setup OS specific network interface */ 641 emx_setup_ifp(sc); 642 643 /* Add sysctl tree, must after em_setup_ifp() */ 644 emx_add_sysctl(sc); 645 646 /* Reset the hardware */ 647 error = emx_reset(sc); 648 if (error) { 649 device_printf(dev, "Unable to reset the hardware\n"); 650 goto fail; 651 } 652 653 /* Initialize statistics */ 654 emx_update_stats(sc); 655 656 sc->hw.mac.get_link_status = 1; 657 emx_update_link_status(sc); 658 659 sc->spare_tx_desc = EMX_TX_SPARE; 660 661 /* 662 * Keep following relationship between spare_tx_desc, oact_tx_desc 663 * and tx_int_nsegs: 664 * (spare_tx_desc + EMX_TX_RESERVED) <= 665 * oact_tx_desc <= EMX_TX_OACTIVE_MAX <= tx_int_nsegs 666 */ 667 sc->oact_tx_desc = sc->num_tx_desc / 8; 668 if (sc->oact_tx_desc > EMX_TX_OACTIVE_MAX) 669 sc->oact_tx_desc = EMX_TX_OACTIVE_MAX; 670 if (sc->oact_tx_desc < sc->spare_tx_desc + EMX_TX_RESERVED) 671 sc->oact_tx_desc = sc->spare_tx_desc + EMX_TX_RESERVED; 672 673 sc->tx_int_nsegs = sc->num_tx_desc / 16; 674 if (sc->tx_int_nsegs < sc->oact_tx_desc) 675 sc->tx_int_nsegs = sc->oact_tx_desc; 676 677 /* Non-AMT based hardware can now take control from firmware */ 678 if (sc->has_manage && !sc->has_amt) 679 emx_get_hw_control(sc); 680 681 error = bus_setup_intr(dev, sc->intr_res, INTR_MPSAFE, emx_intr, sc, 682 &sc->intr_tag, &sc->main_serialize); 683 if (error) { 684 device_printf(dev, "Failed to register interrupt handler"); 685 ether_ifdetach(&sc->arpcom.ac_if); 686 goto fail; 687 } 688 689 ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->intr_res)); 690 KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus); 691 return (0); 692 fail: 693 emx_detach(dev); 694 return (error); 695 } 696 697 static int 698 emx_detach(device_t dev) 699 { 700 struct emx_softc *sc = device_get_softc(dev); 701 702 if (device_is_attached(dev)) { 703 struct ifnet *ifp = &sc->arpcom.ac_if; 704 705 ifnet_serialize_all(ifp); 706 707 emx_stop(sc); 708 709 e1000_phy_hw_reset(&sc->hw); 710 711 emx_rel_mgmt(sc); 712 emx_rel_hw_control(sc); 713 714 if (sc->wol) { 715 E1000_WRITE_REG(&sc->hw, E1000_WUC, E1000_WUC_PME_EN); 716 E1000_WRITE_REG(&sc->hw, E1000_WUFC, sc->wol); 717 emx_enable_wol(dev); 718 } 719 720 bus_teardown_intr(dev, sc->intr_res, sc->intr_tag); 721 722 ifnet_deserialize_all(ifp); 723 724 ether_ifdetach(ifp); 725 } else { 726 emx_rel_hw_control(sc); 727 } 728 bus_generic_detach(dev); 729 730 if (sc->intr_res != NULL) { 731 bus_release_resource(dev, SYS_RES_IRQ, sc->intr_rid, 732 sc->intr_res); 733 } 734 735 if (sc->memory != NULL) { 736 bus_release_resource(dev, SYS_RES_MEMORY, sc->memory_rid, 737 sc->memory); 738 } 739 740 emx_dma_free(sc); 741 742 /* Free sysctl tree */ 743 if (sc->sysctl_tree != NULL) 744 sysctl_ctx_free(&sc->sysctl_ctx); 745 746 return (0); 747 } 748 749 static int 750 emx_shutdown(device_t dev) 751 { 752 return emx_suspend(dev); 753 } 754 755 static int 756 emx_suspend(device_t dev) 757 { 758 struct emx_softc *sc = device_get_softc(dev); 759 struct ifnet *ifp = &sc->arpcom.ac_if; 760 761 ifnet_serialize_all(ifp); 762 763 emx_stop(sc); 764 765 emx_rel_mgmt(sc); 766 emx_rel_hw_control(sc); 767 768 if (sc->wol) { 769 E1000_WRITE_REG(&sc->hw, E1000_WUC, E1000_WUC_PME_EN); 770 E1000_WRITE_REG(&sc->hw, E1000_WUFC, sc->wol); 771 emx_enable_wol(dev); 772 } 773 774 ifnet_deserialize_all(ifp); 775 776 return bus_generic_suspend(dev); 777 } 778 779 static int 780 emx_resume(device_t dev) 781 { 782 struct emx_softc *sc = device_get_softc(dev); 783 struct ifnet *ifp = &sc->arpcom.ac_if; 784 785 ifnet_serialize_all(ifp); 786 787 emx_init(sc); 788 emx_get_mgmt(sc); 789 if_devstart(ifp); 790 791 ifnet_deserialize_all(ifp); 792 793 return bus_generic_resume(dev); 794 } 795 796 static void 797 emx_start(struct ifnet *ifp) 798 { 799 struct emx_softc *sc = ifp->if_softc; 800 struct mbuf *m_head; 801 802 ASSERT_SERIALIZED(&sc->tx_serialize); 803 804 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) 805 return; 806 807 if (!sc->link_active) { 808 ifq_purge(&ifp->if_snd); 809 return; 810 } 811 812 while (!ifq_is_empty(&ifp->if_snd)) { 813 /* Now do we at least have a minimal? */ 814 if (EMX_IS_OACTIVE(sc)) { 815 emx_tx_collect(sc); 816 if (EMX_IS_OACTIVE(sc)) { 817 ifp->if_flags |= IFF_OACTIVE; 818 sc->no_tx_desc_avail1++; 819 break; 820 } 821 } 822 823 logif(pkt_txqueue); 824 m_head = ifq_dequeue(&ifp->if_snd, NULL); 825 if (m_head == NULL) 826 break; 827 828 if (emx_encap(sc, &m_head)) { 829 ifp->if_oerrors++; 830 emx_tx_collect(sc); 831 continue; 832 } 833 834 /* Send a copy of the frame to the BPF listener */ 835 ETHER_BPF_MTAP(ifp, m_head); 836 837 /* Set timeout in case hardware has problems transmitting. */ 838 ifp->if_timer = EMX_TX_TIMEOUT; 839 } 840 } 841 842 static int 843 emx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) 844 { 845 struct emx_softc *sc = ifp->if_softc; 846 struct ifreq *ifr = (struct ifreq *)data; 847 uint16_t eeprom_data = 0; 848 int max_frame_size, mask, reinit; 849 int error = 0; 850 851 ASSERT_IFNET_SERIALIZED_ALL(ifp); 852 853 switch (command) { 854 case SIOCSIFMTU: 855 switch (sc->hw.mac.type) { 856 case e1000_82573: 857 /* 858 * 82573 only supports jumbo frames 859 * if ASPM is disabled. 860 */ 861 e1000_read_nvm(&sc->hw, NVM_INIT_3GIO_3, 1, 862 &eeprom_data); 863 if (eeprom_data & NVM_WORD1A_ASPM_MASK) { 864 max_frame_size = ETHER_MAX_LEN; 865 break; 866 } 867 /* FALL THROUGH */ 868 869 /* Limit Jumbo Frame size */ 870 case e1000_82571: 871 case e1000_82572: 872 case e1000_82574: 873 case e1000_80003es2lan: 874 max_frame_size = 9234; 875 break; 876 877 default: 878 max_frame_size = MAX_JUMBO_FRAME_SIZE; 879 break; 880 } 881 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 882 ETHER_CRC_LEN) { 883 error = EINVAL; 884 break; 885 } 886 887 ifp->if_mtu = ifr->ifr_mtu; 888 sc->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + 889 ETHER_CRC_LEN; 890 891 if (ifp->if_flags & IFF_RUNNING) 892 emx_init(sc); 893 break; 894 895 case SIOCSIFFLAGS: 896 if (ifp->if_flags & IFF_UP) { 897 if ((ifp->if_flags & IFF_RUNNING)) { 898 if ((ifp->if_flags ^ sc->if_flags) & 899 (IFF_PROMISC | IFF_ALLMULTI)) { 900 emx_disable_promisc(sc); 901 emx_set_promisc(sc); 902 } 903 } else { 904 emx_init(sc); 905 } 906 } else if (ifp->if_flags & IFF_RUNNING) { 907 emx_stop(sc); 908 } 909 sc->if_flags = ifp->if_flags; 910 break; 911 912 case SIOCADDMULTI: 913 case SIOCDELMULTI: 914 if (ifp->if_flags & IFF_RUNNING) { 915 emx_disable_intr(sc); 916 emx_set_multi(sc); 917 #ifdef IFPOLL_ENABLE 918 if (!(ifp->if_flags & IFF_NPOLLING)) 919 #endif 920 emx_enable_intr(sc); 921 } 922 break; 923 924 case SIOCSIFMEDIA: 925 /* Check SOL/IDER usage */ 926 if (e1000_check_reset_block(&sc->hw)) { 927 device_printf(sc->dev, "Media change is" 928 " blocked due to SOL/IDER session.\n"); 929 break; 930 } 931 /* FALL THROUGH */ 932 933 case SIOCGIFMEDIA: 934 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 935 break; 936 937 case SIOCSIFCAP: 938 reinit = 0; 939 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 940 if (mask & IFCAP_HWCSUM) { 941 ifp->if_capenable ^= (mask & IFCAP_HWCSUM); 942 reinit = 1; 943 } 944 if (mask & IFCAP_VLAN_HWTAGGING) { 945 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 946 reinit = 1; 947 } 948 if (mask & IFCAP_RSS) { 949 ifp->if_capenable ^= IFCAP_RSS; 950 reinit = 1; 951 } 952 if (reinit && (ifp->if_flags & IFF_RUNNING)) 953 emx_init(sc); 954 break; 955 956 default: 957 error = ether_ioctl(ifp, command, data); 958 break; 959 } 960 return (error); 961 } 962 963 static void 964 emx_watchdog(struct ifnet *ifp) 965 { 966 struct emx_softc *sc = ifp->if_softc; 967 968 ASSERT_IFNET_SERIALIZED_ALL(ifp); 969 970 /* 971 * The timer is set to 5 every time start queues a packet. 972 * Then txeof keeps resetting it as long as it cleans at 973 * least one descriptor. 974 * Finally, anytime all descriptors are clean the timer is 975 * set to 0. 976 */ 977 978 if (E1000_READ_REG(&sc->hw, E1000_TDT(0)) == 979 E1000_READ_REG(&sc->hw, E1000_TDH(0))) { 980 /* 981 * If we reach here, all TX jobs are completed and 982 * the TX engine should have been idled for some time. 983 * We don't need to call if_devstart() here. 984 */ 985 ifp->if_flags &= ~IFF_OACTIVE; 986 ifp->if_timer = 0; 987 return; 988 } 989 990 /* 991 * If we are in this routine because of pause frames, then 992 * don't reset the hardware. 993 */ 994 if (E1000_READ_REG(&sc->hw, E1000_STATUS) & E1000_STATUS_TXOFF) { 995 ifp->if_timer = EMX_TX_TIMEOUT; 996 return; 997 } 998 999 if (e1000_check_for_link(&sc->hw) == 0) 1000 if_printf(ifp, "watchdog timeout -- resetting\n"); 1001 1002 ifp->if_oerrors++; 1003 sc->watchdog_events++; 1004 1005 emx_init(sc); 1006 1007 if (!ifq_is_empty(&ifp->if_snd)) 1008 if_devstart(ifp); 1009 } 1010 1011 static void 1012 emx_init(void *xsc) 1013 { 1014 struct emx_softc *sc = xsc; 1015 struct ifnet *ifp = &sc->arpcom.ac_if; 1016 device_t dev = sc->dev; 1017 uint32_t pba; 1018 int i; 1019 1020 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1021 1022 emx_stop(sc); 1023 1024 /* 1025 * Packet Buffer Allocation (PBA) 1026 * Writing PBA sets the receive portion of the buffer 1027 * the remainder is used for the transmit buffer. 1028 */ 1029 switch (sc->hw.mac.type) { 1030 /* Total Packet Buffer on these is 48K */ 1031 case e1000_82571: 1032 case e1000_82572: 1033 case e1000_80003es2lan: 1034 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1035 break; 1036 1037 case e1000_82573: /* 82573: Total Packet Buffer is 32K */ 1038 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 1039 break; 1040 1041 case e1000_82574: 1042 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */ 1043 break; 1044 1045 default: 1046 /* Devices before 82547 had a Packet Buffer of 64K. */ 1047 if (sc->max_frame_size > 8192) 1048 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 1049 else 1050 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 1051 } 1052 E1000_WRITE_REG(&sc->hw, E1000_PBA, pba); 1053 1054 /* Get the latest mac address, User can use a LAA */ 1055 bcopy(IF_LLADDR(ifp), sc->hw.mac.addr, ETHER_ADDR_LEN); 1056 1057 /* Put the address into the Receive Address Array */ 1058 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1059 1060 /* 1061 * With the 82571 sc, RAR[0] may be overwritten 1062 * when the other port is reset, we make a duplicate 1063 * in RAR[14] for that eventuality, this assures 1064 * the interface continues to function. 1065 */ 1066 if (sc->hw.mac.type == e1000_82571) { 1067 e1000_set_laa_state_82571(&sc->hw, TRUE); 1068 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 1069 E1000_RAR_ENTRIES - 1); 1070 } 1071 1072 /* Initialize the hardware */ 1073 if (emx_reset(sc)) { 1074 device_printf(dev, "Unable to reset the hardware\n"); 1075 /* XXX emx_stop()? */ 1076 return; 1077 } 1078 emx_update_link_status(sc); 1079 1080 /* Setup VLAN support, basic and offload if available */ 1081 E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN); 1082 1083 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1084 uint32_t ctrl; 1085 1086 ctrl = E1000_READ_REG(&sc->hw, E1000_CTRL); 1087 ctrl |= E1000_CTRL_VME; 1088 E1000_WRITE_REG(&sc->hw, E1000_CTRL, ctrl); 1089 } 1090 1091 /* Set hardware offload abilities */ 1092 if (ifp->if_capenable & IFCAP_TXCSUM) 1093 ifp->if_hwassist = EMX_CSUM_FEATURES; 1094 else 1095 ifp->if_hwassist = 0; 1096 1097 /* Configure for OS presence */ 1098 emx_get_mgmt(sc); 1099 1100 /* Prepare transmit descriptors and buffers */ 1101 emx_init_tx_ring(sc); 1102 emx_init_tx_unit(sc); 1103 1104 /* Setup Multicast table */ 1105 emx_set_multi(sc); 1106 1107 /* 1108 * Adjust # of RX ring to be used based on IFCAP_RSS 1109 */ 1110 if (ifp->if_capenable & IFCAP_RSS) 1111 sc->rx_ring_inuse = sc->rx_ring_cnt; 1112 else 1113 sc->rx_ring_inuse = 1; 1114 1115 /* Prepare receive descriptors and buffers */ 1116 for (i = 0; i < sc->rx_ring_inuse; ++i) { 1117 if (emx_init_rx_ring(sc, &sc->rx_data[i])) { 1118 device_printf(dev, 1119 "Could not setup receive structures\n"); 1120 emx_stop(sc); 1121 return; 1122 } 1123 } 1124 emx_init_rx_unit(sc); 1125 1126 /* Don't lose promiscuous settings */ 1127 emx_set_promisc(sc); 1128 1129 ifp->if_flags |= IFF_RUNNING; 1130 ifp->if_flags &= ~IFF_OACTIVE; 1131 1132 callout_reset(&sc->timer, hz, emx_timer, sc); 1133 e1000_clear_hw_cntrs_base_generic(&sc->hw); 1134 1135 /* MSI/X configuration for 82574 */ 1136 if (sc->hw.mac.type == e1000_82574) { 1137 int tmp; 1138 1139 tmp = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 1140 tmp |= E1000_CTRL_EXT_PBA_CLR; 1141 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, tmp); 1142 /* 1143 * XXX MSIX 1144 * Set the IVAR - interrupt vector routing. 1145 * Each nibble represents a vector, high bit 1146 * is enable, other 3 bits are the MSIX table 1147 * entry, we map RXQ0 to 0, TXQ0 to 1, and 1148 * Link (other) to 2, hence the magic number. 1149 */ 1150 E1000_WRITE_REG(&sc->hw, E1000_IVAR, 0x800A0908); 1151 } 1152 1153 #ifdef IFPOLL_ENABLE 1154 /* 1155 * Only enable interrupts if we are not polling, make sure 1156 * they are off otherwise. 1157 */ 1158 if (ifp->if_flags & IFF_NPOLLING) 1159 emx_disable_intr(sc); 1160 else 1161 #endif /* IFPOLL_ENABLE */ 1162 emx_enable_intr(sc); 1163 1164 /* AMT based hardware can now take control from firmware */ 1165 if (sc->has_manage && sc->has_amt) 1166 emx_get_hw_control(sc); 1167 1168 /* Don't reset the phy next time init gets called */ 1169 sc->hw.phy.reset_disable = TRUE; 1170 } 1171 1172 static void 1173 emx_intr(void *xsc) 1174 { 1175 struct emx_softc *sc = xsc; 1176 struct ifnet *ifp = &sc->arpcom.ac_if; 1177 uint32_t reg_icr; 1178 1179 logif(intr_beg); 1180 ASSERT_SERIALIZED(&sc->main_serialize); 1181 1182 reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); 1183 1184 if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0) { 1185 logif(intr_end); 1186 return; 1187 } 1188 1189 /* 1190 * XXX: some laptops trigger several spurious interrupts 1191 * on emx(4) when in the resume cycle. The ICR register 1192 * reports all-ones value in this case. Processing such 1193 * interrupts would lead to a freeze. I don't know why. 1194 */ 1195 if (reg_icr == 0xffffffff) { 1196 logif(intr_end); 1197 return; 1198 } 1199 1200 if (ifp->if_flags & IFF_RUNNING) { 1201 if (reg_icr & 1202 (E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_RXO)) { 1203 int i; 1204 1205 for (i = 0; i < sc->rx_ring_inuse; ++i) { 1206 lwkt_serialize_enter( 1207 &sc->rx_data[i].rx_serialize); 1208 emx_rxeof(sc, i, -1); 1209 lwkt_serialize_exit( 1210 &sc->rx_data[i].rx_serialize); 1211 } 1212 } 1213 if (reg_icr & E1000_ICR_TXDW) { 1214 lwkt_serialize_enter(&sc->tx_serialize); 1215 emx_txeof(sc); 1216 if (!ifq_is_empty(&ifp->if_snd)) 1217 if_devstart(ifp); 1218 lwkt_serialize_exit(&sc->tx_serialize); 1219 } 1220 } 1221 1222 /* Link status change */ 1223 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1224 emx_serialize_skipmain(sc); 1225 1226 callout_stop(&sc->timer); 1227 sc->hw.mac.get_link_status = 1; 1228 emx_update_link_status(sc); 1229 1230 /* Deal with TX cruft when link lost */ 1231 emx_tx_purge(sc); 1232 1233 callout_reset(&sc->timer, hz, emx_timer, sc); 1234 1235 emx_deserialize_skipmain(sc); 1236 } 1237 1238 if (reg_icr & E1000_ICR_RXO) 1239 sc->rx_overruns++; 1240 1241 logif(intr_end); 1242 } 1243 1244 static void 1245 emx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1246 { 1247 struct emx_softc *sc = ifp->if_softc; 1248 1249 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1250 1251 emx_update_link_status(sc); 1252 1253 ifmr->ifm_status = IFM_AVALID; 1254 ifmr->ifm_active = IFM_ETHER; 1255 1256 if (!sc->link_active) 1257 return; 1258 1259 ifmr->ifm_status |= IFM_ACTIVE; 1260 1261 if (sc->hw.phy.media_type == e1000_media_type_fiber || 1262 sc->hw.phy.media_type == e1000_media_type_internal_serdes) { 1263 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 1264 } else { 1265 switch (sc->link_speed) { 1266 case 10: 1267 ifmr->ifm_active |= IFM_10_T; 1268 break; 1269 case 100: 1270 ifmr->ifm_active |= IFM_100_TX; 1271 break; 1272 1273 case 1000: 1274 ifmr->ifm_active |= IFM_1000_T; 1275 break; 1276 } 1277 if (sc->link_duplex == FULL_DUPLEX) 1278 ifmr->ifm_active |= IFM_FDX; 1279 else 1280 ifmr->ifm_active |= IFM_HDX; 1281 } 1282 } 1283 1284 static int 1285 emx_media_change(struct ifnet *ifp) 1286 { 1287 struct emx_softc *sc = ifp->if_softc; 1288 struct ifmedia *ifm = &sc->media; 1289 1290 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1291 1292 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1293 return (EINVAL); 1294 1295 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1296 case IFM_AUTO: 1297 sc->hw.mac.autoneg = EMX_DO_AUTO_NEG; 1298 sc->hw.phy.autoneg_advertised = EMX_AUTONEG_ADV_DEFAULT; 1299 break; 1300 1301 case IFM_1000_LX: 1302 case IFM_1000_SX: 1303 case IFM_1000_T: 1304 sc->hw.mac.autoneg = EMX_DO_AUTO_NEG; 1305 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1306 break; 1307 1308 case IFM_100_TX: 1309 sc->hw.mac.autoneg = FALSE; 1310 sc->hw.phy.autoneg_advertised = 0; 1311 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1312 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1313 else 1314 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1315 break; 1316 1317 case IFM_10_T: 1318 sc->hw.mac.autoneg = FALSE; 1319 sc->hw.phy.autoneg_advertised = 0; 1320 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1321 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1322 else 1323 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1324 break; 1325 1326 default: 1327 if_printf(ifp, "Unsupported media type\n"); 1328 break; 1329 } 1330 1331 /* 1332 * As the speed/duplex settings my have changed we need to 1333 * reset the PHY. 1334 */ 1335 sc->hw.phy.reset_disable = FALSE; 1336 1337 emx_init(sc); 1338 1339 return (0); 1340 } 1341 1342 static int 1343 emx_encap(struct emx_softc *sc, struct mbuf **m_headp) 1344 { 1345 bus_dma_segment_t segs[EMX_MAX_SCATTER]; 1346 bus_dmamap_t map; 1347 struct emx_txbuf *tx_buffer, *tx_buffer_mapped; 1348 struct e1000_tx_desc *ctxd = NULL; 1349 struct mbuf *m_head = *m_headp; 1350 uint32_t txd_upper, txd_lower, cmd = 0; 1351 int maxsegs, nsegs, i, j, first, last = 0, error; 1352 1353 if (m_head->m_len < EMX_TXCSUM_MINHL && 1354 (m_head->m_flags & EMX_CSUM_FEATURES)) { 1355 /* 1356 * Make sure that ethernet header and ip.ip_hl are in 1357 * contiguous memory, since if TXCSUM is enabled, later 1358 * TX context descriptor's setup need to access ip.ip_hl. 1359 */ 1360 error = emx_txcsum_pullup(sc, m_headp); 1361 if (error) { 1362 KKASSERT(*m_headp == NULL); 1363 return error; 1364 } 1365 m_head = *m_headp; 1366 } 1367 1368 txd_upper = txd_lower = 0; 1369 1370 /* 1371 * Capture the first descriptor index, this descriptor 1372 * will have the index of the EOP which is the only one 1373 * that now gets a DONE bit writeback. 1374 */ 1375 first = sc->next_avail_tx_desc; 1376 tx_buffer = &sc->tx_buf[first]; 1377 tx_buffer_mapped = tx_buffer; 1378 map = tx_buffer->map; 1379 1380 maxsegs = sc->num_tx_desc_avail - EMX_TX_RESERVED; 1381 KASSERT(maxsegs >= sc->spare_tx_desc, ("not enough spare TX desc\n")); 1382 if (maxsegs > EMX_MAX_SCATTER) 1383 maxsegs = EMX_MAX_SCATTER; 1384 1385 error = bus_dmamap_load_mbuf_defrag(sc->txtag, map, m_headp, 1386 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 1387 if (error) { 1388 if (error == ENOBUFS) 1389 sc->mbuf_alloc_failed++; 1390 else 1391 sc->no_tx_dma_setup++; 1392 1393 m_freem(*m_headp); 1394 *m_headp = NULL; 1395 return error; 1396 } 1397 bus_dmamap_sync(sc->txtag, map, BUS_DMASYNC_PREWRITE); 1398 1399 m_head = *m_headp; 1400 sc->tx_nsegs += nsegs; 1401 1402 if (m_head->m_pkthdr.csum_flags & EMX_CSUM_FEATURES) { 1403 /* TX csum offloading will consume one TX desc */ 1404 sc->tx_nsegs += emx_txcsum(sc, m_head, &txd_upper, &txd_lower); 1405 } 1406 i = sc->next_avail_tx_desc; 1407 1408 /* Set up our transmit descriptors */ 1409 for (j = 0; j < nsegs; j++) { 1410 tx_buffer = &sc->tx_buf[i]; 1411 ctxd = &sc->tx_desc_base[i]; 1412 1413 ctxd->buffer_addr = htole64(segs[j].ds_addr); 1414 ctxd->lower.data = htole32(E1000_TXD_CMD_IFCS | 1415 txd_lower | segs[j].ds_len); 1416 ctxd->upper.data = htole32(txd_upper); 1417 1418 last = i; 1419 if (++i == sc->num_tx_desc) 1420 i = 0; 1421 } 1422 1423 sc->next_avail_tx_desc = i; 1424 1425 KKASSERT(sc->num_tx_desc_avail > nsegs); 1426 sc->num_tx_desc_avail -= nsegs; 1427 1428 /* Handle VLAN tag */ 1429 if (m_head->m_flags & M_VLANTAG) { 1430 /* Set the vlan id. */ 1431 ctxd->upper.fields.special = 1432 htole16(m_head->m_pkthdr.ether_vlantag); 1433 1434 /* Tell hardware to add tag */ 1435 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE); 1436 } 1437 1438 tx_buffer->m_head = m_head; 1439 tx_buffer_mapped->map = tx_buffer->map; 1440 tx_buffer->map = map; 1441 1442 if (sc->tx_nsegs >= sc->tx_int_nsegs) { 1443 sc->tx_nsegs = 0; 1444 1445 /* 1446 * Report Status (RS) is turned on 1447 * every tx_int_nsegs descriptors. 1448 */ 1449 cmd = E1000_TXD_CMD_RS; 1450 1451 /* 1452 * Keep track of the descriptor, which will 1453 * be written back by hardware. 1454 */ 1455 sc->tx_dd[sc->tx_dd_tail] = last; 1456 EMX_INC_TXDD_IDX(sc->tx_dd_tail); 1457 KKASSERT(sc->tx_dd_tail != sc->tx_dd_head); 1458 } 1459 1460 /* 1461 * Last Descriptor of Packet needs End Of Packet (EOP) 1462 */ 1463 ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | cmd); 1464 1465 /* 1466 * Advance the Transmit Descriptor Tail (TDT), this tells 1467 * the E1000 that this frame is available to transmit. 1468 */ 1469 E1000_WRITE_REG(&sc->hw, E1000_TDT(0), i); 1470 1471 return (0); 1472 } 1473 1474 static void 1475 emx_set_promisc(struct emx_softc *sc) 1476 { 1477 struct ifnet *ifp = &sc->arpcom.ac_if; 1478 uint32_t reg_rctl; 1479 1480 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 1481 1482 if (ifp->if_flags & IFF_PROMISC) { 1483 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 1484 /* Turn this on if you want to see bad packets */ 1485 if (emx_debug_sbp) 1486 reg_rctl |= E1000_RCTL_SBP; 1487 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 1488 } else if (ifp->if_flags & IFF_ALLMULTI) { 1489 reg_rctl |= E1000_RCTL_MPE; 1490 reg_rctl &= ~E1000_RCTL_UPE; 1491 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 1492 } 1493 } 1494 1495 static void 1496 emx_disable_promisc(struct emx_softc *sc) 1497 { 1498 uint32_t reg_rctl; 1499 1500 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 1501 1502 reg_rctl &= ~E1000_RCTL_UPE; 1503 reg_rctl &= ~E1000_RCTL_MPE; 1504 reg_rctl &= ~E1000_RCTL_SBP; 1505 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 1506 } 1507 1508 static void 1509 emx_set_multi(struct emx_softc *sc) 1510 { 1511 struct ifnet *ifp = &sc->arpcom.ac_if; 1512 struct ifmultiaddr *ifma; 1513 uint32_t reg_rctl = 0; 1514 uint8_t *mta; 1515 int mcnt = 0; 1516 1517 mta = sc->mta; 1518 bzero(mta, ETH_ADDR_LEN * EMX_MCAST_ADDR_MAX); 1519 1520 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1521 if (ifma->ifma_addr->sa_family != AF_LINK) 1522 continue; 1523 1524 if (mcnt == EMX_MCAST_ADDR_MAX) 1525 break; 1526 1527 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1528 &mta[mcnt * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 1529 mcnt++; 1530 } 1531 1532 if (mcnt >= EMX_MCAST_ADDR_MAX) { 1533 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 1534 reg_rctl |= E1000_RCTL_MPE; 1535 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 1536 } else { 1537 e1000_update_mc_addr_list(&sc->hw, mta, mcnt); 1538 } 1539 } 1540 1541 /* 1542 * This routine checks for link status and updates statistics. 1543 */ 1544 static void 1545 emx_timer(void *xsc) 1546 { 1547 struct emx_softc *sc = xsc; 1548 struct ifnet *ifp = &sc->arpcom.ac_if; 1549 1550 ifnet_serialize_all(ifp); 1551 1552 emx_update_link_status(sc); 1553 emx_update_stats(sc); 1554 1555 /* Reset LAA into RAR[0] on 82571 */ 1556 if (e1000_get_laa_state_82571(&sc->hw) == TRUE) 1557 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1558 1559 if (emx_display_debug_stats && (ifp->if_flags & IFF_RUNNING)) 1560 emx_print_hw_stats(sc); 1561 1562 emx_smartspeed(sc); 1563 1564 callout_reset(&sc->timer, hz, emx_timer, sc); 1565 1566 ifnet_deserialize_all(ifp); 1567 } 1568 1569 static void 1570 emx_update_link_status(struct emx_softc *sc) 1571 { 1572 struct e1000_hw *hw = &sc->hw; 1573 struct ifnet *ifp = &sc->arpcom.ac_if; 1574 device_t dev = sc->dev; 1575 uint32_t link_check = 0; 1576 1577 /* Get the cached link value or read phy for real */ 1578 switch (hw->phy.media_type) { 1579 case e1000_media_type_copper: 1580 if (hw->mac.get_link_status) { 1581 /* Do the work to read phy */ 1582 e1000_check_for_link(hw); 1583 link_check = !hw->mac.get_link_status; 1584 if (link_check) /* ESB2 fix */ 1585 e1000_cfg_on_link_up(hw); 1586 } else { 1587 link_check = TRUE; 1588 } 1589 break; 1590 1591 case e1000_media_type_fiber: 1592 e1000_check_for_link(hw); 1593 link_check = E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU; 1594 break; 1595 1596 case e1000_media_type_internal_serdes: 1597 e1000_check_for_link(hw); 1598 link_check = sc->hw.mac.serdes_has_link; 1599 break; 1600 1601 case e1000_media_type_unknown: 1602 default: 1603 break; 1604 } 1605 1606 /* Now check for a transition */ 1607 if (link_check && sc->link_active == 0) { 1608 e1000_get_speed_and_duplex(hw, &sc->link_speed, 1609 &sc->link_duplex); 1610 1611 /* 1612 * Check if we should enable/disable SPEED_MODE bit on 1613 * 82571EB/82572EI 1614 */ 1615 if (sc->link_speed != SPEED_1000 && 1616 (hw->mac.type == e1000_82571 || 1617 hw->mac.type == e1000_82572)) { 1618 int tarc0; 1619 1620 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 1621 tarc0 &= ~EMX_TARC_SPEED_MODE; 1622 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 1623 } 1624 if (bootverbose) { 1625 device_printf(dev, "Link is up %d Mbps %s\n", 1626 sc->link_speed, 1627 ((sc->link_duplex == FULL_DUPLEX) ? 1628 "Full Duplex" : "Half Duplex")); 1629 } 1630 sc->link_active = 1; 1631 sc->smartspeed = 0; 1632 ifp->if_baudrate = sc->link_speed * 1000000; 1633 ifp->if_link_state = LINK_STATE_UP; 1634 if_link_state_change(ifp); 1635 } else if (!link_check && sc->link_active == 1) { 1636 ifp->if_baudrate = sc->link_speed = 0; 1637 sc->link_duplex = 0; 1638 if (bootverbose) 1639 device_printf(dev, "Link is Down\n"); 1640 sc->link_active = 0; 1641 #if 0 1642 /* Link down, disable watchdog */ 1643 if->if_timer = 0; 1644 #endif 1645 ifp->if_link_state = LINK_STATE_DOWN; 1646 if_link_state_change(ifp); 1647 } 1648 } 1649 1650 static void 1651 emx_stop(struct emx_softc *sc) 1652 { 1653 struct ifnet *ifp = &sc->arpcom.ac_if; 1654 int i; 1655 1656 ASSERT_IFNET_SERIALIZED_ALL(ifp); 1657 1658 emx_disable_intr(sc); 1659 1660 callout_stop(&sc->timer); 1661 1662 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1663 ifp->if_timer = 0; 1664 1665 /* 1666 * Disable multiple receive queues. 1667 * 1668 * NOTE: 1669 * We should disable multiple receive queues before 1670 * resetting the hardware. 1671 */ 1672 E1000_WRITE_REG(&sc->hw, E1000_MRQC, 0); 1673 1674 e1000_reset_hw(&sc->hw); 1675 E1000_WRITE_REG(&sc->hw, E1000_WUC, 0); 1676 1677 for (i = 0; i < sc->num_tx_desc; i++) { 1678 struct emx_txbuf *tx_buffer = &sc->tx_buf[i]; 1679 1680 if (tx_buffer->m_head != NULL) { 1681 bus_dmamap_unload(sc->txtag, tx_buffer->map); 1682 m_freem(tx_buffer->m_head); 1683 tx_buffer->m_head = NULL; 1684 } 1685 } 1686 1687 for (i = 0; i < sc->rx_ring_inuse; ++i) 1688 emx_free_rx_ring(sc, &sc->rx_data[i]); 1689 1690 sc->csum_flags = 0; 1691 sc->csum_ehlen = 0; 1692 sc->csum_iphlen = 0; 1693 1694 sc->tx_dd_head = 0; 1695 sc->tx_dd_tail = 0; 1696 sc->tx_nsegs = 0; 1697 } 1698 1699 static int 1700 emx_reset(struct emx_softc *sc) 1701 { 1702 device_t dev = sc->dev; 1703 uint16_t rx_buffer_size; 1704 1705 /* Set up smart power down as default off on newer adapters. */ 1706 if (!emx_smart_pwr_down && 1707 (sc->hw.mac.type == e1000_82571 || 1708 sc->hw.mac.type == e1000_82572)) { 1709 uint16_t phy_tmp = 0; 1710 1711 /* Speed up time to link by disabling smart power down. */ 1712 e1000_read_phy_reg(&sc->hw, 1713 IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 1714 phy_tmp &= ~IGP02E1000_PM_SPD; 1715 e1000_write_phy_reg(&sc->hw, 1716 IGP02E1000_PHY_POWER_MGMT, phy_tmp); 1717 } 1718 1719 /* 1720 * These parameters control the automatic generation (Tx) and 1721 * response (Rx) to Ethernet PAUSE frames. 1722 * - High water mark should allow for at least two frames to be 1723 * received after sending an XOFF. 1724 * - Low water mark works best when it is very near the high water mark. 1725 * This allows the receiver to restart by sending XON when it has 1726 * drained a bit. Here we use an arbitary value of 1500 which will 1727 * restart after one full frame is pulled from the buffer. There 1728 * could be several smaller frames in the buffer and if so they will 1729 * not trigger the XON until their total number reduces the buffer 1730 * by 1500. 1731 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 1732 */ 1733 rx_buffer_size = (E1000_READ_REG(&sc->hw, E1000_PBA) & 0xffff) << 10; 1734 1735 sc->hw.fc.high_water = rx_buffer_size - 1736 roundup2(sc->max_frame_size, 1024); 1737 sc->hw.fc.low_water = sc->hw.fc.high_water - 1500; 1738 1739 if (sc->hw.mac.type == e1000_80003es2lan) 1740 sc->hw.fc.pause_time = 0xFFFF; 1741 else 1742 sc->hw.fc.pause_time = EMX_FC_PAUSE_TIME; 1743 sc->hw.fc.send_xon = TRUE; 1744 sc->hw.fc.requested_mode = e1000_fc_full; 1745 1746 /* Issue a global reset */ 1747 e1000_reset_hw(&sc->hw); 1748 E1000_WRITE_REG(&sc->hw, E1000_WUC, 0); 1749 1750 if (e1000_init_hw(&sc->hw) < 0) { 1751 device_printf(dev, "Hardware Initialization Failed\n"); 1752 return (EIO); 1753 } 1754 1755 E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN); 1756 e1000_get_phy_info(&sc->hw); 1757 e1000_check_for_link(&sc->hw); 1758 1759 return (0); 1760 } 1761 1762 static void 1763 emx_setup_ifp(struct emx_softc *sc) 1764 { 1765 struct ifnet *ifp = &sc->arpcom.ac_if; 1766 1767 if_initname(ifp, device_get_name(sc->dev), 1768 device_get_unit(sc->dev)); 1769 ifp->if_softc = sc; 1770 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1771 ifp->if_init = emx_init; 1772 ifp->if_ioctl = emx_ioctl; 1773 ifp->if_start = emx_start; 1774 #ifdef IFPOLL_ENABLE 1775 ifp->if_qpoll = emx_qpoll; 1776 #endif 1777 ifp->if_watchdog = emx_watchdog; 1778 ifp->if_serialize = emx_serialize; 1779 ifp->if_deserialize = emx_deserialize; 1780 ifp->if_tryserialize = emx_tryserialize; 1781 #ifdef INVARIANTS 1782 ifp->if_serialize_assert = emx_serialize_assert; 1783 #endif 1784 ifq_set_maxlen(&ifp->if_snd, sc->num_tx_desc - 1); 1785 ifq_set_ready(&ifp->if_snd); 1786 1787 ether_ifattach(ifp, sc->hw.mac.addr, NULL); 1788 1789 ifp->if_capabilities = IFCAP_HWCSUM | 1790 IFCAP_VLAN_HWTAGGING | 1791 IFCAP_VLAN_MTU; 1792 if (sc->rx_ring_cnt > 1) 1793 ifp->if_capabilities |= IFCAP_RSS; 1794 ifp->if_capenable = ifp->if_capabilities; 1795 ifp->if_hwassist = EMX_CSUM_FEATURES; 1796 1797 /* 1798 * Tell the upper layer(s) we support long frames. 1799 */ 1800 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 1801 1802 /* 1803 * Specify the media types supported by this sc and register 1804 * callbacks to update media and link information 1805 */ 1806 ifmedia_init(&sc->media, IFM_IMASK, 1807 emx_media_change, emx_media_status); 1808 if (sc->hw.phy.media_type == e1000_media_type_fiber || 1809 sc->hw.phy.media_type == e1000_media_type_internal_serdes) { 1810 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 1811 0, NULL); 1812 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 1813 } else { 1814 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 1815 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 1816 0, NULL); 1817 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1818 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 1819 0, NULL); 1820 if (sc->hw.phy.type != e1000_phy_ife) { 1821 ifmedia_add(&sc->media, 1822 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1823 ifmedia_add(&sc->media, 1824 IFM_ETHER | IFM_1000_T, 0, NULL); 1825 } 1826 } 1827 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1828 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1829 } 1830 1831 /* 1832 * Workaround for SmartSpeed on 82541 and 82547 controllers 1833 */ 1834 static void 1835 emx_smartspeed(struct emx_softc *sc) 1836 { 1837 uint16_t phy_tmp; 1838 1839 if (sc->link_active || sc->hw.phy.type != e1000_phy_igp || 1840 sc->hw.mac.autoneg == 0 || 1841 (sc->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) 1842 return; 1843 1844 if (sc->smartspeed == 0) { 1845 /* 1846 * If Master/Slave config fault is asserted twice, 1847 * we assume back-to-back 1848 */ 1849 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 1850 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) 1851 return; 1852 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 1853 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { 1854 e1000_read_phy_reg(&sc->hw, 1855 PHY_1000T_CTRL, &phy_tmp); 1856 if (phy_tmp & CR_1000T_MS_ENABLE) { 1857 phy_tmp &= ~CR_1000T_MS_ENABLE; 1858 e1000_write_phy_reg(&sc->hw, 1859 PHY_1000T_CTRL, phy_tmp); 1860 sc->smartspeed++; 1861 if (sc->hw.mac.autoneg && 1862 !e1000_phy_setup_autoneg(&sc->hw) && 1863 !e1000_read_phy_reg(&sc->hw, 1864 PHY_CONTROL, &phy_tmp)) { 1865 phy_tmp |= MII_CR_AUTO_NEG_EN | 1866 MII_CR_RESTART_AUTO_NEG; 1867 e1000_write_phy_reg(&sc->hw, 1868 PHY_CONTROL, phy_tmp); 1869 } 1870 } 1871 } 1872 return; 1873 } else if (sc->smartspeed == EMX_SMARTSPEED_DOWNSHIFT) { 1874 /* If still no link, perhaps using 2/3 pair cable */ 1875 e1000_read_phy_reg(&sc->hw, PHY_1000T_CTRL, &phy_tmp); 1876 phy_tmp |= CR_1000T_MS_ENABLE; 1877 e1000_write_phy_reg(&sc->hw, PHY_1000T_CTRL, phy_tmp); 1878 if (sc->hw.mac.autoneg && 1879 !e1000_phy_setup_autoneg(&sc->hw) && 1880 !e1000_read_phy_reg(&sc->hw, PHY_CONTROL, &phy_tmp)) { 1881 phy_tmp |= MII_CR_AUTO_NEG_EN | MII_CR_RESTART_AUTO_NEG; 1882 e1000_write_phy_reg(&sc->hw, PHY_CONTROL, phy_tmp); 1883 } 1884 } 1885 1886 /* Restart process after EMX_SMARTSPEED_MAX iterations */ 1887 if (sc->smartspeed++ == EMX_SMARTSPEED_MAX) 1888 sc->smartspeed = 0; 1889 } 1890 1891 static int 1892 emx_create_tx_ring(struct emx_softc *sc) 1893 { 1894 device_t dev = sc->dev; 1895 struct emx_txbuf *tx_buffer; 1896 int error, i, tsize; 1897 1898 /* 1899 * Validate number of transmit descriptors. It must not exceed 1900 * hardware maximum, and must be multiple of E1000_DBA_ALIGN. 1901 */ 1902 if ((emx_txd * sizeof(struct e1000_tx_desc)) % EMX_DBA_ALIGN != 0 || 1903 emx_txd > EMX_MAX_TXD || emx_txd < EMX_MIN_TXD) { 1904 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 1905 EMX_DEFAULT_TXD, emx_txd); 1906 sc->num_tx_desc = EMX_DEFAULT_TXD; 1907 } else { 1908 sc->num_tx_desc = emx_txd; 1909 } 1910 1911 /* 1912 * Allocate Transmit Descriptor ring 1913 */ 1914 tsize = roundup2(sc->num_tx_desc * sizeof(struct e1000_tx_desc), 1915 EMX_DBA_ALIGN); 1916 sc->tx_desc_base = bus_dmamem_coherent_any(sc->parent_dtag, 1917 EMX_DBA_ALIGN, tsize, BUS_DMA_WAITOK, 1918 &sc->tx_desc_dtag, &sc->tx_desc_dmap, 1919 &sc->tx_desc_paddr); 1920 if (sc->tx_desc_base == NULL) { 1921 device_printf(dev, "Unable to allocate tx_desc memory\n"); 1922 return ENOMEM; 1923 } 1924 1925 sc->tx_buf = kmalloc(sizeof(struct emx_txbuf) * sc->num_tx_desc, 1926 M_DEVBUF, M_WAITOK | M_ZERO); 1927 1928 /* 1929 * Create DMA tags for tx buffers 1930 */ 1931 error = bus_dma_tag_create(sc->parent_dtag, /* parent */ 1932 1, 0, /* alignment, bounds */ 1933 BUS_SPACE_MAXADDR, /* lowaddr */ 1934 BUS_SPACE_MAXADDR, /* highaddr */ 1935 NULL, NULL, /* filter, filterarg */ 1936 EMX_TSO_SIZE, /* maxsize */ 1937 EMX_MAX_SCATTER, /* nsegments */ 1938 EMX_MAX_SEGSIZE, /* maxsegsize */ 1939 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | 1940 BUS_DMA_ONEBPAGE, /* flags */ 1941 &sc->txtag); 1942 if (error) { 1943 device_printf(dev, "Unable to allocate TX DMA tag\n"); 1944 kfree(sc->tx_buf, M_DEVBUF); 1945 sc->tx_buf = NULL; 1946 return error; 1947 } 1948 1949 /* 1950 * Create DMA maps for tx buffers 1951 */ 1952 for (i = 0; i < sc->num_tx_desc; i++) { 1953 tx_buffer = &sc->tx_buf[i]; 1954 1955 error = bus_dmamap_create(sc->txtag, 1956 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 1957 &tx_buffer->map); 1958 if (error) { 1959 device_printf(dev, "Unable to create TX DMA map\n"); 1960 emx_destroy_tx_ring(sc, i); 1961 return error; 1962 } 1963 } 1964 return (0); 1965 } 1966 1967 static void 1968 emx_init_tx_ring(struct emx_softc *sc) 1969 { 1970 /* Clear the old ring contents */ 1971 bzero(sc->tx_desc_base, 1972 sizeof(struct e1000_tx_desc) * sc->num_tx_desc); 1973 1974 /* Reset state */ 1975 sc->next_avail_tx_desc = 0; 1976 sc->next_tx_to_clean = 0; 1977 sc->num_tx_desc_avail = sc->num_tx_desc; 1978 } 1979 1980 static void 1981 emx_init_tx_unit(struct emx_softc *sc) 1982 { 1983 uint32_t tctl, tarc, tipg = 0; 1984 uint64_t bus_addr; 1985 1986 /* Setup the Base and Length of the Tx Descriptor Ring */ 1987 bus_addr = sc->tx_desc_paddr; 1988 E1000_WRITE_REG(&sc->hw, E1000_TDLEN(0), 1989 sc->num_tx_desc * sizeof(struct e1000_tx_desc)); 1990 E1000_WRITE_REG(&sc->hw, E1000_TDBAH(0), 1991 (uint32_t)(bus_addr >> 32)); 1992 E1000_WRITE_REG(&sc->hw, E1000_TDBAL(0), 1993 (uint32_t)bus_addr); 1994 /* Setup the HW Tx Head and Tail descriptor pointers */ 1995 E1000_WRITE_REG(&sc->hw, E1000_TDT(0), 0); 1996 E1000_WRITE_REG(&sc->hw, E1000_TDH(0), 0); 1997 1998 /* Set the default values for the Tx Inter Packet Gap timer */ 1999 switch (sc->hw.mac.type) { 2000 case e1000_80003es2lan: 2001 tipg = DEFAULT_82543_TIPG_IPGR1; 2002 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 2003 E1000_TIPG_IPGR2_SHIFT; 2004 break; 2005 2006 default: 2007 if (sc->hw.phy.media_type == e1000_media_type_fiber || 2008 sc->hw.phy.media_type == e1000_media_type_internal_serdes) 2009 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 2010 else 2011 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 2012 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 2013 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 2014 break; 2015 } 2016 2017 E1000_WRITE_REG(&sc->hw, E1000_TIPG, tipg); 2018 2019 /* NOTE: 0 is not allowed for TIDV */ 2020 E1000_WRITE_REG(&sc->hw, E1000_TIDV, 1); 2021 E1000_WRITE_REG(&sc->hw, E1000_TADV, 0); 2022 2023 if (sc->hw.mac.type == e1000_82571 || 2024 sc->hw.mac.type == e1000_82572) { 2025 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(0)); 2026 tarc |= EMX_TARC_SPEED_MODE; 2027 E1000_WRITE_REG(&sc->hw, E1000_TARC(0), tarc); 2028 } else if (sc->hw.mac.type == e1000_80003es2lan) { 2029 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(0)); 2030 tarc |= 1; 2031 E1000_WRITE_REG(&sc->hw, E1000_TARC(0), tarc); 2032 tarc = E1000_READ_REG(&sc->hw, E1000_TARC(1)); 2033 tarc |= 1; 2034 E1000_WRITE_REG(&sc->hw, E1000_TARC(1), tarc); 2035 } 2036 2037 /* Program the Transmit Control Register */ 2038 tctl = E1000_READ_REG(&sc->hw, E1000_TCTL); 2039 tctl &= ~E1000_TCTL_CT; 2040 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 2041 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2042 tctl |= E1000_TCTL_MULR; 2043 2044 /* This write will effectively turn on the transmit unit. */ 2045 E1000_WRITE_REG(&sc->hw, E1000_TCTL, tctl); 2046 } 2047 2048 static void 2049 emx_destroy_tx_ring(struct emx_softc *sc, int ndesc) 2050 { 2051 struct emx_txbuf *tx_buffer; 2052 int i; 2053 2054 /* Free Transmit Descriptor ring */ 2055 if (sc->tx_desc_base) { 2056 bus_dmamap_unload(sc->tx_desc_dtag, sc->tx_desc_dmap); 2057 bus_dmamem_free(sc->tx_desc_dtag, sc->tx_desc_base, 2058 sc->tx_desc_dmap); 2059 bus_dma_tag_destroy(sc->tx_desc_dtag); 2060 2061 sc->tx_desc_base = NULL; 2062 } 2063 2064 if (sc->tx_buf == NULL) 2065 return; 2066 2067 for (i = 0; i < ndesc; i++) { 2068 tx_buffer = &sc->tx_buf[i]; 2069 2070 KKASSERT(tx_buffer->m_head == NULL); 2071 bus_dmamap_destroy(sc->txtag, tx_buffer->map); 2072 } 2073 bus_dma_tag_destroy(sc->txtag); 2074 2075 kfree(sc->tx_buf, M_DEVBUF); 2076 sc->tx_buf = NULL; 2077 } 2078 2079 /* 2080 * The offload context needs to be set when we transfer the first 2081 * packet of a particular protocol (TCP/UDP). This routine has been 2082 * enhanced to deal with inserted VLAN headers. 2083 * 2084 * If the new packet's ether header length, ip header length and 2085 * csum offloading type are same as the previous packet, we should 2086 * avoid allocating a new csum context descriptor; mainly to take 2087 * advantage of the pipeline effect of the TX data read request. 2088 * 2089 * This function returns number of TX descrptors allocated for 2090 * csum context. 2091 */ 2092 static int 2093 emx_txcsum(struct emx_softc *sc, struct mbuf *mp, 2094 uint32_t *txd_upper, uint32_t *txd_lower) 2095 { 2096 struct e1000_context_desc *TXD; 2097 struct emx_txbuf *tx_buffer; 2098 struct ether_vlan_header *eh; 2099 struct ip *ip; 2100 int curr_txd, ehdrlen, csum_flags; 2101 uint32_t cmd, hdr_len, ip_hlen; 2102 uint16_t etype; 2103 2104 /* 2105 * Determine where frame payload starts. 2106 * Jump over vlan headers if already present, 2107 * helpful for QinQ too. 2108 */ 2109 KASSERT(mp->m_len >= ETHER_HDR_LEN, 2110 ("emx_txcsum_pullup is not called (eh)?\n")); 2111 eh = mtod(mp, struct ether_vlan_header *); 2112 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2113 KASSERT(mp->m_len >= ETHER_HDR_LEN + EVL_ENCAPLEN, 2114 ("emx_txcsum_pullup is not called (evh)?\n")); 2115 etype = ntohs(eh->evl_proto); 2116 ehdrlen = ETHER_HDR_LEN + EVL_ENCAPLEN; 2117 } else { 2118 etype = ntohs(eh->evl_encap_proto); 2119 ehdrlen = ETHER_HDR_LEN; 2120 } 2121 2122 /* 2123 * We only support TCP/UDP for IPv4 for the moment. 2124 * TODO: Support SCTP too when it hits the tree. 2125 */ 2126 if (etype != ETHERTYPE_IP) 2127 return 0; 2128 2129 KASSERT(mp->m_len >= ehdrlen + EMX_IPVHL_SIZE, 2130 ("emx_txcsum_pullup is not called (eh+ip_vhl)?\n")); 2131 2132 /* NOTE: We could only safely access ip.ip_vhl part */ 2133 ip = (struct ip *)(mp->m_data + ehdrlen); 2134 ip_hlen = ip->ip_hl << 2; 2135 2136 csum_flags = mp->m_pkthdr.csum_flags & EMX_CSUM_FEATURES; 2137 2138 if (sc->csum_ehlen == ehdrlen && sc->csum_iphlen == ip_hlen && 2139 sc->csum_flags == csum_flags) { 2140 /* 2141 * Same csum offload context as the previous packets; 2142 * just return. 2143 */ 2144 *txd_upper = sc->csum_txd_upper; 2145 *txd_lower = sc->csum_txd_lower; 2146 return 0; 2147 } 2148 2149 /* 2150 * Setup a new csum offload context. 2151 */ 2152 2153 curr_txd = sc->next_avail_tx_desc; 2154 tx_buffer = &sc->tx_buf[curr_txd]; 2155 TXD = (struct e1000_context_desc *)&sc->tx_desc_base[curr_txd]; 2156 2157 cmd = 0; 2158 2159 /* Setup of IP header checksum. */ 2160 if (csum_flags & CSUM_IP) { 2161 /* 2162 * Start offset for header checksum calculation. 2163 * End offset for header checksum calculation. 2164 * Offset of place to put the checksum. 2165 */ 2166 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 2167 TXD->lower_setup.ip_fields.ipcse = 2168 htole16(ehdrlen + ip_hlen - 1); 2169 TXD->lower_setup.ip_fields.ipcso = 2170 ehdrlen + offsetof(struct ip, ip_sum); 2171 cmd |= E1000_TXD_CMD_IP; 2172 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 2173 } 2174 hdr_len = ehdrlen + ip_hlen; 2175 2176 if (csum_flags & CSUM_TCP) { 2177 /* 2178 * Start offset for payload checksum calculation. 2179 * End offset for payload checksum calculation. 2180 * Offset of place to put the checksum. 2181 */ 2182 TXD->upper_setup.tcp_fields.tucss = hdr_len; 2183 TXD->upper_setup.tcp_fields.tucse = htole16(0); 2184 TXD->upper_setup.tcp_fields.tucso = 2185 hdr_len + offsetof(struct tcphdr, th_sum); 2186 cmd |= E1000_TXD_CMD_TCP; 2187 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 2188 } else if (csum_flags & CSUM_UDP) { 2189 /* 2190 * Start offset for header checksum calculation. 2191 * End offset for header checksum calculation. 2192 * Offset of place to put the checksum. 2193 */ 2194 TXD->upper_setup.tcp_fields.tucss = hdr_len; 2195 TXD->upper_setup.tcp_fields.tucse = htole16(0); 2196 TXD->upper_setup.tcp_fields.tucso = 2197 hdr_len + offsetof(struct udphdr, uh_sum); 2198 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 2199 } 2200 2201 *txd_lower = E1000_TXD_CMD_DEXT | /* Extended descr type */ 2202 E1000_TXD_DTYP_D; /* Data descr */ 2203 2204 /* Save the information for this csum offloading context */ 2205 sc->csum_ehlen = ehdrlen; 2206 sc->csum_iphlen = ip_hlen; 2207 sc->csum_flags = csum_flags; 2208 sc->csum_txd_upper = *txd_upper; 2209 sc->csum_txd_lower = *txd_lower; 2210 2211 TXD->tcp_seg_setup.data = htole32(0); 2212 TXD->cmd_and_length = 2213 htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); 2214 2215 if (++curr_txd == sc->num_tx_desc) 2216 curr_txd = 0; 2217 2218 KKASSERT(sc->num_tx_desc_avail > 0); 2219 sc->num_tx_desc_avail--; 2220 2221 sc->next_avail_tx_desc = curr_txd; 2222 return 1; 2223 } 2224 2225 static int 2226 emx_txcsum_pullup(struct emx_softc *sc, struct mbuf **m0) 2227 { 2228 struct mbuf *m = *m0; 2229 struct ether_header *eh; 2230 int len; 2231 2232 sc->tx_csum_try_pullup++; 2233 2234 len = ETHER_HDR_LEN + EMX_IPVHL_SIZE; 2235 2236 if (__predict_false(!M_WRITABLE(m))) { 2237 if (__predict_false(m->m_len < ETHER_HDR_LEN)) { 2238 sc->tx_csum_drop1++; 2239 m_freem(m); 2240 *m0 = NULL; 2241 return ENOBUFS; 2242 } 2243 eh = mtod(m, struct ether_header *); 2244 2245 if (eh->ether_type == htons(ETHERTYPE_VLAN)) 2246 len += EVL_ENCAPLEN; 2247 2248 if (m->m_len < len) { 2249 sc->tx_csum_drop2++; 2250 m_freem(m); 2251 *m0 = NULL; 2252 return ENOBUFS; 2253 } 2254 return 0; 2255 } 2256 2257 if (__predict_false(m->m_len < ETHER_HDR_LEN)) { 2258 sc->tx_csum_pullup1++; 2259 m = m_pullup(m, ETHER_HDR_LEN); 2260 if (m == NULL) { 2261 sc->tx_csum_pullup1_failed++; 2262 *m0 = NULL; 2263 return ENOBUFS; 2264 } 2265 *m0 = m; 2266 } 2267 eh = mtod(m, struct ether_header *); 2268 2269 if (eh->ether_type == htons(ETHERTYPE_VLAN)) 2270 len += EVL_ENCAPLEN; 2271 2272 if (m->m_len < len) { 2273 sc->tx_csum_pullup2++; 2274 m = m_pullup(m, len); 2275 if (m == NULL) { 2276 sc->tx_csum_pullup2_failed++; 2277 *m0 = NULL; 2278 return ENOBUFS; 2279 } 2280 *m0 = m; 2281 } 2282 return 0; 2283 } 2284 2285 static void 2286 emx_txeof(struct emx_softc *sc) 2287 { 2288 struct ifnet *ifp = &sc->arpcom.ac_if; 2289 struct emx_txbuf *tx_buffer; 2290 int first, num_avail; 2291 2292 if (sc->tx_dd_head == sc->tx_dd_tail) 2293 return; 2294 2295 if (sc->num_tx_desc_avail == sc->num_tx_desc) 2296 return; 2297 2298 num_avail = sc->num_tx_desc_avail; 2299 first = sc->next_tx_to_clean; 2300 2301 while (sc->tx_dd_head != sc->tx_dd_tail) { 2302 int dd_idx = sc->tx_dd[sc->tx_dd_head]; 2303 struct e1000_tx_desc *tx_desc; 2304 2305 tx_desc = &sc->tx_desc_base[dd_idx]; 2306 if (tx_desc->upper.fields.status & E1000_TXD_STAT_DD) { 2307 EMX_INC_TXDD_IDX(sc->tx_dd_head); 2308 2309 if (++dd_idx == sc->num_tx_desc) 2310 dd_idx = 0; 2311 2312 while (first != dd_idx) { 2313 logif(pkt_txclean); 2314 2315 num_avail++; 2316 2317 tx_buffer = &sc->tx_buf[first]; 2318 if (tx_buffer->m_head) { 2319 ifp->if_opackets++; 2320 bus_dmamap_unload(sc->txtag, 2321 tx_buffer->map); 2322 m_freem(tx_buffer->m_head); 2323 tx_buffer->m_head = NULL; 2324 } 2325 2326 if (++first == sc->num_tx_desc) 2327 first = 0; 2328 } 2329 } else { 2330 break; 2331 } 2332 } 2333 sc->next_tx_to_clean = first; 2334 sc->num_tx_desc_avail = num_avail; 2335 2336 if (sc->tx_dd_head == sc->tx_dd_tail) { 2337 sc->tx_dd_head = 0; 2338 sc->tx_dd_tail = 0; 2339 } 2340 2341 if (!EMX_IS_OACTIVE(sc)) { 2342 ifp->if_flags &= ~IFF_OACTIVE; 2343 2344 /* All clean, turn off the timer */ 2345 if (sc->num_tx_desc_avail == sc->num_tx_desc) 2346 ifp->if_timer = 0; 2347 } 2348 } 2349 2350 static void 2351 emx_tx_collect(struct emx_softc *sc) 2352 { 2353 struct ifnet *ifp = &sc->arpcom.ac_if; 2354 struct emx_txbuf *tx_buffer; 2355 int tdh, first, num_avail, dd_idx = -1; 2356 2357 if (sc->num_tx_desc_avail == sc->num_tx_desc) 2358 return; 2359 2360 tdh = E1000_READ_REG(&sc->hw, E1000_TDH(0)); 2361 if (tdh == sc->next_tx_to_clean) 2362 return; 2363 2364 if (sc->tx_dd_head != sc->tx_dd_tail) 2365 dd_idx = sc->tx_dd[sc->tx_dd_head]; 2366 2367 num_avail = sc->num_tx_desc_avail; 2368 first = sc->next_tx_to_clean; 2369 2370 while (first != tdh) { 2371 logif(pkt_txclean); 2372 2373 num_avail++; 2374 2375 tx_buffer = &sc->tx_buf[first]; 2376 if (tx_buffer->m_head) { 2377 ifp->if_opackets++; 2378 bus_dmamap_unload(sc->txtag, 2379 tx_buffer->map); 2380 m_freem(tx_buffer->m_head); 2381 tx_buffer->m_head = NULL; 2382 } 2383 2384 if (first == dd_idx) { 2385 EMX_INC_TXDD_IDX(sc->tx_dd_head); 2386 if (sc->tx_dd_head == sc->tx_dd_tail) { 2387 sc->tx_dd_head = 0; 2388 sc->tx_dd_tail = 0; 2389 dd_idx = -1; 2390 } else { 2391 dd_idx = sc->tx_dd[sc->tx_dd_head]; 2392 } 2393 } 2394 2395 if (++first == sc->num_tx_desc) 2396 first = 0; 2397 } 2398 sc->next_tx_to_clean = first; 2399 sc->num_tx_desc_avail = num_avail; 2400 2401 if (!EMX_IS_OACTIVE(sc)) { 2402 ifp->if_flags &= ~IFF_OACTIVE; 2403 2404 /* All clean, turn off the timer */ 2405 if (sc->num_tx_desc_avail == sc->num_tx_desc) 2406 ifp->if_timer = 0; 2407 } 2408 } 2409 2410 /* 2411 * When Link is lost sometimes there is work still in the TX ring 2412 * which will result in a watchdog, rather than allow that do an 2413 * attempted cleanup and then reinit here. Note that this has been 2414 * seens mostly with fiber adapters. 2415 */ 2416 static void 2417 emx_tx_purge(struct emx_softc *sc) 2418 { 2419 struct ifnet *ifp = &sc->arpcom.ac_if; 2420 2421 if (!sc->link_active && ifp->if_timer) { 2422 emx_tx_collect(sc); 2423 if (ifp->if_timer) { 2424 if_printf(ifp, "Link lost, TX pending, reinit\n"); 2425 ifp->if_timer = 0; 2426 emx_init(sc); 2427 } 2428 } 2429 } 2430 2431 static int 2432 emx_newbuf(struct emx_softc *sc, struct emx_rxdata *rdata, int i, int init) 2433 { 2434 struct mbuf *m; 2435 bus_dma_segment_t seg; 2436 bus_dmamap_t map; 2437 struct emx_rxbuf *rx_buffer; 2438 int error, nseg; 2439 2440 m = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR); 2441 if (m == NULL) { 2442 rdata->mbuf_cluster_failed++; 2443 if (init) { 2444 if_printf(&sc->arpcom.ac_if, 2445 "Unable to allocate RX mbuf\n"); 2446 } 2447 return (ENOBUFS); 2448 } 2449 m->m_len = m->m_pkthdr.len = MCLBYTES; 2450 2451 if (sc->max_frame_size <= MCLBYTES - ETHER_ALIGN) 2452 m_adj(m, ETHER_ALIGN); 2453 2454 error = bus_dmamap_load_mbuf_segment(rdata->rxtag, 2455 rdata->rx_sparemap, m, 2456 &seg, 1, &nseg, BUS_DMA_NOWAIT); 2457 if (error) { 2458 m_freem(m); 2459 if (init) { 2460 if_printf(&sc->arpcom.ac_if, 2461 "Unable to load RX mbuf\n"); 2462 } 2463 return (error); 2464 } 2465 2466 rx_buffer = &rdata->rx_buf[i]; 2467 if (rx_buffer->m_head != NULL) 2468 bus_dmamap_unload(rdata->rxtag, rx_buffer->map); 2469 2470 map = rx_buffer->map; 2471 rx_buffer->map = rdata->rx_sparemap; 2472 rdata->rx_sparemap = map; 2473 2474 rx_buffer->m_head = m; 2475 rx_buffer->paddr = seg.ds_addr; 2476 2477 emx_setup_rxdesc(&rdata->rx_desc[i], rx_buffer); 2478 return (0); 2479 } 2480 2481 static int 2482 emx_create_rx_ring(struct emx_softc *sc, struct emx_rxdata *rdata) 2483 { 2484 device_t dev = sc->dev; 2485 struct emx_rxbuf *rx_buffer; 2486 int i, error, rsize; 2487 2488 /* 2489 * Validate number of receive descriptors. It must not exceed 2490 * hardware maximum, and must be multiple of E1000_DBA_ALIGN. 2491 */ 2492 if ((emx_rxd * sizeof(emx_rxdesc_t)) % EMX_DBA_ALIGN != 0 || 2493 emx_rxd > EMX_MAX_RXD || emx_rxd < EMX_MIN_RXD) { 2494 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 2495 EMX_DEFAULT_RXD, emx_rxd); 2496 rdata->num_rx_desc = EMX_DEFAULT_RXD; 2497 } else { 2498 rdata->num_rx_desc = emx_rxd; 2499 } 2500 2501 /* 2502 * Allocate Receive Descriptor ring 2503 */ 2504 rsize = roundup2(rdata->num_rx_desc * sizeof(emx_rxdesc_t), 2505 EMX_DBA_ALIGN); 2506 rdata->rx_desc = bus_dmamem_coherent_any(sc->parent_dtag, 2507 EMX_DBA_ALIGN, rsize, BUS_DMA_WAITOK, 2508 &rdata->rx_desc_dtag, &rdata->rx_desc_dmap, 2509 &rdata->rx_desc_paddr); 2510 if (rdata->rx_desc == NULL) { 2511 device_printf(dev, "Unable to allocate rx_desc memory\n"); 2512 return ENOMEM; 2513 } 2514 2515 rdata->rx_buf = kmalloc(sizeof(struct emx_rxbuf) * rdata->num_rx_desc, 2516 M_DEVBUF, M_WAITOK | M_ZERO); 2517 2518 /* 2519 * Create DMA tag for rx buffers 2520 */ 2521 error = bus_dma_tag_create(sc->parent_dtag, /* parent */ 2522 1, 0, /* alignment, bounds */ 2523 BUS_SPACE_MAXADDR, /* lowaddr */ 2524 BUS_SPACE_MAXADDR, /* highaddr */ 2525 NULL, NULL, /* filter, filterarg */ 2526 MCLBYTES, /* maxsize */ 2527 1, /* nsegments */ 2528 MCLBYTES, /* maxsegsize */ 2529 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, /* flags */ 2530 &rdata->rxtag); 2531 if (error) { 2532 device_printf(dev, "Unable to allocate RX DMA tag\n"); 2533 kfree(rdata->rx_buf, M_DEVBUF); 2534 rdata->rx_buf = NULL; 2535 return error; 2536 } 2537 2538 /* 2539 * Create spare DMA map for rx buffers 2540 */ 2541 error = bus_dmamap_create(rdata->rxtag, BUS_DMA_WAITOK, 2542 &rdata->rx_sparemap); 2543 if (error) { 2544 device_printf(dev, "Unable to create spare RX DMA map\n"); 2545 bus_dma_tag_destroy(rdata->rxtag); 2546 kfree(rdata->rx_buf, M_DEVBUF); 2547 rdata->rx_buf = NULL; 2548 return error; 2549 } 2550 2551 /* 2552 * Create DMA maps for rx buffers 2553 */ 2554 for (i = 0; i < rdata->num_rx_desc; i++) { 2555 rx_buffer = &rdata->rx_buf[i]; 2556 2557 error = bus_dmamap_create(rdata->rxtag, BUS_DMA_WAITOK, 2558 &rx_buffer->map); 2559 if (error) { 2560 device_printf(dev, "Unable to create RX DMA map\n"); 2561 emx_destroy_rx_ring(sc, rdata, i); 2562 return error; 2563 } 2564 } 2565 return (0); 2566 } 2567 2568 static void 2569 emx_free_rx_ring(struct emx_softc *sc, struct emx_rxdata *rdata) 2570 { 2571 int i; 2572 2573 for (i = 0; i < rdata->num_rx_desc; i++) { 2574 struct emx_rxbuf *rx_buffer = &rdata->rx_buf[i]; 2575 2576 if (rx_buffer->m_head != NULL) { 2577 bus_dmamap_unload(rdata->rxtag, rx_buffer->map); 2578 m_freem(rx_buffer->m_head); 2579 rx_buffer->m_head = NULL; 2580 } 2581 } 2582 2583 if (rdata->fmp != NULL) 2584 m_freem(rdata->fmp); 2585 rdata->fmp = NULL; 2586 rdata->lmp = NULL; 2587 } 2588 2589 static int 2590 emx_init_rx_ring(struct emx_softc *sc, struct emx_rxdata *rdata) 2591 { 2592 int i, error; 2593 2594 /* Reset descriptor ring */ 2595 bzero(rdata->rx_desc, sizeof(emx_rxdesc_t) * rdata->num_rx_desc); 2596 2597 /* Allocate new ones. */ 2598 for (i = 0; i < rdata->num_rx_desc; i++) { 2599 error = emx_newbuf(sc, rdata, i, 1); 2600 if (error) 2601 return (error); 2602 } 2603 2604 /* Setup our descriptor pointers */ 2605 rdata->next_rx_desc_to_check = 0; 2606 2607 return (0); 2608 } 2609 2610 static void 2611 emx_init_rx_unit(struct emx_softc *sc) 2612 { 2613 struct ifnet *ifp = &sc->arpcom.ac_if; 2614 uint64_t bus_addr; 2615 uint32_t rctl, itr, rfctl; 2616 int i; 2617 2618 /* 2619 * Make sure receives are disabled while setting 2620 * up the descriptor ring 2621 */ 2622 rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 2623 E1000_WRITE_REG(&sc->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 2624 2625 /* 2626 * Set the interrupt throttling rate. Value is calculated 2627 * as ITR = 1 / (INT_THROTTLE_CEIL * 256ns) 2628 */ 2629 if (sc->int_throttle_ceil) 2630 itr = 1000000000 / 256 / sc->int_throttle_ceil; 2631 else 2632 itr = 0; 2633 emx_set_itr(sc, itr); 2634 2635 /* Use extended RX descriptor */ 2636 rfctl = E1000_RFCTL_EXTEN; 2637 2638 /* Disable accelerated ackknowledge */ 2639 if (sc->hw.mac.type == e1000_82574) 2640 rfctl |= E1000_RFCTL_ACK_DIS; 2641 2642 E1000_WRITE_REG(&sc->hw, E1000_RFCTL, rfctl); 2643 2644 /* 2645 * Receive Checksum Offload for TCP and UDP 2646 * 2647 * Checksum offloading is also enabled if multiple receive 2648 * queue is to be supported, since we need it to figure out 2649 * packet type. 2650 */ 2651 if (ifp->if_capenable & (IFCAP_RSS | IFCAP_RXCSUM)) { 2652 uint32_t rxcsum; 2653 2654 rxcsum = E1000_READ_REG(&sc->hw, E1000_RXCSUM); 2655 2656 /* 2657 * NOTE: 2658 * PCSD must be enabled to enable multiple 2659 * receive queues. 2660 */ 2661 rxcsum |= E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL | 2662 E1000_RXCSUM_PCSD; 2663 E1000_WRITE_REG(&sc->hw, E1000_RXCSUM, rxcsum); 2664 } 2665 2666 /* 2667 * Configure multiple receive queue (RSS) 2668 */ 2669 if (ifp->if_capenable & IFCAP_RSS) { 2670 uint8_t key[EMX_NRSSRK * EMX_RSSRK_SIZE]; 2671 uint32_t reta; 2672 2673 KASSERT(sc->rx_ring_inuse == EMX_NRX_RING, 2674 ("invalid number of RX ring (%d)", 2675 sc->rx_ring_inuse)); 2676 2677 /* 2678 * NOTE: 2679 * When we reach here, RSS has already been disabled 2680 * in emx_stop(), so we could safely configure RSS key 2681 * and redirect table. 2682 */ 2683 2684 /* 2685 * Configure RSS key 2686 */ 2687 toeplitz_get_key(key, sizeof(key)); 2688 for (i = 0; i < EMX_NRSSRK; ++i) { 2689 uint32_t rssrk; 2690 2691 rssrk = EMX_RSSRK_VAL(key, i); 2692 EMX_RSS_DPRINTF(sc, 1, "rssrk%d 0x%08x\n", i, rssrk); 2693 2694 E1000_WRITE_REG(&sc->hw, E1000_RSSRK(i), rssrk); 2695 } 2696 2697 /* 2698 * Configure RSS redirect table in following fashion: 2699 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] 2700 */ 2701 reta = 0; 2702 for (i = 0; i < EMX_RETA_SIZE; ++i) { 2703 uint32_t q; 2704 2705 q = (i % sc->rx_ring_inuse) << EMX_RETA_RINGIDX_SHIFT; 2706 reta |= q << (8 * i); 2707 } 2708 EMX_RSS_DPRINTF(sc, 1, "reta 0x%08x\n", reta); 2709 2710 for (i = 0; i < EMX_NRETA; ++i) 2711 E1000_WRITE_REG(&sc->hw, E1000_RETA(i), reta); 2712 2713 /* 2714 * Enable multiple receive queues. 2715 * Enable IPv4 RSS standard hash functions. 2716 * Disable RSS interrupt. 2717 */ 2718 E1000_WRITE_REG(&sc->hw, E1000_MRQC, 2719 E1000_MRQC_ENABLE_RSS_2Q | 2720 E1000_MRQC_RSS_FIELD_IPV4_TCP | 2721 E1000_MRQC_RSS_FIELD_IPV4); 2722 } 2723 2724 /* 2725 * XXX TEMPORARY WORKAROUND: on some systems with 82573 2726 * long latencies are observed, like Lenovo X60. This 2727 * change eliminates the problem, but since having positive 2728 * values in RDTR is a known source of problems on other 2729 * platforms another solution is being sought. 2730 */ 2731 if (emx_82573_workaround && sc->hw.mac.type == e1000_82573) { 2732 E1000_WRITE_REG(&sc->hw, E1000_RADV, EMX_RADV_82573); 2733 E1000_WRITE_REG(&sc->hw, E1000_RDTR, EMX_RDTR_82573); 2734 } 2735 2736 for (i = 0; i < sc->rx_ring_inuse; ++i) { 2737 struct emx_rxdata *rdata = &sc->rx_data[i]; 2738 2739 /* 2740 * Setup the Base and Length of the Rx Descriptor Ring 2741 */ 2742 bus_addr = rdata->rx_desc_paddr; 2743 E1000_WRITE_REG(&sc->hw, E1000_RDLEN(i), 2744 rdata->num_rx_desc * sizeof(emx_rxdesc_t)); 2745 E1000_WRITE_REG(&sc->hw, E1000_RDBAH(i), 2746 (uint32_t)(bus_addr >> 32)); 2747 E1000_WRITE_REG(&sc->hw, E1000_RDBAL(i), 2748 (uint32_t)bus_addr); 2749 2750 /* 2751 * Setup the HW Rx Head and Tail Descriptor Pointers 2752 */ 2753 E1000_WRITE_REG(&sc->hw, E1000_RDH(i), 0); 2754 E1000_WRITE_REG(&sc->hw, E1000_RDT(i), 2755 sc->rx_data[i].num_rx_desc - 1); 2756 } 2757 2758 /* Setup the Receive Control Register */ 2759 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 2760 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 2761 E1000_RCTL_RDMTS_HALF | E1000_RCTL_SECRC | 2762 (sc->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 2763 2764 /* Make sure VLAN Filters are off */ 2765 rctl &= ~E1000_RCTL_VFE; 2766 2767 /* Don't store bad paket */ 2768 rctl &= ~E1000_RCTL_SBP; 2769 2770 /* MCLBYTES */ 2771 rctl |= E1000_RCTL_SZ_2048; 2772 2773 if (ifp->if_mtu > ETHERMTU) 2774 rctl |= E1000_RCTL_LPE; 2775 else 2776 rctl &= ~E1000_RCTL_LPE; 2777 2778 /* Enable Receives */ 2779 E1000_WRITE_REG(&sc->hw, E1000_RCTL, rctl); 2780 } 2781 2782 static void 2783 emx_destroy_rx_ring(struct emx_softc *sc, struct emx_rxdata *rdata, int ndesc) 2784 { 2785 struct emx_rxbuf *rx_buffer; 2786 int i; 2787 2788 /* Free Receive Descriptor ring */ 2789 if (rdata->rx_desc) { 2790 bus_dmamap_unload(rdata->rx_desc_dtag, rdata->rx_desc_dmap); 2791 bus_dmamem_free(rdata->rx_desc_dtag, rdata->rx_desc, 2792 rdata->rx_desc_dmap); 2793 bus_dma_tag_destroy(rdata->rx_desc_dtag); 2794 2795 rdata->rx_desc = NULL; 2796 } 2797 2798 if (rdata->rx_buf == NULL) 2799 return; 2800 2801 for (i = 0; i < ndesc; i++) { 2802 rx_buffer = &rdata->rx_buf[i]; 2803 2804 KKASSERT(rx_buffer->m_head == NULL); 2805 bus_dmamap_destroy(rdata->rxtag, rx_buffer->map); 2806 } 2807 bus_dmamap_destroy(rdata->rxtag, rdata->rx_sparemap); 2808 bus_dma_tag_destroy(rdata->rxtag); 2809 2810 kfree(rdata->rx_buf, M_DEVBUF); 2811 rdata->rx_buf = NULL; 2812 } 2813 2814 static void 2815 emx_rxeof(struct emx_softc *sc, int ring_idx, int count) 2816 { 2817 struct emx_rxdata *rdata = &sc->rx_data[ring_idx]; 2818 struct ifnet *ifp = &sc->arpcom.ac_if; 2819 uint32_t staterr; 2820 emx_rxdesc_t *current_desc; 2821 struct mbuf *mp; 2822 int i; 2823 struct mbuf_chain chain[MAXCPU]; 2824 2825 i = rdata->next_rx_desc_to_check; 2826 current_desc = &rdata->rx_desc[i]; 2827 staterr = le32toh(current_desc->rxd_staterr); 2828 2829 if (!(staterr & E1000_RXD_STAT_DD)) 2830 return; 2831 2832 ether_input_chain_init(chain); 2833 2834 while ((staterr & E1000_RXD_STAT_DD) && count != 0) { 2835 struct pktinfo *pi = NULL, pi0; 2836 struct emx_rxbuf *rx_buf = &rdata->rx_buf[i]; 2837 struct mbuf *m = NULL; 2838 int eop, len; 2839 2840 logif(pkt_receive); 2841 2842 mp = rx_buf->m_head; 2843 2844 /* 2845 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT 2846 * needs to access the last received byte in the mbuf. 2847 */ 2848 bus_dmamap_sync(rdata->rxtag, rx_buf->map, 2849 BUS_DMASYNC_POSTREAD); 2850 2851 len = le16toh(current_desc->rxd_length); 2852 if (staterr & E1000_RXD_STAT_EOP) { 2853 count--; 2854 eop = 1; 2855 } else { 2856 eop = 0; 2857 } 2858 2859 if (!(staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK)) { 2860 uint16_t vlan = 0; 2861 uint32_t mrq, rss_hash; 2862 2863 /* 2864 * Save several necessary information, 2865 * before emx_newbuf() destroy it. 2866 */ 2867 if ((staterr & E1000_RXD_STAT_VP) && eop) 2868 vlan = le16toh(current_desc->rxd_vlan); 2869 2870 mrq = le32toh(current_desc->rxd_mrq); 2871 rss_hash = le32toh(current_desc->rxd_rss); 2872 2873 EMX_RSS_DPRINTF(sc, 10, 2874 "ring%d, mrq 0x%08x, rss_hash 0x%08x\n", 2875 ring_idx, mrq, rss_hash); 2876 2877 if (emx_newbuf(sc, rdata, i, 0) != 0) { 2878 ifp->if_iqdrops++; 2879 goto discard; 2880 } 2881 2882 /* Assign correct length to the current fragment */ 2883 mp->m_len = len; 2884 2885 if (rdata->fmp == NULL) { 2886 mp->m_pkthdr.len = len; 2887 rdata->fmp = mp; /* Store the first mbuf */ 2888 rdata->lmp = mp; 2889 } else { 2890 /* 2891 * Chain mbuf's together 2892 */ 2893 rdata->lmp->m_next = mp; 2894 rdata->lmp = rdata->lmp->m_next; 2895 rdata->fmp->m_pkthdr.len += len; 2896 } 2897 2898 if (eop) { 2899 rdata->fmp->m_pkthdr.rcvif = ifp; 2900 ifp->if_ipackets++; 2901 2902 if (ifp->if_capenable & IFCAP_RXCSUM) 2903 emx_rxcsum(staterr, rdata->fmp); 2904 2905 if (staterr & E1000_RXD_STAT_VP) { 2906 rdata->fmp->m_pkthdr.ether_vlantag = 2907 vlan; 2908 rdata->fmp->m_flags |= M_VLANTAG; 2909 } 2910 m = rdata->fmp; 2911 rdata->fmp = NULL; 2912 rdata->lmp = NULL; 2913 2914 if (ifp->if_capenable & IFCAP_RSS) { 2915 pi = emx_rssinfo(m, &pi0, mrq, 2916 rss_hash, staterr); 2917 } 2918 #ifdef EMX_RSS_DEBUG 2919 rdata->rx_pkts++; 2920 #endif 2921 } 2922 } else { 2923 ifp->if_ierrors++; 2924 discard: 2925 emx_setup_rxdesc(current_desc, rx_buf); 2926 if (rdata->fmp != NULL) { 2927 m_freem(rdata->fmp); 2928 rdata->fmp = NULL; 2929 rdata->lmp = NULL; 2930 } 2931 m = NULL; 2932 } 2933 2934 if (m != NULL) 2935 ether_input_chain(ifp, m, pi, chain); 2936 2937 /* Advance our pointers to the next descriptor. */ 2938 if (++i == rdata->num_rx_desc) 2939 i = 0; 2940 2941 current_desc = &rdata->rx_desc[i]; 2942 staterr = le32toh(current_desc->rxd_staterr); 2943 } 2944 rdata->next_rx_desc_to_check = i; 2945 2946 ether_input_dispatch(chain); 2947 2948 /* Advance the E1000's Receive Queue "Tail Pointer". */ 2949 if (--i < 0) 2950 i = rdata->num_rx_desc - 1; 2951 E1000_WRITE_REG(&sc->hw, E1000_RDT(ring_idx), i); 2952 } 2953 2954 static void 2955 emx_enable_intr(struct emx_softc *sc) 2956 { 2957 uint32_t ims_mask = IMS_ENABLE_MASK; 2958 2959 lwkt_serialize_handler_enable(&sc->main_serialize); 2960 2961 #if 0 2962 if (sc->hw.mac.type == e1000_82574) { 2963 E1000_WRITE_REG(hw, EMX_EIAC, EM_MSIX_MASK); 2964 ims_mask |= EM_MSIX_MASK; 2965 } 2966 #endif 2967 E1000_WRITE_REG(&sc->hw, E1000_IMS, ims_mask); 2968 } 2969 2970 static void 2971 emx_disable_intr(struct emx_softc *sc) 2972 { 2973 if (sc->hw.mac.type == e1000_82574) 2974 E1000_WRITE_REG(&sc->hw, EMX_EIAC, 0); 2975 E1000_WRITE_REG(&sc->hw, E1000_IMC, 0xffffffff); 2976 2977 lwkt_serialize_handler_disable(&sc->main_serialize); 2978 } 2979 2980 /* 2981 * Bit of a misnomer, what this really means is 2982 * to enable OS management of the system... aka 2983 * to disable special hardware management features 2984 */ 2985 static void 2986 emx_get_mgmt(struct emx_softc *sc) 2987 { 2988 /* A shared code workaround */ 2989 if (sc->has_manage) { 2990 int manc2h = E1000_READ_REG(&sc->hw, E1000_MANC2H); 2991 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 2992 2993 /* disable hardware interception of ARP */ 2994 manc &= ~(E1000_MANC_ARP_EN); 2995 2996 /* enable receiving management packets to the host */ 2997 manc |= E1000_MANC_EN_MNG2HOST; 2998 #define E1000_MNG2HOST_PORT_623 (1 << 5) 2999 #define E1000_MNG2HOST_PORT_664 (1 << 6) 3000 manc2h |= E1000_MNG2HOST_PORT_623; 3001 manc2h |= E1000_MNG2HOST_PORT_664; 3002 E1000_WRITE_REG(&sc->hw, E1000_MANC2H, manc2h); 3003 3004 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 3005 } 3006 } 3007 3008 /* 3009 * Give control back to hardware management 3010 * controller if there is one. 3011 */ 3012 static void 3013 emx_rel_mgmt(struct emx_softc *sc) 3014 { 3015 if (sc->has_manage) { 3016 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 3017 3018 /* re-enable hardware interception of ARP */ 3019 manc |= E1000_MANC_ARP_EN; 3020 manc &= ~E1000_MANC_EN_MNG2HOST; 3021 3022 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 3023 } 3024 } 3025 3026 /* 3027 * emx_get_hw_control() sets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3028 * For ASF and Pass Through versions of f/w this means that 3029 * the driver is loaded. For AMT version (only with 82573) 3030 * of the f/w this means that the network i/f is open. 3031 */ 3032 static void 3033 emx_get_hw_control(struct emx_softc *sc) 3034 { 3035 /* Let firmware know the driver has taken over */ 3036 if (sc->hw.mac.type == e1000_82573) { 3037 uint32_t swsm; 3038 3039 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 3040 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 3041 swsm | E1000_SWSM_DRV_LOAD); 3042 } else { 3043 uint32_t ctrl_ext; 3044 3045 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 3046 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 3047 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 3048 } 3049 sc->control_hw = 1; 3050 } 3051 3052 /* 3053 * emx_rel_hw_control() resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3054 * For ASF and Pass Through versions of f/w this means that the 3055 * driver is no longer loaded. For AMT version (only with 82573) 3056 * of the f/w this means that the network i/f is closed. 3057 */ 3058 static void 3059 emx_rel_hw_control(struct emx_softc *sc) 3060 { 3061 if (!sc->control_hw) 3062 return; 3063 sc->control_hw = 0; 3064 3065 /* Let firmware taken over control of h/w */ 3066 if (sc->hw.mac.type == e1000_82573) { 3067 uint32_t swsm; 3068 3069 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 3070 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 3071 swsm & ~E1000_SWSM_DRV_LOAD); 3072 } else { 3073 uint32_t ctrl_ext; 3074 3075 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 3076 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 3077 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 3078 } 3079 } 3080 3081 static int 3082 emx_is_valid_eaddr(const uint8_t *addr) 3083 { 3084 char zero_addr[ETHER_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 }; 3085 3086 if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3087 return (FALSE); 3088 3089 return (TRUE); 3090 } 3091 3092 /* 3093 * Enable PCI Wake On Lan capability 3094 */ 3095 void 3096 emx_enable_wol(device_t dev) 3097 { 3098 uint16_t cap, status; 3099 uint8_t id; 3100 3101 /* First find the capabilities pointer*/ 3102 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 3103 3104 /* Read the PM Capabilities */ 3105 id = pci_read_config(dev, cap, 1); 3106 if (id != PCIY_PMG) /* Something wrong */ 3107 return; 3108 3109 /* 3110 * OK, we have the power capabilities, 3111 * so now get the status register 3112 */ 3113 cap += PCIR_POWER_STATUS; 3114 status = pci_read_config(dev, cap, 2); 3115 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 3116 pci_write_config(dev, cap, status, 2); 3117 } 3118 3119 static void 3120 emx_update_stats(struct emx_softc *sc) 3121 { 3122 struct ifnet *ifp = &sc->arpcom.ac_if; 3123 3124 if (sc->hw.phy.media_type == e1000_media_type_copper || 3125 (E1000_READ_REG(&sc->hw, E1000_STATUS) & E1000_STATUS_LU)) { 3126 sc->stats.symerrs += E1000_READ_REG(&sc->hw, E1000_SYMERRS); 3127 sc->stats.sec += E1000_READ_REG(&sc->hw, E1000_SEC); 3128 } 3129 sc->stats.crcerrs += E1000_READ_REG(&sc->hw, E1000_CRCERRS); 3130 sc->stats.mpc += E1000_READ_REG(&sc->hw, E1000_MPC); 3131 sc->stats.scc += E1000_READ_REG(&sc->hw, E1000_SCC); 3132 sc->stats.ecol += E1000_READ_REG(&sc->hw, E1000_ECOL); 3133 3134 sc->stats.mcc += E1000_READ_REG(&sc->hw, E1000_MCC); 3135 sc->stats.latecol += E1000_READ_REG(&sc->hw, E1000_LATECOL); 3136 sc->stats.colc += E1000_READ_REG(&sc->hw, E1000_COLC); 3137 sc->stats.dc += E1000_READ_REG(&sc->hw, E1000_DC); 3138 sc->stats.rlec += E1000_READ_REG(&sc->hw, E1000_RLEC); 3139 sc->stats.xonrxc += E1000_READ_REG(&sc->hw, E1000_XONRXC); 3140 sc->stats.xontxc += E1000_READ_REG(&sc->hw, E1000_XONTXC); 3141 sc->stats.xoffrxc += E1000_READ_REG(&sc->hw, E1000_XOFFRXC); 3142 sc->stats.xofftxc += E1000_READ_REG(&sc->hw, E1000_XOFFTXC); 3143 sc->stats.fcruc += E1000_READ_REG(&sc->hw, E1000_FCRUC); 3144 sc->stats.prc64 += E1000_READ_REG(&sc->hw, E1000_PRC64); 3145 sc->stats.prc127 += E1000_READ_REG(&sc->hw, E1000_PRC127); 3146 sc->stats.prc255 += E1000_READ_REG(&sc->hw, E1000_PRC255); 3147 sc->stats.prc511 += E1000_READ_REG(&sc->hw, E1000_PRC511); 3148 sc->stats.prc1023 += E1000_READ_REG(&sc->hw, E1000_PRC1023); 3149 sc->stats.prc1522 += E1000_READ_REG(&sc->hw, E1000_PRC1522); 3150 sc->stats.gprc += E1000_READ_REG(&sc->hw, E1000_GPRC); 3151 sc->stats.bprc += E1000_READ_REG(&sc->hw, E1000_BPRC); 3152 sc->stats.mprc += E1000_READ_REG(&sc->hw, E1000_MPRC); 3153 sc->stats.gptc += E1000_READ_REG(&sc->hw, E1000_GPTC); 3154 3155 /* For the 64-bit byte counters the low dword must be read first. */ 3156 /* Both registers clear on the read of the high dword */ 3157 3158 sc->stats.gorc += E1000_READ_REG(&sc->hw, E1000_GORCH); 3159 sc->stats.gotc += E1000_READ_REG(&sc->hw, E1000_GOTCH); 3160 3161 sc->stats.rnbc += E1000_READ_REG(&sc->hw, E1000_RNBC); 3162 sc->stats.ruc += E1000_READ_REG(&sc->hw, E1000_RUC); 3163 sc->stats.rfc += E1000_READ_REG(&sc->hw, E1000_RFC); 3164 sc->stats.roc += E1000_READ_REG(&sc->hw, E1000_ROC); 3165 sc->stats.rjc += E1000_READ_REG(&sc->hw, E1000_RJC); 3166 3167 sc->stats.tor += E1000_READ_REG(&sc->hw, E1000_TORH); 3168 sc->stats.tot += E1000_READ_REG(&sc->hw, E1000_TOTH); 3169 3170 sc->stats.tpr += E1000_READ_REG(&sc->hw, E1000_TPR); 3171 sc->stats.tpt += E1000_READ_REG(&sc->hw, E1000_TPT); 3172 sc->stats.ptc64 += E1000_READ_REG(&sc->hw, E1000_PTC64); 3173 sc->stats.ptc127 += E1000_READ_REG(&sc->hw, E1000_PTC127); 3174 sc->stats.ptc255 += E1000_READ_REG(&sc->hw, E1000_PTC255); 3175 sc->stats.ptc511 += E1000_READ_REG(&sc->hw, E1000_PTC511); 3176 sc->stats.ptc1023 += E1000_READ_REG(&sc->hw, E1000_PTC1023); 3177 sc->stats.ptc1522 += E1000_READ_REG(&sc->hw, E1000_PTC1522); 3178 sc->stats.mptc += E1000_READ_REG(&sc->hw, E1000_MPTC); 3179 sc->stats.bptc += E1000_READ_REG(&sc->hw, E1000_BPTC); 3180 3181 sc->stats.algnerrc += E1000_READ_REG(&sc->hw, E1000_ALGNERRC); 3182 sc->stats.rxerrc += E1000_READ_REG(&sc->hw, E1000_RXERRC); 3183 sc->stats.tncrs += E1000_READ_REG(&sc->hw, E1000_TNCRS); 3184 sc->stats.cexterr += E1000_READ_REG(&sc->hw, E1000_CEXTERR); 3185 sc->stats.tsctc += E1000_READ_REG(&sc->hw, E1000_TSCTC); 3186 sc->stats.tsctfc += E1000_READ_REG(&sc->hw, E1000_TSCTFC); 3187 3188 ifp->if_collisions = sc->stats.colc; 3189 3190 /* Rx Errors */ 3191 ifp->if_ierrors = sc->dropped_pkts + sc->stats.rxerrc + 3192 sc->stats.crcerrs + sc->stats.algnerrc + 3193 sc->stats.ruc + sc->stats.roc + 3194 sc->stats.mpc + sc->stats.cexterr; 3195 3196 /* Tx Errors */ 3197 ifp->if_oerrors = sc->stats.ecol + sc->stats.latecol + 3198 sc->watchdog_events; 3199 } 3200 3201 static void 3202 emx_print_debug_info(struct emx_softc *sc) 3203 { 3204 device_t dev = sc->dev; 3205 uint8_t *hw_addr = sc->hw.hw_addr; 3206 3207 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 3208 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 3209 E1000_READ_REG(&sc->hw, E1000_CTRL), 3210 E1000_READ_REG(&sc->hw, E1000_RCTL)); 3211 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 3212 ((E1000_READ_REG(&sc->hw, E1000_PBA) & 0xffff0000) >> 16),\ 3213 (E1000_READ_REG(&sc->hw, E1000_PBA) & 0xffff) ); 3214 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 3215 sc->hw.fc.high_water, sc->hw.fc.low_water); 3216 device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n", 3217 E1000_READ_REG(&sc->hw, E1000_TIDV), 3218 E1000_READ_REG(&sc->hw, E1000_TADV)); 3219 device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n", 3220 E1000_READ_REG(&sc->hw, E1000_RDTR), 3221 E1000_READ_REG(&sc->hw, E1000_RADV)); 3222 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 3223 E1000_READ_REG(&sc->hw, E1000_TDH(0)), 3224 E1000_READ_REG(&sc->hw, E1000_TDT(0))); 3225 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 3226 E1000_READ_REG(&sc->hw, E1000_RDH(0)), 3227 E1000_READ_REG(&sc->hw, E1000_RDT(0))); 3228 device_printf(dev, "Num Tx descriptors avail = %d\n", 3229 sc->num_tx_desc_avail); 3230 device_printf(dev, "Tx Descriptors not avail1 = %ld\n", 3231 sc->no_tx_desc_avail1); 3232 device_printf(dev, "Tx Descriptors not avail2 = %ld\n", 3233 sc->no_tx_desc_avail2); 3234 device_printf(dev, "Std mbuf failed = %ld\n", 3235 sc->mbuf_alloc_failed); 3236 device_printf(dev, "Std mbuf cluster failed = %ld\n", 3237 sc->rx_data[0].mbuf_cluster_failed); 3238 device_printf(dev, "Driver dropped packets = %ld\n", 3239 sc->dropped_pkts); 3240 device_printf(dev, "Driver tx dma failure in encap = %ld\n", 3241 sc->no_tx_dma_setup); 3242 3243 device_printf(dev, "TXCSUM try pullup = %lu\n", 3244 sc->tx_csum_try_pullup); 3245 device_printf(dev, "TXCSUM m_pullup(eh) called = %lu\n", 3246 sc->tx_csum_pullup1); 3247 device_printf(dev, "TXCSUM m_pullup(eh) failed = %lu\n", 3248 sc->tx_csum_pullup1_failed); 3249 device_printf(dev, "TXCSUM m_pullup(eh+ip) called = %lu\n", 3250 sc->tx_csum_pullup2); 3251 device_printf(dev, "TXCSUM m_pullup(eh+ip) failed = %lu\n", 3252 sc->tx_csum_pullup2_failed); 3253 device_printf(dev, "TXCSUM non-writable(eh) droped = %lu\n", 3254 sc->tx_csum_drop1); 3255 device_printf(dev, "TXCSUM non-writable(eh+ip) droped = %lu\n", 3256 sc->tx_csum_drop2); 3257 } 3258 3259 static void 3260 emx_print_hw_stats(struct emx_softc *sc) 3261 { 3262 device_t dev = sc->dev; 3263 3264 device_printf(dev, "Excessive collisions = %lld\n", 3265 (long long)sc->stats.ecol); 3266 #if (DEBUG_HW > 0) /* Dont output these errors normally */ 3267 device_printf(dev, "Symbol errors = %lld\n", 3268 (long long)sc->stats.symerrs); 3269 #endif 3270 device_printf(dev, "Sequence errors = %lld\n", 3271 (long long)sc->stats.sec); 3272 device_printf(dev, "Defer count = %lld\n", 3273 (long long)sc->stats.dc); 3274 device_printf(dev, "Missed Packets = %lld\n", 3275 (long long)sc->stats.mpc); 3276 device_printf(dev, "Receive No Buffers = %lld\n", 3277 (long long)sc->stats.rnbc); 3278 /* RLEC is inaccurate on some hardware, calculate our own. */ 3279 device_printf(dev, "Receive Length Errors = %lld\n", 3280 ((long long)sc->stats.roc + (long long)sc->stats.ruc)); 3281 device_printf(dev, "Receive errors = %lld\n", 3282 (long long)sc->stats.rxerrc); 3283 device_printf(dev, "Crc errors = %lld\n", 3284 (long long)sc->stats.crcerrs); 3285 device_printf(dev, "Alignment errors = %lld\n", 3286 (long long)sc->stats.algnerrc); 3287 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 3288 (long long)sc->stats.cexterr); 3289 device_printf(dev, "RX overruns = %ld\n", sc->rx_overruns); 3290 device_printf(dev, "watchdog timeouts = %ld\n", 3291 sc->watchdog_events); 3292 device_printf(dev, "XON Rcvd = %lld\n", 3293 (long long)sc->stats.xonrxc); 3294 device_printf(dev, "XON Xmtd = %lld\n", 3295 (long long)sc->stats.xontxc); 3296 device_printf(dev, "XOFF Rcvd = %lld\n", 3297 (long long)sc->stats.xoffrxc); 3298 device_printf(dev, "XOFF Xmtd = %lld\n", 3299 (long long)sc->stats.xofftxc); 3300 device_printf(dev, "Good Packets Rcvd = %lld\n", 3301 (long long)sc->stats.gprc); 3302 device_printf(dev, "Good Packets Xmtd = %lld\n", 3303 (long long)sc->stats.gptc); 3304 } 3305 3306 static void 3307 emx_print_nvm_info(struct emx_softc *sc) 3308 { 3309 uint16_t eeprom_data; 3310 int i, j, row = 0; 3311 3312 /* Its a bit crude, but it gets the job done */ 3313 kprintf("\nInterface EEPROM Dump:\n"); 3314 kprintf("Offset\n0x0000 "); 3315 for (i = 0, j = 0; i < 32; i++, j++) { 3316 if (j == 8) { /* Make the offset block */ 3317 j = 0; ++row; 3318 kprintf("\n0x00%x0 ",row); 3319 } 3320 e1000_read_nvm(&sc->hw, i, 1, &eeprom_data); 3321 kprintf("%04x ", eeprom_data); 3322 } 3323 kprintf("\n"); 3324 } 3325 3326 static int 3327 emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 3328 { 3329 struct emx_softc *sc; 3330 struct ifnet *ifp; 3331 int error, result; 3332 3333 result = -1; 3334 error = sysctl_handle_int(oidp, &result, 0, req); 3335 if (error || !req->newptr) 3336 return (error); 3337 3338 sc = (struct emx_softc *)arg1; 3339 ifp = &sc->arpcom.ac_if; 3340 3341 ifnet_serialize_all(ifp); 3342 3343 if (result == 1) 3344 emx_print_debug_info(sc); 3345 3346 /* 3347 * This value will cause a hex dump of the 3348 * first 32 16-bit words of the EEPROM to 3349 * the screen. 3350 */ 3351 if (result == 2) 3352 emx_print_nvm_info(sc); 3353 3354 ifnet_deserialize_all(ifp); 3355 3356 return (error); 3357 } 3358 3359 static int 3360 emx_sysctl_stats(SYSCTL_HANDLER_ARGS) 3361 { 3362 int error, result; 3363 3364 result = -1; 3365 error = sysctl_handle_int(oidp, &result, 0, req); 3366 if (error || !req->newptr) 3367 return (error); 3368 3369 if (result == 1) { 3370 struct emx_softc *sc = (struct emx_softc *)arg1; 3371 struct ifnet *ifp = &sc->arpcom.ac_if; 3372 3373 ifnet_serialize_all(ifp); 3374 emx_print_hw_stats(sc); 3375 ifnet_deserialize_all(ifp); 3376 } 3377 return (error); 3378 } 3379 3380 static void 3381 emx_add_sysctl(struct emx_softc *sc) 3382 { 3383 #ifdef EMX_RSS_DEBUG 3384 char rx_pkt[32]; 3385 int i; 3386 #endif 3387 3388 sysctl_ctx_init(&sc->sysctl_ctx); 3389 sc->sysctl_tree = SYSCTL_ADD_NODE(&sc->sysctl_ctx, 3390 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO, 3391 device_get_nameunit(sc->dev), 3392 CTLFLAG_RD, 0, ""); 3393 if (sc->sysctl_tree == NULL) { 3394 device_printf(sc->dev, "can't add sysctl node\n"); 3395 return; 3396 } 3397 3398 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3399 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 3400 emx_sysctl_debug_info, "I", "Debug Information"); 3401 3402 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3403 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, sc, 0, 3404 emx_sysctl_stats, "I", "Statistics"); 3405 3406 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3407 OID_AUTO, "rxd", CTLFLAG_RD, 3408 &sc->rx_data[0].num_rx_desc, 0, NULL); 3409 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3410 OID_AUTO, "txd", CTLFLAG_RD, &sc->num_tx_desc, 0, NULL); 3411 3412 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3413 OID_AUTO, "int_throttle_ceil", CTLTYPE_INT|CTLFLAG_RW, 3414 sc, 0, emx_sysctl_int_throttle, "I", 3415 "interrupt throttling rate"); 3416 SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3417 OID_AUTO, "int_tx_nsegs", CTLTYPE_INT|CTLFLAG_RW, 3418 sc, 0, emx_sysctl_int_tx_nsegs, "I", 3419 "# segments per TX interrupt"); 3420 3421 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3422 OID_AUTO, "rx_ring_inuse", CTLFLAG_RD, 3423 &sc->rx_ring_inuse, 0, "RX ring in use"); 3424 3425 #ifdef EMX_RSS_DEBUG 3426 SYSCTL_ADD_INT(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree), 3427 OID_AUTO, "rss_debug", CTLFLAG_RW, &sc->rss_debug, 3428 0, "RSS debug level"); 3429 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3430 ksnprintf(rx_pkt, sizeof(rx_pkt), "rx%d_pkt", i); 3431 SYSCTL_ADD_UINT(&sc->sysctl_ctx, 3432 SYSCTL_CHILDREN(sc->sysctl_tree), OID_AUTO, 3433 rx_pkt, CTLFLAG_RW, 3434 &sc->rx_data[i].rx_pkts, 0, "RXed packets"); 3435 } 3436 #endif 3437 } 3438 3439 static int 3440 emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS) 3441 { 3442 struct emx_softc *sc = (void *)arg1; 3443 struct ifnet *ifp = &sc->arpcom.ac_if; 3444 int error, throttle; 3445 3446 throttle = sc->int_throttle_ceil; 3447 error = sysctl_handle_int(oidp, &throttle, 0, req); 3448 if (error || req->newptr == NULL) 3449 return error; 3450 if (throttle < 0 || throttle > 1000000000 / 256) 3451 return EINVAL; 3452 3453 if (throttle) { 3454 /* 3455 * Set the interrupt throttling rate in 256ns increments, 3456 * recalculate sysctl value assignment to get exact frequency. 3457 */ 3458 throttle = 1000000000 / 256 / throttle; 3459 3460 /* Upper 16bits of ITR is reserved and should be zero */ 3461 if (throttle & 0xffff0000) 3462 return EINVAL; 3463 } 3464 3465 ifnet_serialize_all(ifp); 3466 3467 if (throttle) 3468 sc->int_throttle_ceil = 1000000000 / 256 / throttle; 3469 else 3470 sc->int_throttle_ceil = 0; 3471 3472 if (ifp->if_flags & IFF_RUNNING) 3473 emx_set_itr(sc, throttle); 3474 3475 ifnet_deserialize_all(ifp); 3476 3477 if (bootverbose) { 3478 if_printf(ifp, "Interrupt moderation set to %d/sec\n", 3479 sc->int_throttle_ceil); 3480 } 3481 return 0; 3482 } 3483 3484 static int 3485 emx_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS) 3486 { 3487 struct emx_softc *sc = (void *)arg1; 3488 struct ifnet *ifp = &sc->arpcom.ac_if; 3489 int error, segs; 3490 3491 segs = sc->tx_int_nsegs; 3492 error = sysctl_handle_int(oidp, &segs, 0, req); 3493 if (error || req->newptr == NULL) 3494 return error; 3495 if (segs <= 0) 3496 return EINVAL; 3497 3498 ifnet_serialize_all(ifp); 3499 3500 /* 3501 * Don't allow int_tx_nsegs to become: 3502 * o Less the oact_tx_desc 3503 * o Too large that no TX desc will cause TX interrupt to 3504 * be generated (OACTIVE will never recover) 3505 * o Too small that will cause tx_dd[] overflow 3506 */ 3507 if (segs < sc->oact_tx_desc || 3508 segs >= sc->num_tx_desc - sc->oact_tx_desc || 3509 segs < sc->num_tx_desc / EMX_TXDD_SAFE) { 3510 error = EINVAL; 3511 } else { 3512 error = 0; 3513 sc->tx_int_nsegs = segs; 3514 } 3515 3516 ifnet_deserialize_all(ifp); 3517 3518 return error; 3519 } 3520 3521 static int 3522 emx_dma_alloc(struct emx_softc *sc) 3523 { 3524 int error, i; 3525 3526 /* 3527 * Create top level busdma tag 3528 */ 3529 error = bus_dma_tag_create(NULL, 1, 0, 3530 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 3531 NULL, NULL, 3532 BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 3533 0, &sc->parent_dtag); 3534 if (error) { 3535 device_printf(sc->dev, "could not create top level DMA tag\n"); 3536 return error; 3537 } 3538 3539 /* 3540 * Allocate transmit descriptors ring and buffers 3541 */ 3542 error = emx_create_tx_ring(sc); 3543 if (error) { 3544 device_printf(sc->dev, "Could not setup transmit structures\n"); 3545 return error; 3546 } 3547 3548 /* 3549 * Allocate receive descriptors ring and buffers 3550 */ 3551 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3552 error = emx_create_rx_ring(sc, &sc->rx_data[i]); 3553 if (error) { 3554 device_printf(sc->dev, 3555 "Could not setup receive structures\n"); 3556 return error; 3557 } 3558 } 3559 return 0; 3560 } 3561 3562 static void 3563 emx_dma_free(struct emx_softc *sc) 3564 { 3565 int i; 3566 3567 emx_destroy_tx_ring(sc, sc->num_tx_desc); 3568 3569 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3570 emx_destroy_rx_ring(sc, &sc->rx_data[i], 3571 sc->rx_data[i].num_rx_desc); 3572 } 3573 3574 /* Free top level busdma tag */ 3575 if (sc->parent_dtag != NULL) 3576 bus_dma_tag_destroy(sc->parent_dtag); 3577 } 3578 3579 static void 3580 emx_serialize(struct ifnet *ifp, enum ifnet_serialize slz) 3581 { 3582 struct emx_softc *sc = ifp->if_softc; 3583 3584 switch (slz) { 3585 case IFNET_SERIALIZE_ALL: 3586 lwkt_serialize_array_enter(sc->serializes, EMX_NSERIALIZE, 0); 3587 break; 3588 3589 case IFNET_SERIALIZE_MAIN: 3590 lwkt_serialize_enter(&sc->main_serialize); 3591 break; 3592 3593 case IFNET_SERIALIZE_TX: 3594 lwkt_serialize_enter(&sc->tx_serialize); 3595 break; 3596 3597 case IFNET_SERIALIZE_RX(0): 3598 lwkt_serialize_enter(&sc->rx_data[0].rx_serialize); 3599 break; 3600 3601 case IFNET_SERIALIZE_RX(1): 3602 lwkt_serialize_enter(&sc->rx_data[1].rx_serialize); 3603 break; 3604 3605 default: 3606 panic("%s unsupported serialize type\n", ifp->if_xname); 3607 } 3608 } 3609 3610 static void 3611 emx_deserialize(struct ifnet *ifp, enum ifnet_serialize slz) 3612 { 3613 struct emx_softc *sc = ifp->if_softc; 3614 3615 switch (slz) { 3616 case IFNET_SERIALIZE_ALL: 3617 lwkt_serialize_array_exit(sc->serializes, EMX_NSERIALIZE, 0); 3618 break; 3619 3620 case IFNET_SERIALIZE_MAIN: 3621 lwkt_serialize_exit(&sc->main_serialize); 3622 break; 3623 3624 case IFNET_SERIALIZE_TX: 3625 lwkt_serialize_exit(&sc->tx_serialize); 3626 break; 3627 3628 case IFNET_SERIALIZE_RX(0): 3629 lwkt_serialize_exit(&sc->rx_data[0].rx_serialize); 3630 break; 3631 3632 case IFNET_SERIALIZE_RX(1): 3633 lwkt_serialize_exit(&sc->rx_data[1].rx_serialize); 3634 break; 3635 3636 default: 3637 panic("%s unsupported serialize type\n", ifp->if_xname); 3638 } 3639 } 3640 3641 static int 3642 emx_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz) 3643 { 3644 struct emx_softc *sc = ifp->if_softc; 3645 3646 switch (slz) { 3647 case IFNET_SERIALIZE_ALL: 3648 return lwkt_serialize_array_try(sc->serializes, 3649 EMX_NSERIALIZE, 0); 3650 3651 case IFNET_SERIALIZE_MAIN: 3652 return lwkt_serialize_try(&sc->main_serialize); 3653 3654 case IFNET_SERIALIZE_TX: 3655 return lwkt_serialize_try(&sc->tx_serialize); 3656 3657 case IFNET_SERIALIZE_RX(0): 3658 return lwkt_serialize_try(&sc->rx_data[0].rx_serialize); 3659 3660 case IFNET_SERIALIZE_RX(1): 3661 return lwkt_serialize_try(&sc->rx_data[1].rx_serialize); 3662 3663 default: 3664 panic("%s unsupported serialize type\n", ifp->if_xname); 3665 } 3666 } 3667 3668 static void 3669 emx_serialize_skipmain(struct emx_softc *sc) 3670 { 3671 lwkt_serialize_array_enter(sc->serializes, EMX_NSERIALIZE, 1); 3672 } 3673 3674 static void 3675 emx_deserialize_skipmain(struct emx_softc *sc) 3676 { 3677 lwkt_serialize_array_exit(sc->serializes, EMX_NSERIALIZE, 1); 3678 } 3679 3680 #ifdef INVARIANTS 3681 3682 static void 3683 emx_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz, 3684 boolean_t serialized) 3685 { 3686 struct emx_softc *sc = ifp->if_softc; 3687 int i; 3688 3689 switch (slz) { 3690 case IFNET_SERIALIZE_ALL: 3691 if (serialized) { 3692 for (i = 0; i < EMX_NSERIALIZE; ++i) 3693 ASSERT_SERIALIZED(sc->serializes[i]); 3694 } else { 3695 for (i = 0; i < EMX_NSERIALIZE; ++i) 3696 ASSERT_NOT_SERIALIZED(sc->serializes[i]); 3697 } 3698 break; 3699 3700 case IFNET_SERIALIZE_MAIN: 3701 if (serialized) 3702 ASSERT_SERIALIZED(&sc->main_serialize); 3703 else 3704 ASSERT_NOT_SERIALIZED(&sc->main_serialize); 3705 break; 3706 3707 case IFNET_SERIALIZE_TX: 3708 if (serialized) 3709 ASSERT_SERIALIZED(&sc->tx_serialize); 3710 else 3711 ASSERT_NOT_SERIALIZED(&sc->tx_serialize); 3712 break; 3713 3714 case IFNET_SERIALIZE_RX(0): 3715 if (serialized) 3716 ASSERT_SERIALIZED(&sc->rx_data[0].rx_serialize); 3717 else 3718 ASSERT_NOT_SERIALIZED(&sc->rx_data[0].rx_serialize); 3719 break; 3720 3721 case IFNET_SERIALIZE_RX(1): 3722 if (serialized) 3723 ASSERT_SERIALIZED(&sc->rx_data[1].rx_serialize); 3724 else 3725 ASSERT_NOT_SERIALIZED(&sc->rx_data[1].rx_serialize); 3726 break; 3727 3728 default: 3729 panic("%s unsupported serialize type\n", ifp->if_xname); 3730 } 3731 } 3732 3733 #endif /* INVARIANTS */ 3734 3735 #ifdef IFPOLL_ENABLE 3736 3737 static void 3738 emx_qpoll_status(struct ifnet *ifp, int pollhz __unused) 3739 { 3740 struct emx_softc *sc = ifp->if_softc; 3741 uint32_t reg_icr; 3742 3743 ASSERT_SERIALIZED(&sc->main_serialize); 3744 3745 reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); 3746 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 3747 emx_serialize_skipmain(sc); 3748 3749 callout_stop(&sc->timer); 3750 sc->hw.mac.get_link_status = 1; 3751 emx_update_link_status(sc); 3752 callout_reset(&sc->timer, hz, emx_timer, sc); 3753 3754 emx_deserialize_skipmain(sc); 3755 } 3756 } 3757 3758 static void 3759 emx_qpoll_tx(struct ifnet *ifp, void *arg __unused, int cycle __unused) 3760 { 3761 struct emx_softc *sc = ifp->if_softc; 3762 3763 ASSERT_SERIALIZED(&sc->tx_serialize); 3764 3765 emx_txeof(sc); 3766 if (!ifq_is_empty(&ifp->if_snd)) 3767 if_devstart(ifp); 3768 } 3769 3770 static void 3771 emx_qpoll_rx(struct ifnet *ifp, void *arg, int cycle) 3772 { 3773 struct emx_softc *sc = ifp->if_softc; 3774 struct emx_rxdata *rdata = arg; 3775 3776 ASSERT_SERIALIZED(&rdata->rx_serialize); 3777 3778 emx_rxeof(sc, rdata - sc->rx_data, cycle); 3779 } 3780 3781 static void 3782 emx_qpoll(struct ifnet *ifp, struct ifpoll_info *info) 3783 { 3784 struct emx_softc *sc = ifp->if_softc; 3785 3786 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3787 3788 if (info) { 3789 int i; 3790 3791 info->ifpi_status.status_func = emx_qpoll_status; 3792 info->ifpi_status.serializer = &sc->main_serialize; 3793 3794 info->ifpi_tx[0].poll_func = emx_qpoll_tx; 3795 info->ifpi_tx[0].arg = NULL; 3796 info->ifpi_tx[0].serializer = &sc->tx_serialize; 3797 3798 for (i = 0; i < sc->rx_ring_cnt; ++i) { 3799 info->ifpi_rx[i].poll_func = emx_qpoll_rx; 3800 info->ifpi_rx[i].arg = &sc->rx_data[i]; 3801 info->ifpi_rx[i].serializer = 3802 &sc->rx_data[i].rx_serialize; 3803 } 3804 3805 if (ifp->if_flags & IFF_RUNNING) 3806 emx_disable_intr(sc); 3807 } else if (ifp->if_flags & IFF_RUNNING) { 3808 emx_enable_intr(sc); 3809 } 3810 } 3811 3812 #endif /* IFPOLL_ENABLE */ 3813 3814 static void 3815 emx_set_itr(struct emx_softc *sc, uint32_t itr) 3816 { 3817 E1000_WRITE_REG(&sc->hw, E1000_ITR, itr); 3818 if (sc->hw.mac.type == e1000_82574) { 3819 int i; 3820 3821 /* 3822 * When using MSIX interrupts we need to 3823 * throttle using the EITR register 3824 */ 3825 for (i = 0; i < 4; ++i) 3826 E1000_WRITE_REG(&sc->hw, E1000_EITR_82574(i), itr); 3827 } 3828 } 3829