1 /****************************************************************************** 2 3 Copyright (c) 2001-2010, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD$*/ 34 35 #ifdef HAVE_KERNEL_OPTION_HEADERS 36 #include "opt_device_polling.h" 37 #include "opt_inet.h" 38 #endif 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #if __FreeBSD_version >= 800000 43 #include <sys/buf_ring.h> 44 #endif 45 #include <sys/bus.h> 46 #include <sys/endian.h> 47 #include <sys/kernel.h> 48 #include <sys/kthread.h> 49 #include <sys/malloc.h> 50 #include <sys/mbuf.h> 51 #include <sys/module.h> 52 #include <sys/rman.h> 53 #include <sys/socket.h> 54 #include <sys/sockio.h> 55 #include <sys/sysctl.h> 56 #include <sys/taskqueue.h> 57 #include <sys/eventhandler.h> 58 #include <machine/bus.h> 59 #include <machine/resource.h> 60 61 #include <net/bpf.h> 62 #include <net/ethernet.h> 63 #include <net/if.h> 64 #include <net/if_arp.h> 65 #include <net/if_dl.h> 66 #include <net/if_media.h> 67 68 #include <net/if_types.h> 69 #include <net/if_vlan_var.h> 70 71 #include <netinet/in_systm.h> 72 #include <netinet/in.h> 73 #include <netinet/if_ether.h> 74 #include <netinet/ip.h> 75 #include <netinet/ip6.h> 76 #include <netinet/tcp.h> 77 #include <netinet/udp.h> 78 79 #include <machine/in_cksum.h> 80 #include <dev/led/led.h> 81 #include <dev/pci/pcivar.h> 82 #include <dev/pci/pcireg.h> 83 84 #include "e1000_api.h" 85 #include "e1000_82571.h" 86 #include "if_em.h" 87 88 /********************************************************************* 89 * Set this to one to display debug statistics 90 *********************************************************************/ 91 int em_display_debug_stats = 0; 92 93 /********************************************************************* 94 * Driver version: 95 *********************************************************************/ 96 char em_driver_version[] = "7.1.7"; 97 98 /********************************************************************* 99 * PCI Device ID Table 100 * 101 * Used by probe to select devices to load on 102 * Last field stores an index into e1000_strings 103 * Last entry must be all 0s 104 * 105 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 106 *********************************************************************/ 107 108 static em_vendor_info_t em_vendor_info_array[] = 109 { 110 /* Intel(R) PRO/1000 Network Connection */ 111 { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 112 { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 113 { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 114 { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, 115 PCI_ANY_ID, PCI_ANY_ID, 0}, 116 { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, 117 PCI_ANY_ID, PCI_ANY_ID, 0}, 118 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, 119 PCI_ANY_ID, PCI_ANY_ID, 0}, 120 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, 121 PCI_ANY_ID, PCI_ANY_ID, 0}, 122 { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, 123 PCI_ANY_ID, PCI_ANY_ID, 0}, 124 { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, 125 PCI_ANY_ID, PCI_ANY_ID, 0}, 126 { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 127 { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 128 { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 129 { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 130 131 { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0}, 132 { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 133 { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0}, 134 { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0}, 135 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, 136 PCI_ANY_ID, PCI_ANY_ID, 0}, 137 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, 138 PCI_ANY_ID, PCI_ANY_ID, 0}, 139 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, 140 PCI_ANY_ID, PCI_ANY_ID, 0}, 141 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, 142 PCI_ANY_ID, PCI_ANY_ID, 0}, 143 { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 144 { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 145 { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 146 { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 147 { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 148 { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 149 { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 150 { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0}, 151 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 152 { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 153 { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 154 { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 155 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 156 { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 157 { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 158 { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 159 { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0}, 160 { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0}, 161 { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0}, 162 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 163 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 164 { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 165 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 166 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 167 { 0x8086, E1000_DEV_ID_ICH10_D_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 168 { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 169 { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0}, 170 { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0}, 171 { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0}, 172 { 0x8086, E1000_DEV_ID_PCH2_LV_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 173 { 0x8086, E1000_DEV_ID_PCH2_LV_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 174 /* required last entry */ 175 { 0, 0, 0, 0, 0} 176 }; 177 178 /********************************************************************* 179 * Table of branding strings for all supported NICs. 180 *********************************************************************/ 181 182 static char *em_strings[] = { 183 "Intel(R) PRO/1000 Network Connection" 184 }; 185 186 /********************************************************************* 187 * Function prototypes 188 *********************************************************************/ 189 static int em_probe(device_t); 190 static int em_attach(device_t); 191 static int em_detach(device_t); 192 static int em_shutdown(device_t); 193 static int em_suspend(device_t); 194 static int em_resume(device_t); 195 static void em_start(struct ifnet *); 196 static void em_start_locked(struct ifnet *, struct tx_ring *); 197 #ifdef EM_MULTIQUEUE 198 static int em_mq_start(struct ifnet *, struct mbuf *); 199 static int em_mq_start_locked(struct ifnet *, 200 struct tx_ring *, struct mbuf *); 201 static void em_qflush(struct ifnet *); 202 #endif 203 static int em_ioctl(struct ifnet *, u_long, caddr_t); 204 static void em_init(void *); 205 static void em_init_locked(struct adapter *); 206 static void em_stop(void *); 207 static void em_media_status(struct ifnet *, struct ifmediareq *); 208 static int em_media_change(struct ifnet *); 209 static void em_identify_hardware(struct adapter *); 210 static int em_allocate_pci_resources(struct adapter *); 211 static int em_allocate_legacy(struct adapter *); 212 static int em_allocate_msix(struct adapter *); 213 static int em_allocate_queues(struct adapter *); 214 static int em_setup_msix(struct adapter *); 215 static void em_free_pci_resources(struct adapter *); 216 static void em_local_timer(void *); 217 static void em_reset(struct adapter *); 218 static int em_setup_interface(device_t, struct adapter *); 219 220 static void em_setup_transmit_structures(struct adapter *); 221 static void em_initialize_transmit_unit(struct adapter *); 222 static int em_allocate_transmit_buffers(struct tx_ring *); 223 static void em_free_transmit_structures(struct adapter *); 224 static void em_free_transmit_buffers(struct tx_ring *); 225 226 static int em_setup_receive_structures(struct adapter *); 227 static int em_allocate_receive_buffers(struct rx_ring *); 228 static void em_initialize_receive_unit(struct adapter *); 229 static void em_free_receive_structures(struct adapter *); 230 static void em_free_receive_buffers(struct rx_ring *); 231 232 static void em_enable_intr(struct adapter *); 233 static void em_disable_intr(struct adapter *); 234 static void em_update_stats_counters(struct adapter *); 235 static void em_add_hw_stats(struct adapter *adapter); 236 static bool em_txeof(struct tx_ring *); 237 static bool em_rxeof(struct rx_ring *, int, int *); 238 #ifndef __NO_STRICT_ALIGNMENT 239 static int em_fixup_rx(struct rx_ring *); 240 #endif 241 static void em_receive_checksum(struct e1000_rx_desc *, struct mbuf *); 242 static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, int, 243 struct ip *, u32 *, u32 *); 244 static void em_tso_setup(struct tx_ring *, struct mbuf *, int, struct ip *, 245 struct tcphdr *, u32 *, u32 *); 246 static void em_set_promisc(struct adapter *); 247 static void em_disable_promisc(struct adapter *); 248 static void em_set_multi(struct adapter *); 249 static void em_update_link_status(struct adapter *); 250 static void em_refresh_mbufs(struct rx_ring *, int); 251 static void em_register_vlan(void *, struct ifnet *, u16); 252 static void em_unregister_vlan(void *, struct ifnet *, u16); 253 static void em_setup_vlan_hw_support(struct adapter *); 254 static int em_xmit(struct tx_ring *, struct mbuf **); 255 static int em_dma_malloc(struct adapter *, bus_size_t, 256 struct em_dma_alloc *, int); 257 static void em_dma_free(struct adapter *, struct em_dma_alloc *); 258 static int em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); 259 static void em_print_nvm_info(struct adapter *); 260 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 261 static void em_print_debug_info(struct adapter *); 262 static int em_is_valid_ether_addr(u8 *); 263 static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); 264 static void em_add_int_delay_sysctl(struct adapter *, const char *, 265 const char *, struct em_int_delay_info *, int, int); 266 /* Management and WOL Support */ 267 static void em_init_manageability(struct adapter *); 268 static void em_release_manageability(struct adapter *); 269 static void em_get_hw_control(struct adapter *); 270 static void em_release_hw_control(struct adapter *); 271 static void em_get_wakeup(device_t); 272 static void em_enable_wakeup(device_t); 273 static int em_enable_phy_wakeup(struct adapter *); 274 static void em_led_func(void *, int); 275 276 static int em_irq_fast(void *); 277 278 /* MSIX handlers */ 279 static void em_msix_tx(void *); 280 static void em_msix_rx(void *); 281 static void em_msix_link(void *); 282 static void em_handle_tx(void *context, int pending); 283 static void em_handle_rx(void *context, int pending); 284 static void em_handle_link(void *context, int pending); 285 286 static void em_add_rx_process_limit(struct adapter *, const char *, 287 const char *, int *, int); 288 static void em_set_flow_cntrl(struct adapter *, const char *, 289 const char *, int *, int); 290 291 static __inline void em_rx_discard(struct rx_ring *, int); 292 293 #ifdef DEVICE_POLLING 294 static poll_handler_t em_poll; 295 #endif /* POLLING */ 296 297 /********************************************************************* 298 * FreeBSD Device Interface Entry Points 299 *********************************************************************/ 300 301 static device_method_t em_methods[] = { 302 /* Device interface */ 303 DEVMETHOD(device_probe, em_probe), 304 DEVMETHOD(device_attach, em_attach), 305 DEVMETHOD(device_detach, em_detach), 306 DEVMETHOD(device_shutdown, em_shutdown), 307 DEVMETHOD(device_suspend, em_suspend), 308 DEVMETHOD(device_resume, em_resume), 309 {0, 0} 310 }; 311 312 static driver_t em_driver = { 313 "em", em_methods, sizeof(struct adapter), 314 }; 315 316 devclass_t em_devclass; 317 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); 318 MODULE_DEPEND(em, pci, 1, 1, 1); 319 MODULE_DEPEND(em, ether, 1, 1, 1); 320 321 /********************************************************************* 322 * Tunable default values. 323 *********************************************************************/ 324 325 #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) 326 #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) 327 #define M_TSO_LEN 66 328 329 /* Allow common code without TSO */ 330 #ifndef CSUM_TSO 331 #define CSUM_TSO 0 332 #endif 333 334 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); 335 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); 336 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt); 337 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt); 338 339 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); 340 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); 341 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt); 342 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt); 343 344 static int em_rxd = EM_DEFAULT_RXD; 345 static int em_txd = EM_DEFAULT_TXD; 346 TUNABLE_INT("hw.em.rxd", &em_rxd); 347 TUNABLE_INT("hw.em.txd", &em_txd); 348 349 static int em_smart_pwr_down = FALSE; 350 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down); 351 352 /* Controls whether promiscuous also shows bad packets */ 353 static int em_debug_sbp = FALSE; 354 TUNABLE_INT("hw.em.sbp", &em_debug_sbp); 355 356 static int em_enable_msix = TRUE; 357 TUNABLE_INT("hw.em.enable_msix", &em_enable_msix); 358 359 /* How many packets rxeof tries to clean at a time */ 360 static int em_rx_process_limit = 100; 361 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit); 362 363 /* Flow control setting - default to FULL */ 364 static int em_fc_setting = e1000_fc_full; 365 TUNABLE_INT("hw.em.fc_setting", &em_fc_setting); 366 367 /* Global used in WOL setup with multiport cards */ 368 static int global_quad_port_a = 0; 369 370 /********************************************************************* 371 * Device identification routine 372 * 373 * em_probe determines if the driver should be loaded on 374 * adapter based on PCI vendor/device id of the adapter. 375 * 376 * return BUS_PROBE_DEFAULT on success, positive on failure 377 *********************************************************************/ 378 379 static int 380 em_probe(device_t dev) 381 { 382 char adapter_name[60]; 383 u16 pci_vendor_id = 0; 384 u16 pci_device_id = 0; 385 u16 pci_subvendor_id = 0; 386 u16 pci_subdevice_id = 0; 387 em_vendor_info_t *ent; 388 389 INIT_DEBUGOUT("em_probe: begin"); 390 391 pci_vendor_id = pci_get_vendor(dev); 392 if (pci_vendor_id != EM_VENDOR_ID) 393 return (ENXIO); 394 395 pci_device_id = pci_get_device(dev); 396 pci_subvendor_id = pci_get_subvendor(dev); 397 pci_subdevice_id = pci_get_subdevice(dev); 398 399 ent = em_vendor_info_array; 400 while (ent->vendor_id != 0) { 401 if ((pci_vendor_id == ent->vendor_id) && 402 (pci_device_id == ent->device_id) && 403 404 ((pci_subvendor_id == ent->subvendor_id) || 405 (ent->subvendor_id == PCI_ANY_ID)) && 406 407 ((pci_subdevice_id == ent->subdevice_id) || 408 (ent->subdevice_id == PCI_ANY_ID))) { 409 sprintf(adapter_name, "%s %s", 410 em_strings[ent->index], 411 em_driver_version); 412 device_set_desc_copy(dev, adapter_name); 413 return (BUS_PROBE_DEFAULT); 414 } 415 ent++; 416 } 417 418 return (ENXIO); 419 } 420 421 /********************************************************************* 422 * Device initialization routine 423 * 424 * The attach entry point is called when the driver is being loaded. 425 * This routine identifies the type of hardware, allocates all resources 426 * and initializes the hardware. 427 * 428 * return 0 on success, positive on failure 429 *********************************************************************/ 430 431 static int 432 em_attach(device_t dev) 433 { 434 struct adapter *adapter; 435 int error = 0; 436 437 INIT_DEBUGOUT("em_attach: begin"); 438 439 adapter = device_get_softc(dev); 440 adapter->dev = adapter->osdep.dev = dev; 441 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 442 443 /* SYSCTL stuff */ 444 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 445 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 446 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 447 em_sysctl_nvm_info, "I", "NVM Information"); 448 449 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 450 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 451 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 452 em_sysctl_debug_info, "I", "Debug Information"); 453 454 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 455 456 /* Determine hardware and mac info */ 457 em_identify_hardware(adapter); 458 459 /* Setup PCI resources */ 460 if (em_allocate_pci_resources(adapter)) { 461 device_printf(dev, "Allocation of PCI resources failed\n"); 462 error = ENXIO; 463 goto err_pci; 464 } 465 466 /* 467 ** For ICH8 and family we need to 468 ** map the flash memory, and this 469 ** must happen after the MAC is 470 ** identified 471 */ 472 if ((adapter->hw.mac.type == e1000_ich8lan) || 473 (adapter->hw.mac.type == e1000_ich9lan) || 474 (adapter->hw.mac.type == e1000_ich10lan) || 475 (adapter->hw.mac.type == e1000_pchlan) || 476 (adapter->hw.mac.type == e1000_pch2lan)) { 477 int rid = EM_BAR_TYPE_FLASH; 478 adapter->flash = bus_alloc_resource_any(dev, 479 SYS_RES_MEMORY, &rid, RF_ACTIVE); 480 if (adapter->flash == NULL) { 481 device_printf(dev, "Mapping of Flash failed\n"); 482 error = ENXIO; 483 goto err_pci; 484 } 485 /* This is used in the shared code */ 486 adapter->hw.flash_address = (u8 *)adapter->flash; 487 adapter->osdep.flash_bus_space_tag = 488 rman_get_bustag(adapter->flash); 489 adapter->osdep.flash_bus_space_handle = 490 rman_get_bushandle(adapter->flash); 491 } 492 493 /* Do Shared Code initialization */ 494 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 495 device_printf(dev, "Setup of Shared code failed\n"); 496 error = ENXIO; 497 goto err_pci; 498 } 499 500 e1000_get_bus_info(&adapter->hw); 501 502 /* Set up some sysctls for the tunable interrupt delays */ 503 em_add_int_delay_sysctl(adapter, "rx_int_delay", 504 "receive interrupt delay in usecs", &adapter->rx_int_delay, 505 E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt); 506 em_add_int_delay_sysctl(adapter, "tx_int_delay", 507 "transmit interrupt delay in usecs", &adapter->tx_int_delay, 508 E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt); 509 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay", 510 "receive interrupt delay limit in usecs", 511 &adapter->rx_abs_int_delay, 512 E1000_REGISTER(&adapter->hw, E1000_RADV), 513 em_rx_abs_int_delay_dflt); 514 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay", 515 "transmit interrupt delay limit in usecs", 516 &adapter->tx_abs_int_delay, 517 E1000_REGISTER(&adapter->hw, E1000_TADV), 518 em_tx_abs_int_delay_dflt); 519 520 /* Sysctl for limiting the amount of work done in the taskqueue */ 521 em_add_rx_process_limit(adapter, "rx_processing_limit", 522 "max number of rx packets to process", &adapter->rx_process_limit, 523 em_rx_process_limit); 524 525 /* Sysctl for setting the interface flow control */ 526 em_set_flow_cntrl(adapter, "flow_control", 527 "max number of rx packets to process", 528 &adapter->fc_setting, em_fc_setting); 529 530 /* 531 * Validate number of transmit and receive descriptors. It 532 * must not exceed hardware maximum, and must be multiple 533 * of E1000_DBA_ALIGN. 534 */ 535 if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || 536 (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) { 537 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 538 EM_DEFAULT_TXD, em_txd); 539 adapter->num_tx_desc = EM_DEFAULT_TXD; 540 } else 541 adapter->num_tx_desc = em_txd; 542 543 if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || 544 (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) { 545 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 546 EM_DEFAULT_RXD, em_rxd); 547 adapter->num_rx_desc = EM_DEFAULT_RXD; 548 } else 549 adapter->num_rx_desc = em_rxd; 550 551 adapter->hw.mac.autoneg = DO_AUTO_NEG; 552 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 553 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 554 555 /* Copper options */ 556 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 557 adapter->hw.phy.mdix = AUTO_ALL_MODES; 558 adapter->hw.phy.disable_polarity_correction = FALSE; 559 adapter->hw.phy.ms_type = EM_MASTER_SLAVE; 560 } 561 562 /* 563 * Set the frame limits assuming 564 * standard ethernet sized frames. 565 */ 566 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 567 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 568 569 /* 570 * This controls when hardware reports transmit completion 571 * status. 572 */ 573 adapter->hw.mac.report_tx_early = 1; 574 575 /* 576 ** Get queue/ring memory 577 */ 578 if (em_allocate_queues(adapter)) { 579 error = ENOMEM; 580 goto err_pci; 581 } 582 583 /* Allocate multicast array memory. */ 584 adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN * 585 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 586 if (adapter->mta == NULL) { 587 device_printf(dev, "Can not allocate multicast setup array\n"); 588 error = ENOMEM; 589 goto err_late; 590 } 591 592 /* Check SOL/IDER usage */ 593 if (e1000_check_reset_block(&adapter->hw)) 594 device_printf(dev, "PHY reset is blocked" 595 " due to SOL/IDER session.\n"); 596 597 /* 598 ** Start from a known state, this is 599 ** important in reading the nvm and 600 ** mac from that. 601 */ 602 e1000_reset_hw(&adapter->hw); 603 604 /* Make sure we have a good EEPROM before we read from it */ 605 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 606 /* 607 ** Some PCI-E parts fail the first check due to 608 ** the link being in sleep state, call it again, 609 ** if it fails a second time its a real issue. 610 */ 611 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 612 device_printf(dev, 613 "The EEPROM Checksum Is Not Valid\n"); 614 error = EIO; 615 goto err_late; 616 } 617 } 618 619 /* Copy the permanent MAC address out of the EEPROM */ 620 if (e1000_read_mac_addr(&adapter->hw) < 0) { 621 device_printf(dev, "EEPROM read error while reading MAC" 622 " address\n"); 623 error = EIO; 624 goto err_late; 625 } 626 627 if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) { 628 device_printf(dev, "Invalid MAC address\n"); 629 error = EIO; 630 goto err_late; 631 } 632 633 /* 634 ** Do interrupt configuration 635 */ 636 if (adapter->msix > 1) /* Do MSIX */ 637 error = em_allocate_msix(adapter); 638 else /* MSI or Legacy */ 639 error = em_allocate_legacy(adapter); 640 if (error) 641 goto err_late; 642 643 /* 644 * Get Wake-on-Lan and Management info for later use 645 */ 646 em_get_wakeup(dev); 647 648 /* Setup OS specific network interface */ 649 if (em_setup_interface(dev, adapter) != 0) 650 goto err_late; 651 652 em_reset(adapter); 653 654 /* Initialize statistics */ 655 em_update_stats_counters(adapter); 656 657 adapter->hw.mac.get_link_status = 1; 658 em_update_link_status(adapter); 659 660 /* Register for VLAN events */ 661 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 662 em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 663 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 664 em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 665 666 em_add_hw_stats(adapter); 667 668 /* Non-AMT based hardware can now take control from firmware */ 669 if (adapter->has_manage && !adapter->has_amt) 670 em_get_hw_control(adapter); 671 672 /* Tell the stack that the interface is not active */ 673 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 674 675 adapter->led_dev = led_create(em_led_func, adapter, 676 device_get_nameunit(dev)); 677 678 INIT_DEBUGOUT("em_attach: end"); 679 680 return (0); 681 682 err_late: 683 em_free_transmit_structures(adapter); 684 em_free_receive_structures(adapter); 685 em_release_hw_control(adapter); 686 if (adapter->ifp != NULL) 687 if_free(adapter->ifp); 688 err_pci: 689 em_free_pci_resources(adapter); 690 free(adapter->mta, M_DEVBUF); 691 EM_CORE_LOCK_DESTROY(adapter); 692 693 return (error); 694 } 695 696 /********************************************************************* 697 * Device removal routine 698 * 699 * The detach entry point is called when the driver is being removed. 700 * This routine stops the adapter and deallocates all the resources 701 * that were allocated for driver operation. 702 * 703 * return 0 on success, positive on failure 704 *********************************************************************/ 705 706 static int 707 em_detach(device_t dev) 708 { 709 struct adapter *adapter = device_get_softc(dev); 710 struct ifnet *ifp = adapter->ifp; 711 712 INIT_DEBUGOUT("em_detach: begin"); 713 714 /* Make sure VLANS are not using driver */ 715 if (adapter->ifp->if_vlantrunk != NULL) { 716 device_printf(dev,"Vlan in use, detach first\n"); 717 return (EBUSY); 718 } 719 720 #ifdef DEVICE_POLLING 721 if (ifp->if_capenable & IFCAP_POLLING) 722 ether_poll_deregister(ifp); 723 #endif 724 725 if (adapter->led_dev != NULL) 726 led_destroy(adapter->led_dev); 727 728 EM_CORE_LOCK(adapter); 729 adapter->in_detach = 1; 730 em_stop(adapter); 731 EM_CORE_UNLOCK(adapter); 732 EM_CORE_LOCK_DESTROY(adapter); 733 734 e1000_phy_hw_reset(&adapter->hw); 735 736 em_release_manageability(adapter); 737 em_release_hw_control(adapter); 738 739 /* Unregister VLAN events */ 740 if (adapter->vlan_attach != NULL) 741 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 742 if (adapter->vlan_detach != NULL) 743 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 744 745 ether_ifdetach(adapter->ifp); 746 callout_drain(&adapter->timer); 747 748 em_free_pci_resources(adapter); 749 bus_generic_detach(dev); 750 if_free(ifp); 751 752 em_free_transmit_structures(adapter); 753 em_free_receive_structures(adapter); 754 755 em_release_hw_control(adapter); 756 free(adapter->mta, M_DEVBUF); 757 758 return (0); 759 } 760 761 /********************************************************************* 762 * 763 * Shutdown entry point 764 * 765 **********************************************************************/ 766 767 static int 768 em_shutdown(device_t dev) 769 { 770 return em_suspend(dev); 771 } 772 773 /* 774 * Suspend/resume device methods. 775 */ 776 static int 777 em_suspend(device_t dev) 778 { 779 struct adapter *adapter = device_get_softc(dev); 780 781 EM_CORE_LOCK(adapter); 782 783 em_release_manageability(adapter); 784 em_release_hw_control(adapter); 785 em_enable_wakeup(dev); 786 787 EM_CORE_UNLOCK(adapter); 788 789 return bus_generic_suspend(dev); 790 } 791 792 static int 793 em_resume(device_t dev) 794 { 795 struct adapter *adapter = device_get_softc(dev); 796 struct ifnet *ifp = adapter->ifp; 797 798 EM_CORE_LOCK(adapter); 799 em_init_locked(adapter); 800 em_init_manageability(adapter); 801 EM_CORE_UNLOCK(adapter); 802 em_start(ifp); 803 804 return bus_generic_resume(dev); 805 } 806 807 808 /********************************************************************* 809 * Transmit entry point 810 * 811 * em_start is called by the stack to initiate a transmit. 812 * The driver will remain in this routine as long as there are 813 * packets to transmit and transmit resources are available. 814 * In case resources are not available stack is notified and 815 * the packet is requeued. 816 **********************************************************************/ 817 818 #ifdef EM_MULTIQUEUE 819 static int 820 em_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 821 { 822 struct adapter *adapter = txr->adapter; 823 struct mbuf *next; 824 int err = 0, enq = 0; 825 826 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 827 IFF_DRV_RUNNING || adapter->link_active == 0) { 828 if (m != NULL) 829 err = drbr_enqueue(ifp, txr->br, m); 830 return (err); 831 } 832 833 /* Call cleanup if number of TX descriptors low */ 834 if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD) 835 em_txeof(txr); 836 837 enq = 0; 838 if (m == NULL) { 839 next = drbr_dequeue(ifp, txr->br); 840 } else if (drbr_needs_enqueue(ifp, txr->br)) { 841 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 842 return (err); 843 next = drbr_dequeue(ifp, txr->br); 844 } else 845 next = m; 846 847 /* Process the queue */ 848 while (next != NULL) { 849 if ((err = em_xmit(txr, &next)) != 0) { 850 if (next != NULL) 851 err = drbr_enqueue(ifp, txr->br, next); 852 break; 853 } 854 enq++; 855 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 856 ETHER_BPF_MTAP(ifp, next); 857 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 858 break; 859 if (txr->tx_avail < EM_MAX_SCATTER) { 860 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 861 break; 862 } 863 next = drbr_dequeue(ifp, txr->br); 864 } 865 866 if (enq > 0) { 867 /* Set the watchdog */ 868 txr->queue_status = EM_QUEUE_WORKING; 869 txr->watchdog_time = ticks; 870 } 871 return (err); 872 } 873 874 /* 875 ** Multiqueue capable stack interface 876 */ 877 static int 878 em_mq_start(struct ifnet *ifp, struct mbuf *m) 879 { 880 struct adapter *adapter = ifp->if_softc; 881 struct tx_ring *txr = adapter->tx_rings; 882 int error; 883 884 if (EM_TX_TRYLOCK(txr)) { 885 error = em_mq_start_locked(ifp, txr, m); 886 EM_TX_UNLOCK(txr); 887 } else 888 error = drbr_enqueue(ifp, txr->br, m); 889 890 return (error); 891 } 892 893 /* 894 ** Flush all ring buffers 895 */ 896 static void 897 em_qflush(struct ifnet *ifp) 898 { 899 struct adapter *adapter = ifp->if_softc; 900 struct tx_ring *txr = adapter->tx_rings; 901 struct mbuf *m; 902 903 for (int i = 0; i < adapter->num_queues; i++, txr++) { 904 EM_TX_LOCK(txr); 905 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 906 m_freem(m); 907 EM_TX_UNLOCK(txr); 908 } 909 if_qflush(ifp); 910 } 911 912 #endif /* EM_MULTIQUEUE */ 913 914 static void 915 em_start_locked(struct ifnet *ifp, struct tx_ring *txr) 916 { 917 struct adapter *adapter = ifp->if_softc; 918 struct mbuf *m_head; 919 920 EM_TX_LOCK_ASSERT(txr); 921 922 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 923 IFF_DRV_RUNNING) 924 return; 925 926 if (!adapter->link_active) 927 return; 928 929 /* Call cleanup if number of TX descriptors low */ 930 if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD) 931 em_txeof(txr); 932 933 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 934 if (txr->tx_avail < EM_MAX_SCATTER) { 935 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 936 break; 937 } 938 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 939 if (m_head == NULL) 940 break; 941 /* 942 * Encapsulation can modify our pointer, and or make it 943 * NULL on failure. In that event, we can't requeue. 944 */ 945 if (em_xmit(txr, &m_head)) { 946 if (m_head == NULL) 947 break; 948 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 949 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 950 break; 951 } 952 953 /* Send a copy of the frame to the BPF listener */ 954 ETHER_BPF_MTAP(ifp, m_head); 955 956 /* Set timeout in case hardware has problems transmitting. */ 957 txr->watchdog_time = ticks; 958 txr->queue_status = EM_QUEUE_WORKING; 959 } 960 961 return; 962 } 963 964 static void 965 em_start(struct ifnet *ifp) 966 { 967 struct adapter *adapter = ifp->if_softc; 968 struct tx_ring *txr = adapter->tx_rings; 969 970 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 971 EM_TX_LOCK(txr); 972 em_start_locked(ifp, txr); 973 EM_TX_UNLOCK(txr); 974 } 975 return; 976 } 977 978 /********************************************************************* 979 * Ioctl entry point 980 * 981 * em_ioctl is called when the user wants to configure the 982 * interface. 983 * 984 * return 0 on success, positive on failure 985 **********************************************************************/ 986 987 static int 988 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 989 { 990 struct adapter *adapter = ifp->if_softc; 991 struct ifreq *ifr = (struct ifreq *)data; 992 #ifdef INET 993 struct ifaddr *ifa = (struct ifaddr *)data; 994 #endif 995 int error = 0; 996 997 if (adapter->in_detach) 998 return (error); 999 1000 switch (command) { 1001 case SIOCSIFADDR: 1002 #ifdef INET 1003 if (ifa->ifa_addr->sa_family == AF_INET) { 1004 /* 1005 * XXX 1006 * Since resetting hardware takes a very long time 1007 * and results in link renegotiation we only 1008 * initialize the hardware only when it is absolutely 1009 * required. 1010 */ 1011 ifp->if_flags |= IFF_UP; 1012 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1013 EM_CORE_LOCK(adapter); 1014 em_init_locked(adapter); 1015 EM_CORE_UNLOCK(adapter); 1016 } 1017 arp_ifinit(ifp, ifa); 1018 } else 1019 #endif 1020 error = ether_ioctl(ifp, command, data); 1021 break; 1022 case SIOCSIFMTU: 1023 { 1024 int max_frame_size; 1025 1026 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 1027 1028 EM_CORE_LOCK(adapter); 1029 switch (adapter->hw.mac.type) { 1030 case e1000_82571: 1031 case e1000_82572: 1032 case e1000_ich9lan: 1033 case e1000_ich10lan: 1034 case e1000_pch2lan: 1035 case e1000_82574: 1036 case e1000_80003es2lan: /* 9K Jumbo Frame size */ 1037 max_frame_size = 9234; 1038 break; 1039 case e1000_pchlan: 1040 max_frame_size = 4096; 1041 break; 1042 /* Adapters that do not support jumbo frames */ 1043 case e1000_82583: 1044 case e1000_ich8lan: 1045 max_frame_size = ETHER_MAX_LEN; 1046 break; 1047 default: 1048 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1049 } 1050 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1051 ETHER_CRC_LEN) { 1052 EM_CORE_UNLOCK(adapter); 1053 error = EINVAL; 1054 break; 1055 } 1056 1057 ifp->if_mtu = ifr->ifr_mtu; 1058 adapter->max_frame_size = 1059 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1060 em_init_locked(adapter); 1061 EM_CORE_UNLOCK(adapter); 1062 break; 1063 } 1064 case SIOCSIFFLAGS: 1065 IOCTL_DEBUGOUT("ioctl rcv'd:\ 1066 SIOCSIFFLAGS (Set Interface Flags)"); 1067 EM_CORE_LOCK(adapter); 1068 if (ifp->if_flags & IFF_UP) { 1069 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1070 if ((ifp->if_flags ^ adapter->if_flags) & 1071 (IFF_PROMISC | IFF_ALLMULTI)) { 1072 em_disable_promisc(adapter); 1073 em_set_promisc(adapter); 1074 } 1075 } else 1076 em_init_locked(adapter); 1077 } else 1078 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1079 em_stop(adapter); 1080 adapter->if_flags = ifp->if_flags; 1081 EM_CORE_UNLOCK(adapter); 1082 break; 1083 case SIOCADDMULTI: 1084 case SIOCDELMULTI: 1085 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 1086 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1087 EM_CORE_LOCK(adapter); 1088 em_disable_intr(adapter); 1089 em_set_multi(adapter); 1090 #ifdef DEVICE_POLLING 1091 if (!(ifp->if_capenable & IFCAP_POLLING)) 1092 #endif 1093 em_enable_intr(adapter); 1094 EM_CORE_UNLOCK(adapter); 1095 } 1096 break; 1097 case SIOCSIFMEDIA: 1098 /* 1099 ** As the speed/duplex settings are being 1100 ** changed, we need to reset the PHY. 1101 */ 1102 adapter->hw.phy.reset_disable = FALSE; 1103 /* Check SOL/IDER usage */ 1104 EM_CORE_LOCK(adapter); 1105 if (e1000_check_reset_block(&adapter->hw)) { 1106 EM_CORE_UNLOCK(adapter); 1107 device_printf(adapter->dev, "Media change is" 1108 " blocked due to SOL/IDER session.\n"); 1109 break; 1110 } 1111 EM_CORE_UNLOCK(adapter); 1112 /* falls thru */ 1113 case SIOCGIFMEDIA: 1114 IOCTL_DEBUGOUT("ioctl rcv'd: \ 1115 SIOCxIFMEDIA (Get/Set Interface Media)"); 1116 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1117 break; 1118 case SIOCSIFCAP: 1119 { 1120 int mask, reinit; 1121 1122 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 1123 reinit = 0; 1124 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1125 #ifdef DEVICE_POLLING 1126 if (mask & IFCAP_POLLING) { 1127 if (ifr->ifr_reqcap & IFCAP_POLLING) { 1128 error = ether_poll_register(em_poll, ifp); 1129 if (error) 1130 return (error); 1131 EM_CORE_LOCK(adapter); 1132 em_disable_intr(adapter); 1133 ifp->if_capenable |= IFCAP_POLLING; 1134 EM_CORE_UNLOCK(adapter); 1135 } else { 1136 error = ether_poll_deregister(ifp); 1137 /* Enable interrupt even in error case */ 1138 EM_CORE_LOCK(adapter); 1139 em_enable_intr(adapter); 1140 ifp->if_capenable &= ~IFCAP_POLLING; 1141 EM_CORE_UNLOCK(adapter); 1142 } 1143 } 1144 #endif 1145 if (mask & IFCAP_HWCSUM) { 1146 ifp->if_capenable ^= IFCAP_HWCSUM; 1147 reinit = 1; 1148 } 1149 if (mask & IFCAP_TSO4) { 1150 ifp->if_capenable ^= IFCAP_TSO4; 1151 reinit = 1; 1152 } 1153 if (mask & IFCAP_VLAN_HWTAGGING) { 1154 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1155 reinit = 1; 1156 } 1157 if (mask & IFCAP_VLAN_HWFILTER) { 1158 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 1159 reinit = 1; 1160 } 1161 if ((mask & IFCAP_WOL) && 1162 (ifp->if_capabilities & IFCAP_WOL) != 0) { 1163 if (mask & IFCAP_WOL_MCAST) 1164 ifp->if_capenable ^= IFCAP_WOL_MCAST; 1165 if (mask & IFCAP_WOL_MAGIC) 1166 ifp->if_capenable ^= IFCAP_WOL_MAGIC; 1167 } 1168 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 1169 em_init(adapter); 1170 VLAN_CAPABILITIES(ifp); 1171 break; 1172 } 1173 1174 default: 1175 error = ether_ioctl(ifp, command, data); 1176 break; 1177 } 1178 1179 return (error); 1180 } 1181 1182 1183 /********************************************************************* 1184 * Init entry point 1185 * 1186 * This routine is used in two ways. It is used by the stack as 1187 * init entry point in network interface structure. It is also used 1188 * by the driver as a hw/sw initialization routine to get to a 1189 * consistent state. 1190 * 1191 * return 0 on success, positive on failure 1192 **********************************************************************/ 1193 1194 static void 1195 em_init_locked(struct adapter *adapter) 1196 { 1197 struct ifnet *ifp = adapter->ifp; 1198 device_t dev = adapter->dev; 1199 u32 pba; 1200 1201 INIT_DEBUGOUT("em_init: begin"); 1202 1203 EM_CORE_LOCK_ASSERT(adapter); 1204 1205 em_disable_intr(adapter); 1206 callout_stop(&adapter->timer); 1207 1208 /* 1209 * Packet Buffer Allocation (PBA) 1210 * Writing PBA sets the receive portion of the buffer 1211 * the remainder is used for the transmit buffer. 1212 */ 1213 switch (adapter->hw.mac.type) { 1214 /* Total Packet Buffer on these is 48K */ 1215 case e1000_82571: 1216 case e1000_82572: 1217 case e1000_80003es2lan: 1218 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1219 break; 1220 case e1000_82573: /* 82573: Total Packet Buffer is 32K */ 1221 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 1222 break; 1223 case e1000_82574: 1224 case e1000_82583: 1225 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */ 1226 break; 1227 case e1000_ich8lan: 1228 pba = E1000_PBA_8K; 1229 break; 1230 case e1000_ich9lan: 1231 case e1000_ich10lan: 1232 case e1000_pchlan: 1233 pba = E1000_PBA_10K; 1234 break; 1235 case e1000_pch2lan: 1236 pba = E1000_PBA_26K; 1237 break; 1238 default: 1239 if (adapter->max_frame_size > 8192) 1240 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 1241 else 1242 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 1243 } 1244 1245 INIT_DEBUGOUT1("em_init: pba=%dK",pba); 1246 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); 1247 1248 /* Get the latest mac address, User can use a LAA */ 1249 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1250 ETHER_ADDR_LEN); 1251 1252 /* Put the address into the Receive Address Array */ 1253 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1254 1255 /* 1256 * With the 82571 adapter, RAR[0] may be overwritten 1257 * when the other port is reset, we make a duplicate 1258 * in RAR[14] for that eventuality, this assures 1259 * the interface continues to function. 1260 */ 1261 if (adapter->hw.mac.type == e1000_82571) { 1262 e1000_set_laa_state_82571(&adapter->hw, TRUE); 1263 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 1264 E1000_RAR_ENTRIES - 1); 1265 } 1266 1267 /* Initialize the hardware */ 1268 em_reset(adapter); 1269 em_update_link_status(adapter); 1270 1271 /* Setup VLAN support, basic and offload if available */ 1272 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1273 1274 /* Set hardware offload abilities */ 1275 ifp->if_hwassist = 0; 1276 if (ifp->if_capenable & IFCAP_TXCSUM) 1277 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1278 if (ifp->if_capenable & IFCAP_TSO4) 1279 ifp->if_hwassist |= CSUM_TSO; 1280 1281 /* Configure for OS presence */ 1282 em_init_manageability(adapter); 1283 1284 /* Prepare transmit descriptors and buffers */ 1285 em_setup_transmit_structures(adapter); 1286 em_initialize_transmit_unit(adapter); 1287 1288 /* Setup Multicast table */ 1289 em_set_multi(adapter); 1290 1291 /* 1292 ** Figure out the desired mbuf 1293 ** pool for doing jumbos 1294 */ 1295 if (adapter->max_frame_size <= 2048) 1296 adapter->rx_mbuf_sz = MCLBYTES; 1297 else if (adapter->max_frame_size <= 4096) 1298 adapter->rx_mbuf_sz = MJUMPAGESIZE; 1299 else 1300 adapter->rx_mbuf_sz = MJUM9BYTES; 1301 1302 /* Prepare receive descriptors and buffers */ 1303 if (em_setup_receive_structures(adapter)) { 1304 device_printf(dev, "Could not setup receive structures\n"); 1305 em_stop(adapter); 1306 return; 1307 } 1308 em_initialize_receive_unit(adapter); 1309 1310 /* Use real VLAN Filter support? */ 1311 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1312 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 1313 /* Use real VLAN Filter support */ 1314 em_setup_vlan_hw_support(adapter); 1315 else { 1316 u32 ctrl; 1317 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 1318 ctrl |= E1000_CTRL_VME; 1319 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 1320 } 1321 } 1322 1323 /* Don't lose promiscuous settings */ 1324 em_set_promisc(adapter); 1325 1326 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1327 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1328 1329 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1330 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1331 1332 /* MSI/X configuration for 82574 */ 1333 if (adapter->hw.mac.type == e1000_82574) { 1334 int tmp; 1335 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 1336 tmp |= E1000_CTRL_EXT_PBA_CLR; 1337 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp); 1338 /* Set the IVAR - interrupt vector routing. */ 1339 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars); 1340 } 1341 1342 #ifdef DEVICE_POLLING 1343 /* 1344 * Only enable interrupts if we are not polling, make sure 1345 * they are off otherwise. 1346 */ 1347 if (ifp->if_capenable & IFCAP_POLLING) 1348 em_disable_intr(adapter); 1349 else 1350 #endif /* DEVICE_POLLING */ 1351 em_enable_intr(adapter); 1352 1353 /* AMT based hardware can now take control from firmware */ 1354 if (adapter->has_manage && adapter->has_amt) 1355 em_get_hw_control(adapter); 1356 1357 /* Don't reset the phy next time init gets called */ 1358 adapter->hw.phy.reset_disable = TRUE; 1359 } 1360 1361 static void 1362 em_init(void *arg) 1363 { 1364 struct adapter *adapter = arg; 1365 1366 EM_CORE_LOCK(adapter); 1367 em_init_locked(adapter); 1368 EM_CORE_UNLOCK(adapter); 1369 } 1370 1371 1372 #ifdef DEVICE_POLLING 1373 /********************************************************************* 1374 * 1375 * Legacy polling routine: note this only works with single queue 1376 * 1377 *********************************************************************/ 1378 static int 1379 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) 1380 { 1381 struct adapter *adapter = ifp->if_softc; 1382 struct tx_ring *txr = adapter->tx_rings; 1383 struct rx_ring *rxr = adapter->rx_rings; 1384 u32 reg_icr; 1385 int rx_done; 1386 1387 EM_CORE_LOCK(adapter); 1388 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1389 EM_CORE_UNLOCK(adapter); 1390 return (0); 1391 } 1392 1393 if (cmd == POLL_AND_CHECK_STATUS) { 1394 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1395 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1396 callout_stop(&adapter->timer); 1397 adapter->hw.mac.get_link_status = 1; 1398 em_update_link_status(adapter); 1399 callout_reset(&adapter->timer, hz, 1400 em_local_timer, adapter); 1401 } 1402 } 1403 EM_CORE_UNLOCK(adapter); 1404 1405 em_rxeof(rxr, count, &rx_done); 1406 1407 EM_TX_LOCK(txr); 1408 em_txeof(txr); 1409 #ifdef EM_MULTIQUEUE 1410 if (!drbr_empty(ifp, txr->br)) 1411 em_mq_start_locked(ifp, txr, NULL); 1412 #else 1413 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1414 em_start_locked(ifp, txr); 1415 #endif 1416 EM_TX_UNLOCK(txr); 1417 1418 return (rx_done); 1419 } 1420 #endif /* DEVICE_POLLING */ 1421 1422 1423 /********************************************************************* 1424 * 1425 * Fast Legacy/MSI Combined Interrupt Service routine 1426 * 1427 *********************************************************************/ 1428 static int 1429 em_irq_fast(void *arg) 1430 { 1431 struct adapter *adapter = arg; 1432 struct ifnet *ifp; 1433 u32 reg_icr; 1434 1435 ifp = adapter->ifp; 1436 1437 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1438 1439 /* Hot eject? */ 1440 if (reg_icr == 0xffffffff) 1441 return FILTER_STRAY; 1442 1443 /* Definitely not our interrupt. */ 1444 if (reg_icr == 0x0) 1445 return FILTER_STRAY; 1446 1447 /* 1448 * Starting with the 82571 chip, bit 31 should be used to 1449 * determine whether the interrupt belongs to us. 1450 */ 1451 if (adapter->hw.mac.type >= e1000_82571 && 1452 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1453 return FILTER_STRAY; 1454 1455 em_disable_intr(adapter); 1456 taskqueue_enqueue(adapter->tq, &adapter->que_task); 1457 1458 /* Link status change */ 1459 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1460 adapter->hw.mac.get_link_status = 1; 1461 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1462 } 1463 1464 if (reg_icr & E1000_ICR_RXO) 1465 adapter->rx_overruns++; 1466 return FILTER_HANDLED; 1467 } 1468 1469 /* Combined RX/TX handler, used by Legacy and MSI */ 1470 static void 1471 em_handle_que(void *context, int pending) 1472 { 1473 struct adapter *adapter = context; 1474 struct ifnet *ifp = adapter->ifp; 1475 struct tx_ring *txr = adapter->tx_rings; 1476 struct rx_ring *rxr = adapter->rx_rings; 1477 bool more; 1478 1479 1480 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1481 more = em_rxeof(rxr, adapter->rx_process_limit, NULL); 1482 1483 EM_TX_LOCK(txr); 1484 em_txeof(txr); 1485 #ifdef EM_MULTIQUEUE 1486 if (!drbr_empty(ifp, txr->br)) 1487 em_mq_start_locked(ifp, txr, NULL); 1488 #else 1489 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1490 em_start_locked(ifp, txr); 1491 #endif 1492 em_txeof(txr); 1493 EM_TX_UNLOCK(txr); 1494 if (more) { 1495 taskqueue_enqueue(adapter->tq, &adapter->que_task); 1496 return; 1497 } 1498 } 1499 1500 em_enable_intr(adapter); 1501 return; 1502 } 1503 1504 1505 /********************************************************************* 1506 * 1507 * MSIX Interrupt Service Routines 1508 * 1509 **********************************************************************/ 1510 static void 1511 em_msix_tx(void *arg) 1512 { 1513 struct tx_ring *txr = arg; 1514 struct adapter *adapter = txr->adapter; 1515 bool more; 1516 1517 ++txr->tx_irq; 1518 EM_TX_LOCK(txr); 1519 more = em_txeof(txr); 1520 EM_TX_UNLOCK(txr); 1521 if (more) 1522 taskqueue_enqueue(txr->tq, &txr->tx_task); 1523 else 1524 /* Reenable this interrupt */ 1525 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); 1526 return; 1527 } 1528 1529 /********************************************************************* 1530 * 1531 * MSIX RX Interrupt Service routine 1532 * 1533 **********************************************************************/ 1534 1535 static void 1536 em_msix_rx(void *arg) 1537 { 1538 struct rx_ring *rxr = arg; 1539 struct adapter *adapter = rxr->adapter; 1540 bool more; 1541 1542 ++rxr->rx_irq; 1543 more = em_rxeof(rxr, adapter->rx_process_limit, NULL); 1544 if (more) 1545 taskqueue_enqueue(rxr->tq, &rxr->rx_task); 1546 else 1547 /* Reenable this interrupt */ 1548 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); 1549 return; 1550 } 1551 1552 /********************************************************************* 1553 * 1554 * MSIX Link Fast Interrupt Service routine 1555 * 1556 **********************************************************************/ 1557 static void 1558 em_msix_link(void *arg) 1559 { 1560 struct adapter *adapter = arg; 1561 u32 reg_icr; 1562 1563 ++adapter->link_irq; 1564 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1565 1566 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1567 adapter->hw.mac.get_link_status = 1; 1568 em_handle_link(adapter, 0); 1569 } else 1570 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 1571 EM_MSIX_LINK | E1000_IMS_LSC); 1572 return; 1573 } 1574 1575 static void 1576 em_handle_rx(void *context, int pending) 1577 { 1578 struct rx_ring *rxr = context; 1579 struct adapter *adapter = rxr->adapter; 1580 bool more; 1581 1582 more = em_rxeof(rxr, adapter->rx_process_limit, NULL); 1583 if (more) 1584 taskqueue_enqueue(rxr->tq, &rxr->rx_task); 1585 else 1586 /* Reenable this interrupt */ 1587 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); 1588 } 1589 1590 static void 1591 em_handle_tx(void *context, int pending) 1592 { 1593 struct tx_ring *txr = context; 1594 struct adapter *adapter = txr->adapter; 1595 struct ifnet *ifp = adapter->ifp; 1596 1597 EM_TX_LOCK(txr); 1598 em_txeof(txr); 1599 #ifdef EM_MULTIQUEUE 1600 if (!drbr_empty(ifp, txr->br)) 1601 em_mq_start_locked(ifp, txr, NULL); 1602 #else 1603 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1604 em_start_locked(ifp, txr); 1605 #endif 1606 em_txeof(txr); 1607 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); 1608 EM_TX_UNLOCK(txr); 1609 } 1610 1611 static void 1612 em_handle_link(void *context, int pending) 1613 { 1614 struct adapter *adapter = context; 1615 struct ifnet *ifp = adapter->ifp; 1616 1617 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1618 return; 1619 1620 EM_CORE_LOCK(adapter); 1621 callout_stop(&adapter->timer); 1622 em_update_link_status(adapter); 1623 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1624 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 1625 EM_MSIX_LINK | E1000_IMS_LSC); 1626 EM_CORE_UNLOCK(adapter); 1627 } 1628 1629 1630 /********************************************************************* 1631 * 1632 * Media Ioctl callback 1633 * 1634 * This routine is called whenever the user queries the status of 1635 * the interface using ifconfig. 1636 * 1637 **********************************************************************/ 1638 static void 1639 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1640 { 1641 struct adapter *adapter = ifp->if_softc; 1642 u_char fiber_type = IFM_1000_SX; 1643 1644 INIT_DEBUGOUT("em_media_status: begin"); 1645 1646 EM_CORE_LOCK(adapter); 1647 em_update_link_status(adapter); 1648 1649 ifmr->ifm_status = IFM_AVALID; 1650 ifmr->ifm_active = IFM_ETHER; 1651 1652 if (!adapter->link_active) { 1653 EM_CORE_UNLOCK(adapter); 1654 return; 1655 } 1656 1657 ifmr->ifm_status |= IFM_ACTIVE; 1658 1659 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 1660 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 1661 ifmr->ifm_active |= fiber_type | IFM_FDX; 1662 } else { 1663 switch (adapter->link_speed) { 1664 case 10: 1665 ifmr->ifm_active |= IFM_10_T; 1666 break; 1667 case 100: 1668 ifmr->ifm_active |= IFM_100_TX; 1669 break; 1670 case 1000: 1671 ifmr->ifm_active |= IFM_1000_T; 1672 break; 1673 } 1674 if (adapter->link_duplex == FULL_DUPLEX) 1675 ifmr->ifm_active |= IFM_FDX; 1676 else 1677 ifmr->ifm_active |= IFM_HDX; 1678 } 1679 EM_CORE_UNLOCK(adapter); 1680 } 1681 1682 /********************************************************************* 1683 * 1684 * Media Ioctl callback 1685 * 1686 * This routine is called when the user changes speed/duplex using 1687 * media/mediopt option with ifconfig. 1688 * 1689 **********************************************************************/ 1690 static int 1691 em_media_change(struct ifnet *ifp) 1692 { 1693 struct adapter *adapter = ifp->if_softc; 1694 struct ifmedia *ifm = &adapter->media; 1695 1696 INIT_DEBUGOUT("em_media_change: begin"); 1697 1698 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1699 return (EINVAL); 1700 1701 EM_CORE_LOCK(adapter); 1702 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1703 case IFM_AUTO: 1704 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1705 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1706 break; 1707 case IFM_1000_LX: 1708 case IFM_1000_SX: 1709 case IFM_1000_T: 1710 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1711 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1712 break; 1713 case IFM_100_TX: 1714 adapter->hw.mac.autoneg = FALSE; 1715 adapter->hw.phy.autoneg_advertised = 0; 1716 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1717 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1718 else 1719 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1720 break; 1721 case IFM_10_T: 1722 adapter->hw.mac.autoneg = FALSE; 1723 adapter->hw.phy.autoneg_advertised = 0; 1724 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1725 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1726 else 1727 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1728 break; 1729 default: 1730 device_printf(adapter->dev, "Unsupported media type\n"); 1731 } 1732 1733 em_init_locked(adapter); 1734 EM_CORE_UNLOCK(adapter); 1735 1736 return (0); 1737 } 1738 1739 /********************************************************************* 1740 * 1741 * This routine maps the mbufs to tx descriptors. 1742 * 1743 * return 0 on success, positive on failure 1744 **********************************************************************/ 1745 1746 static int 1747 em_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1748 { 1749 struct adapter *adapter = txr->adapter; 1750 bus_dma_segment_t segs[EM_MAX_SCATTER]; 1751 bus_dmamap_t map; 1752 struct em_buffer *tx_buffer, *tx_buffer_mapped; 1753 struct e1000_tx_desc *ctxd = NULL; 1754 struct mbuf *m_head; 1755 struct ether_header *eh; 1756 struct ip *ip = NULL; 1757 struct tcphdr *tp = NULL; 1758 u32 txd_upper, txd_lower, txd_used, txd_saved; 1759 int ip_off, poff; 1760 int nsegs, i, j, first, last = 0; 1761 int error, do_tso, tso_desc = 0; 1762 1763 m_head = *m_headp; 1764 txd_upper = txd_lower = txd_used = txd_saved = 0; 1765 do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0); 1766 ip_off = poff = 0; 1767 1768 /* 1769 * Intel recommends entire IP/TCP header length reside in a single 1770 * buffer. If multiple descriptors are used to describe the IP and 1771 * TCP header, each descriptor should describe one or more 1772 * complete headers; descriptors referencing only parts of headers 1773 * are not supported. If all layer headers are not coalesced into 1774 * a single buffer, each buffer should not cross a 4KB boundary, 1775 * or be larger than the maximum read request size. 1776 * Controller also requires modifing IP/TCP header to make TSO work 1777 * so we firstly get a writable mbuf chain then coalesce ethernet/ 1778 * IP/TCP header into a single buffer to meet the requirement of 1779 * controller. This also simplifies IP/TCP/UDP checksum offloading 1780 * which also has similiar restrictions. 1781 */ 1782 if (do_tso || m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) { 1783 if (do_tso || (m_head->m_next != NULL && 1784 m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)) { 1785 if (M_WRITABLE(*m_headp) == 0) { 1786 m_head = m_dup(*m_headp, M_DONTWAIT); 1787 m_freem(*m_headp); 1788 if (m_head == NULL) { 1789 *m_headp = NULL; 1790 return (ENOBUFS); 1791 } 1792 *m_headp = m_head; 1793 } 1794 } 1795 /* 1796 * XXX 1797 * Assume IPv4, we don't have TSO/checksum offload support 1798 * for IPv6 yet. 1799 */ 1800 ip_off = sizeof(struct ether_header); 1801 m_head = m_pullup(m_head, ip_off); 1802 if (m_head == NULL) { 1803 *m_headp = NULL; 1804 return (ENOBUFS); 1805 } 1806 eh = mtod(m_head, struct ether_header *); 1807 if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 1808 ip_off = sizeof(struct ether_vlan_header); 1809 m_head = m_pullup(m_head, ip_off); 1810 if (m_head == NULL) { 1811 *m_headp = NULL; 1812 return (ENOBUFS); 1813 } 1814 } 1815 m_head = m_pullup(m_head, ip_off + sizeof(struct ip)); 1816 if (m_head == NULL) { 1817 *m_headp = NULL; 1818 return (ENOBUFS); 1819 } 1820 ip = (struct ip *)(mtod(m_head, char *) + ip_off); 1821 poff = ip_off + (ip->ip_hl << 2); 1822 m_head = m_pullup(m_head, poff + sizeof(struct tcphdr)); 1823 if (m_head == NULL) { 1824 *m_headp = NULL; 1825 return (ENOBUFS); 1826 } 1827 if (do_tso) { 1828 tp = (struct tcphdr *)(mtod(m_head, char *) + poff); 1829 /* 1830 * TSO workaround: 1831 * pull 4 more bytes of data into it. 1832 */ 1833 m_head = m_pullup(m_head, poff + (tp->th_off << 2) + 4); 1834 if (m_head == NULL) { 1835 *m_headp = NULL; 1836 return (ENOBUFS); 1837 } 1838 ip = (struct ip *)(mtod(m_head, char *) + ip_off); 1839 ip->ip_len = 0; 1840 ip->ip_sum = 0; 1841 /* 1842 * The pseudo TCP checksum does not include TCP payload 1843 * length so driver should recompute the checksum here 1844 * what hardware expect to see. This is adherence of 1845 * Microsoft's Large Send specification. 1846 */ 1847 tp = (struct tcphdr *)(mtod(m_head, char *) + poff); 1848 tp->th_sum = in_pseudo(ip->ip_src.s_addr, 1849 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 1850 } else if (m_head->m_pkthdr.csum_flags & CSUM_TCP) { 1851 tp = (struct tcphdr *)(mtod(m_head, char *) + poff); 1852 m_head = m_pullup(m_head, poff + (tp->th_off << 2)); 1853 if (m_head == NULL) { 1854 *m_headp = NULL; 1855 return (ENOBUFS); 1856 } 1857 ip = (struct ip *)(mtod(m_head, char *) + ip_off); 1858 tp = (struct tcphdr *)(mtod(m_head, char *) + poff); 1859 } else if (m_head->m_pkthdr.csum_flags & CSUM_UDP) { 1860 m_head = m_pullup(m_head, poff + sizeof(struct udphdr)); 1861 if (m_head == NULL) { 1862 *m_headp = NULL; 1863 return (ENOBUFS); 1864 } 1865 ip = (struct ip *)(mtod(m_head, char *) + ip_off); 1866 } 1867 *m_headp = m_head; 1868 } 1869 1870 /* 1871 * Map the packet for DMA 1872 * 1873 * Capture the first descriptor index, 1874 * this descriptor will have the index 1875 * of the EOP which is the only one that 1876 * now gets a DONE bit writeback. 1877 */ 1878 first = txr->next_avail_desc; 1879 tx_buffer = &txr->tx_buffers[first]; 1880 tx_buffer_mapped = tx_buffer; 1881 map = tx_buffer->map; 1882 1883 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1884 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1885 1886 /* 1887 * There are two types of errors we can (try) to handle: 1888 * - EFBIG means the mbuf chain was too long and bus_dma ran 1889 * out of segments. Defragment the mbuf chain and try again. 1890 * - ENOMEM means bus_dma could not obtain enough bounce buffers 1891 * at this point in time. Defer sending and try again later. 1892 * All other errors, in particular EINVAL, are fatal and prevent the 1893 * mbuf chain from ever going through. Drop it and report error. 1894 */ 1895 if (error == EFBIG) { 1896 struct mbuf *m; 1897 1898 m = m_defrag(*m_headp, M_DONTWAIT); 1899 if (m == NULL) { 1900 adapter->mbuf_alloc_failed++; 1901 m_freem(*m_headp); 1902 *m_headp = NULL; 1903 return (ENOBUFS); 1904 } 1905 *m_headp = m; 1906 1907 /* Try it again */ 1908 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1909 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1910 1911 if (error) { 1912 adapter->no_tx_dma_setup++; 1913 m_freem(*m_headp); 1914 *m_headp = NULL; 1915 return (error); 1916 } 1917 } else if (error != 0) { 1918 adapter->no_tx_dma_setup++; 1919 return (error); 1920 } 1921 1922 /* 1923 * TSO Hardware workaround, if this packet is not 1924 * TSO, and is only a single descriptor long, and 1925 * it follows a TSO burst, then we need to add a 1926 * sentinel descriptor to prevent premature writeback. 1927 */ 1928 if ((do_tso == 0) && (txr->tx_tso == TRUE)) { 1929 if (nsegs == 1) 1930 tso_desc = TRUE; 1931 txr->tx_tso = FALSE; 1932 } 1933 1934 if (nsegs > (txr->tx_avail - 2)) { 1935 txr->no_desc_avail++; 1936 bus_dmamap_unload(txr->txtag, map); 1937 return (ENOBUFS); 1938 } 1939 m_head = *m_headp; 1940 1941 /* Do hardware assists */ 1942 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1943 em_tso_setup(txr, m_head, ip_off, ip, tp, 1944 &txd_upper, &txd_lower); 1945 /* we need to make a final sentinel transmit desc */ 1946 tso_desc = TRUE; 1947 } else if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) 1948 em_transmit_checksum_setup(txr, m_head, 1949 ip_off, ip, &txd_upper, &txd_lower); 1950 1951 i = txr->next_avail_desc; 1952 1953 /* Set up our transmit descriptors */ 1954 for (j = 0; j < nsegs; j++) { 1955 bus_size_t seg_len; 1956 bus_addr_t seg_addr; 1957 1958 tx_buffer = &txr->tx_buffers[i]; 1959 ctxd = &txr->tx_base[i]; 1960 seg_addr = segs[j].ds_addr; 1961 seg_len = segs[j].ds_len; 1962 /* 1963 ** TSO Workaround: 1964 ** If this is the last descriptor, we want to 1965 ** split it so we have a small final sentinel 1966 */ 1967 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) { 1968 seg_len -= 4; 1969 ctxd->buffer_addr = htole64(seg_addr); 1970 ctxd->lower.data = htole32( 1971 adapter->txd_cmd | txd_lower | seg_len); 1972 ctxd->upper.data = 1973 htole32(txd_upper); 1974 if (++i == adapter->num_tx_desc) 1975 i = 0; 1976 /* Now make the sentinel */ 1977 ++txd_used; /* using an extra txd */ 1978 ctxd = &txr->tx_base[i]; 1979 tx_buffer = &txr->tx_buffers[i]; 1980 ctxd->buffer_addr = 1981 htole64(seg_addr + seg_len); 1982 ctxd->lower.data = htole32( 1983 adapter->txd_cmd | txd_lower | 4); 1984 ctxd->upper.data = 1985 htole32(txd_upper); 1986 last = i; 1987 if (++i == adapter->num_tx_desc) 1988 i = 0; 1989 } else { 1990 ctxd->buffer_addr = htole64(seg_addr); 1991 ctxd->lower.data = htole32( 1992 adapter->txd_cmd | txd_lower | seg_len); 1993 ctxd->upper.data = 1994 htole32(txd_upper); 1995 last = i; 1996 if (++i == adapter->num_tx_desc) 1997 i = 0; 1998 } 1999 tx_buffer->m_head = NULL; 2000 tx_buffer->next_eop = -1; 2001 } 2002 2003 txr->next_avail_desc = i; 2004 txr->tx_avail -= nsegs; 2005 if (tso_desc) /* TSO used an extra for sentinel */ 2006 txr->tx_avail -= txd_used; 2007 2008 if (m_head->m_flags & M_VLANTAG) { 2009 /* Set the vlan id. */ 2010 ctxd->upper.fields.special = 2011 htole16(m_head->m_pkthdr.ether_vtag); 2012 /* Tell hardware to add tag */ 2013 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE); 2014 } 2015 2016 tx_buffer->m_head = m_head; 2017 tx_buffer_mapped->map = tx_buffer->map; 2018 tx_buffer->map = map; 2019 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 2020 2021 /* 2022 * Last Descriptor of Packet 2023 * needs End Of Packet (EOP) 2024 * and Report Status (RS) 2025 */ 2026 ctxd->lower.data |= 2027 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); 2028 /* 2029 * Keep track in the first buffer which 2030 * descriptor will be written back 2031 */ 2032 tx_buffer = &txr->tx_buffers[first]; 2033 tx_buffer->next_eop = last; 2034 /* Update the watchdog time early and often */ 2035 txr->watchdog_time = ticks; 2036 2037 /* 2038 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 2039 * that this frame is available to transmit. 2040 */ 2041 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2042 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2043 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); 2044 2045 return (0); 2046 } 2047 2048 static void 2049 em_set_promisc(struct adapter *adapter) 2050 { 2051 struct ifnet *ifp = adapter->ifp; 2052 u32 reg_rctl; 2053 2054 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2055 2056 if (ifp->if_flags & IFF_PROMISC) { 2057 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2058 /* Turn this on if you want to see bad packets */ 2059 if (em_debug_sbp) 2060 reg_rctl |= E1000_RCTL_SBP; 2061 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2062 } else if (ifp->if_flags & IFF_ALLMULTI) { 2063 reg_rctl |= E1000_RCTL_MPE; 2064 reg_rctl &= ~E1000_RCTL_UPE; 2065 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2066 } 2067 } 2068 2069 static void 2070 em_disable_promisc(struct adapter *adapter) 2071 { 2072 u32 reg_rctl; 2073 2074 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2075 2076 reg_rctl &= (~E1000_RCTL_UPE); 2077 reg_rctl &= (~E1000_RCTL_MPE); 2078 reg_rctl &= (~E1000_RCTL_SBP); 2079 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2080 } 2081 2082 2083 /********************************************************************* 2084 * Multicast Update 2085 * 2086 * This routine is called whenever multicast address list is updated. 2087 * 2088 **********************************************************************/ 2089 2090 static void 2091 em_set_multi(struct adapter *adapter) 2092 { 2093 struct ifnet *ifp = adapter->ifp; 2094 struct ifmultiaddr *ifma; 2095 u32 reg_rctl = 0; 2096 u8 *mta; /* Multicast array memory */ 2097 int mcnt = 0; 2098 2099 IOCTL_DEBUGOUT("em_set_multi: begin"); 2100 2101 mta = adapter->mta; 2102 bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); 2103 2104 if (adapter->hw.mac.type == e1000_82542 && 2105 adapter->hw.revision_id == E1000_REVISION_2) { 2106 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2107 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2108 e1000_pci_clear_mwi(&adapter->hw); 2109 reg_rctl |= E1000_RCTL_RST; 2110 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2111 msec_delay(5); 2112 } 2113 2114 #if __FreeBSD_version < 800000 2115 IF_ADDR_LOCK(ifp); 2116 #else 2117 if_maddr_rlock(ifp); 2118 #endif 2119 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2120 if (ifma->ifma_addr->sa_family != AF_LINK) 2121 continue; 2122 2123 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 2124 break; 2125 2126 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 2127 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 2128 mcnt++; 2129 } 2130 #if __FreeBSD_version < 800000 2131 IF_ADDR_UNLOCK(ifp); 2132 #else 2133 if_maddr_runlock(ifp); 2134 #endif 2135 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 2136 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2137 reg_rctl |= E1000_RCTL_MPE; 2138 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2139 } else 2140 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 2141 2142 if (adapter->hw.mac.type == e1000_82542 && 2143 adapter->hw.revision_id == E1000_REVISION_2) { 2144 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2145 reg_rctl &= ~E1000_RCTL_RST; 2146 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2147 msec_delay(5); 2148 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2149 e1000_pci_set_mwi(&adapter->hw); 2150 } 2151 } 2152 2153 2154 /********************************************************************* 2155 * Timer routine 2156 * 2157 * This routine checks for link status and updates statistics. 2158 * 2159 **********************************************************************/ 2160 2161 static void 2162 em_local_timer(void *arg) 2163 { 2164 struct adapter *adapter = arg; 2165 struct ifnet *ifp = adapter->ifp; 2166 struct tx_ring *txr = adapter->tx_rings; 2167 2168 EM_CORE_LOCK_ASSERT(adapter); 2169 2170 em_update_link_status(adapter); 2171 em_update_stats_counters(adapter); 2172 2173 /* Reset LAA into RAR[0] on 82571 */ 2174 if ((adapter->hw.mac.type == e1000_82571) && 2175 e1000_get_laa_state_82571(&adapter->hw)) 2176 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 2177 2178 /* 2179 ** Don't do TX watchdog check if we've been paused 2180 */ 2181 if (adapter->pause_frames) { 2182 adapter->pause_frames = 0; 2183 goto out; 2184 } 2185 /* 2186 ** Check on the state of the TX queue(s), this 2187 ** can be done without the lock because its RO 2188 ** and the HUNG state will be static if set. 2189 */ 2190 for (int i = 0; i < adapter->num_queues; i++, txr++) 2191 if (txr->queue_status == EM_QUEUE_HUNG) 2192 goto hung; 2193 out: 2194 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 2195 return; 2196 hung: 2197 /* Looks like we're hung */ 2198 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 2199 device_printf(adapter->dev, 2200 "Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 2201 E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)), 2202 E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me))); 2203 device_printf(adapter->dev,"TX(%d) desc avail = %d," 2204 "Next TX to Clean = %d\n", 2205 txr->me, txr->tx_avail, txr->next_to_clean); 2206 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2207 adapter->watchdog_events++; 2208 EM_TX_UNLOCK(txr); 2209 em_init_locked(adapter); 2210 } 2211 2212 2213 static void 2214 em_update_link_status(struct adapter *adapter) 2215 { 2216 struct e1000_hw *hw = &adapter->hw; 2217 struct ifnet *ifp = adapter->ifp; 2218 device_t dev = adapter->dev; 2219 struct tx_ring *txr = adapter->tx_rings; 2220 u32 link_check = 0; 2221 2222 /* Get the cached link value or read phy for real */ 2223 switch (hw->phy.media_type) { 2224 case e1000_media_type_copper: 2225 if (hw->mac.get_link_status) { 2226 /* Do the work to read phy */ 2227 e1000_check_for_link(hw); 2228 link_check = !hw->mac.get_link_status; 2229 if (link_check) /* ESB2 fix */ 2230 e1000_cfg_on_link_up(hw); 2231 } else 2232 link_check = TRUE; 2233 break; 2234 case e1000_media_type_fiber: 2235 e1000_check_for_link(hw); 2236 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2237 E1000_STATUS_LU); 2238 break; 2239 case e1000_media_type_internal_serdes: 2240 e1000_check_for_link(hw); 2241 link_check = adapter->hw.mac.serdes_has_link; 2242 break; 2243 default: 2244 case e1000_media_type_unknown: 2245 break; 2246 } 2247 2248 /* Now check for a transition */ 2249 if (link_check && (adapter->link_active == 0)) { 2250 e1000_get_speed_and_duplex(hw, &adapter->link_speed, 2251 &adapter->link_duplex); 2252 /* Check if we must disable SPEED_MODE bit on PCI-E */ 2253 if ((adapter->link_speed != SPEED_1000) && 2254 ((hw->mac.type == e1000_82571) || 2255 (hw->mac.type == e1000_82572))) { 2256 int tarc0; 2257 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 2258 tarc0 &= ~SPEED_MODE_BIT; 2259 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 2260 } 2261 if (bootverbose) 2262 device_printf(dev, "Link is up %d Mbps %s\n", 2263 adapter->link_speed, 2264 ((adapter->link_duplex == FULL_DUPLEX) ? 2265 "Full Duplex" : "Half Duplex")); 2266 adapter->link_active = 1; 2267 adapter->smartspeed = 0; 2268 ifp->if_baudrate = adapter->link_speed * 1000000; 2269 if_link_state_change(ifp, LINK_STATE_UP); 2270 } else if (!link_check && (adapter->link_active == 1)) { 2271 ifp->if_baudrate = adapter->link_speed = 0; 2272 adapter->link_duplex = 0; 2273 if (bootverbose) 2274 device_printf(dev, "Link is Down\n"); 2275 adapter->link_active = 0; 2276 /* Link down, disable watchdog */ 2277 for (int i = 0; i < adapter->num_queues; i++, txr++) 2278 txr->queue_status = EM_QUEUE_IDLE; 2279 if_link_state_change(ifp, LINK_STATE_DOWN); 2280 } 2281 } 2282 2283 /********************************************************************* 2284 * 2285 * This routine disables all traffic on the adapter by issuing a 2286 * global reset on the MAC and deallocates TX/RX buffers. 2287 * 2288 * This routine should always be called with BOTH the CORE 2289 * and TX locks. 2290 **********************************************************************/ 2291 2292 static void 2293 em_stop(void *arg) 2294 { 2295 struct adapter *adapter = arg; 2296 struct ifnet *ifp = adapter->ifp; 2297 struct tx_ring *txr = adapter->tx_rings; 2298 2299 EM_CORE_LOCK_ASSERT(adapter); 2300 2301 INIT_DEBUGOUT("em_stop: begin"); 2302 2303 em_disable_intr(adapter); 2304 callout_stop(&adapter->timer); 2305 2306 /* Tell the stack that the interface is no longer active */ 2307 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2308 2309 /* Unarm watchdog timer. */ 2310 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2311 EM_TX_LOCK(txr); 2312 txr->queue_status = EM_QUEUE_IDLE; 2313 EM_TX_UNLOCK(txr); 2314 } 2315 2316 e1000_reset_hw(&adapter->hw); 2317 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2318 2319 e1000_led_off(&adapter->hw); 2320 e1000_cleanup_led(&adapter->hw); 2321 } 2322 2323 2324 /********************************************************************* 2325 * 2326 * Determine hardware revision. 2327 * 2328 **********************************************************************/ 2329 static void 2330 em_identify_hardware(struct adapter *adapter) 2331 { 2332 device_t dev = adapter->dev; 2333 2334 /* Make sure our PCI config space has the necessary stuff set */ 2335 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2336 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 2337 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 2338 device_printf(dev, "Memory Access and/or Bus Master bits " 2339 "were not set!\n"); 2340 adapter->hw.bus.pci_cmd_word |= 2341 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 2342 pci_write_config(dev, PCIR_COMMAND, 2343 adapter->hw.bus.pci_cmd_word, 2); 2344 } 2345 2346 /* Save off the information about this board */ 2347 adapter->hw.vendor_id = pci_get_vendor(dev); 2348 adapter->hw.device_id = pci_get_device(dev); 2349 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2350 adapter->hw.subsystem_vendor_id = 2351 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2352 adapter->hw.subsystem_device_id = 2353 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2354 2355 /* Do Shared Code Init and Setup */ 2356 if (e1000_set_mac_type(&adapter->hw)) { 2357 device_printf(dev, "Setup init failure\n"); 2358 return; 2359 } 2360 } 2361 2362 static int 2363 em_allocate_pci_resources(struct adapter *adapter) 2364 { 2365 device_t dev = adapter->dev; 2366 int rid; 2367 2368 rid = PCIR_BAR(0); 2369 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2370 &rid, RF_ACTIVE); 2371 if (adapter->memory == NULL) { 2372 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2373 return (ENXIO); 2374 } 2375 adapter->osdep.mem_bus_space_tag = 2376 rman_get_bustag(adapter->memory); 2377 adapter->osdep.mem_bus_space_handle = 2378 rman_get_bushandle(adapter->memory); 2379 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; 2380 2381 /* Default to a single queue */ 2382 adapter->num_queues = 1; 2383 2384 /* 2385 * Setup MSI/X or MSI if PCI Express 2386 */ 2387 adapter->msix = em_setup_msix(adapter); 2388 2389 adapter->hw.back = &adapter->osdep; 2390 2391 return (0); 2392 } 2393 2394 /********************************************************************* 2395 * 2396 * Setup the Legacy or MSI Interrupt handler 2397 * 2398 **********************************************************************/ 2399 int 2400 em_allocate_legacy(struct adapter *adapter) 2401 { 2402 device_t dev = adapter->dev; 2403 int error, rid = 0; 2404 2405 /* Manually turn off all interrupts */ 2406 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2407 2408 if (adapter->msix == 1) /* using MSI */ 2409 rid = 1; 2410 /* We allocate a single interrupt resource */ 2411 adapter->res = bus_alloc_resource_any(dev, 2412 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2413 if (adapter->res == NULL) { 2414 device_printf(dev, "Unable to allocate bus resource: " 2415 "interrupt\n"); 2416 return (ENXIO); 2417 } 2418 2419 /* 2420 * Allocate a fast interrupt and the associated 2421 * deferred processing contexts. 2422 */ 2423 TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter); 2424 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2425 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 2426 taskqueue_thread_enqueue, &adapter->tq); 2427 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2428 device_get_nameunit(adapter->dev)); 2429 if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET, 2430 em_irq_fast, NULL, adapter, &adapter->tag)) != 0) { 2431 device_printf(dev, "Failed to register fast interrupt " 2432 "handler: %d\n", error); 2433 taskqueue_free(adapter->tq); 2434 adapter->tq = NULL; 2435 return (error); 2436 } 2437 2438 return (0); 2439 } 2440 2441 /********************************************************************* 2442 * 2443 * Setup the MSIX Interrupt handlers 2444 * This is not really Multiqueue, rather 2445 * its just multiple interrupt vectors. 2446 * 2447 **********************************************************************/ 2448 int 2449 em_allocate_msix(struct adapter *adapter) 2450 { 2451 device_t dev = adapter->dev; 2452 struct tx_ring *txr = adapter->tx_rings; 2453 struct rx_ring *rxr = adapter->rx_rings; 2454 int error, rid, vector = 0; 2455 2456 2457 /* Make sure all interrupts are disabled */ 2458 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2459 2460 /* First set up ring resources */ 2461 for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { 2462 2463 /* RX ring */ 2464 rid = vector + 1; 2465 2466 rxr->res = bus_alloc_resource_any(dev, 2467 SYS_RES_IRQ, &rid, RF_ACTIVE); 2468 if (rxr->res == NULL) { 2469 device_printf(dev, 2470 "Unable to allocate bus resource: " 2471 "RX MSIX Interrupt %d\n", i); 2472 return (ENXIO); 2473 } 2474 if ((error = bus_setup_intr(dev, rxr->res, 2475 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, 2476 rxr, &rxr->tag)) != 0) { 2477 device_printf(dev, "Failed to register RX handler"); 2478 return (error); 2479 } 2480 #if __FreeBSD_version >= 800504 2481 bus_describe_intr(dev, rxr->res, rxr->tag, "rx %d", i); 2482 #endif 2483 rxr->msix = vector++; /* NOTE increment vector for TX */ 2484 TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr); 2485 rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT, 2486 taskqueue_thread_enqueue, &rxr->tq); 2487 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", 2488 device_get_nameunit(adapter->dev)); 2489 /* 2490 ** Set the bit to enable interrupt 2491 ** in E1000_IMS -- bits 20 and 21 2492 ** are for RX0 and RX1, note this has 2493 ** NOTHING to do with the MSIX vector 2494 */ 2495 rxr->ims = 1 << (20 + i); 2496 adapter->ivars |= (8 | rxr->msix) << (i * 4); 2497 2498 /* TX ring */ 2499 rid = vector + 1; 2500 txr->res = bus_alloc_resource_any(dev, 2501 SYS_RES_IRQ, &rid, RF_ACTIVE); 2502 if (txr->res == NULL) { 2503 device_printf(dev, 2504 "Unable to allocate bus resource: " 2505 "TX MSIX Interrupt %d\n", i); 2506 return (ENXIO); 2507 } 2508 if ((error = bus_setup_intr(dev, txr->res, 2509 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, 2510 txr, &txr->tag)) != 0) { 2511 device_printf(dev, "Failed to register TX handler"); 2512 return (error); 2513 } 2514 #if __FreeBSD_version >= 800504 2515 bus_describe_intr(dev, txr->res, txr->tag, "tx %d", i); 2516 #endif 2517 txr->msix = vector++; /* Increment vector for next pass */ 2518 TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr); 2519 txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT, 2520 taskqueue_thread_enqueue, &txr->tq); 2521 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", 2522 device_get_nameunit(adapter->dev)); 2523 /* 2524 ** Set the bit to enable interrupt 2525 ** in E1000_IMS -- bits 22 and 23 2526 ** are for TX0 and TX1, note this has 2527 ** NOTHING to do with the MSIX vector 2528 */ 2529 txr->ims = 1 << (22 + i); 2530 adapter->ivars |= (8 | txr->msix) << (8 + (i * 4)); 2531 } 2532 2533 /* Link interrupt */ 2534 ++rid; 2535 adapter->res = bus_alloc_resource_any(dev, 2536 SYS_RES_IRQ, &rid, RF_ACTIVE); 2537 if (!adapter->res) { 2538 device_printf(dev,"Unable to allocate " 2539 "bus resource: Link interrupt [%d]\n", rid); 2540 return (ENXIO); 2541 } 2542 /* Set the link handler function */ 2543 error = bus_setup_intr(dev, adapter->res, 2544 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2545 em_msix_link, adapter, &adapter->tag); 2546 if (error) { 2547 adapter->res = NULL; 2548 device_printf(dev, "Failed to register LINK handler"); 2549 return (error); 2550 } 2551 #if __FreeBSD_version >= 800504 2552 bus_describe_intr(dev, adapter->res, adapter->tag, "link"); 2553 #endif 2554 adapter->linkvec = vector; 2555 adapter->ivars |= (8 | vector) << 16; 2556 adapter->ivars |= 0x80000000; 2557 2558 return (0); 2559 } 2560 2561 2562 static void 2563 em_free_pci_resources(struct adapter *adapter) 2564 { 2565 device_t dev = adapter->dev; 2566 struct tx_ring *txr; 2567 struct rx_ring *rxr; 2568 int rid; 2569 2570 2571 /* 2572 ** Release all the queue interrupt resources: 2573 */ 2574 for (int i = 0; i < adapter->num_queues; i++) { 2575 txr = &adapter->tx_rings[i]; 2576 rxr = &adapter->rx_rings[i]; 2577 /* an early abort? */ 2578 if ((txr == NULL) || (rxr == NULL)) 2579 break; 2580 rid = txr->msix +1; 2581 if (txr->tag != NULL) { 2582 bus_teardown_intr(dev, txr->res, txr->tag); 2583 txr->tag = NULL; 2584 } 2585 if (txr->res != NULL) 2586 bus_release_resource(dev, SYS_RES_IRQ, 2587 rid, txr->res); 2588 rid = rxr->msix +1; 2589 if (rxr->tag != NULL) { 2590 bus_teardown_intr(dev, rxr->res, rxr->tag); 2591 rxr->tag = NULL; 2592 } 2593 if (rxr->res != NULL) 2594 bus_release_resource(dev, SYS_RES_IRQ, 2595 rid, rxr->res); 2596 } 2597 2598 if (adapter->linkvec) /* we are doing MSIX */ 2599 rid = adapter->linkvec + 1; 2600 else 2601 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2602 2603 if (adapter->tag != NULL) { 2604 bus_teardown_intr(dev, adapter->res, adapter->tag); 2605 adapter->tag = NULL; 2606 } 2607 2608 if (adapter->res != NULL) 2609 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 2610 2611 2612 if (adapter->msix) 2613 pci_release_msi(dev); 2614 2615 if (adapter->msix_mem != NULL) 2616 bus_release_resource(dev, SYS_RES_MEMORY, 2617 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); 2618 2619 if (adapter->memory != NULL) 2620 bus_release_resource(dev, SYS_RES_MEMORY, 2621 PCIR_BAR(0), adapter->memory); 2622 2623 if (adapter->flash != NULL) 2624 bus_release_resource(dev, SYS_RES_MEMORY, 2625 EM_FLASH, adapter->flash); 2626 } 2627 2628 /* 2629 * Setup MSI or MSI/X 2630 */ 2631 static int 2632 em_setup_msix(struct adapter *adapter) 2633 { 2634 device_t dev = adapter->dev; 2635 int val = 0; 2636 2637 2638 /* 2639 ** Setup MSI/X for Hartwell: tests have shown 2640 ** use of two queues to be unstable, and to 2641 ** provide no great gain anyway, so we simply 2642 ** seperate the interrupts and use a single queue. 2643 */ 2644 if ((adapter->hw.mac.type == e1000_82574) && 2645 (em_enable_msix == TRUE)) { 2646 /* Map the MSIX BAR */ 2647 int rid = PCIR_BAR(EM_MSIX_BAR); 2648 adapter->msix_mem = bus_alloc_resource_any(dev, 2649 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2650 if (!adapter->msix_mem) { 2651 /* May not be enabled */ 2652 device_printf(adapter->dev, 2653 "Unable to map MSIX table \n"); 2654 goto msi; 2655 } 2656 val = pci_msix_count(dev); 2657 if (val < 3) { 2658 bus_release_resource(dev, SYS_RES_MEMORY, 2659 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); 2660 adapter->msix_mem = NULL; 2661 device_printf(adapter->dev, 2662 "MSIX: insufficient vectors, using MSI\n"); 2663 goto msi; 2664 } 2665 val = 3; 2666 adapter->num_queues = 1; 2667 if (pci_alloc_msix(dev, &val) == 0) { 2668 device_printf(adapter->dev, 2669 "Using MSIX interrupts " 2670 "with %d vectors\n", val); 2671 } 2672 2673 return (val); 2674 } 2675 msi: 2676 val = pci_msi_count(dev); 2677 if (val == 1 && pci_alloc_msi(dev, &val) == 0) { 2678 adapter->msix = 1; 2679 device_printf(adapter->dev,"Using an MSI interrupt\n"); 2680 return (val); 2681 } 2682 /* Should only happen due to manual configuration */ 2683 device_printf(adapter->dev,"No MSI/MSIX using a Legacy IRQ\n"); 2684 return (0); 2685 } 2686 2687 2688 /********************************************************************* 2689 * 2690 * Initialize the hardware to a configuration 2691 * as specified by the adapter structure. 2692 * 2693 **********************************************************************/ 2694 static void 2695 em_reset(struct adapter *adapter) 2696 { 2697 device_t dev = adapter->dev; 2698 struct ifnet *ifp = adapter->ifp; 2699 struct e1000_hw *hw = &adapter->hw; 2700 u16 rx_buffer_size; 2701 2702 INIT_DEBUGOUT("em_reset: begin"); 2703 2704 /* Set up smart power down as default off on newer adapters. */ 2705 if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 || 2706 hw->mac.type == e1000_82572)) { 2707 u16 phy_tmp = 0; 2708 2709 /* Speed up time to link by disabling smart power down. */ 2710 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 2711 phy_tmp &= ~IGP02E1000_PM_SPD; 2712 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp); 2713 } 2714 2715 /* 2716 * These parameters control the automatic generation (Tx) and 2717 * response (Rx) to Ethernet PAUSE frames. 2718 * - High water mark should allow for at least two frames to be 2719 * received after sending an XOFF. 2720 * - Low water mark works best when it is very near the high water mark. 2721 * This allows the receiver to restart by sending XON when it has 2722 * drained a bit. Here we use an arbitary value of 1500 which will 2723 * restart after one full frame is pulled from the buffer. There 2724 * could be several smaller frames in the buffer and if so they will 2725 * not trigger the XON until their total number reduces the buffer 2726 * by 1500. 2727 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 2728 */ 2729 rx_buffer_size = ((E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10 ); 2730 2731 hw->fc.high_water = rx_buffer_size - 2732 roundup2(adapter->max_frame_size, 1024); 2733 hw->fc.low_water = hw->fc.high_water - 1500; 2734 2735 if (hw->mac.type == e1000_80003es2lan) 2736 hw->fc.pause_time = 0xFFFF; 2737 else 2738 hw->fc.pause_time = EM_FC_PAUSE_TIME; 2739 2740 hw->fc.send_xon = TRUE; 2741 2742 /* Set Flow control, use the tunable location if sane */ 2743 hw->fc.requested_mode = adapter->fc_setting; 2744 2745 /* Workaround: no TX flow ctrl for PCH */ 2746 if (hw->mac.type == e1000_pchlan) 2747 hw->fc.requested_mode = e1000_fc_rx_pause; 2748 2749 /* Override - settings for PCH2LAN, ya its magic :) */ 2750 if (hw->mac.type == e1000_pch2lan) { 2751 hw->fc.high_water = 0x5C20; 2752 hw->fc.low_water = 0x5048; 2753 hw->fc.pause_time = 0x0650; 2754 hw->fc.refresh_time = 0x0400; 2755 /* Jumbos need adjusted PBA */ 2756 if (ifp->if_mtu > ETHERMTU) 2757 E1000_WRITE_REG(hw, E1000_PBA, 12); 2758 else 2759 E1000_WRITE_REG(hw, E1000_PBA, 26); 2760 } 2761 2762 /* Issue a global reset */ 2763 e1000_reset_hw(hw); 2764 E1000_WRITE_REG(hw, E1000_WUC, 0); 2765 2766 if (e1000_init_hw(hw) < 0) { 2767 device_printf(dev, "Hardware Initialization Failed\n"); 2768 return; 2769 } 2770 2771 E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN); 2772 e1000_get_phy_info(hw); 2773 e1000_check_for_link(hw); 2774 return; 2775 } 2776 2777 /********************************************************************* 2778 * 2779 * Setup networking device structure and register an interface. 2780 * 2781 **********************************************************************/ 2782 static int 2783 em_setup_interface(device_t dev, struct adapter *adapter) 2784 { 2785 struct ifnet *ifp; 2786 2787 INIT_DEBUGOUT("em_setup_interface: begin"); 2788 2789 ifp = adapter->ifp = if_alloc(IFT_ETHER); 2790 if (ifp == NULL) { 2791 device_printf(dev, "can not allocate ifnet structure\n"); 2792 return (-1); 2793 } 2794 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2795 ifp->if_mtu = ETHERMTU; 2796 ifp->if_init = em_init; 2797 ifp->if_softc = adapter; 2798 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2799 ifp->if_ioctl = em_ioctl; 2800 ifp->if_start = em_start; 2801 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 2802 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 2803 IFQ_SET_READY(&ifp->if_snd); 2804 2805 ether_ifattach(ifp, adapter->hw.mac.addr); 2806 2807 ifp->if_capabilities = ifp->if_capenable = 0; 2808 2809 #ifdef EM_MULTIQUEUE 2810 /* Multiqueue tx functions */ 2811 ifp->if_transmit = em_mq_start; 2812 ifp->if_qflush = em_qflush; 2813 #endif 2814 2815 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 2816 ifp->if_capenable |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 2817 2818 /* Enable TSO by default, can disable with ifconfig */ 2819 ifp->if_capabilities |= IFCAP_TSO4; 2820 ifp->if_capenable |= IFCAP_TSO4; 2821 2822 /* 2823 * Tell the upper layer(s) we 2824 * support full VLAN capability 2825 */ 2826 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2827 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2828 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2829 2830 /* 2831 ** Dont turn this on by default, if vlans are 2832 ** created on another pseudo device (eg. lagg) 2833 ** then vlan events are not passed thru, breaking 2834 ** operation, but with HW FILTER off it works. If 2835 ** using vlans directly on the em driver you can 2836 ** enable this and get full hardware tag filtering. 2837 */ 2838 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2839 2840 #ifdef DEVICE_POLLING 2841 ifp->if_capabilities |= IFCAP_POLLING; 2842 #endif 2843 2844 /* Enable only WOL MAGIC by default */ 2845 if (adapter->wol) { 2846 ifp->if_capabilities |= IFCAP_WOL; 2847 ifp->if_capenable |= IFCAP_WOL_MAGIC; 2848 } 2849 2850 /* 2851 * Specify the media types supported by this adapter and register 2852 * callbacks to update media and link information 2853 */ 2854 ifmedia_init(&adapter->media, IFM_IMASK, 2855 em_media_change, em_media_status); 2856 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 2857 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 2858 u_char fiber_type = IFM_1000_SX; /* default type */ 2859 2860 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 2861 0, NULL); 2862 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); 2863 } else { 2864 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 2865 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 2866 0, NULL); 2867 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 2868 0, NULL); 2869 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 2870 0, NULL); 2871 if (adapter->hw.phy.type != e1000_phy_ife) { 2872 ifmedia_add(&adapter->media, 2873 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2874 ifmedia_add(&adapter->media, 2875 IFM_ETHER | IFM_1000_T, 0, NULL); 2876 } 2877 } 2878 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2879 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2880 return (0); 2881 } 2882 2883 2884 /* 2885 * Manage DMA'able memory. 2886 */ 2887 static void 2888 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 2889 { 2890 if (error) 2891 return; 2892 *(bus_addr_t *) arg = segs[0].ds_addr; 2893 } 2894 2895 static int 2896 em_dma_malloc(struct adapter *adapter, bus_size_t size, 2897 struct em_dma_alloc *dma, int mapflags) 2898 { 2899 int error; 2900 2901 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 2902 EM_DBA_ALIGN, 0, /* alignment, bounds */ 2903 BUS_SPACE_MAXADDR, /* lowaddr */ 2904 BUS_SPACE_MAXADDR, /* highaddr */ 2905 NULL, NULL, /* filter, filterarg */ 2906 size, /* maxsize */ 2907 1, /* nsegments */ 2908 size, /* maxsegsize */ 2909 0, /* flags */ 2910 NULL, /* lockfunc */ 2911 NULL, /* lockarg */ 2912 &dma->dma_tag); 2913 if (error) { 2914 device_printf(adapter->dev, 2915 "%s: bus_dma_tag_create failed: %d\n", 2916 __func__, error); 2917 goto fail_0; 2918 } 2919 2920 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 2921 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); 2922 if (error) { 2923 device_printf(adapter->dev, 2924 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 2925 __func__, (uintmax_t)size, error); 2926 goto fail_2; 2927 } 2928 2929 dma->dma_paddr = 0; 2930 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2931 size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 2932 if (error || dma->dma_paddr == 0) { 2933 device_printf(adapter->dev, 2934 "%s: bus_dmamap_load failed: %d\n", 2935 __func__, error); 2936 goto fail_3; 2937 } 2938 2939 return (0); 2940 2941 fail_3: 2942 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2943 fail_2: 2944 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2945 bus_dma_tag_destroy(dma->dma_tag); 2946 fail_0: 2947 dma->dma_map = NULL; 2948 dma->dma_tag = NULL; 2949 2950 return (error); 2951 } 2952 2953 static void 2954 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) 2955 { 2956 if (dma->dma_tag == NULL) 2957 return; 2958 if (dma->dma_map != NULL) { 2959 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 2960 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2961 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2962 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2963 dma->dma_map = NULL; 2964 } 2965 bus_dma_tag_destroy(dma->dma_tag); 2966 dma->dma_tag = NULL; 2967 } 2968 2969 2970 /********************************************************************* 2971 * 2972 * Allocate memory for the transmit and receive rings, and then 2973 * the descriptors associated with each, called only once at attach. 2974 * 2975 **********************************************************************/ 2976 static int 2977 em_allocate_queues(struct adapter *adapter) 2978 { 2979 device_t dev = adapter->dev; 2980 struct tx_ring *txr = NULL; 2981 struct rx_ring *rxr = NULL; 2982 int rsize, tsize, error = E1000_SUCCESS; 2983 int txconf = 0, rxconf = 0; 2984 2985 2986 /* Allocate the TX ring struct memory */ 2987 if (!(adapter->tx_rings = 2988 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2989 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2990 device_printf(dev, "Unable to allocate TX ring memory\n"); 2991 error = ENOMEM; 2992 goto fail; 2993 } 2994 2995 /* Now allocate the RX */ 2996 if (!(adapter->rx_rings = 2997 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2998 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2999 device_printf(dev, "Unable to allocate RX ring memory\n"); 3000 error = ENOMEM; 3001 goto rx_fail; 3002 } 3003 3004 tsize = roundup2(adapter->num_tx_desc * 3005 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); 3006 /* 3007 * Now set up the TX queues, txconf is needed to handle the 3008 * possibility that things fail midcourse and we need to 3009 * undo memory gracefully 3010 */ 3011 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 3012 /* Set up some basics */ 3013 txr = &adapter->tx_rings[i]; 3014 txr->adapter = adapter; 3015 txr->me = i; 3016 3017 /* Initialize the TX lock */ 3018 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 3019 device_get_nameunit(dev), txr->me); 3020 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 3021 3022 if (em_dma_malloc(adapter, tsize, 3023 &txr->txdma, BUS_DMA_NOWAIT)) { 3024 device_printf(dev, 3025 "Unable to allocate TX Descriptor memory\n"); 3026 error = ENOMEM; 3027 goto err_tx_desc; 3028 } 3029 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr; 3030 bzero((void *)txr->tx_base, tsize); 3031 3032 if (em_allocate_transmit_buffers(txr)) { 3033 device_printf(dev, 3034 "Critical Failure setting up transmit buffers\n"); 3035 error = ENOMEM; 3036 goto err_tx_desc; 3037 } 3038 #if __FreeBSD_version >= 800000 3039 /* Allocate a buf ring */ 3040 txr->br = buf_ring_alloc(4096, M_DEVBUF, 3041 M_WAITOK, &txr->tx_mtx); 3042 #endif 3043 } 3044 3045 /* 3046 * Next the RX queues... 3047 */ 3048 rsize = roundup2(adapter->num_rx_desc * 3049 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); 3050 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 3051 rxr = &adapter->rx_rings[i]; 3052 rxr->adapter = adapter; 3053 rxr->me = i; 3054 3055 /* Initialize the RX lock */ 3056 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 3057 device_get_nameunit(dev), txr->me); 3058 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 3059 3060 if (em_dma_malloc(adapter, rsize, 3061 &rxr->rxdma, BUS_DMA_NOWAIT)) { 3062 device_printf(dev, 3063 "Unable to allocate RxDescriptor memory\n"); 3064 error = ENOMEM; 3065 goto err_rx_desc; 3066 } 3067 rxr->rx_base = (struct e1000_rx_desc *)rxr->rxdma.dma_vaddr; 3068 bzero((void *)rxr->rx_base, rsize); 3069 3070 /* Allocate receive buffers for the ring*/ 3071 if (em_allocate_receive_buffers(rxr)) { 3072 device_printf(dev, 3073 "Critical Failure setting up receive buffers\n"); 3074 error = ENOMEM; 3075 goto err_rx_desc; 3076 } 3077 } 3078 3079 return (0); 3080 3081 err_rx_desc: 3082 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 3083 em_dma_free(adapter, &rxr->rxdma); 3084 err_tx_desc: 3085 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 3086 em_dma_free(adapter, &txr->txdma); 3087 free(adapter->rx_rings, M_DEVBUF); 3088 rx_fail: 3089 #if __FreeBSD_version >= 800000 3090 buf_ring_free(txr->br, M_DEVBUF); 3091 #endif 3092 free(adapter->tx_rings, M_DEVBUF); 3093 fail: 3094 return (error); 3095 } 3096 3097 3098 /********************************************************************* 3099 * 3100 * Allocate memory for tx_buffer structures. The tx_buffer stores all 3101 * the information needed to transmit a packet on the wire. This is 3102 * called only once at attach, setup is done every reset. 3103 * 3104 **********************************************************************/ 3105 static int 3106 em_allocate_transmit_buffers(struct tx_ring *txr) 3107 { 3108 struct adapter *adapter = txr->adapter; 3109 device_t dev = adapter->dev; 3110 struct em_buffer *txbuf; 3111 int error, i; 3112 3113 /* 3114 * Setup DMA descriptor areas. 3115 */ 3116 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 3117 1, 0, /* alignment, bounds */ 3118 BUS_SPACE_MAXADDR, /* lowaddr */ 3119 BUS_SPACE_MAXADDR, /* highaddr */ 3120 NULL, NULL, /* filter, filterarg */ 3121 EM_TSO_SIZE, /* maxsize */ 3122 EM_MAX_SCATTER, /* nsegments */ 3123 PAGE_SIZE, /* maxsegsize */ 3124 0, /* flags */ 3125 NULL, /* lockfunc */ 3126 NULL, /* lockfuncarg */ 3127 &txr->txtag))) { 3128 device_printf(dev,"Unable to allocate TX DMA tag\n"); 3129 goto fail; 3130 } 3131 3132 if (!(txr->tx_buffers = 3133 (struct em_buffer *) malloc(sizeof(struct em_buffer) * 3134 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3135 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 3136 error = ENOMEM; 3137 goto fail; 3138 } 3139 3140 /* Create the descriptor buffer dma maps */ 3141 txbuf = txr->tx_buffers; 3142 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3143 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 3144 if (error != 0) { 3145 device_printf(dev, "Unable to create TX DMA map\n"); 3146 goto fail; 3147 } 3148 } 3149 3150 return 0; 3151 fail: 3152 /* We free all, it handles case where we are in the middle */ 3153 em_free_transmit_structures(adapter); 3154 return (error); 3155 } 3156 3157 /********************************************************************* 3158 * 3159 * Initialize a transmit ring. 3160 * 3161 **********************************************************************/ 3162 static void 3163 em_setup_transmit_ring(struct tx_ring *txr) 3164 { 3165 struct adapter *adapter = txr->adapter; 3166 struct em_buffer *txbuf; 3167 int i; 3168 3169 /* Clear the old descriptor contents */ 3170 EM_TX_LOCK(txr); 3171 bzero((void *)txr->tx_base, 3172 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); 3173 /* Reset indices */ 3174 txr->next_avail_desc = 0; 3175 txr->next_to_clean = 0; 3176 3177 /* Free any existing tx buffers. */ 3178 txbuf = txr->tx_buffers; 3179 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3180 if (txbuf->m_head != NULL) { 3181 bus_dmamap_sync(txr->txtag, txbuf->map, 3182 BUS_DMASYNC_POSTWRITE); 3183 bus_dmamap_unload(txr->txtag, txbuf->map); 3184 m_freem(txbuf->m_head); 3185 txbuf->m_head = NULL; 3186 } 3187 /* clear the watch index */ 3188 txbuf->next_eop = -1; 3189 } 3190 3191 /* Set number of descriptors available */ 3192 txr->tx_avail = adapter->num_tx_desc; 3193 txr->queue_status = EM_QUEUE_IDLE; 3194 3195 /* Clear checksum offload context. */ 3196 txr->last_hw_offload = 0; 3197 txr->last_hw_ipcss = 0; 3198 txr->last_hw_ipcso = 0; 3199 txr->last_hw_tucss = 0; 3200 txr->last_hw_tucso = 0; 3201 3202 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3203 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3204 EM_TX_UNLOCK(txr); 3205 } 3206 3207 /********************************************************************* 3208 * 3209 * Initialize all transmit rings. 3210 * 3211 **********************************************************************/ 3212 static void 3213 em_setup_transmit_structures(struct adapter *adapter) 3214 { 3215 struct tx_ring *txr = adapter->tx_rings; 3216 3217 for (int i = 0; i < adapter->num_queues; i++, txr++) 3218 em_setup_transmit_ring(txr); 3219 3220 return; 3221 } 3222 3223 /********************************************************************* 3224 * 3225 * Enable transmit unit. 3226 * 3227 **********************************************************************/ 3228 static void 3229 em_initialize_transmit_unit(struct adapter *adapter) 3230 { 3231 struct tx_ring *txr = adapter->tx_rings; 3232 struct e1000_hw *hw = &adapter->hw; 3233 u32 tctl, tarc, tipg = 0; 3234 3235 INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); 3236 3237 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3238 u64 bus_addr = txr->txdma.dma_paddr; 3239 /* Base and Len of TX Ring */ 3240 E1000_WRITE_REG(hw, E1000_TDLEN(i), 3241 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 3242 E1000_WRITE_REG(hw, E1000_TDBAH(i), 3243 (u32)(bus_addr >> 32)); 3244 E1000_WRITE_REG(hw, E1000_TDBAL(i), 3245 (u32)bus_addr); 3246 /* Init the HEAD/TAIL indices */ 3247 E1000_WRITE_REG(hw, E1000_TDT(i), 0); 3248 E1000_WRITE_REG(hw, E1000_TDH(i), 0); 3249 3250 HW_DEBUGOUT2("Base = %x, Length = %x\n", 3251 E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)), 3252 E1000_READ_REG(&adapter->hw, E1000_TDLEN(i))); 3253 3254 txr->queue_status = EM_QUEUE_IDLE; 3255 } 3256 3257 /* Set the default values for the Tx Inter Packet Gap timer */ 3258 switch (adapter->hw.mac.type) { 3259 case e1000_82542: 3260 tipg = DEFAULT_82542_TIPG_IPGT; 3261 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3262 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3263 break; 3264 case e1000_80003es2lan: 3265 tipg = DEFAULT_82543_TIPG_IPGR1; 3266 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 3267 E1000_TIPG_IPGR2_SHIFT; 3268 break; 3269 default: 3270 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3271 (adapter->hw.phy.media_type == 3272 e1000_media_type_internal_serdes)) 3273 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 3274 else 3275 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 3276 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3277 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3278 } 3279 3280 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg); 3281 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value); 3282 3283 if(adapter->hw.mac.type >= e1000_82540) 3284 E1000_WRITE_REG(&adapter->hw, E1000_TADV, 3285 adapter->tx_abs_int_delay.value); 3286 3287 if ((adapter->hw.mac.type == e1000_82571) || 3288 (adapter->hw.mac.type == e1000_82572)) { 3289 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3290 tarc |= SPEED_MODE_BIT; 3291 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3292 } else if (adapter->hw.mac.type == e1000_80003es2lan) { 3293 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3294 tarc |= 1; 3295 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3296 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1)); 3297 tarc |= 1; 3298 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); 3299 } 3300 3301 adapter->txd_cmd = E1000_TXD_CMD_IFCS; 3302 if (adapter->tx_int_delay.value > 0) 3303 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 3304 3305 /* Program the Transmit Control Register */ 3306 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 3307 tctl &= ~E1000_TCTL_CT; 3308 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 3309 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 3310 3311 if (adapter->hw.mac.type >= e1000_82571) 3312 tctl |= E1000_TCTL_MULR; 3313 3314 /* This write will effectively turn on the transmit unit. */ 3315 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 3316 3317 } 3318 3319 3320 /********************************************************************* 3321 * 3322 * Free all transmit rings. 3323 * 3324 **********************************************************************/ 3325 static void 3326 em_free_transmit_structures(struct adapter *adapter) 3327 { 3328 struct tx_ring *txr = adapter->tx_rings; 3329 3330 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3331 EM_TX_LOCK(txr); 3332 em_free_transmit_buffers(txr); 3333 em_dma_free(adapter, &txr->txdma); 3334 EM_TX_UNLOCK(txr); 3335 EM_TX_LOCK_DESTROY(txr); 3336 } 3337 3338 free(adapter->tx_rings, M_DEVBUF); 3339 } 3340 3341 /********************************************************************* 3342 * 3343 * Free transmit ring related data structures. 3344 * 3345 **********************************************************************/ 3346 static void 3347 em_free_transmit_buffers(struct tx_ring *txr) 3348 { 3349 struct adapter *adapter = txr->adapter; 3350 struct em_buffer *txbuf; 3351 3352 INIT_DEBUGOUT("free_transmit_ring: begin"); 3353 3354 if (txr->tx_buffers == NULL) 3355 return; 3356 3357 for (int i = 0; i < adapter->num_tx_desc; i++) { 3358 txbuf = &txr->tx_buffers[i]; 3359 if (txbuf->m_head != NULL) { 3360 bus_dmamap_sync(txr->txtag, txbuf->map, 3361 BUS_DMASYNC_POSTWRITE); 3362 bus_dmamap_unload(txr->txtag, 3363 txbuf->map); 3364 m_freem(txbuf->m_head); 3365 txbuf->m_head = NULL; 3366 if (txbuf->map != NULL) { 3367 bus_dmamap_destroy(txr->txtag, 3368 txbuf->map); 3369 txbuf->map = NULL; 3370 } 3371 } else if (txbuf->map != NULL) { 3372 bus_dmamap_unload(txr->txtag, 3373 txbuf->map); 3374 bus_dmamap_destroy(txr->txtag, 3375 txbuf->map); 3376 txbuf->map = NULL; 3377 } 3378 } 3379 #if __FreeBSD_version >= 800000 3380 if (txr->br != NULL) 3381 buf_ring_free(txr->br, M_DEVBUF); 3382 #endif 3383 if (txr->tx_buffers != NULL) { 3384 free(txr->tx_buffers, M_DEVBUF); 3385 txr->tx_buffers = NULL; 3386 } 3387 if (txr->txtag != NULL) { 3388 bus_dma_tag_destroy(txr->txtag); 3389 txr->txtag = NULL; 3390 } 3391 return; 3392 } 3393 3394 3395 /********************************************************************* 3396 * The offload context is protocol specific (TCP/UDP) and thus 3397 * only needs to be set when the protocol changes. The occasion 3398 * of a context change can be a performance detriment, and 3399 * might be better just disabled. The reason arises in the way 3400 * in which the controller supports pipelined requests from the 3401 * Tx data DMA. Up to four requests can be pipelined, and they may 3402 * belong to the same packet or to multiple packets. However all 3403 * requests for one packet are issued before a request is issued 3404 * for a subsequent packet and if a request for the next packet 3405 * requires a context change, that request will be stalled 3406 * until the previous request completes. This means setting up 3407 * a new context effectively disables pipelined Tx data DMA which 3408 * in turn greatly slow down performance to send small sized 3409 * frames. 3410 **********************************************************************/ 3411 static void 3412 em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, 3413 struct ip *ip, u32 *txd_upper, u32 *txd_lower) 3414 { 3415 struct adapter *adapter = txr->adapter; 3416 struct e1000_context_desc *TXD = NULL; 3417 struct em_buffer *tx_buffer; 3418 int cur, hdr_len; 3419 u32 cmd = 0; 3420 u16 offload = 0; 3421 u8 ipcso, ipcss, tucso, tucss; 3422 3423 ipcss = ipcso = tucss = tucso = 0; 3424 hdr_len = ip_off + (ip->ip_hl << 2); 3425 cur = txr->next_avail_desc; 3426 3427 /* Setup of IP header checksum. */ 3428 if (mp->m_pkthdr.csum_flags & CSUM_IP) { 3429 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 3430 offload |= CSUM_IP; 3431 ipcss = ip_off; 3432 ipcso = ip_off + offsetof(struct ip, ip_sum); 3433 /* 3434 * Start offset for header checksum calculation. 3435 * End offset for header checksum calculation. 3436 * Offset of place to put the checksum. 3437 */ 3438 TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 3439 TXD->lower_setup.ip_fields.ipcss = ipcss; 3440 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len); 3441 TXD->lower_setup.ip_fields.ipcso = ipcso; 3442 cmd |= E1000_TXD_CMD_IP; 3443 } 3444 3445 if (mp->m_pkthdr.csum_flags & CSUM_TCP) { 3446 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 3447 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3448 offload |= CSUM_TCP; 3449 tucss = hdr_len; 3450 tucso = hdr_len + offsetof(struct tcphdr, th_sum); 3451 /* 3452 * Setting up new checksum offload context for every frames 3453 * takes a lot of processing time for hardware. This also 3454 * reduces performance a lot for small sized frames so avoid 3455 * it if driver can use previously configured checksum 3456 * offload context. 3457 */ 3458 if (txr->last_hw_offload == offload) { 3459 if (offload & CSUM_IP) { 3460 if (txr->last_hw_ipcss == ipcss && 3461 txr->last_hw_ipcso == ipcso && 3462 txr->last_hw_tucss == tucss && 3463 txr->last_hw_tucso == tucso) 3464 return; 3465 } else { 3466 if (txr->last_hw_tucss == tucss && 3467 txr->last_hw_tucso == tucso) 3468 return; 3469 } 3470 } 3471 txr->last_hw_offload = offload; 3472 txr->last_hw_tucss = tucss; 3473 txr->last_hw_tucso = tucso; 3474 /* 3475 * Start offset for payload checksum calculation. 3476 * End offset for payload checksum calculation. 3477 * Offset of place to put the checksum. 3478 */ 3479 TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 3480 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3481 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3482 TXD->upper_setup.tcp_fields.tucso = tucso; 3483 cmd |= E1000_TXD_CMD_TCP; 3484 } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) { 3485 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 3486 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3487 tucss = hdr_len; 3488 tucso = hdr_len + offsetof(struct udphdr, uh_sum); 3489 /* 3490 * Setting up new checksum offload context for every frames 3491 * takes a lot of processing time for hardware. This also 3492 * reduces performance a lot for small sized frames so avoid 3493 * it if driver can use previously configured checksum 3494 * offload context. 3495 */ 3496 if (txr->last_hw_offload == offload) { 3497 if (offload & CSUM_IP) { 3498 if (txr->last_hw_ipcss == ipcss && 3499 txr->last_hw_ipcso == ipcso && 3500 txr->last_hw_tucss == tucss && 3501 txr->last_hw_tucso == tucso) 3502 return; 3503 } else { 3504 if (txr->last_hw_tucss == tucss && 3505 txr->last_hw_tucso == tucso) 3506 return; 3507 } 3508 } 3509 txr->last_hw_offload = offload; 3510 txr->last_hw_tucss = tucss; 3511 txr->last_hw_tucso = tucso; 3512 /* 3513 * Start offset for header checksum calculation. 3514 * End offset for header checksum calculation. 3515 * Offset of place to put the checksum. 3516 */ 3517 TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 3518 TXD->upper_setup.tcp_fields.tucss = tucss; 3519 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3520 TXD->upper_setup.tcp_fields.tucso = tucso; 3521 } 3522 3523 if (offload & CSUM_IP) { 3524 txr->last_hw_ipcss = ipcss; 3525 txr->last_hw_ipcso = ipcso; 3526 } 3527 3528 TXD->tcp_seg_setup.data = htole32(0); 3529 TXD->cmd_and_length = 3530 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); 3531 tx_buffer = &txr->tx_buffers[cur]; 3532 tx_buffer->m_head = NULL; 3533 tx_buffer->next_eop = -1; 3534 3535 if (++cur == adapter->num_tx_desc) 3536 cur = 0; 3537 3538 txr->tx_avail--; 3539 txr->next_avail_desc = cur; 3540 } 3541 3542 3543 /********************************************************************** 3544 * 3545 * Setup work for hardware segmentation offload (TSO) 3546 * 3547 **********************************************************************/ 3548 static void 3549 em_tso_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, 3550 struct ip *ip, struct tcphdr *tp, u32 *txd_upper, u32 *txd_lower) 3551 { 3552 struct adapter *adapter = txr->adapter; 3553 struct e1000_context_desc *TXD; 3554 struct em_buffer *tx_buffer; 3555 int cur, hdr_len; 3556 3557 /* 3558 * In theory we can use the same TSO context if and only if 3559 * frame is the same type(IP/TCP) and the same MSS. However 3560 * checking whether a frame has the same IP/TCP structure is 3561 * hard thing so just ignore that and always restablish a 3562 * new TSO context. 3563 */ 3564 hdr_len = ip_off + (ip->ip_hl << 2) + (tp->th_off << 2); 3565 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 3566 E1000_TXD_DTYP_D | /* Data descr type */ 3567 E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 3568 3569 /* IP and/or TCP header checksum calculation and insertion. */ 3570 *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; 3571 3572 cur = txr->next_avail_desc; 3573 tx_buffer = &txr->tx_buffers[cur]; 3574 TXD = (struct e1000_context_desc *) &txr->tx_base[cur]; 3575 3576 /* 3577 * Start offset for header checksum calculation. 3578 * End offset for header checksum calculation. 3579 * Offset of place put the checksum. 3580 */ 3581 TXD->lower_setup.ip_fields.ipcss = ip_off; 3582 TXD->lower_setup.ip_fields.ipcse = 3583 htole16(ip_off + (ip->ip_hl << 2) - 1); 3584 TXD->lower_setup.ip_fields.ipcso = ip_off + offsetof(struct ip, ip_sum); 3585 /* 3586 * Start offset for payload checksum calculation. 3587 * End offset for payload checksum calculation. 3588 * Offset of place to put the checksum. 3589 */ 3590 TXD->upper_setup.tcp_fields.tucss = ip_off + (ip->ip_hl << 2); 3591 TXD->upper_setup.tcp_fields.tucse = 0; 3592 TXD->upper_setup.tcp_fields.tucso = 3593 ip_off + (ip->ip_hl << 2) + offsetof(struct tcphdr, th_sum); 3594 /* 3595 * Payload size per packet w/o any headers. 3596 * Length of all headers up to payload. 3597 */ 3598 TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz); 3599 TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 3600 3601 TXD->cmd_and_length = htole32(adapter->txd_cmd | 3602 E1000_TXD_CMD_DEXT | /* Extended descr */ 3603 E1000_TXD_CMD_TSE | /* TSE context */ 3604 E1000_TXD_CMD_IP | /* Do IP csum */ 3605 E1000_TXD_CMD_TCP | /* Do TCP checksum */ 3606 (mp->m_pkthdr.len - (hdr_len))); /* Total len */ 3607 3608 tx_buffer->m_head = NULL; 3609 tx_buffer->next_eop = -1; 3610 3611 if (++cur == adapter->num_tx_desc) 3612 cur = 0; 3613 3614 txr->tx_avail--; 3615 txr->next_avail_desc = cur; 3616 txr->tx_tso = TRUE; 3617 } 3618 3619 3620 /********************************************************************** 3621 * 3622 * Examine each tx_buffer in the used queue. If the hardware is done 3623 * processing the packet then free associated resources. The 3624 * tx_buffer is put back on the free queue. 3625 * 3626 **********************************************************************/ 3627 static bool 3628 em_txeof(struct tx_ring *txr) 3629 { 3630 struct adapter *adapter = txr->adapter; 3631 int first, last, done, processed; 3632 struct em_buffer *tx_buffer; 3633 struct e1000_tx_desc *tx_desc, *eop_desc; 3634 struct ifnet *ifp = adapter->ifp; 3635 3636 EM_TX_LOCK_ASSERT(txr); 3637 3638 /* No work, make sure watchdog is off */ 3639 if (txr->tx_avail == adapter->num_tx_desc) { 3640 txr->queue_status = EM_QUEUE_IDLE; 3641 return (FALSE); 3642 } 3643 3644 processed = 0; 3645 first = txr->next_to_clean; 3646 tx_desc = &txr->tx_base[first]; 3647 tx_buffer = &txr->tx_buffers[first]; 3648 last = tx_buffer->next_eop; 3649 eop_desc = &txr->tx_base[last]; 3650 3651 /* 3652 * What this does is get the index of the 3653 * first descriptor AFTER the EOP of the 3654 * first packet, that way we can do the 3655 * simple comparison on the inner while loop. 3656 */ 3657 if (++last == adapter->num_tx_desc) 3658 last = 0; 3659 done = last; 3660 3661 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3662 BUS_DMASYNC_POSTREAD); 3663 3664 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 3665 /* We clean the range of the packet */ 3666 while (first != done) { 3667 tx_desc->upper.data = 0; 3668 tx_desc->lower.data = 0; 3669 tx_desc->buffer_addr = 0; 3670 ++txr->tx_avail; 3671 ++processed; 3672 3673 if (tx_buffer->m_head) { 3674 bus_dmamap_sync(txr->txtag, 3675 tx_buffer->map, 3676 BUS_DMASYNC_POSTWRITE); 3677 bus_dmamap_unload(txr->txtag, 3678 tx_buffer->map); 3679 m_freem(tx_buffer->m_head); 3680 tx_buffer->m_head = NULL; 3681 } 3682 tx_buffer->next_eop = -1; 3683 txr->watchdog_time = ticks; 3684 3685 if (++first == adapter->num_tx_desc) 3686 first = 0; 3687 3688 tx_buffer = &txr->tx_buffers[first]; 3689 tx_desc = &txr->tx_base[first]; 3690 } 3691 ++ifp->if_opackets; 3692 /* See if we can continue to the next packet */ 3693 last = tx_buffer->next_eop; 3694 if (last != -1) { 3695 eop_desc = &txr->tx_base[last]; 3696 /* Get new done point */ 3697 if (++last == adapter->num_tx_desc) last = 0; 3698 done = last; 3699 } else 3700 break; 3701 } 3702 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3703 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3704 3705 txr->next_to_clean = first; 3706 3707 /* 3708 ** Watchdog calculation, we know there's 3709 ** work outstanding or the first return 3710 ** would have been taken, so none processed 3711 ** for too long indicates a hang. local timer 3712 ** will examine this and do a reset if needed. 3713 */ 3714 if ((!processed) && ((ticks - txr->watchdog_time) > EM_WATCHDOG)) 3715 txr->queue_status = EM_QUEUE_HUNG; 3716 3717 /* 3718 * If we have enough room, clear IFF_DRV_OACTIVE 3719 * to tell the stack that it is OK to send packets. 3720 */ 3721 if (txr->tx_avail > EM_TX_CLEANUP_THRESHOLD) { 3722 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3723 /* Disable watchdog if all clean */ 3724 if (txr->tx_avail == adapter->num_tx_desc) { 3725 txr->queue_status = EM_QUEUE_IDLE; 3726 return (FALSE); 3727 } 3728 } 3729 3730 return (TRUE); 3731 } 3732 3733 3734 /********************************************************************* 3735 * 3736 * Refresh RX descriptor mbufs from system mbuf buffer pool. 3737 * 3738 **********************************************************************/ 3739 static void 3740 em_refresh_mbufs(struct rx_ring *rxr, int limit) 3741 { 3742 struct adapter *adapter = rxr->adapter; 3743 struct mbuf *m; 3744 bus_dma_segment_t segs[1]; 3745 struct em_buffer *rxbuf; 3746 int i, error, nsegs, cleaned; 3747 3748 i = rxr->next_to_refresh; 3749 cleaned = -1; 3750 while (i != limit) { 3751 rxbuf = &rxr->rx_buffers[i]; 3752 /* 3753 ** Just skip entries with a buffer, 3754 ** they can only be due to an error 3755 ** and are to be reused. 3756 */ 3757 if (rxbuf->m_head != NULL) 3758 goto reuse; 3759 m = m_getjcl(M_DONTWAIT, MT_DATA, 3760 M_PKTHDR, adapter->rx_mbuf_sz); 3761 /* 3762 ** If we have a temporary resource shortage 3763 ** that causes a failure, just abort refresh 3764 ** for now, we will return to this point when 3765 ** reinvoked from em_rxeof. 3766 */ 3767 if (m == NULL) 3768 goto update; 3769 m->m_len = m->m_pkthdr.len = adapter->rx_mbuf_sz; 3770 3771 /* Use bus_dma machinery to setup the memory mapping */ 3772 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map, 3773 m, segs, &nsegs, BUS_DMA_NOWAIT); 3774 if (error != 0) { 3775 m_free(m); 3776 goto update; 3777 } 3778 3779 /* If nsegs is wrong then the stack is corrupt. */ 3780 KASSERT(nsegs == 1, ("Too many segments returned!")); 3781 3782 bus_dmamap_sync(rxr->rxtag, 3783 rxbuf->map, BUS_DMASYNC_PREREAD); 3784 rxbuf->m_head = m; 3785 rxr->rx_base[i].buffer_addr = htole64(segs[0].ds_addr); 3786 reuse: 3787 cleaned = i; 3788 /* Calculate next index */ 3789 if (++i == adapter->num_rx_desc) 3790 i = 0; 3791 /* This is the work marker for refresh */ 3792 rxr->next_to_refresh = i; 3793 } 3794 update: 3795 /* 3796 ** Update the tail pointer only if, 3797 ** and as far as we have refreshed. 3798 */ 3799 if (cleaned != -1) /* Update tail index */ 3800 E1000_WRITE_REG(&adapter->hw, 3801 E1000_RDT(rxr->me), cleaned); 3802 3803 return; 3804 } 3805 3806 3807 /********************************************************************* 3808 * 3809 * Allocate memory for rx_buffer structures. Since we use one 3810 * rx_buffer per received packet, the maximum number of rx_buffer's 3811 * that we'll need is equal to the number of receive descriptors 3812 * that we've allocated. 3813 * 3814 **********************************************************************/ 3815 static int 3816 em_allocate_receive_buffers(struct rx_ring *rxr) 3817 { 3818 struct adapter *adapter = rxr->adapter; 3819 device_t dev = adapter->dev; 3820 struct em_buffer *rxbuf; 3821 int error; 3822 3823 rxr->rx_buffers = malloc(sizeof(struct em_buffer) * 3824 adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 3825 if (rxr->rx_buffers == NULL) { 3826 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 3827 return (ENOMEM); 3828 } 3829 3830 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 3831 1, 0, /* alignment, bounds */ 3832 BUS_SPACE_MAXADDR, /* lowaddr */ 3833 BUS_SPACE_MAXADDR, /* highaddr */ 3834 NULL, NULL, /* filter, filterarg */ 3835 MJUM9BYTES, /* maxsize */ 3836 1, /* nsegments */ 3837 MJUM9BYTES, /* maxsegsize */ 3838 0, /* flags */ 3839 NULL, /* lockfunc */ 3840 NULL, /* lockarg */ 3841 &rxr->rxtag); 3842 if (error) { 3843 device_printf(dev, "%s: bus_dma_tag_create failed %d\n", 3844 __func__, error); 3845 goto fail; 3846 } 3847 3848 rxbuf = rxr->rx_buffers; 3849 for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3850 rxbuf = &rxr->rx_buffers[i]; 3851 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT, 3852 &rxbuf->map); 3853 if (error) { 3854 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 3855 __func__, error); 3856 goto fail; 3857 } 3858 } 3859 3860 return (0); 3861 3862 fail: 3863 em_free_receive_structures(adapter); 3864 return (error); 3865 } 3866 3867 3868 /********************************************************************* 3869 * 3870 * Initialize a receive ring and its buffers. 3871 * 3872 **********************************************************************/ 3873 static int 3874 em_setup_receive_ring(struct rx_ring *rxr) 3875 { 3876 struct adapter *adapter = rxr->adapter; 3877 struct em_buffer *rxbuf; 3878 bus_dma_segment_t seg[1]; 3879 int rsize, nsegs, error; 3880 3881 3882 /* Clear the ring contents */ 3883 EM_RX_LOCK(rxr); 3884 rsize = roundup2(adapter->num_rx_desc * 3885 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); 3886 bzero((void *)rxr->rx_base, rsize); 3887 3888 /* 3889 ** Free current RX buffer structs and their mbufs 3890 */ 3891 for (int i = 0; i < adapter->num_rx_desc; i++) { 3892 rxbuf = &rxr->rx_buffers[i]; 3893 if (rxbuf->m_head != NULL) { 3894 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3895 BUS_DMASYNC_POSTREAD); 3896 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3897 m_freem(rxbuf->m_head); 3898 } 3899 } 3900 3901 /* Now replenish the mbufs */ 3902 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3903 3904 rxbuf = &rxr->rx_buffers[j]; 3905 rxbuf->m_head = m_getjcl(M_DONTWAIT, MT_DATA, 3906 M_PKTHDR, adapter->rx_mbuf_sz); 3907 if (rxbuf->m_head == NULL) 3908 return (ENOBUFS); 3909 rxbuf->m_head->m_len = adapter->rx_mbuf_sz; 3910 rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */ 3911 rxbuf->m_head->m_pkthdr.len = adapter->rx_mbuf_sz; 3912 3913 /* Get the memory mapping */ 3914 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, 3915 rxbuf->map, rxbuf->m_head, seg, 3916 &nsegs, BUS_DMA_NOWAIT); 3917 if (error != 0) { 3918 m_freem(rxbuf->m_head); 3919 rxbuf->m_head = NULL; 3920 return (error); 3921 } 3922 bus_dmamap_sync(rxr->rxtag, 3923 rxbuf->map, BUS_DMASYNC_PREREAD); 3924 3925 /* Update descriptor */ 3926 rxr->rx_base[j].buffer_addr = htole64(seg[0].ds_addr); 3927 } 3928 3929 3930 /* Setup our descriptor indices */ 3931 rxr->next_to_check = 0; 3932 rxr->next_to_refresh = 0; 3933 3934 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3935 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3936 3937 EM_RX_UNLOCK(rxr); 3938 return (0); 3939 } 3940 3941 /********************************************************************* 3942 * 3943 * Initialize all receive rings. 3944 * 3945 **********************************************************************/ 3946 static int 3947 em_setup_receive_structures(struct adapter *adapter) 3948 { 3949 struct rx_ring *rxr = adapter->rx_rings; 3950 int j; 3951 3952 for (j = 0; j < adapter->num_queues; j++, rxr++) 3953 if (em_setup_receive_ring(rxr)) 3954 goto fail; 3955 3956 return (0); 3957 fail: 3958 /* 3959 * Free RX buffers allocated so far, we will only handle 3960 * the rings that completed, the failing case will have 3961 * cleaned up for itself. 'j' failed, so its the terminus. 3962 */ 3963 for (int i = 0; i < j; ++i) { 3964 rxr = &adapter->rx_rings[i]; 3965 for (int n = 0; n < adapter->num_rx_desc; n++) { 3966 struct em_buffer *rxbuf; 3967 rxbuf = &rxr->rx_buffers[n]; 3968 if (rxbuf->m_head != NULL) { 3969 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3970 BUS_DMASYNC_POSTREAD); 3971 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3972 m_freem(rxbuf->m_head); 3973 rxbuf->m_head = NULL; 3974 } 3975 } 3976 } 3977 3978 return (ENOBUFS); 3979 } 3980 3981 /********************************************************************* 3982 * 3983 * Free all receive rings. 3984 * 3985 **********************************************************************/ 3986 static void 3987 em_free_receive_structures(struct adapter *adapter) 3988 { 3989 struct rx_ring *rxr = adapter->rx_rings; 3990 3991 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3992 em_free_receive_buffers(rxr); 3993 /* Free the ring memory as well */ 3994 em_dma_free(adapter, &rxr->rxdma); 3995 EM_RX_LOCK_DESTROY(rxr); 3996 } 3997 3998 free(adapter->rx_rings, M_DEVBUF); 3999 } 4000 4001 4002 /********************************************************************* 4003 * 4004 * Free receive ring data structures 4005 * 4006 **********************************************************************/ 4007 static void 4008 em_free_receive_buffers(struct rx_ring *rxr) 4009 { 4010 struct adapter *adapter = rxr->adapter; 4011 struct em_buffer *rxbuf = NULL; 4012 4013 INIT_DEBUGOUT("free_receive_buffers: begin"); 4014 4015 if (rxr->rx_buffers != NULL) { 4016 for (int i = 0; i < adapter->num_rx_desc; i++) { 4017 rxbuf = &rxr->rx_buffers[i]; 4018 if (rxbuf->map != NULL) { 4019 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 4020 BUS_DMASYNC_POSTREAD); 4021 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 4022 bus_dmamap_destroy(rxr->rxtag, rxbuf->map); 4023 } 4024 if (rxbuf->m_head != NULL) { 4025 m_freem(rxbuf->m_head); 4026 rxbuf->m_head = NULL; 4027 } 4028 } 4029 free(rxr->rx_buffers, M_DEVBUF); 4030 rxr->rx_buffers = NULL; 4031 } 4032 4033 if (rxr->rxtag != NULL) { 4034 bus_dma_tag_destroy(rxr->rxtag); 4035 rxr->rxtag = NULL; 4036 } 4037 4038 return; 4039 } 4040 4041 4042 /********************************************************************* 4043 * 4044 * Enable receive unit. 4045 * 4046 **********************************************************************/ 4047 #define MAX_INTS_PER_SEC 8000 4048 #define DEFAULT_ITR 1000000000/(MAX_INTS_PER_SEC * 256) 4049 4050 static void 4051 em_initialize_receive_unit(struct adapter *adapter) 4052 { 4053 struct rx_ring *rxr = adapter->rx_rings; 4054 struct ifnet *ifp = adapter->ifp; 4055 struct e1000_hw *hw = &adapter->hw; 4056 u64 bus_addr; 4057 u32 rctl, rxcsum; 4058 4059 INIT_DEBUGOUT("em_initialize_receive_units: begin"); 4060 4061 /* 4062 * Make sure receives are disabled while setting 4063 * up the descriptor ring 4064 */ 4065 rctl = E1000_READ_REG(hw, E1000_RCTL); 4066 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 4067 4068 E1000_WRITE_REG(&adapter->hw, E1000_RADV, 4069 adapter->rx_abs_int_delay.value); 4070 /* 4071 * Set the interrupt throttling rate. Value is calculated 4072 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 4073 */ 4074 E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR); 4075 4076 /* 4077 ** When using MSIX interrupts we need to throttle 4078 ** using the EITR register (82574 only) 4079 */ 4080 if (hw->mac.type == e1000_82574) 4081 for (int i = 0; i < 4; i++) 4082 E1000_WRITE_REG(hw, E1000_EITR_82574(i), 4083 DEFAULT_ITR); 4084 4085 /* Disable accelerated ackknowledge */ 4086 if (adapter->hw.mac.type == e1000_82574) 4087 E1000_WRITE_REG(hw, E1000_RFCTL, E1000_RFCTL_ACK_DIS); 4088 4089 if (ifp->if_capenable & IFCAP_RXCSUM) { 4090 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 4091 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 4092 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 4093 } 4094 4095 /* 4096 ** XXX TEMPORARY WORKAROUND: on some systems with 82573 4097 ** long latencies are observed, like Lenovo X60. This 4098 ** change eliminates the problem, but since having positive 4099 ** values in RDTR is a known source of problems on other 4100 ** platforms another solution is being sought. 4101 */ 4102 if (hw->mac.type == e1000_82573) 4103 E1000_WRITE_REG(hw, E1000_RDTR, 0x20); 4104 4105 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4106 /* Setup the Base and Length of the Rx Descriptor Ring */ 4107 bus_addr = rxr->rxdma.dma_paddr; 4108 E1000_WRITE_REG(hw, E1000_RDLEN(i), 4109 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 4110 E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32)); 4111 E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr); 4112 /* Setup the Head and Tail Descriptor Pointers */ 4113 E1000_WRITE_REG(hw, E1000_RDH(i), 0); 4114 E1000_WRITE_REG(hw, E1000_RDT(i), adapter->num_rx_desc - 1); 4115 } 4116 4117 /* Set early receive threshold on appropriate hw */ 4118 if (((adapter->hw.mac.type == e1000_ich9lan) || 4119 (adapter->hw.mac.type == e1000_pch2lan) || 4120 (adapter->hw.mac.type == e1000_ich10lan)) && 4121 (ifp->if_mtu > ETHERMTU)) { 4122 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); 4123 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3); 4124 E1000_WRITE_REG(hw, E1000_ERT, 0x100 | (1 << 13)); 4125 } 4126 4127 if (adapter->hw.mac.type == e1000_pch2lan) { 4128 if (ifp->if_mtu > ETHERMTU) 4129 e1000_lv_jumbo_workaround_ich8lan(hw, TRUE); 4130 else 4131 e1000_lv_jumbo_workaround_ich8lan(hw, FALSE); 4132 } 4133 4134 /* Setup the Receive Control Register */ 4135 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4136 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 4137 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | 4138 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4139 4140 /* Strip the CRC */ 4141 rctl |= E1000_RCTL_SECRC; 4142 4143 /* Make sure VLAN Filters are off */ 4144 rctl &= ~E1000_RCTL_VFE; 4145 rctl &= ~E1000_RCTL_SBP; 4146 4147 if (adapter->rx_mbuf_sz == MCLBYTES) 4148 rctl |= E1000_RCTL_SZ_2048; 4149 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) 4150 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; 4151 else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) 4152 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; 4153 4154 if (ifp->if_mtu > ETHERMTU) 4155 rctl |= E1000_RCTL_LPE; 4156 else 4157 rctl &= ~E1000_RCTL_LPE; 4158 4159 /* Write out the settings */ 4160 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4161 4162 return; 4163 } 4164 4165 4166 /********************************************************************* 4167 * 4168 * This routine executes in interrupt context. It replenishes 4169 * the mbufs in the descriptor and sends data which has been 4170 * dma'ed into host memory to upper layer. 4171 * 4172 * We loop at most count times if count is > 0, or until done if 4173 * count < 0. 4174 * 4175 * For polling we also now return the number of cleaned packets 4176 *********************************************************************/ 4177 static bool 4178 em_rxeof(struct rx_ring *rxr, int count, int *done) 4179 { 4180 struct adapter *adapter = rxr->adapter; 4181 struct ifnet *ifp = adapter->ifp; 4182 struct mbuf *mp, *sendmp; 4183 u8 status = 0; 4184 u16 len; 4185 int i, processed, rxdone = 0; 4186 bool eop; 4187 struct e1000_rx_desc *cur; 4188 4189 EM_RX_LOCK(rxr); 4190 4191 for (i = rxr->next_to_check, processed = 0; count != 0;) { 4192 4193 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 4194 break; 4195 4196 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4197 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4198 4199 cur = &rxr->rx_base[i]; 4200 status = cur->status; 4201 mp = sendmp = NULL; 4202 4203 if ((status & E1000_RXD_STAT_DD) == 0) 4204 break; 4205 4206 len = le16toh(cur->length); 4207 eop = (status & E1000_RXD_STAT_EOP) != 0; 4208 count--; 4209 4210 if (((cur->errors & E1000_RXD_ERR_FRAME_ERR_MASK) == 0) && 4211 (rxr->discard == FALSE)) { 4212 4213 /* Assign correct length to the current fragment */ 4214 mp = rxr->rx_buffers[i].m_head; 4215 mp->m_len = len; 4216 4217 /* Trigger for refresh */ 4218 rxr->rx_buffers[i].m_head = NULL; 4219 4220 if (rxr->fmp == NULL) { 4221 mp->m_pkthdr.len = len; 4222 rxr->fmp = mp; /* Store the first mbuf */ 4223 rxr->lmp = mp; 4224 } else { 4225 /* Chain mbuf's together */ 4226 mp->m_flags &= ~M_PKTHDR; 4227 rxr->lmp->m_next = mp; 4228 rxr->lmp = rxr->lmp->m_next; 4229 rxr->fmp->m_pkthdr.len += len; 4230 } 4231 4232 if (eop) { 4233 rxr->fmp->m_pkthdr.rcvif = ifp; 4234 ifp->if_ipackets++; 4235 em_receive_checksum(cur, rxr->fmp); 4236 #ifndef __NO_STRICT_ALIGNMENT 4237 if (adapter->max_frame_size > 4238 (MCLBYTES - ETHER_ALIGN) && 4239 em_fixup_rx(rxr) != 0) 4240 goto skip; 4241 #endif 4242 if (status & E1000_RXD_STAT_VP) { 4243 rxr->fmp->m_pkthdr.ether_vtag = 4244 (le16toh(cur->special) & 4245 E1000_RXD_SPC_VLAN_MASK); 4246 rxr->fmp->m_flags |= M_VLANTAG; 4247 } 4248 #ifdef EM_MULTIQUEUE 4249 rxr->fmp->m_pkthdr.flowid = rxr->msix; 4250 rxr->fmp->m_flags |= M_FLOWID; 4251 #endif 4252 #ifndef __NO_STRICT_ALIGNMENT 4253 skip: 4254 #endif 4255 sendmp = rxr->fmp; 4256 rxr->fmp = NULL; 4257 rxr->lmp = NULL; 4258 } 4259 } else { 4260 ifp->if_ierrors++; 4261 ++rxr->rx_discarded; 4262 if (!eop) /* Catch subsequent segs */ 4263 rxr->discard = TRUE; 4264 else 4265 rxr->discard = FALSE; 4266 em_rx_discard(rxr, i); 4267 sendmp = NULL; 4268 } 4269 4270 /* Zero out the receive descriptors status. */ 4271 cur->status = 0; 4272 ++rxdone; /* cumulative for POLL */ 4273 ++processed; 4274 4275 /* Advance our pointers to the next descriptor. */ 4276 if (++i == adapter->num_rx_desc) 4277 i = 0; 4278 4279 /* Send to the stack */ 4280 if (sendmp != NULL) { 4281 rxr->next_to_check = i; 4282 EM_RX_UNLOCK(rxr); 4283 (*ifp->if_input)(ifp, sendmp); 4284 EM_RX_LOCK(rxr); 4285 i = rxr->next_to_check; 4286 } 4287 4288 /* Only refresh mbufs every 8 descriptors */ 4289 if (processed == 8) { 4290 em_refresh_mbufs(rxr, i); 4291 processed = 0; 4292 } 4293 } 4294 4295 /* Catch any remaining refresh work */ 4296 if (processed != 0) { 4297 em_refresh_mbufs(rxr, i); 4298 processed = 0; 4299 } 4300 4301 rxr->next_to_check = i; 4302 if (done != NULL) 4303 *done = rxdone; 4304 EM_RX_UNLOCK(rxr); 4305 4306 return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE); 4307 } 4308 4309 static __inline void 4310 em_rx_discard(struct rx_ring *rxr, int i) 4311 { 4312 struct adapter *adapter = rxr->adapter; 4313 struct em_buffer *rbuf; 4314 struct mbuf *m; 4315 4316 rbuf = &rxr->rx_buffers[i]; 4317 /* Free any previous pieces */ 4318 if (rxr->fmp != NULL) { 4319 rxr->fmp->m_flags |= M_PKTHDR; 4320 m_freem(rxr->fmp); 4321 rxr->fmp = NULL; 4322 rxr->lmp = NULL; 4323 } 4324 4325 /* Reset state, keep loaded DMA map and reuse */ 4326 m = rbuf->m_head; 4327 m->m_len = m->m_pkthdr.len = adapter->rx_mbuf_sz; 4328 m->m_flags |= M_PKTHDR; 4329 m->m_data = m->m_ext.ext_buf; 4330 m->m_next = NULL; 4331 4332 return; 4333 } 4334 4335 #ifndef __NO_STRICT_ALIGNMENT 4336 /* 4337 * When jumbo frames are enabled we should realign entire payload on 4338 * architecures with strict alignment. This is serious design mistake of 8254x 4339 * as it nullifies DMA operations. 8254x just allows RX buffer size to be 4340 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its 4341 * payload. On architecures without strict alignment restrictions 8254x still 4342 * performs unaligned memory access which would reduce the performance too. 4343 * To avoid copying over an entire frame to align, we allocate a new mbuf and 4344 * copy ethernet header to the new mbuf. The new mbuf is prepended into the 4345 * existing mbuf chain. 4346 * 4347 * Be aware, best performance of the 8254x is achived only when jumbo frame is 4348 * not used at all on architectures with strict alignment. 4349 */ 4350 static int 4351 em_fixup_rx(struct rx_ring *rxr) 4352 { 4353 struct adapter *adapter = rxr->adapter; 4354 struct mbuf *m, *n; 4355 int error; 4356 4357 error = 0; 4358 m = rxr->fmp; 4359 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { 4360 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); 4361 m->m_data += ETHER_HDR_LEN; 4362 } else { 4363 MGETHDR(n, M_DONTWAIT, MT_DATA); 4364 if (n != NULL) { 4365 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); 4366 m->m_data += ETHER_HDR_LEN; 4367 m->m_len -= ETHER_HDR_LEN; 4368 n->m_len = ETHER_HDR_LEN; 4369 M_MOVE_PKTHDR(n, m); 4370 n->m_next = m; 4371 rxr->fmp = n; 4372 } else { 4373 adapter->dropped_pkts++; 4374 m_freem(rxr->fmp); 4375 rxr->fmp = NULL; 4376 error = ENOMEM; 4377 } 4378 } 4379 4380 return (error); 4381 } 4382 #endif 4383 4384 /********************************************************************* 4385 * 4386 * Verify that the hardware indicated that the checksum is valid. 4387 * Inform the stack about the status of checksum so that stack 4388 * doesn't spend time verifying the checksum. 4389 * 4390 *********************************************************************/ 4391 static void 4392 em_receive_checksum(struct e1000_rx_desc *rx_desc, struct mbuf *mp) 4393 { 4394 /* Ignore Checksum bit is set */ 4395 if (rx_desc->status & E1000_RXD_STAT_IXSM) { 4396 mp->m_pkthdr.csum_flags = 0; 4397 return; 4398 } 4399 4400 if (rx_desc->status & E1000_RXD_STAT_IPCS) { 4401 /* Did it pass? */ 4402 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) { 4403 /* IP Checksum Good */ 4404 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4405 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4406 4407 } else { 4408 mp->m_pkthdr.csum_flags = 0; 4409 } 4410 } 4411 4412 if (rx_desc->status & E1000_RXD_STAT_TCPCS) { 4413 /* Did it pass? */ 4414 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) { 4415 mp->m_pkthdr.csum_flags |= 4416 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4417 mp->m_pkthdr.csum_data = htons(0xffff); 4418 } 4419 } 4420 } 4421 4422 /* 4423 * This routine is run via an vlan 4424 * config EVENT 4425 */ 4426 static void 4427 em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4428 { 4429 struct adapter *adapter = ifp->if_softc; 4430 u32 index, bit; 4431 4432 if (ifp->if_softc != arg) /* Not our event */ 4433 return; 4434 4435 if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */ 4436 return; 4437 4438 EM_CORE_LOCK(adapter); 4439 index = (vtag >> 5) & 0x7F; 4440 bit = vtag & 0x1F; 4441 adapter->shadow_vfta[index] |= (1 << bit); 4442 ++adapter->num_vlans; 4443 /* Re-init to load the changes */ 4444 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 4445 em_init_locked(adapter); 4446 EM_CORE_UNLOCK(adapter); 4447 } 4448 4449 /* 4450 * This routine is run via an vlan 4451 * unconfig EVENT 4452 */ 4453 static void 4454 em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4455 { 4456 struct adapter *adapter = ifp->if_softc; 4457 u32 index, bit; 4458 4459 if (ifp->if_softc != arg) 4460 return; 4461 4462 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4463 return; 4464 4465 EM_CORE_LOCK(adapter); 4466 index = (vtag >> 5) & 0x7F; 4467 bit = vtag & 0x1F; 4468 adapter->shadow_vfta[index] &= ~(1 << bit); 4469 --adapter->num_vlans; 4470 /* Re-init to load the changes */ 4471 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 4472 em_init_locked(adapter); 4473 EM_CORE_UNLOCK(adapter); 4474 } 4475 4476 static void 4477 em_setup_vlan_hw_support(struct adapter *adapter) 4478 { 4479 struct e1000_hw *hw = &adapter->hw; 4480 u32 reg; 4481 4482 /* 4483 ** We get here thru init_locked, meaning 4484 ** a soft reset, this has already cleared 4485 ** the VFTA and other state, so if there 4486 ** have been no vlan's registered do nothing. 4487 */ 4488 if (adapter->num_vlans == 0) 4489 return; 4490 4491 /* 4492 ** A soft reset zero's out the VFTA, so 4493 ** we need to repopulate it now. 4494 */ 4495 for (int i = 0; i < EM_VFTA_SIZE; i++) 4496 if (adapter->shadow_vfta[i] != 0) 4497 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, 4498 i, adapter->shadow_vfta[i]); 4499 4500 reg = E1000_READ_REG(hw, E1000_CTRL); 4501 reg |= E1000_CTRL_VME; 4502 E1000_WRITE_REG(hw, E1000_CTRL, reg); 4503 4504 /* Enable the Filter Table */ 4505 reg = E1000_READ_REG(hw, E1000_RCTL); 4506 reg &= ~E1000_RCTL_CFIEN; 4507 reg |= E1000_RCTL_VFE; 4508 E1000_WRITE_REG(hw, E1000_RCTL, reg); 4509 } 4510 4511 static void 4512 em_enable_intr(struct adapter *adapter) 4513 { 4514 struct e1000_hw *hw = &adapter->hw; 4515 u32 ims_mask = IMS_ENABLE_MASK; 4516 4517 if (hw->mac.type == e1000_82574) { 4518 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK); 4519 ims_mask |= EM_MSIX_MASK; 4520 } 4521 E1000_WRITE_REG(hw, E1000_IMS, ims_mask); 4522 } 4523 4524 static void 4525 em_disable_intr(struct adapter *adapter) 4526 { 4527 struct e1000_hw *hw = &adapter->hw; 4528 4529 if (hw->mac.type == e1000_82574) 4530 E1000_WRITE_REG(hw, EM_EIAC, 0); 4531 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 4532 } 4533 4534 /* 4535 * Bit of a misnomer, what this really means is 4536 * to enable OS management of the system... aka 4537 * to disable special hardware management features 4538 */ 4539 static void 4540 em_init_manageability(struct adapter *adapter) 4541 { 4542 /* A shared code workaround */ 4543 #define E1000_82542_MANC2H E1000_MANC2H 4544 if (adapter->has_manage) { 4545 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4546 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4547 4548 /* disable hardware interception of ARP */ 4549 manc &= ~(E1000_MANC_ARP_EN); 4550 4551 /* enable receiving management packets to the host */ 4552 manc |= E1000_MANC_EN_MNG2HOST; 4553 #define E1000_MNG2HOST_PORT_623 (1 << 5) 4554 #define E1000_MNG2HOST_PORT_664 (1 << 6) 4555 manc2h |= E1000_MNG2HOST_PORT_623; 4556 manc2h |= E1000_MNG2HOST_PORT_664; 4557 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4558 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4559 } 4560 } 4561 4562 /* 4563 * Give control back to hardware management 4564 * controller if there is one. 4565 */ 4566 static void 4567 em_release_manageability(struct adapter *adapter) 4568 { 4569 if (adapter->has_manage) { 4570 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4571 4572 /* re-enable hardware interception of ARP */ 4573 manc |= E1000_MANC_ARP_EN; 4574 manc &= ~E1000_MANC_EN_MNG2HOST; 4575 4576 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4577 } 4578 } 4579 4580 /* 4581 * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 4582 * For ASF and Pass Through versions of f/w this means 4583 * that the driver is loaded. For AMT version type f/w 4584 * this means that the network i/f is open. 4585 */ 4586 static void 4587 em_get_hw_control(struct adapter *adapter) 4588 { 4589 u32 ctrl_ext, swsm; 4590 4591 if (adapter->hw.mac.type == e1000_82573) { 4592 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4593 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4594 swsm | E1000_SWSM_DRV_LOAD); 4595 return; 4596 } 4597 /* else */ 4598 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4599 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4600 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4601 return; 4602 } 4603 4604 /* 4605 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 4606 * For ASF and Pass Through versions of f/w this means that 4607 * the driver is no longer loaded. For AMT versions of the 4608 * f/w this means that the network i/f is closed. 4609 */ 4610 static void 4611 em_release_hw_control(struct adapter *adapter) 4612 { 4613 u32 ctrl_ext, swsm; 4614 4615 if (!adapter->has_manage) 4616 return; 4617 4618 if (adapter->hw.mac.type == e1000_82573) { 4619 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4620 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4621 swsm & ~E1000_SWSM_DRV_LOAD); 4622 return; 4623 } 4624 /* else */ 4625 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4626 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4627 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4628 return; 4629 } 4630 4631 static int 4632 em_is_valid_ether_addr(u8 *addr) 4633 { 4634 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4635 4636 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4637 return (FALSE); 4638 } 4639 4640 return (TRUE); 4641 } 4642 4643 /* 4644 ** Parse the interface capabilities with regard 4645 ** to both system management and wake-on-lan for 4646 ** later use. 4647 */ 4648 static void 4649 em_get_wakeup(device_t dev) 4650 { 4651 struct adapter *adapter = device_get_softc(dev); 4652 u16 eeprom_data = 0, device_id, apme_mask; 4653 4654 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 4655 apme_mask = EM_EEPROM_APME; 4656 4657 switch (adapter->hw.mac.type) { 4658 case e1000_82573: 4659 case e1000_82583: 4660 adapter->has_amt = TRUE; 4661 /* Falls thru */ 4662 case e1000_82571: 4663 case e1000_82572: 4664 case e1000_80003es2lan: 4665 if (adapter->hw.bus.func == 1) { 4666 e1000_read_nvm(&adapter->hw, 4667 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 4668 break; 4669 } else 4670 e1000_read_nvm(&adapter->hw, 4671 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 4672 break; 4673 case e1000_ich8lan: 4674 case e1000_ich9lan: 4675 case e1000_ich10lan: 4676 case e1000_pchlan: 4677 case e1000_pch2lan: 4678 apme_mask = E1000_WUC_APME; 4679 adapter->has_amt = TRUE; 4680 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC); 4681 break; 4682 default: 4683 e1000_read_nvm(&adapter->hw, 4684 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 4685 break; 4686 } 4687 if (eeprom_data & apme_mask) 4688 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC); 4689 /* 4690 * We have the eeprom settings, now apply the special cases 4691 * where the eeprom may be wrong or the board won't support 4692 * wake on lan on a particular port 4693 */ 4694 device_id = pci_get_device(dev); 4695 switch (device_id) { 4696 case E1000_DEV_ID_82571EB_FIBER: 4697 /* Wake events only supported on port A for dual fiber 4698 * regardless of eeprom setting */ 4699 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 4700 E1000_STATUS_FUNC_1) 4701 adapter->wol = 0; 4702 break; 4703 case E1000_DEV_ID_82571EB_QUAD_COPPER: 4704 case E1000_DEV_ID_82571EB_QUAD_FIBER: 4705 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 4706 /* if quad port adapter, disable WoL on all but port A */ 4707 if (global_quad_port_a != 0) 4708 adapter->wol = 0; 4709 /* Reset for multiple quad port adapters */ 4710 if (++global_quad_port_a == 4) 4711 global_quad_port_a = 0; 4712 break; 4713 } 4714 return; 4715 } 4716 4717 4718 /* 4719 * Enable PCI Wake On Lan capability 4720 */ 4721 static void 4722 em_enable_wakeup(device_t dev) 4723 { 4724 struct adapter *adapter = device_get_softc(dev); 4725 struct ifnet *ifp = adapter->ifp; 4726 u32 pmc, ctrl, ctrl_ext, rctl; 4727 u16 status; 4728 4729 if ((pci_find_extcap(dev, PCIY_PMG, &pmc) != 0)) 4730 return; 4731 4732 /* Advertise the wakeup capability */ 4733 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 4734 ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3); 4735 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 4736 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 4737 4738 if ((adapter->hw.mac.type == e1000_ich8lan) || 4739 (adapter->hw.mac.type == e1000_pchlan) || 4740 (adapter->hw.mac.type == e1000_ich9lan) || 4741 (adapter->hw.mac.type == e1000_ich10lan)) { 4742 e1000_disable_gig_wol_ich8lan(&adapter->hw); 4743 e1000_hv_phy_powerdown_workaround_ich8lan(&adapter->hw); 4744 } 4745 4746 /* Keep the laser running on Fiber adapters */ 4747 if (adapter->hw.phy.media_type == e1000_media_type_fiber || 4748 adapter->hw.phy.media_type == e1000_media_type_internal_serdes) { 4749 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4750 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; 4751 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext); 4752 } 4753 4754 /* 4755 ** Determine type of Wakeup: note that wol 4756 ** is set with all bits on by default. 4757 */ 4758 if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0) 4759 adapter->wol &= ~E1000_WUFC_MAG; 4760 4761 if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0) 4762 adapter->wol &= ~E1000_WUFC_MC; 4763 else { 4764 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4765 rctl |= E1000_RCTL_MPE; 4766 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4767 } 4768 4769 if ((adapter->hw.mac.type == e1000_pchlan) || 4770 (adapter->hw.mac.type == e1000_pch2lan)) { 4771 if (em_enable_phy_wakeup(adapter)) 4772 return; 4773 } else { 4774 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 4775 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 4776 } 4777 4778 if (adapter->hw.phy.type == e1000_phy_igp_3) 4779 e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw); 4780 4781 /* Request PME */ 4782 status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); 4783 status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE); 4784 if (ifp->if_capenable & IFCAP_WOL) 4785 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 4786 pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); 4787 4788 return; 4789 } 4790 4791 /* 4792 ** WOL in the newer chipset interfaces (pchlan) 4793 ** require thing to be copied into the phy 4794 */ 4795 static int 4796 em_enable_phy_wakeup(struct adapter *adapter) 4797 { 4798 struct e1000_hw *hw = &adapter->hw; 4799 u32 mreg, ret = 0; 4800 u16 preg; 4801 4802 /* copy MAC RARs to PHY RARs */ 4803 e1000_copy_rx_addrs_to_phy_ich8lan(hw); 4804 4805 /* copy MAC MTA to PHY MTA */ 4806 for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) { 4807 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); 4808 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF)); 4809 e1000_write_phy_reg(hw, BM_MTA(i) + 1, 4810 (u16)((mreg >> 16) & 0xFFFF)); 4811 } 4812 4813 /* configure PHY Rx Control register */ 4814 e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg); 4815 mreg = E1000_READ_REG(hw, E1000_RCTL); 4816 if (mreg & E1000_RCTL_UPE) 4817 preg |= BM_RCTL_UPE; 4818 if (mreg & E1000_RCTL_MPE) 4819 preg |= BM_RCTL_MPE; 4820 preg &= ~(BM_RCTL_MO_MASK); 4821 if (mreg & E1000_RCTL_MO_3) 4822 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) 4823 << BM_RCTL_MO_SHIFT); 4824 if (mreg & E1000_RCTL_BAM) 4825 preg |= BM_RCTL_BAM; 4826 if (mreg & E1000_RCTL_PMCF) 4827 preg |= BM_RCTL_PMCF; 4828 mreg = E1000_READ_REG(hw, E1000_CTRL); 4829 if (mreg & E1000_CTRL_RFCE) 4830 preg |= BM_RCTL_RFCE; 4831 e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg); 4832 4833 /* enable PHY wakeup in MAC register */ 4834 E1000_WRITE_REG(hw, E1000_WUC, 4835 E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN); 4836 E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol); 4837 4838 /* configure and enable PHY wakeup in PHY registers */ 4839 e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol); 4840 e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN); 4841 4842 /* activate PHY wakeup */ 4843 ret = hw->phy.ops.acquire(hw); 4844 if (ret) { 4845 printf("Could not acquire PHY\n"); 4846 return ret; 4847 } 4848 e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT, 4849 (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT)); 4850 ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg); 4851 if (ret) { 4852 printf("Could not read PHY page 769\n"); 4853 goto out; 4854 } 4855 preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; 4856 ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg); 4857 if (ret) 4858 printf("Could not set PHY Host Wakeup bit\n"); 4859 out: 4860 hw->phy.ops.release(hw); 4861 4862 return ret; 4863 } 4864 4865 static void 4866 em_led_func(void *arg, int onoff) 4867 { 4868 struct adapter *adapter = arg; 4869 4870 EM_CORE_LOCK(adapter); 4871 if (onoff) { 4872 e1000_setup_led(&adapter->hw); 4873 e1000_led_on(&adapter->hw); 4874 } else { 4875 e1000_led_off(&adapter->hw); 4876 e1000_cleanup_led(&adapter->hw); 4877 } 4878 EM_CORE_UNLOCK(adapter); 4879 } 4880 4881 /********************************************************************** 4882 * 4883 * Update the board statistics counters. 4884 * 4885 **********************************************************************/ 4886 static void 4887 em_update_stats_counters(struct adapter *adapter) 4888 { 4889 struct ifnet *ifp; 4890 4891 if(adapter->hw.phy.media_type == e1000_media_type_copper || 4892 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 4893 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 4894 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); 4895 } 4896 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 4897 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 4898 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 4899 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 4900 4901 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 4902 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 4903 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 4904 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 4905 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 4906 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 4907 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 4908 /* 4909 ** For watchdog management we need to know if we have been 4910 ** paused during the last interval, so capture that here. 4911 */ 4912 adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 4913 adapter->stats.xoffrxc += adapter->pause_frames; 4914 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 4915 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 4916 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 4917 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 4918 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 4919 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 4920 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 4921 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 4922 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 4923 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 4924 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 4925 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 4926 4927 /* For the 64-bit byte counters the low dword must be read first. */ 4928 /* Both registers clear on the read of the high dword */ 4929 4930 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) + 4931 ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32); 4932 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) + 4933 ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32); 4934 4935 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 4936 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 4937 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 4938 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 4939 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 4940 4941 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 4942 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 4943 4944 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 4945 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 4946 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 4947 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 4948 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 4949 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 4950 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 4951 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 4952 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 4953 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 4954 4955 /* Interrupt Counts */ 4956 4957 adapter->stats.iac += E1000_READ_REG(&adapter->hw, E1000_IAC); 4958 adapter->stats.icrxptc += E1000_READ_REG(&adapter->hw, E1000_ICRXPTC); 4959 adapter->stats.icrxatc += E1000_READ_REG(&adapter->hw, E1000_ICRXATC); 4960 adapter->stats.ictxptc += E1000_READ_REG(&adapter->hw, E1000_ICTXPTC); 4961 adapter->stats.ictxatc += E1000_READ_REG(&adapter->hw, E1000_ICTXATC); 4962 adapter->stats.ictxqec += E1000_READ_REG(&adapter->hw, E1000_ICTXQEC); 4963 adapter->stats.ictxqmtc += E1000_READ_REG(&adapter->hw, E1000_ICTXQMTC); 4964 adapter->stats.icrxdmtc += E1000_READ_REG(&adapter->hw, E1000_ICRXDMTC); 4965 adapter->stats.icrxoc += E1000_READ_REG(&adapter->hw, E1000_ICRXOC); 4966 4967 if (adapter->hw.mac.type >= e1000_82543) { 4968 adapter->stats.algnerrc += 4969 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 4970 adapter->stats.rxerrc += 4971 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 4972 adapter->stats.tncrs += 4973 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 4974 adapter->stats.cexterr += 4975 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 4976 adapter->stats.tsctc += 4977 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 4978 adapter->stats.tsctfc += 4979 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 4980 } 4981 ifp = adapter->ifp; 4982 4983 ifp->if_collisions = adapter->stats.colc; 4984 4985 /* Rx Errors */ 4986 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + 4987 adapter->stats.crcerrs + adapter->stats.algnerrc + 4988 adapter->stats.ruc + adapter->stats.roc + 4989 adapter->stats.mpc + adapter->stats.cexterr; 4990 4991 /* Tx Errors */ 4992 ifp->if_oerrors = adapter->stats.ecol + 4993 adapter->stats.latecol + adapter->watchdog_events; 4994 } 4995 4996 /* Export a single 32-bit register via a read-only sysctl. */ 4997 static int 4998 em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) 4999 { 5000 struct adapter *adapter; 5001 u_int val; 5002 5003 adapter = oidp->oid_arg1; 5004 val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2); 5005 return (sysctl_handle_int(oidp, &val, 0, req)); 5006 } 5007 5008 /* 5009 * Add sysctl variables, one per statistic, to the system. 5010 */ 5011 static void 5012 em_add_hw_stats(struct adapter *adapter) 5013 { 5014 device_t dev = adapter->dev; 5015 5016 struct tx_ring *txr = adapter->tx_rings; 5017 struct rx_ring *rxr = adapter->rx_rings; 5018 5019 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 5020 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 5021 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 5022 struct e1000_hw_stats *stats = &adapter->stats; 5023 5024 struct sysctl_oid *stat_node, *queue_node, *int_node; 5025 struct sysctl_oid_list *stat_list, *queue_list, *int_list; 5026 5027 #define QUEUE_NAME_LEN 32 5028 char namebuf[QUEUE_NAME_LEN]; 5029 5030 /* Driver Statistics */ 5031 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "link_irq", 5032 CTLFLAG_RD, &adapter->link_irq, 0, 5033 "Link MSIX IRQ Handled"); 5034 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_alloc_fail", 5035 CTLFLAG_RD, &adapter->mbuf_alloc_failed, 5036 "Std mbuf failed"); 5037 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "cluster_alloc_fail", 5038 CTLFLAG_RD, &adapter->mbuf_cluster_failed, 5039 "Std mbuf cluster failed"); 5040 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 5041 CTLFLAG_RD, &adapter->dropped_pkts, 5042 "Driver dropped packets"); 5043 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", 5044 CTLFLAG_RD, &adapter->no_tx_dma_setup, 5045 "Driver tx dma failure in xmit"); 5046 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", 5047 CTLFLAG_RD, &adapter->rx_overruns, 5048 "RX overruns"); 5049 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts", 5050 CTLFLAG_RD, &adapter->watchdog_events, 5051 "Watchdog timeouts"); 5052 5053 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control", 5054 CTLFLAG_RD, adapter, E1000_CTRL, 5055 em_sysctl_reg_handler, "IU", 5056 "Device Control Register"); 5057 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control", 5058 CTLFLAG_RD, adapter, E1000_RCTL, 5059 em_sysctl_reg_handler, "IU", 5060 "Receiver Control Register"); 5061 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", 5062 CTLFLAG_RD, &adapter->hw.fc.high_water, 0, 5063 "Flow Control High Watermark"); 5064 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", 5065 CTLFLAG_RD, &adapter->hw.fc.low_water, 0, 5066 "Flow Control Low Watermark"); 5067 5068 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 5069 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 5070 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 5071 CTLFLAG_RD, NULL, "Queue Name"); 5072 queue_list = SYSCTL_CHILDREN(queue_node); 5073 5074 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", 5075 CTLFLAG_RD, adapter, E1000_TDH(txr->me), 5076 em_sysctl_reg_handler, "IU", 5077 "Transmit Descriptor Head"); 5078 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", 5079 CTLFLAG_RD, adapter, E1000_TDT(txr->me), 5080 em_sysctl_reg_handler, "IU", 5081 "Transmit Descriptor Tail"); 5082 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq", 5083 CTLFLAG_RD, &txr->tx_irq, 5084 "Queue MSI-X Transmit Interrupts"); 5085 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail", 5086 CTLFLAG_RD, &txr->no_desc_avail, 5087 "Queue No Descriptor Available"); 5088 5089 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", 5090 CTLFLAG_RD, adapter, E1000_RDH(rxr->me), 5091 em_sysctl_reg_handler, "IU", 5092 "Receive Descriptor Head"); 5093 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", 5094 CTLFLAG_RD, adapter, E1000_RDT(rxr->me), 5095 em_sysctl_reg_handler, "IU", 5096 "Receive Descriptor Tail"); 5097 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq", 5098 CTLFLAG_RD, &rxr->rx_irq, 5099 "Queue MSI-X Receive Interrupts"); 5100 } 5101 5102 /* MAC stats get their own sub node */ 5103 5104 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 5105 CTLFLAG_RD, NULL, "Statistics"); 5106 stat_list = SYSCTL_CHILDREN(stat_node); 5107 5108 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll", 5109 CTLFLAG_RD, &stats->ecol, 5110 "Excessive collisions"); 5111 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll", 5112 CTLFLAG_RD, &stats->scc, 5113 "Single collisions"); 5114 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll", 5115 CTLFLAG_RD, &stats->mcc, 5116 "Multiple collisions"); 5117 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll", 5118 CTLFLAG_RD, &stats->latecol, 5119 "Late collisions"); 5120 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count", 5121 CTLFLAG_RD, &stats->colc, 5122 "Collision Count"); 5123 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors", 5124 CTLFLAG_RD, &adapter->stats.symerrs, 5125 "Symbol Errors"); 5126 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors", 5127 CTLFLAG_RD, &adapter->stats.sec, 5128 "Sequence Errors"); 5129 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count", 5130 CTLFLAG_RD, &adapter->stats.dc, 5131 "Defer Count"); 5132 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets", 5133 CTLFLAG_RD, &adapter->stats.mpc, 5134 "Missed Packets"); 5135 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", 5136 CTLFLAG_RD, &adapter->stats.rnbc, 5137 "Receive No Buffers"); 5138 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize", 5139 CTLFLAG_RD, &adapter->stats.ruc, 5140 "Receive Undersize"); 5141 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 5142 CTLFLAG_RD, &adapter->stats.rfc, 5143 "Fragmented Packets Received "); 5144 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize", 5145 CTLFLAG_RD, &adapter->stats.roc, 5146 "Oversized Packets Received"); 5147 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber", 5148 CTLFLAG_RD, &adapter->stats.rjc, 5149 "Recevied Jabber"); 5150 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs", 5151 CTLFLAG_RD, &adapter->stats.rxerrc, 5152 "Receive Errors"); 5153 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs", 5154 CTLFLAG_RD, &adapter->stats.crcerrs, 5155 "CRC errors"); 5156 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs", 5157 CTLFLAG_RD, &adapter->stats.algnerrc, 5158 "Alignment Errors"); 5159 /* On 82575 these are collision counts */ 5160 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs", 5161 CTLFLAG_RD, &adapter->stats.cexterr, 5162 "Collision/Carrier extension errors"); 5163 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd", 5164 CTLFLAG_RD, &adapter->stats.xonrxc, 5165 "XON Received"); 5166 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd", 5167 CTLFLAG_RD, &adapter->stats.xontxc, 5168 "XON Transmitted"); 5169 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", 5170 CTLFLAG_RD, &adapter->stats.xoffrxc, 5171 "XOFF Received"); 5172 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd", 5173 CTLFLAG_RD, &adapter->stats.xofftxc, 5174 "XOFF Transmitted"); 5175 5176 /* Packet Reception Stats */ 5177 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", 5178 CTLFLAG_RD, &adapter->stats.tpr, 5179 "Total Packets Received "); 5180 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 5181 CTLFLAG_RD, &adapter->stats.gprc, 5182 "Good Packets Received"); 5183 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", 5184 CTLFLAG_RD, &adapter->stats.bprc, 5185 "Broadcast Packets Received"); 5186 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", 5187 CTLFLAG_RD, &adapter->stats.mprc, 5188 "Multicast Packets Received"); 5189 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 5190 CTLFLAG_RD, &adapter->stats.prc64, 5191 "64 byte frames received "); 5192 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 5193 CTLFLAG_RD, &adapter->stats.prc127, 5194 "65-127 byte frames received"); 5195 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 5196 CTLFLAG_RD, &adapter->stats.prc255, 5197 "128-255 byte frames received"); 5198 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 5199 CTLFLAG_RD, &adapter->stats.prc511, 5200 "256-511 byte frames received"); 5201 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 5202 CTLFLAG_RD, &adapter->stats.prc1023, 5203 "512-1023 byte frames received"); 5204 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 5205 CTLFLAG_RD, &adapter->stats.prc1522, 5206 "1023-1522 byte frames received"); 5207 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 5208 CTLFLAG_RD, &adapter->stats.gorc, 5209 "Good Octets Received"); 5210 5211 /* Packet Transmission Stats */ 5212 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 5213 CTLFLAG_RD, &adapter->stats.gotc, 5214 "Good Octets Transmitted"); 5215 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 5216 CTLFLAG_RD, &adapter->stats.tpt, 5217 "Total Packets Transmitted"); 5218 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 5219 CTLFLAG_RD, &adapter->stats.gptc, 5220 "Good Packets Transmitted"); 5221 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 5222 CTLFLAG_RD, &adapter->stats.bptc, 5223 "Broadcast Packets Transmitted"); 5224 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 5225 CTLFLAG_RD, &adapter->stats.mptc, 5226 "Multicast Packets Transmitted"); 5227 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 5228 CTLFLAG_RD, &adapter->stats.ptc64, 5229 "64 byte frames transmitted "); 5230 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 5231 CTLFLAG_RD, &adapter->stats.ptc127, 5232 "65-127 byte frames transmitted"); 5233 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 5234 CTLFLAG_RD, &adapter->stats.ptc255, 5235 "128-255 byte frames transmitted"); 5236 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 5237 CTLFLAG_RD, &adapter->stats.ptc511, 5238 "256-511 byte frames transmitted"); 5239 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 5240 CTLFLAG_RD, &adapter->stats.ptc1023, 5241 "512-1023 byte frames transmitted"); 5242 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 5243 CTLFLAG_RD, &adapter->stats.ptc1522, 5244 "1024-1522 byte frames transmitted"); 5245 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd", 5246 CTLFLAG_RD, &adapter->stats.tsctc, 5247 "TSO Contexts Transmitted"); 5248 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail", 5249 CTLFLAG_RD, &adapter->stats.tsctfc, 5250 "TSO Contexts Failed"); 5251 5252 5253 /* Interrupt Stats */ 5254 5255 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 5256 CTLFLAG_RD, NULL, "Interrupt Statistics"); 5257 int_list = SYSCTL_CHILDREN(int_node); 5258 5259 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts", 5260 CTLFLAG_RD, &adapter->stats.iac, 5261 "Interrupt Assertion Count"); 5262 5263 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer", 5264 CTLFLAG_RD, &adapter->stats.icrxptc, 5265 "Interrupt Cause Rx Pkt Timer Expire Count"); 5266 5267 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer", 5268 CTLFLAG_RD, &adapter->stats.icrxatc, 5269 "Interrupt Cause Rx Abs Timer Expire Count"); 5270 5271 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer", 5272 CTLFLAG_RD, &adapter->stats.ictxptc, 5273 "Interrupt Cause Tx Pkt Timer Expire Count"); 5274 5275 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer", 5276 CTLFLAG_RD, &adapter->stats.ictxatc, 5277 "Interrupt Cause Tx Abs Timer Expire Count"); 5278 5279 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty", 5280 CTLFLAG_RD, &adapter->stats.ictxqec, 5281 "Interrupt Cause Tx Queue Empty Count"); 5282 5283 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh", 5284 CTLFLAG_RD, &adapter->stats.ictxqmtc, 5285 "Interrupt Cause Tx Queue Min Thresh Count"); 5286 5287 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh", 5288 CTLFLAG_RD, &adapter->stats.icrxdmtc, 5289 "Interrupt Cause Rx Desc Min Thresh Count"); 5290 5291 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun", 5292 CTLFLAG_RD, &adapter->stats.icrxoc, 5293 "Interrupt Cause Receiver Overrun Count"); 5294 } 5295 5296 /********************************************************************** 5297 * 5298 * This routine provides a way to dump out the adapter eeprom, 5299 * often a useful debug/service tool. This only dumps the first 5300 * 32 words, stuff that matters is in that extent. 5301 * 5302 **********************************************************************/ 5303 static int 5304 em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) 5305 { 5306 struct adapter *adapter; 5307 int error; 5308 int result; 5309 5310 result = -1; 5311 error = sysctl_handle_int(oidp, &result, 0, req); 5312 5313 if (error || !req->newptr) 5314 return (error); 5315 5316 /* 5317 * This value will cause a hex dump of the 5318 * first 32 16-bit words of the EEPROM to 5319 * the screen. 5320 */ 5321 if (result == 1) { 5322 adapter = (struct adapter *)arg1; 5323 em_print_nvm_info(adapter); 5324 } 5325 5326 return (error); 5327 } 5328 5329 static void 5330 em_print_nvm_info(struct adapter *adapter) 5331 { 5332 u16 eeprom_data; 5333 int i, j, row = 0; 5334 5335 /* Its a bit crude, but it gets the job done */ 5336 printf("\nInterface EEPROM Dump:\n"); 5337 printf("Offset\n0x0000 "); 5338 for (i = 0, j = 0; i < 32; i++, j++) { 5339 if (j == 8) { /* Make the offset block */ 5340 j = 0; ++row; 5341 printf("\n0x00%x0 ",row); 5342 } 5343 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 5344 printf("%04x ", eeprom_data); 5345 } 5346 printf("\n"); 5347 } 5348 5349 static int 5350 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) 5351 { 5352 struct em_int_delay_info *info; 5353 struct adapter *adapter; 5354 u32 regval; 5355 int error, usecs, ticks; 5356 5357 info = (struct em_int_delay_info *)arg1; 5358 usecs = info->value; 5359 error = sysctl_handle_int(oidp, &usecs, 0, req); 5360 if (error != 0 || req->newptr == NULL) 5361 return (error); 5362 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) 5363 return (EINVAL); 5364 info->value = usecs; 5365 ticks = EM_USECS_TO_TICKS(usecs); 5366 5367 adapter = info->adapter; 5368 5369 EM_CORE_LOCK(adapter); 5370 regval = E1000_READ_OFFSET(&adapter->hw, info->offset); 5371 regval = (regval & ~0xffff) | (ticks & 0xffff); 5372 /* Handle a few special cases. */ 5373 switch (info->offset) { 5374 case E1000_RDTR: 5375 break; 5376 case E1000_TIDV: 5377 if (ticks == 0) { 5378 adapter->txd_cmd &= ~E1000_TXD_CMD_IDE; 5379 /* Don't write 0 into the TIDV register. */ 5380 regval++; 5381 } else 5382 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 5383 break; 5384 } 5385 E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); 5386 EM_CORE_UNLOCK(adapter); 5387 return (0); 5388 } 5389 5390 static void 5391 em_add_int_delay_sysctl(struct adapter *adapter, const char *name, 5392 const char *description, struct em_int_delay_info *info, 5393 int offset, int value) 5394 { 5395 info->adapter = adapter; 5396 info->offset = offset; 5397 info->value = value; 5398 SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev), 5399 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5400 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, 5401 info, 0, em_sysctl_int_delay, "I", description); 5402 } 5403 5404 static void 5405 em_add_rx_process_limit(struct adapter *adapter, const char *name, 5406 const char *description, int *limit, int value) 5407 { 5408 *limit = value; 5409 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 5410 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5411 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5412 } 5413 5414 static void 5415 em_set_flow_cntrl(struct adapter *adapter, const char *name, 5416 const char *description, int *limit, int value) 5417 { 5418 *limit = value; 5419 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 5420 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5421 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5422 } 5423 5424 static int 5425 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 5426 { 5427 struct adapter *adapter; 5428 int error; 5429 int result; 5430 5431 result = -1; 5432 error = sysctl_handle_int(oidp, &result, 0, req); 5433 5434 if (error || !req->newptr) 5435 return (error); 5436 5437 if (result == 1) { 5438 adapter = (struct adapter *)arg1; 5439 em_print_debug_info(adapter); 5440 } 5441 5442 return (error); 5443 } 5444 5445 /* 5446 ** This routine is meant to be fluid, add whatever is 5447 ** needed for debugging a problem. -jfv 5448 */ 5449 static void 5450 em_print_debug_info(struct adapter *adapter) 5451 { 5452 device_t dev = adapter->dev; 5453 struct tx_ring *txr = adapter->tx_rings; 5454 struct rx_ring *rxr = adapter->rx_rings; 5455 5456 if (adapter->ifp->if_drv_flags & IFF_DRV_RUNNING) 5457 printf("Interface is RUNNING "); 5458 else 5459 printf("Interface is NOT RUNNING\n"); 5460 if (adapter->ifp->if_drv_flags & IFF_DRV_OACTIVE) 5461 printf("and ACTIVE\n"); 5462 else 5463 printf("and INACTIVE\n"); 5464 5465 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 5466 E1000_READ_REG(&adapter->hw, E1000_TDH(0)), 5467 E1000_READ_REG(&adapter->hw, E1000_TDT(0))); 5468 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 5469 E1000_READ_REG(&adapter->hw, E1000_RDH(0)), 5470 E1000_READ_REG(&adapter->hw, E1000_RDT(0))); 5471 device_printf(dev, "Tx Queue Status = %d\n", txr->queue_status); 5472 device_printf(dev, "TX descriptors avail = %d\n", 5473 txr->tx_avail); 5474 device_printf(dev, "Tx Descriptors avail failure = %ld\n", 5475 txr->no_desc_avail); 5476 device_printf(dev, "RX discarded packets = %ld\n", 5477 rxr->rx_discarded); 5478 device_printf(dev, "RX Next to Check = %d\n", rxr->next_to_check); 5479 device_printf(dev, "RX Next to Refresh = %d\n", rxr->next_to_refresh); 5480 } 5481