1 /* 2 * Copyright (c) 2001 Wind River Systems 3 * Copyright (c) 1997, 1998, 1999, 2001 4 * Bill Paul <wpaul@windriver.com>. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. All advertising materials mentioning features or use of this software 15 * must display the following acknowledgement: 16 * This product includes software developed by Bill Paul. 17 * 4. Neither the name of the author nor the names of any co-contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD 25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 31 * THE POSSIBILITY OF SUCH DAMAGE. 32 * 33 * $FreeBSD: src/sys/dev/bge/if_bge.c,v 1.3.2.39 2005/07/03 03:41:18 silby Exp $ 34 */ 35 36 #include "opt_bnx.h" 37 #include "opt_ifpoll.h" 38 39 #include <sys/param.h> 40 #include <sys/bus.h> 41 #include <sys/endian.h> 42 #include <sys/kernel.h> 43 #include <sys/interrupt.h> 44 #include <sys/mbuf.h> 45 #include <sys/malloc.h> 46 #include <sys/queue.h> 47 #include <sys/rman.h> 48 #include <sys/serialize.h> 49 #include <sys/socket.h> 50 #include <sys/sockio.h> 51 #include <sys/sysctl.h> 52 53 #include <netinet/ip.h> 54 #include <netinet/tcp.h> 55 56 #include <net/bpf.h> 57 #include <net/ethernet.h> 58 #include <net/if.h> 59 #include <net/if_arp.h> 60 #include <net/if_dl.h> 61 #include <net/if_media.h> 62 #include <net/if_poll.h> 63 #include <net/if_types.h> 64 #include <net/ifq_var.h> 65 #include <net/toeplitz.h> 66 #include <net/toeplitz2.h> 67 #include <net/vlan/if_vlan_var.h> 68 #include <net/vlan/if_vlan_ether.h> 69 70 #include <dev/netif/mii_layer/mii.h> 71 #include <dev/netif/mii_layer/miivar.h> 72 #include <dev/netif/mii_layer/brgphyreg.h> 73 74 #include "pcidevs.h" 75 #include <bus/pci/pcireg.h> 76 #include <bus/pci/pcivar.h> 77 78 #include <dev/netif/bge/if_bgereg.h> 79 #include <dev/netif/bnx/if_bnxvar.h> 80 81 /* "device miibus" required. See GENERIC if you get errors here. */ 82 #include "miibus_if.h" 83 84 #define BNX_CSUM_FEATURES (CSUM_IP | CSUM_TCP | CSUM_UDP) 85 86 #define BNX_RESET_SHUTDOWN 0 87 #define BNX_RESET_START 1 88 #define BNX_RESET_SUSPEND 2 89 90 #define BNX_INTR_CKINTVL ((10 * hz) / 1000) /* 10ms */ 91 92 #ifdef BNX_RSS_DEBUG 93 #define BNX_RSS_DPRINTF(sc, lvl, fmt, ...) \ 94 do { \ 95 if (sc->bnx_rss_debug >= lvl) \ 96 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \ 97 } while (0) 98 #else /* !BNX_RSS_DEBUG */ 99 #define BNX_RSS_DPRINTF(sc, lvl, fmt, ...) ((void)0) 100 #endif /* BNX_RSS_DEBUG */ 101 102 static const struct bnx_type { 103 uint16_t bnx_vid; 104 uint16_t bnx_did; 105 char *bnx_name; 106 } bnx_devs[] = { 107 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717, 108 "Broadcom BCM5717 Gigabit Ethernet" }, 109 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717C, 110 "Broadcom BCM5717C Gigabit Ethernet" }, 111 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5718, 112 "Broadcom BCM5718 Gigabit Ethernet" }, 113 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5719, 114 "Broadcom BCM5719 Gigabit Ethernet" }, 115 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5720_ALT, 116 "Broadcom BCM5720 Gigabit Ethernet" }, 117 118 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5725, 119 "Broadcom BCM5725 Gigabit Ethernet" }, 120 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5727, 121 "Broadcom BCM5727 Gigabit Ethernet" }, 122 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5762, 123 "Broadcom BCM5762 Gigabit Ethernet" }, 124 125 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57761, 126 "Broadcom BCM57761 Gigabit Ethernet" }, 127 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57762, 128 "Broadcom BCM57762 Gigabit Ethernet" }, 129 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57765, 130 "Broadcom BCM57765 Gigabit Ethernet" }, 131 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57766, 132 "Broadcom BCM57766 Gigabit Ethernet" }, 133 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57781, 134 "Broadcom BCM57781 Gigabit Ethernet" }, 135 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57782, 136 "Broadcom BCM57782 Gigabit Ethernet" }, 137 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57785, 138 "Broadcom BCM57785 Gigabit Ethernet" }, 139 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57786, 140 "Broadcom BCM57786 Gigabit Ethernet" }, 141 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57791, 142 "Broadcom BCM57791 Fast Ethernet" }, 143 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57795, 144 "Broadcom BCM57795 Fast Ethernet" }, 145 146 { 0, 0, NULL } 147 }; 148 149 static const int bnx_tx_mailbox[BNX_TX_RING_MAX] = { 150 BGE_MBX_TX_HOST_PROD0_LO, 151 BGE_MBX_TX_HOST_PROD0_HI, 152 BGE_MBX_TX_HOST_PROD1_LO, 153 BGE_MBX_TX_HOST_PROD1_HI 154 }; 155 156 #define BNX_IS_JUMBO_CAPABLE(sc) ((sc)->bnx_flags & BNX_FLAG_JUMBO) 157 #define BNX_IS_5717_PLUS(sc) ((sc)->bnx_flags & BNX_FLAG_5717_PLUS) 158 #define BNX_IS_57765_PLUS(sc) ((sc)->bnx_flags & BNX_FLAG_57765_PLUS) 159 #define BNX_IS_57765_FAMILY(sc) \ 160 ((sc)->bnx_flags & BNX_FLAG_57765_FAMILY) 161 162 typedef int (*bnx_eaddr_fcn_t)(struct bnx_softc *, uint8_t[]); 163 164 static int bnx_probe(device_t); 165 static int bnx_attach(device_t); 166 static int bnx_detach(device_t); 167 static void bnx_shutdown(device_t); 168 static int bnx_suspend(device_t); 169 static int bnx_resume(device_t); 170 static int bnx_miibus_readreg(device_t, int, int); 171 static int bnx_miibus_writereg(device_t, int, int, int); 172 static void bnx_miibus_statchg(device_t); 173 174 static int bnx_handle_status(struct bnx_softc *); 175 #ifdef IFPOLL_ENABLE 176 static void bnx_npoll(struct ifnet *, struct ifpoll_info *); 177 static void bnx_npoll_rx(struct ifnet *, void *, int); 178 static void bnx_npoll_tx(struct ifnet *, void *, int); 179 static void bnx_npoll_tx_notag(struct ifnet *, void *, int); 180 static void bnx_npoll_status(struct ifnet *); 181 static void bnx_npoll_status_notag(struct ifnet *); 182 #endif 183 static void bnx_intr_legacy(void *); 184 static void bnx_msi(void *); 185 static void bnx_intr(struct bnx_softc *); 186 static void bnx_msix_status(void *); 187 static void bnx_msix_tx_status(void *); 188 static void bnx_msix_rx(void *); 189 static void bnx_msix_rxtx(void *); 190 static void bnx_enable_intr(struct bnx_softc *); 191 static void bnx_disable_intr(struct bnx_softc *); 192 static void bnx_txeof(struct bnx_tx_ring *, uint16_t); 193 static void bnx_rxeof(struct bnx_rx_ret_ring *, uint16_t, int); 194 static int bnx_alloc_intr(struct bnx_softc *); 195 static int bnx_setup_intr(struct bnx_softc *); 196 static void bnx_free_intr(struct bnx_softc *); 197 static void bnx_teardown_intr(struct bnx_softc *, int); 198 static int bnx_alloc_msix(struct bnx_softc *); 199 static void bnx_free_msix(struct bnx_softc *, boolean_t); 200 static void bnx_check_intr_rxtx(void *); 201 static void bnx_check_intr_rx(void *); 202 static void bnx_check_intr_tx(void *); 203 static void bnx_rx_std_refill_ithread(void *); 204 static void bnx_rx_std_refill(void *, void *); 205 static void bnx_rx_std_refill_sched_ipi(void *); 206 static void bnx_rx_std_refill_stop(void *); 207 static void bnx_rx_std_refill_sched(struct bnx_rx_ret_ring *, 208 struct bnx_rx_std_ring *); 209 210 static void bnx_start(struct ifnet *, struct ifaltq_subque *); 211 static int bnx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 212 static void bnx_init(void *); 213 static void bnx_stop(struct bnx_softc *); 214 static void bnx_watchdog(struct ifaltq_subque *); 215 static int bnx_ifmedia_upd(struct ifnet *); 216 static void bnx_ifmedia_sts(struct ifnet *, struct ifmediareq *); 217 static void bnx_tick(void *); 218 static void bnx_serialize(struct ifnet *, enum ifnet_serialize); 219 static void bnx_deserialize(struct ifnet *, enum ifnet_serialize); 220 static int bnx_tryserialize(struct ifnet *, enum ifnet_serialize); 221 #ifdef INVARIANTS 222 static void bnx_serialize_assert(struct ifnet *, enum ifnet_serialize, 223 boolean_t); 224 #endif 225 static void bnx_serialize_skipmain(struct bnx_softc *); 226 static void bnx_deserialize_skipmain(struct bnx_softc *sc); 227 228 static int bnx_alloc_jumbo_mem(struct bnx_softc *); 229 static void bnx_free_jumbo_mem(struct bnx_softc *); 230 static struct bnx_jslot 231 *bnx_jalloc(struct bnx_softc *); 232 static void bnx_jfree(void *); 233 static void bnx_jref(void *); 234 static int bnx_newbuf_std(struct bnx_rx_ret_ring *, int, int); 235 static int bnx_newbuf_jumbo(struct bnx_softc *, int, int); 236 static void bnx_setup_rxdesc_std(struct bnx_rx_std_ring *, int); 237 static void bnx_setup_rxdesc_jumbo(struct bnx_softc *, int); 238 static int bnx_init_rx_ring_std(struct bnx_rx_std_ring *); 239 static void bnx_free_rx_ring_std(struct bnx_rx_std_ring *); 240 static int bnx_init_rx_ring_jumbo(struct bnx_softc *); 241 static void bnx_free_rx_ring_jumbo(struct bnx_softc *); 242 static void bnx_free_tx_ring(struct bnx_tx_ring *); 243 static int bnx_init_tx_ring(struct bnx_tx_ring *); 244 static int bnx_create_tx_ring(struct bnx_tx_ring *); 245 static void bnx_destroy_tx_ring(struct bnx_tx_ring *); 246 static int bnx_create_rx_ret_ring(struct bnx_rx_ret_ring *); 247 static void bnx_destroy_rx_ret_ring(struct bnx_rx_ret_ring *); 248 static int bnx_dma_alloc(device_t); 249 static void bnx_dma_free(struct bnx_softc *); 250 static int bnx_dma_block_alloc(struct bnx_softc *, bus_size_t, 251 bus_dma_tag_t *, bus_dmamap_t *, void **, bus_addr_t *); 252 static void bnx_dma_block_free(bus_dma_tag_t, bus_dmamap_t, void *); 253 static struct mbuf * 254 bnx_defrag_shortdma(struct mbuf *); 255 static int bnx_encap(struct bnx_tx_ring *, struct mbuf **, 256 uint32_t *, int *); 257 static int bnx_setup_tso(struct bnx_tx_ring *, struct mbuf **, 258 uint16_t *, uint16_t *); 259 static void bnx_setup_serialize(struct bnx_softc *); 260 static void bnx_set_tick_cpuid(struct bnx_softc *, boolean_t); 261 static void bnx_setup_ring_cnt(struct bnx_softc *); 262 263 static struct pktinfo *bnx_rss_info(struct pktinfo *, 264 const struct bge_rx_bd *); 265 static void bnx_init_rss(struct bnx_softc *); 266 static void bnx_reset(struct bnx_softc *); 267 static int bnx_chipinit(struct bnx_softc *); 268 static int bnx_blockinit(struct bnx_softc *); 269 static void bnx_stop_block(struct bnx_softc *, bus_size_t, uint32_t); 270 static void bnx_enable_msi(struct bnx_softc *, boolean_t); 271 static void bnx_setmulti(struct bnx_softc *); 272 static void bnx_setpromisc(struct bnx_softc *); 273 static void bnx_stats_update_regs(struct bnx_softc *); 274 static uint32_t bnx_dma_swap_options(struct bnx_softc *); 275 276 static uint32_t bnx_readmem_ind(struct bnx_softc *, uint32_t); 277 static void bnx_writemem_ind(struct bnx_softc *, uint32_t, uint32_t); 278 #ifdef notdef 279 static uint32_t bnx_readreg_ind(struct bnx_softc *, uint32_t); 280 #endif 281 static void bnx_writemem_direct(struct bnx_softc *, uint32_t, uint32_t); 282 static void bnx_writembx(struct bnx_softc *, int, int); 283 static int bnx_read_nvram(struct bnx_softc *, caddr_t, int, int); 284 static uint8_t bnx_eeprom_getbyte(struct bnx_softc *, uint32_t, uint8_t *); 285 static int bnx_read_eeprom(struct bnx_softc *, caddr_t, uint32_t, size_t); 286 287 static void bnx_tbi_link_upd(struct bnx_softc *, uint32_t); 288 static void bnx_copper_link_upd(struct bnx_softc *, uint32_t); 289 static void bnx_autopoll_link_upd(struct bnx_softc *, uint32_t); 290 static void bnx_link_poll(struct bnx_softc *); 291 292 static int bnx_get_eaddr_mem(struct bnx_softc *, uint8_t[]); 293 static int bnx_get_eaddr_nvram(struct bnx_softc *, uint8_t[]); 294 static int bnx_get_eaddr_eeprom(struct bnx_softc *, uint8_t[]); 295 static int bnx_get_eaddr(struct bnx_softc *, uint8_t[]); 296 297 static void bnx_coal_change(struct bnx_softc *); 298 static int bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS); 299 static int bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS); 300 static int bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS); 301 static int bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS); 302 static int bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS); 303 static int bnx_sysctl_rx_coal_bds_poll(SYSCTL_HANDLER_ARGS); 304 static int bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS); 305 static int bnx_sysctl_tx_coal_bds_poll(SYSCTL_HANDLER_ARGS); 306 static int bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS); 307 static int bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS); 308 static int bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *, 309 int, int, uint32_t); 310 static int bnx_sysctl_std_refill(SYSCTL_HANDLER_ARGS); 311 312 static void bnx_sig_post_reset(struct bnx_softc *, int); 313 static void bnx_sig_pre_reset(struct bnx_softc *, int); 314 static void bnx_ape_lock_init(struct bnx_softc *); 315 static void bnx_ape_read_fw_ver(struct bnx_softc *); 316 static int bnx_ape_lock(struct bnx_softc *, int); 317 static void bnx_ape_unlock(struct bnx_softc *, int); 318 static void bnx_ape_send_event(struct bnx_softc *, uint32_t); 319 static void bnx_ape_driver_state_change(struct bnx_softc *, int); 320 321 static int bnx_msi_enable = 1; 322 static int bnx_msix_enable = 1; 323 324 static int bnx_rx_rings = 0; /* auto */ 325 static int bnx_tx_rings = 0; /* auto */ 326 327 TUNABLE_INT("hw.bnx.msi.enable", &bnx_msi_enable); 328 TUNABLE_INT("hw.bnx.msix.enable", &bnx_msix_enable); 329 TUNABLE_INT("hw.bnx.rx_rings", &bnx_rx_rings); 330 TUNABLE_INT("hw.bnx.tx_rings", &bnx_tx_rings); 331 332 static device_method_t bnx_methods[] = { 333 /* Device interface */ 334 DEVMETHOD(device_probe, bnx_probe), 335 DEVMETHOD(device_attach, bnx_attach), 336 DEVMETHOD(device_detach, bnx_detach), 337 DEVMETHOD(device_shutdown, bnx_shutdown), 338 DEVMETHOD(device_suspend, bnx_suspend), 339 DEVMETHOD(device_resume, bnx_resume), 340 341 /* bus interface */ 342 DEVMETHOD(bus_print_child, bus_generic_print_child), 343 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 344 345 /* MII interface */ 346 DEVMETHOD(miibus_readreg, bnx_miibus_readreg), 347 DEVMETHOD(miibus_writereg, bnx_miibus_writereg), 348 DEVMETHOD(miibus_statchg, bnx_miibus_statchg), 349 350 DEVMETHOD_END 351 }; 352 353 static DEFINE_CLASS_0(bnx, bnx_driver, bnx_methods, sizeof(struct bnx_softc)); 354 static devclass_t bnx_devclass; 355 356 DECLARE_DUMMY_MODULE(if_bnx); 357 MODULE_DEPEND(if_bnx, miibus, 1, 1, 1); 358 DRIVER_MODULE(if_bnx, pci, bnx_driver, bnx_devclass, NULL, NULL); 359 DRIVER_MODULE(miibus, bnx, miibus_driver, miibus_devclass, NULL, NULL); 360 361 static uint32_t 362 bnx_readmem_ind(struct bnx_softc *sc, uint32_t off) 363 { 364 device_t dev = sc->bnx_dev; 365 uint32_t val; 366 367 pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4); 368 val = pci_read_config(dev, BGE_PCI_MEMWIN_DATA, 4); 369 pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4); 370 return (val); 371 } 372 373 static void 374 bnx_writemem_ind(struct bnx_softc *sc, uint32_t off, uint32_t val) 375 { 376 device_t dev = sc->bnx_dev; 377 378 pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4); 379 pci_write_config(dev, BGE_PCI_MEMWIN_DATA, val, 4); 380 pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4); 381 } 382 383 static void 384 bnx_writemem_direct(struct bnx_softc *sc, uint32_t off, uint32_t val) 385 { 386 CSR_WRITE_4(sc, off, val); 387 } 388 389 static void 390 bnx_writembx(struct bnx_softc *sc, int off, int val) 391 { 392 CSR_WRITE_4(sc, off, val); 393 } 394 395 /* 396 * Read a sequence of bytes from NVRAM. 397 */ 398 static int 399 bnx_read_nvram(struct bnx_softc *sc, caddr_t dest, int off, int cnt) 400 { 401 return (1); 402 } 403 404 /* 405 * Read a byte of data stored in the EEPROM at address 'addr.' The 406 * BCM570x supports both the traditional bitbang interface and an 407 * auto access interface for reading the EEPROM. We use the auto 408 * access method. 409 */ 410 static uint8_t 411 bnx_eeprom_getbyte(struct bnx_softc *sc, uint32_t addr, uint8_t *dest) 412 { 413 int i; 414 uint32_t byte = 0; 415 416 /* 417 * Enable use of auto EEPROM access so we can avoid 418 * having to use the bitbang method. 419 */ 420 BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_AUTO_EEPROM); 421 422 /* Reset the EEPROM, load the clock period. */ 423 CSR_WRITE_4(sc, BGE_EE_ADDR, 424 BGE_EEADDR_RESET|BGE_EEHALFCLK(BGE_HALFCLK_384SCL)); 425 DELAY(20); 426 427 /* Issue the read EEPROM command. */ 428 CSR_WRITE_4(sc, BGE_EE_ADDR, BGE_EE_READCMD | addr); 429 430 /* Wait for completion */ 431 for(i = 0; i < BNX_TIMEOUT * 10; i++) { 432 DELAY(10); 433 if (CSR_READ_4(sc, BGE_EE_ADDR) & BGE_EEADDR_DONE) 434 break; 435 } 436 437 if (i == BNX_TIMEOUT) { 438 if_printf(&sc->arpcom.ac_if, "eeprom read timed out\n"); 439 return(1); 440 } 441 442 /* Get result. */ 443 byte = CSR_READ_4(sc, BGE_EE_DATA); 444 445 *dest = (byte >> ((addr % 4) * 8)) & 0xFF; 446 447 return(0); 448 } 449 450 /* 451 * Read a sequence of bytes from the EEPROM. 452 */ 453 static int 454 bnx_read_eeprom(struct bnx_softc *sc, caddr_t dest, uint32_t off, size_t len) 455 { 456 size_t i; 457 int err; 458 uint8_t byte; 459 460 for (byte = 0, err = 0, i = 0; i < len; i++) { 461 err = bnx_eeprom_getbyte(sc, off + i, &byte); 462 if (err) 463 break; 464 *(dest + i) = byte; 465 } 466 467 return(err ? 1 : 0); 468 } 469 470 static int 471 bnx_miibus_readreg(device_t dev, int phy, int reg) 472 { 473 struct bnx_softc *sc = device_get_softc(dev); 474 uint32_t val; 475 int i; 476 477 KASSERT(phy == sc->bnx_phyno, 478 ("invalid phyno %d, should be %d", phy, sc->bnx_phyno)); 479 480 if (bnx_ape_lock(sc, sc->bnx_phy_ape_lock) != 0) 481 return 0; 482 483 /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */ 484 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) { 485 CSR_WRITE_4(sc, BGE_MI_MODE, 486 sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL); 487 DELAY(80); 488 } 489 490 CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_READ | BGE_MICOMM_BUSY | 491 BGE_MIPHY(phy) | BGE_MIREG(reg)); 492 493 /* Poll for the PHY register access to complete. */ 494 for (i = 0; i < BNX_TIMEOUT; i++) { 495 DELAY(10); 496 val = CSR_READ_4(sc, BGE_MI_COMM); 497 if ((val & BGE_MICOMM_BUSY) == 0) { 498 DELAY(5); 499 val = CSR_READ_4(sc, BGE_MI_COMM); 500 break; 501 } 502 } 503 if (i == BNX_TIMEOUT) { 504 if_printf(&sc->arpcom.ac_if, "PHY read timed out " 505 "(phy %d, reg %d, val 0x%08x)\n", phy, reg, val); 506 val = 0; 507 } 508 509 /* Restore the autopoll bit if necessary. */ 510 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) { 511 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode); 512 DELAY(80); 513 } 514 515 bnx_ape_unlock(sc, sc->bnx_phy_ape_lock); 516 517 if (val & BGE_MICOMM_READFAIL) 518 return 0; 519 520 return (val & 0xFFFF); 521 } 522 523 static int 524 bnx_miibus_writereg(device_t dev, int phy, int reg, int val) 525 { 526 struct bnx_softc *sc = device_get_softc(dev); 527 int i; 528 529 KASSERT(phy == sc->bnx_phyno, 530 ("invalid phyno %d, should be %d", phy, sc->bnx_phyno)); 531 532 if (bnx_ape_lock(sc, sc->bnx_phy_ape_lock) != 0) 533 return 0; 534 535 /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */ 536 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) { 537 CSR_WRITE_4(sc, BGE_MI_MODE, 538 sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL); 539 DELAY(80); 540 } 541 542 CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_WRITE | BGE_MICOMM_BUSY | 543 BGE_MIPHY(phy) | BGE_MIREG(reg) | val); 544 545 for (i = 0; i < BNX_TIMEOUT; i++) { 546 DELAY(10); 547 if (!(CSR_READ_4(sc, BGE_MI_COMM) & BGE_MICOMM_BUSY)) { 548 DELAY(5); 549 CSR_READ_4(sc, BGE_MI_COMM); /* dummy read */ 550 break; 551 } 552 } 553 if (i == BNX_TIMEOUT) { 554 if_printf(&sc->arpcom.ac_if, "PHY write timed out " 555 "(phy %d, reg %d, val %d)\n", phy, reg, val); 556 } 557 558 /* Restore the autopoll bit if necessary. */ 559 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) { 560 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode); 561 DELAY(80); 562 } 563 564 bnx_ape_unlock(sc, sc->bnx_phy_ape_lock); 565 566 return 0; 567 } 568 569 static void 570 bnx_miibus_statchg(device_t dev) 571 { 572 struct bnx_softc *sc; 573 struct mii_data *mii; 574 uint32_t mac_mode; 575 576 sc = device_get_softc(dev); 577 if ((sc->arpcom.ac_if.if_flags & IFF_RUNNING) == 0) 578 return; 579 580 mii = device_get_softc(sc->bnx_miibus); 581 582 if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) == 583 (IFM_ACTIVE | IFM_AVALID)) { 584 switch (IFM_SUBTYPE(mii->mii_media_active)) { 585 case IFM_10_T: 586 case IFM_100_TX: 587 sc->bnx_link = 1; 588 break; 589 case IFM_1000_T: 590 case IFM_1000_SX: 591 case IFM_2500_SX: 592 sc->bnx_link = 1; 593 break; 594 default: 595 sc->bnx_link = 0; 596 break; 597 } 598 } else { 599 sc->bnx_link = 0; 600 } 601 if (sc->bnx_link == 0) 602 return; 603 604 /* 605 * APE firmware touches these registers to keep the MAC 606 * connected to the outside world. Try to keep the 607 * accesses atomic. 608 */ 609 610 mac_mode = CSR_READ_4(sc, BGE_MAC_MODE) & 611 ~(BGE_MACMODE_PORTMODE | BGE_MACMODE_HALF_DUPLEX); 612 613 if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T || 614 IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX) 615 mac_mode |= BGE_PORTMODE_GMII; 616 else 617 mac_mode |= BGE_PORTMODE_MII; 618 619 if ((mii->mii_media_active & IFM_GMASK) != IFM_FDX) 620 mac_mode |= BGE_MACMODE_HALF_DUPLEX; 621 622 CSR_WRITE_4(sc, BGE_MAC_MODE, mac_mode); 623 DELAY(40); 624 } 625 626 /* 627 * Memory management for jumbo frames. 628 */ 629 static int 630 bnx_alloc_jumbo_mem(struct bnx_softc *sc) 631 { 632 struct ifnet *ifp = &sc->arpcom.ac_if; 633 struct bnx_jslot *entry; 634 uint8_t *ptr; 635 bus_addr_t paddr; 636 int i, error; 637 638 /* 639 * Create tag for jumbo mbufs. 640 * This is really a bit of a kludge. We allocate a special 641 * jumbo buffer pool which (thanks to the way our DMA 642 * memory allocation works) will consist of contiguous 643 * pages. This means that even though a jumbo buffer might 644 * be larger than a page size, we don't really need to 645 * map it into more than one DMA segment. However, the 646 * default mbuf tag will result in multi-segment mappings, 647 * so we have to create a special jumbo mbuf tag that 648 * lets us get away with mapping the jumbo buffers as 649 * a single segment. I think eventually the driver should 650 * be changed so that it uses ordinary mbufs and cluster 651 * buffers, i.e. jumbo frames can span multiple DMA 652 * descriptors. But that's a project for another day. 653 */ 654 655 /* 656 * Create DMA stuffs for jumbo RX ring. 657 */ 658 error = bnx_dma_block_alloc(sc, BGE_JUMBO_RX_RING_SZ, 659 &sc->bnx_cdata.bnx_rx_jumbo_ring_tag, 660 &sc->bnx_cdata.bnx_rx_jumbo_ring_map, 661 (void *)&sc->bnx_ldata.bnx_rx_jumbo_ring, 662 &sc->bnx_ldata.bnx_rx_jumbo_ring_paddr); 663 if (error) { 664 if_printf(ifp, "could not create jumbo RX ring\n"); 665 return error; 666 } 667 668 /* 669 * Create DMA stuffs for jumbo buffer block. 670 */ 671 error = bnx_dma_block_alloc(sc, BNX_JMEM, 672 &sc->bnx_cdata.bnx_jumbo_tag, 673 &sc->bnx_cdata.bnx_jumbo_map, 674 (void **)&sc->bnx_ldata.bnx_jumbo_buf, 675 &paddr); 676 if (error) { 677 if_printf(ifp, "could not create jumbo buffer\n"); 678 return error; 679 } 680 681 SLIST_INIT(&sc->bnx_jfree_listhead); 682 683 /* 684 * Now divide it up into 9K pieces and save the addresses 685 * in an array. Note that we play an evil trick here by using 686 * the first few bytes in the buffer to hold the the address 687 * of the softc structure for this interface. This is because 688 * bnx_jfree() needs it, but it is called by the mbuf management 689 * code which will not pass it to us explicitly. 690 */ 691 for (i = 0, ptr = sc->bnx_ldata.bnx_jumbo_buf; i < BNX_JSLOTS; i++) { 692 entry = &sc->bnx_cdata.bnx_jslots[i]; 693 entry->bnx_sc = sc; 694 entry->bnx_buf = ptr; 695 entry->bnx_paddr = paddr; 696 entry->bnx_inuse = 0; 697 entry->bnx_slot = i; 698 SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead, entry, jslot_link); 699 700 ptr += BNX_JLEN; 701 paddr += BNX_JLEN; 702 } 703 return 0; 704 } 705 706 static void 707 bnx_free_jumbo_mem(struct bnx_softc *sc) 708 { 709 /* Destroy jumbo RX ring. */ 710 bnx_dma_block_free(sc->bnx_cdata.bnx_rx_jumbo_ring_tag, 711 sc->bnx_cdata.bnx_rx_jumbo_ring_map, 712 sc->bnx_ldata.bnx_rx_jumbo_ring); 713 714 /* Destroy jumbo buffer block. */ 715 bnx_dma_block_free(sc->bnx_cdata.bnx_jumbo_tag, 716 sc->bnx_cdata.bnx_jumbo_map, 717 sc->bnx_ldata.bnx_jumbo_buf); 718 } 719 720 /* 721 * Allocate a jumbo buffer. 722 */ 723 static struct bnx_jslot * 724 bnx_jalloc(struct bnx_softc *sc) 725 { 726 struct bnx_jslot *entry; 727 728 lwkt_serialize_enter(&sc->bnx_jslot_serializer); 729 entry = SLIST_FIRST(&sc->bnx_jfree_listhead); 730 if (entry) { 731 SLIST_REMOVE_HEAD(&sc->bnx_jfree_listhead, jslot_link); 732 entry->bnx_inuse = 1; 733 } else { 734 if_printf(&sc->arpcom.ac_if, "no free jumbo buffers\n"); 735 } 736 lwkt_serialize_exit(&sc->bnx_jslot_serializer); 737 return(entry); 738 } 739 740 /* 741 * Adjust usage count on a jumbo buffer. 742 */ 743 static void 744 bnx_jref(void *arg) 745 { 746 struct bnx_jslot *entry = (struct bnx_jslot *)arg; 747 struct bnx_softc *sc = entry->bnx_sc; 748 749 if (sc == NULL) 750 panic("bnx_jref: can't find softc pointer!"); 751 752 if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) { 753 panic("bnx_jref: asked to reference buffer " 754 "that we don't manage!"); 755 } else if (entry->bnx_inuse == 0) { 756 panic("bnx_jref: buffer already free!"); 757 } else { 758 atomic_add_int(&entry->bnx_inuse, 1); 759 } 760 } 761 762 /* 763 * Release a jumbo buffer. 764 */ 765 static void 766 bnx_jfree(void *arg) 767 { 768 struct bnx_jslot *entry = (struct bnx_jslot *)arg; 769 struct bnx_softc *sc = entry->bnx_sc; 770 771 if (sc == NULL) 772 panic("bnx_jfree: can't find softc pointer!"); 773 774 if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) { 775 panic("bnx_jfree: asked to free buffer that we don't manage!"); 776 } else if (entry->bnx_inuse == 0) { 777 panic("bnx_jfree: buffer already free!"); 778 } else { 779 /* 780 * Possible MP race to 0, use the serializer. The atomic insn 781 * is still needed for races against bnx_jref(). 782 */ 783 lwkt_serialize_enter(&sc->bnx_jslot_serializer); 784 atomic_subtract_int(&entry->bnx_inuse, 1); 785 if (entry->bnx_inuse == 0) { 786 SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead, 787 entry, jslot_link); 788 } 789 lwkt_serialize_exit(&sc->bnx_jslot_serializer); 790 } 791 } 792 793 794 /* 795 * Intialize a standard receive ring descriptor. 796 */ 797 static int 798 bnx_newbuf_std(struct bnx_rx_ret_ring *ret, int i, int init) 799 { 800 struct mbuf *m_new = NULL; 801 bus_dma_segment_t seg; 802 bus_dmamap_t map; 803 int error, nsegs; 804 struct bnx_rx_buf *rb; 805 806 rb = &ret->bnx_std->bnx_rx_std_buf[i]; 807 KASSERT(!rb->bnx_rx_refilled, ("RX buf %dth has been refilled", i)); 808 809 m_new = m_getcl(init ? M_WAITOK : M_NOWAIT, MT_DATA, M_PKTHDR); 810 if (m_new == NULL) { 811 error = ENOBUFS; 812 goto back; 813 } 814 m_new->m_len = m_new->m_pkthdr.len = MCLBYTES; 815 m_adj(m_new, ETHER_ALIGN); 816 817 error = bus_dmamap_load_mbuf_segment(ret->bnx_rx_mtag, 818 ret->bnx_rx_tmpmap, m_new, &seg, 1, &nsegs, BUS_DMA_NOWAIT); 819 if (error) { 820 m_freem(m_new); 821 goto back; 822 } 823 824 if (!init) { 825 bus_dmamap_sync(ret->bnx_rx_mtag, rb->bnx_rx_dmamap, 826 BUS_DMASYNC_POSTREAD); 827 bus_dmamap_unload(ret->bnx_rx_mtag, rb->bnx_rx_dmamap); 828 } 829 830 map = ret->bnx_rx_tmpmap; 831 ret->bnx_rx_tmpmap = rb->bnx_rx_dmamap; 832 833 rb->bnx_rx_dmamap = map; 834 rb->bnx_rx_mbuf = m_new; 835 rb->bnx_rx_paddr = seg.ds_addr; 836 rb->bnx_rx_len = m_new->m_len; 837 back: 838 cpu_sfence(); 839 rb->bnx_rx_refilled = 1; 840 return error; 841 } 842 843 static void 844 bnx_setup_rxdesc_std(struct bnx_rx_std_ring *std, int i) 845 { 846 struct bnx_rx_buf *rb; 847 struct bge_rx_bd *r; 848 bus_addr_t paddr; 849 int len; 850 851 rb = &std->bnx_rx_std_buf[i]; 852 KASSERT(rb->bnx_rx_refilled, ("RX buf %dth is not refilled", i)); 853 854 paddr = rb->bnx_rx_paddr; 855 len = rb->bnx_rx_len; 856 857 cpu_mfence(); 858 859 rb->bnx_rx_refilled = 0; 860 861 r = &std->bnx_rx_std_ring[i]; 862 r->bge_addr.bge_addr_lo = BGE_ADDR_LO(paddr); 863 r->bge_addr.bge_addr_hi = BGE_ADDR_HI(paddr); 864 r->bge_len = len; 865 r->bge_idx = i; 866 r->bge_flags = BGE_RXBDFLAG_END; 867 } 868 869 /* 870 * Initialize a jumbo receive ring descriptor. This allocates 871 * a jumbo buffer from the pool managed internally by the driver. 872 */ 873 static int 874 bnx_newbuf_jumbo(struct bnx_softc *sc, int i, int init) 875 { 876 struct mbuf *m_new = NULL; 877 struct bnx_jslot *buf; 878 bus_addr_t paddr; 879 880 /* Allocate the mbuf. */ 881 MGETHDR(m_new, init ? M_WAITOK : M_NOWAIT, MT_DATA); 882 if (m_new == NULL) 883 return ENOBUFS; 884 885 /* Allocate the jumbo buffer */ 886 buf = bnx_jalloc(sc); 887 if (buf == NULL) { 888 m_freem(m_new); 889 return ENOBUFS; 890 } 891 892 /* Attach the buffer to the mbuf. */ 893 m_new->m_ext.ext_arg = buf; 894 m_new->m_ext.ext_buf = buf->bnx_buf; 895 m_new->m_ext.ext_free = bnx_jfree; 896 m_new->m_ext.ext_ref = bnx_jref; 897 m_new->m_ext.ext_size = BNX_JUMBO_FRAMELEN; 898 899 m_new->m_flags |= M_EXT; 900 901 m_new->m_data = m_new->m_ext.ext_buf; 902 m_new->m_len = m_new->m_pkthdr.len = m_new->m_ext.ext_size; 903 904 paddr = buf->bnx_paddr; 905 m_adj(m_new, ETHER_ALIGN); 906 paddr += ETHER_ALIGN; 907 908 /* Save necessary information */ 909 sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_rx_mbuf = m_new; 910 sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_rx_paddr = paddr; 911 912 /* Set up the descriptor. */ 913 bnx_setup_rxdesc_jumbo(sc, i); 914 return 0; 915 } 916 917 static void 918 bnx_setup_rxdesc_jumbo(struct bnx_softc *sc, int i) 919 { 920 struct bge_rx_bd *r; 921 struct bnx_rx_buf *rc; 922 923 r = &sc->bnx_ldata.bnx_rx_jumbo_ring[i]; 924 rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i]; 925 926 r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_rx_paddr); 927 r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_rx_paddr); 928 r->bge_len = rc->bnx_rx_mbuf->m_len; 929 r->bge_idx = i; 930 r->bge_flags = BGE_RXBDFLAG_END|BGE_RXBDFLAG_JUMBO_RING; 931 } 932 933 static int 934 bnx_init_rx_ring_std(struct bnx_rx_std_ring *std) 935 { 936 int i, error; 937 938 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) { 939 /* Use the first RX return ring's tmp RX mbuf DMA map */ 940 error = bnx_newbuf_std(&std->bnx_sc->bnx_rx_ret_ring[0], i, 1); 941 if (error) 942 return error; 943 bnx_setup_rxdesc_std(std, i); 944 } 945 946 std->bnx_rx_std_used = 0; 947 std->bnx_rx_std_refill = 0; 948 std->bnx_rx_std_running = 0; 949 cpu_sfence(); 950 lwkt_serialize_handler_enable(&std->bnx_rx_std_serialize); 951 952 std->bnx_rx_std = BGE_STD_RX_RING_CNT - 1; 953 bnx_writembx(std->bnx_sc, BGE_MBX_RX_STD_PROD_LO, std->bnx_rx_std); 954 955 return(0); 956 } 957 958 static void 959 bnx_free_rx_ring_std(struct bnx_rx_std_ring *std) 960 { 961 int i; 962 963 lwkt_serialize_handler_disable(&std->bnx_rx_std_serialize); 964 965 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) { 966 struct bnx_rx_buf *rb = &std->bnx_rx_std_buf[i]; 967 968 rb->bnx_rx_refilled = 0; 969 if (rb->bnx_rx_mbuf != NULL) { 970 bus_dmamap_unload(std->bnx_rx_mtag, rb->bnx_rx_dmamap); 971 m_freem(rb->bnx_rx_mbuf); 972 rb->bnx_rx_mbuf = NULL; 973 } 974 bzero(&std->bnx_rx_std_ring[i], sizeof(struct bge_rx_bd)); 975 } 976 } 977 978 static int 979 bnx_init_rx_ring_jumbo(struct bnx_softc *sc) 980 { 981 struct bge_rcb *rcb; 982 int i, error; 983 984 for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) { 985 error = bnx_newbuf_jumbo(sc, i, 1); 986 if (error) 987 return error; 988 } 989 990 sc->bnx_jumbo = BGE_JUMBO_RX_RING_CNT - 1; 991 992 rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb; 993 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0, 0); 994 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags); 995 996 bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bnx_jumbo); 997 998 return(0); 999 } 1000 1001 static void 1002 bnx_free_rx_ring_jumbo(struct bnx_softc *sc) 1003 { 1004 int i; 1005 1006 for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) { 1007 struct bnx_rx_buf *rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i]; 1008 1009 if (rc->bnx_rx_mbuf != NULL) { 1010 m_freem(rc->bnx_rx_mbuf); 1011 rc->bnx_rx_mbuf = NULL; 1012 } 1013 bzero(&sc->bnx_ldata.bnx_rx_jumbo_ring[i], 1014 sizeof(struct bge_rx_bd)); 1015 } 1016 } 1017 1018 static void 1019 bnx_free_tx_ring(struct bnx_tx_ring *txr) 1020 { 1021 int i; 1022 1023 for (i = 0; i < BGE_TX_RING_CNT; i++) { 1024 struct bnx_tx_buf *buf = &txr->bnx_tx_buf[i]; 1025 1026 if (buf->bnx_tx_mbuf != NULL) { 1027 bus_dmamap_unload(txr->bnx_tx_mtag, 1028 buf->bnx_tx_dmamap); 1029 m_freem(buf->bnx_tx_mbuf); 1030 buf->bnx_tx_mbuf = NULL; 1031 } 1032 bzero(&txr->bnx_tx_ring[i], sizeof(struct bge_tx_bd)); 1033 } 1034 txr->bnx_tx_saved_considx = BNX_TXCONS_UNSET; 1035 } 1036 1037 static int 1038 bnx_init_tx_ring(struct bnx_tx_ring *txr) 1039 { 1040 txr->bnx_tx_cnt = 0; 1041 txr->bnx_tx_saved_considx = 0; 1042 txr->bnx_tx_prodidx = 0; 1043 1044 /* Initialize transmit producer index for host-memory send ring. */ 1045 bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, txr->bnx_tx_prodidx); 1046 1047 return(0); 1048 } 1049 1050 static void 1051 bnx_setmulti(struct bnx_softc *sc) 1052 { 1053 struct ifnet *ifp; 1054 struct ifmultiaddr *ifma; 1055 uint32_t hashes[4] = { 0, 0, 0, 0 }; 1056 int h, i; 1057 1058 ifp = &sc->arpcom.ac_if; 1059 1060 if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) { 1061 for (i = 0; i < 4; i++) 1062 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0xFFFFFFFF); 1063 return; 1064 } 1065 1066 /* First, zot all the existing filters. */ 1067 for (i = 0; i < 4; i++) 1068 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0); 1069 1070 /* Now program new ones. */ 1071 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1072 if (ifma->ifma_addr->sa_family != AF_LINK) 1073 continue; 1074 h = ether_crc32_le( 1075 LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1076 ETHER_ADDR_LEN) & 0x7f; 1077 hashes[(h & 0x60) >> 5] |= 1 << (h & 0x1F); 1078 } 1079 1080 for (i = 0; i < 4; i++) 1081 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), hashes[i]); 1082 } 1083 1084 /* 1085 * Do endian, PCI and DMA initialization. Also check the on-board ROM 1086 * self-test results. 1087 */ 1088 static int 1089 bnx_chipinit(struct bnx_softc *sc) 1090 { 1091 uint32_t dma_rw_ctl, mode_ctl; 1092 int i; 1093 1094 /* Set endian type before we access any non-PCI registers. */ 1095 pci_write_config(sc->bnx_dev, BGE_PCI_MISC_CTL, 1096 BGE_INIT | BGE_PCIMISCCTL_TAGGED_STATUS, 4); 1097 1098 /* 1099 * Clear the MAC statistics block in the NIC's 1100 * internal memory. 1101 */ 1102 for (i = BGE_STATS_BLOCK; 1103 i < BGE_STATS_BLOCK_END + 1; i += sizeof(uint32_t)) 1104 BNX_MEMWIN_WRITE(sc, i, 0); 1105 1106 for (i = BGE_STATUS_BLOCK; 1107 i < BGE_STATUS_BLOCK_END + 1; i += sizeof(uint32_t)) 1108 BNX_MEMWIN_WRITE(sc, i, 0); 1109 1110 if (BNX_IS_57765_FAMILY(sc)) { 1111 uint32_t val; 1112 1113 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0) { 1114 mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL); 1115 val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS; 1116 1117 /* Access the lower 1K of PL PCI-E block registers. */ 1118 CSR_WRITE_4(sc, BGE_MODE_CTL, 1119 val | BGE_MODECTL_PCIE_PL_SEL); 1120 1121 val = CSR_READ_4(sc, BGE_PCIE_PL_LO_PHYCTL5); 1122 val |= BGE_PCIE_PL_LO_PHYCTL5_DIS_L2CLKREQ; 1123 CSR_WRITE_4(sc, BGE_PCIE_PL_LO_PHYCTL5, val); 1124 1125 CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl); 1126 } 1127 if (sc->bnx_chiprev != BGE_CHIPREV_57765_AX) { 1128 /* Fix transmit hangs */ 1129 val = CSR_READ_4(sc, BGE_CPMU_PADRNG_CTL); 1130 val |= BGE_CPMU_PADRNG_CTL_RDIV2; 1131 CSR_WRITE_4(sc, BGE_CPMU_PADRNG_CTL, val); 1132 1133 mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL); 1134 val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS; 1135 1136 /* Access the lower 1K of DL PCI-E block registers. */ 1137 CSR_WRITE_4(sc, BGE_MODE_CTL, 1138 val | BGE_MODECTL_PCIE_DL_SEL); 1139 1140 val = CSR_READ_4(sc, BGE_PCIE_DL_LO_FTSMAX); 1141 val &= ~BGE_PCIE_DL_LO_FTSMAX_MASK; 1142 val |= BGE_PCIE_DL_LO_FTSMAX_VAL; 1143 CSR_WRITE_4(sc, BGE_PCIE_DL_LO_FTSMAX, val); 1144 1145 CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl); 1146 } 1147 1148 val = CSR_READ_4(sc, BGE_CPMU_LSPD_10MB_CLK); 1149 val &= ~BGE_CPMU_LSPD_10MB_MACCLK_MASK; 1150 val |= BGE_CPMU_LSPD_10MB_MACCLK_6_25; 1151 CSR_WRITE_4(sc, BGE_CPMU_LSPD_10MB_CLK, val); 1152 } 1153 1154 /* 1155 * Set up the PCI DMA control register. 1156 */ 1157 dma_rw_ctl = pci_read_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, 4); 1158 /* 1159 * Disable 32bytes cache alignment for DMA write to host memory 1160 * 1161 * NOTE: 1162 * 64bytes cache alignment for DMA write to host memory is still 1163 * enabled. 1164 */ 1165 dma_rw_ctl |= BGE_PCIDMARWCTL_DIS_CACHE_ALIGNMENT; 1166 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0) 1167 dma_rw_ctl &= ~BGE_PCIDMARWCTL_CRDRDR_RDMA_MRRS_MSK; 1168 /* 1169 * Enable HW workaround for controllers that misinterpret 1170 * a status tag update and leave interrupts permanently 1171 * disabled. 1172 */ 1173 if (sc->bnx_asicrev != BGE_ASICREV_BCM5717 && 1174 sc->bnx_asicrev != BGE_ASICREV_BCM5762 && 1175 !BNX_IS_57765_FAMILY(sc)) 1176 dma_rw_ctl |= BGE_PCIDMARWCTL_TAGGED_STATUS_WA; 1177 if (bootverbose) { 1178 if_printf(&sc->arpcom.ac_if, "DMA read/write %#x\n", 1179 dma_rw_ctl); 1180 } 1181 pci_write_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, dma_rw_ctl, 4); 1182 1183 /* 1184 * Set up general mode register. 1185 */ 1186 mode_ctl = bnx_dma_swap_options(sc); 1187 if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 || 1188 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 1189 /* Retain Host-2-BMC settings written by APE firmware. */ 1190 mode_ctl |= CSR_READ_4(sc, BGE_MODE_CTL) & 1191 (BGE_MODECTL_BYTESWAP_B2HRX_DATA | 1192 BGE_MODECTL_WORDSWAP_B2HRX_DATA | 1193 BGE_MODECTL_B2HRX_ENABLE | BGE_MODECTL_HTX2B_ENABLE); 1194 } 1195 mode_ctl |= BGE_MODECTL_MAC_ATTN_INTR | 1196 BGE_MODECTL_HOST_SEND_BDS | BGE_MODECTL_TX_NO_PHDR_CSUM; 1197 CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl); 1198 1199 /* 1200 * Disable memory write invalidate. Apparently it is not supported 1201 * properly by these devices. Also ensure that INTx isn't disabled, 1202 * as these chips need it even when using MSI. 1203 */ 1204 PCI_CLRBIT(sc->bnx_dev, BGE_PCI_CMD, 1205 (PCIM_CMD_MWRICEN | PCIM_CMD_INTxDIS), 4); 1206 1207 /* Set the timer prescaler (always 66Mhz) */ 1208 CSR_WRITE_4(sc, BGE_MISC_CFG, 65 << 1/*BGE_32BITTIME_66MHZ*/); 1209 1210 return(0); 1211 } 1212 1213 static int 1214 bnx_blockinit(struct bnx_softc *sc) 1215 { 1216 struct bnx_intr_data *intr; 1217 struct bge_rcb *rcb; 1218 bus_size_t vrcb; 1219 bge_hostaddr taddr; 1220 uint32_t val; 1221 int i, limit; 1222 1223 /* 1224 * Initialize the memory window pointer register so that 1225 * we can access the first 32K of internal NIC RAM. This will 1226 * allow us to set up the TX send ring RCBs and the RX return 1227 * ring RCBs, plus other things which live in NIC memory. 1228 */ 1229 CSR_WRITE_4(sc, BGE_PCI_MEMWIN_BASEADDR, 0); 1230 1231 /* Configure mbuf pool watermarks */ 1232 if (BNX_IS_57765_PLUS(sc)) { 1233 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0); 1234 if (sc->arpcom.ac_if.if_mtu > ETHERMTU) { 1235 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x7e); 1236 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xea); 1237 } else { 1238 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x2a); 1239 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xa0); 1240 } 1241 } else { 1242 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0); 1243 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x10); 1244 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x60); 1245 } 1246 1247 /* Configure DMA resource watermarks */ 1248 CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LOWAT, 5); 1249 CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_HIWAT, 10); 1250 1251 /* Enable buffer manager */ 1252 val = BGE_BMANMODE_ENABLE | BGE_BMANMODE_LOMBUF_ATTN; 1253 /* 1254 * Change the arbitration algorithm of TXMBUF read request to 1255 * round-robin instead of priority based for BCM5719. When 1256 * TXFIFO is almost empty, RDMA will hold its request until 1257 * TXFIFO is not almost empty. 1258 */ 1259 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719) 1260 val |= BGE_BMANMODE_NO_TX_UNDERRUN; 1261 if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 || 1262 sc->bnx_chipid == BGE_CHIPID_BCM5719_A0 || 1263 sc->bnx_chipid == BGE_CHIPID_BCM5720_A0) 1264 val |= BGE_BMANMODE_LOMBUF_ATTN; 1265 CSR_WRITE_4(sc, BGE_BMAN_MODE, val); 1266 1267 /* Poll for buffer manager start indication */ 1268 for (i = 0; i < BNX_TIMEOUT; i++) { 1269 if (CSR_READ_4(sc, BGE_BMAN_MODE) & BGE_BMANMODE_ENABLE) 1270 break; 1271 DELAY(10); 1272 } 1273 1274 if (i == BNX_TIMEOUT) { 1275 if_printf(&sc->arpcom.ac_if, 1276 "buffer manager failed to start\n"); 1277 return(ENXIO); 1278 } 1279 1280 /* Enable flow-through queues */ 1281 CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF); 1282 CSR_WRITE_4(sc, BGE_FTQ_RESET, 0); 1283 1284 /* Wait until queue initialization is complete */ 1285 for (i = 0; i < BNX_TIMEOUT; i++) { 1286 if (CSR_READ_4(sc, BGE_FTQ_RESET) == 0) 1287 break; 1288 DELAY(10); 1289 } 1290 1291 if (i == BNX_TIMEOUT) { 1292 if_printf(&sc->arpcom.ac_if, 1293 "flow-through queue init failed\n"); 1294 return(ENXIO); 1295 } 1296 1297 /* 1298 * Summary of rings supported by the controller: 1299 * 1300 * Standard Receive Producer Ring 1301 * - This ring is used to feed receive buffers for "standard" 1302 * sized frames (typically 1536 bytes) to the controller. 1303 * 1304 * Jumbo Receive Producer Ring 1305 * - This ring is used to feed receive buffers for jumbo sized 1306 * frames (i.e. anything bigger than the "standard" frames) 1307 * to the controller. 1308 * 1309 * Mini Receive Producer Ring 1310 * - This ring is used to feed receive buffers for "mini" 1311 * sized frames to the controller. 1312 * - This feature required external memory for the controller 1313 * but was never used in a production system. Should always 1314 * be disabled. 1315 * 1316 * Receive Return Ring 1317 * - After the controller has placed an incoming frame into a 1318 * receive buffer that buffer is moved into a receive return 1319 * ring. The driver is then responsible to passing the 1320 * buffer up to the stack. BCM5718/BCM57785 families support 1321 * multiple receive return rings. 1322 * 1323 * Send Ring 1324 * - This ring is used for outgoing frames. BCM5719/BCM5720 1325 * support multiple send rings. 1326 */ 1327 1328 /* Initialize the standard receive producer ring control block. */ 1329 rcb = &sc->bnx_ldata.bnx_info.bnx_std_rx_rcb; 1330 rcb->bge_hostaddr.bge_addr_lo = 1331 BGE_ADDR_LO(sc->bnx_rx_std_ring.bnx_rx_std_ring_paddr); 1332 rcb->bge_hostaddr.bge_addr_hi = 1333 BGE_ADDR_HI(sc->bnx_rx_std_ring.bnx_rx_std_ring_paddr); 1334 if (BNX_IS_57765_PLUS(sc)) { 1335 /* 1336 * Bits 31-16: Programmable ring size (2048, 1024, 512, .., 32) 1337 * Bits 15-2 : Maximum RX frame size 1338 * Bit 1 : 1 = Ring Disabled, 0 = Ring ENabled 1339 * Bit 0 : Reserved 1340 */ 1341 rcb->bge_maxlen_flags = 1342 BGE_RCB_MAXLEN_FLAGS(512, BNX_MAX_FRAMELEN << 2); 1343 } else { 1344 /* 1345 * Bits 31-16: Programmable ring size (512, 256, 128, 64, 32) 1346 * Bits 15-2 : Reserved (should be 0) 1347 * Bit 1 : 1 = Ring Disabled, 0 = Ring Enabled 1348 * Bit 0 : Reserved 1349 */ 1350 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(512, 0); 1351 } 1352 if (BNX_IS_5717_PLUS(sc)) 1353 rcb->bge_nicaddr = BGE_STD_RX_RINGS_5717; 1354 else 1355 rcb->bge_nicaddr = BGE_STD_RX_RINGS; 1356 /* Write the standard receive producer ring control block. */ 1357 CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_HI, rcb->bge_hostaddr.bge_addr_hi); 1358 CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_LO, rcb->bge_hostaddr.bge_addr_lo); 1359 CSR_WRITE_4(sc, BGE_RX_STD_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags); 1360 if (!BNX_IS_5717_PLUS(sc)) 1361 CSR_WRITE_4(sc, BGE_RX_STD_RCB_NICADDR, rcb->bge_nicaddr); 1362 /* Reset the standard receive producer ring producer index. */ 1363 bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, 0); 1364 1365 /* 1366 * Initialize the jumbo RX producer ring control 1367 * block. We set the 'ring disabled' bit in the 1368 * flags field until we're actually ready to start 1369 * using this ring (i.e. once we set the MTU 1370 * high enough to require it). 1371 */ 1372 if (BNX_IS_JUMBO_CAPABLE(sc)) { 1373 rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb; 1374 /* Get the jumbo receive producer ring RCB parameters. */ 1375 rcb->bge_hostaddr.bge_addr_lo = 1376 BGE_ADDR_LO(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr); 1377 rcb->bge_hostaddr.bge_addr_hi = 1378 BGE_ADDR_HI(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr); 1379 rcb->bge_maxlen_flags = 1380 BGE_RCB_MAXLEN_FLAGS(BNX_MAX_FRAMELEN, 1381 BGE_RCB_FLAG_RING_DISABLED); 1382 if (BNX_IS_5717_PLUS(sc)) 1383 rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS_5717; 1384 else 1385 rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS; 1386 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_HI, 1387 rcb->bge_hostaddr.bge_addr_hi); 1388 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_LO, 1389 rcb->bge_hostaddr.bge_addr_lo); 1390 /* Program the jumbo receive producer ring RCB parameters. */ 1391 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, 1392 rcb->bge_maxlen_flags); 1393 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_NICADDR, rcb->bge_nicaddr); 1394 /* Reset the jumbo receive producer ring producer index. */ 1395 bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, 0); 1396 } 1397 1398 /* 1399 * The BD ring replenish thresholds control how often the 1400 * hardware fetches new BD's from the producer rings in host 1401 * memory. Setting the value too low on a busy system can 1402 * starve the hardware and recue the throughpout. 1403 * 1404 * Set the BD ring replentish thresholds. The recommended 1405 * values are 1/8th the number of descriptors allocated to 1406 * each ring. 1407 */ 1408 val = 8; 1409 CSR_WRITE_4(sc, BGE_RBDI_STD_REPL_THRESH, val); 1410 if (BNX_IS_JUMBO_CAPABLE(sc)) { 1411 CSR_WRITE_4(sc, BGE_RBDI_JUMBO_REPL_THRESH, 1412 BGE_JUMBO_RX_RING_CNT/8); 1413 } 1414 if (BNX_IS_57765_PLUS(sc)) { 1415 CSR_WRITE_4(sc, BGE_STD_REPLENISH_LWM, 32); 1416 CSR_WRITE_4(sc, BGE_JMB_REPLENISH_LWM, 16); 1417 } 1418 1419 /* 1420 * Disable all send rings by setting the 'ring disabled' bit 1421 * in the flags field of all the TX send ring control blocks, 1422 * located in NIC memory. 1423 */ 1424 if (BNX_IS_5717_PLUS(sc)) 1425 limit = 4; 1426 else if (BNX_IS_57765_FAMILY(sc) || 1427 sc->bnx_asicrev == BGE_ASICREV_BCM5762) 1428 limit = 2; 1429 else 1430 limit = 1; 1431 vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB; 1432 for (i = 0; i < limit; i++) { 1433 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags, 1434 BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED)); 1435 vrcb += sizeof(struct bge_rcb); 1436 } 1437 1438 /* 1439 * Configure send ring RCBs 1440 */ 1441 vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB; 1442 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 1443 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 1444 1445 BGE_HOSTADDR(taddr, txr->bnx_tx_ring_paddr); 1446 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 1447 taddr.bge_addr_hi); 1448 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 1449 taddr.bge_addr_lo); 1450 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags, 1451 BGE_RCB_MAXLEN_FLAGS(BGE_TX_RING_CNT, 0)); 1452 vrcb += sizeof(struct bge_rcb); 1453 } 1454 1455 /* 1456 * Disable all receive return rings by setting the 1457 * 'ring disabled' bit in the flags field of all the receive 1458 * return ring control blocks, located in NIC memory. 1459 */ 1460 if (BNX_IS_5717_PLUS(sc)) { 1461 /* Should be 17, use 16 until we get an SRAM map. */ 1462 limit = 16; 1463 } else if (BNX_IS_57765_FAMILY(sc) || 1464 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 1465 limit = 4; 1466 } else { 1467 limit = 1; 1468 } 1469 /* Disable all receive return rings. */ 1470 vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB; 1471 for (i = 0; i < limit; i++) { 1472 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 0); 1473 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 0); 1474 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags, 1475 BGE_RCB_FLAG_RING_DISABLED); 1476 bnx_writembx(sc, BGE_MBX_RX_CONS0_LO + 1477 (i * (sizeof(uint64_t))), 0); 1478 vrcb += sizeof(struct bge_rcb); 1479 } 1480 1481 /* 1482 * Set up receive return rings. 1483 */ 1484 vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB; 1485 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 1486 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i]; 1487 1488 BGE_HOSTADDR(taddr, ret->bnx_rx_ret_ring_paddr); 1489 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 1490 taddr.bge_addr_hi); 1491 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 1492 taddr.bge_addr_lo); 1493 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags, 1494 BGE_RCB_MAXLEN_FLAGS(BNX_RETURN_RING_CNT, 0)); 1495 vrcb += sizeof(struct bge_rcb); 1496 } 1497 1498 /* Set random backoff seed for TX */ 1499 CSR_WRITE_4(sc, BGE_TX_RANDOM_BACKOFF, 1500 (sc->arpcom.ac_enaddr[0] + sc->arpcom.ac_enaddr[1] + 1501 sc->arpcom.ac_enaddr[2] + sc->arpcom.ac_enaddr[3] + 1502 sc->arpcom.ac_enaddr[4] + sc->arpcom.ac_enaddr[5]) & 1503 BGE_TX_BACKOFF_SEED_MASK); 1504 1505 /* Set inter-packet gap */ 1506 val = 0x2620; 1507 if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 || 1508 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 1509 val |= CSR_READ_4(sc, BGE_TX_LENGTHS) & 1510 (BGE_TXLEN_JMB_FRM_LEN_MSK | BGE_TXLEN_CNT_DN_VAL_MSK); 1511 } 1512 CSR_WRITE_4(sc, BGE_TX_LENGTHS, val); 1513 1514 /* 1515 * Specify which ring to use for packets that don't match 1516 * any RX rules. 1517 */ 1518 CSR_WRITE_4(sc, BGE_RX_RULES_CFG, 0x08); 1519 1520 /* 1521 * Configure number of RX lists. One interrupt distribution 1522 * list, sixteen active lists, one bad frames class. 1523 */ 1524 CSR_WRITE_4(sc, BGE_RXLP_CFG, 0x181); 1525 1526 /* Inialize RX list placement stats mask. */ 1527 CSR_WRITE_4(sc, BGE_RXLP_STATS_ENABLE_MASK, 0x007FFFFF); 1528 CSR_WRITE_4(sc, BGE_RXLP_STATS_CTL, 0x1); 1529 1530 /* Disable host coalescing until we get it set up */ 1531 CSR_WRITE_4(sc, BGE_HCC_MODE, 0x00000000); 1532 1533 /* Poll to make sure it's shut down. */ 1534 for (i = 0; i < BNX_TIMEOUT; i++) { 1535 if (!(CSR_READ_4(sc, BGE_HCC_MODE) & BGE_HCCMODE_ENABLE)) 1536 break; 1537 DELAY(10); 1538 } 1539 1540 if (i == BNX_TIMEOUT) { 1541 if_printf(&sc->arpcom.ac_if, 1542 "host coalescing engine failed to idle\n"); 1543 return(ENXIO); 1544 } 1545 1546 /* Set up host coalescing defaults */ 1547 sc->bnx_coal_chg = BNX_RX_COAL_TICKS_CHG | 1548 BNX_TX_COAL_TICKS_CHG | 1549 BNX_RX_COAL_BDS_CHG | 1550 BNX_TX_COAL_BDS_CHG | 1551 BNX_RX_COAL_BDS_INT_CHG | 1552 BNX_TX_COAL_BDS_INT_CHG; 1553 bnx_coal_change(sc); 1554 1555 /* 1556 * Set up addresses of status blocks 1557 */ 1558 intr = &sc->bnx_intr_data[0]; 1559 bzero(intr->bnx_status_block, BGE_STATUS_BLK_SZ); 1560 CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_HI, 1561 BGE_ADDR_HI(intr->bnx_status_block_paddr)); 1562 CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_LO, 1563 BGE_ADDR_LO(intr->bnx_status_block_paddr)); 1564 for (i = 1; i < sc->bnx_intr_cnt; ++i) { 1565 intr = &sc->bnx_intr_data[i]; 1566 bzero(intr->bnx_status_block, BGE_STATUS_BLK_SZ); 1567 CSR_WRITE_4(sc, BGE_VEC1_STATUSBLK_ADDR_HI + ((i - 1) * 8), 1568 BGE_ADDR_HI(intr->bnx_status_block_paddr)); 1569 CSR_WRITE_4(sc, BGE_VEC1_STATUSBLK_ADDR_LO + ((i - 1) * 8), 1570 BGE_ADDR_LO(intr->bnx_status_block_paddr)); 1571 } 1572 1573 /* Set up status block partail update size. */ 1574 val = BGE_STATBLKSZ_32BYTE; 1575 #if 0 1576 /* 1577 * Does not seem to have visible effect in both 1578 * bulk data (1472B UDP datagram) and tiny data 1579 * (18B UDP datagram) TX tests. 1580 */ 1581 val |= BGE_HCCMODE_CLRTICK_TX; 1582 #endif 1583 /* Turn on host coalescing state machine */ 1584 CSR_WRITE_4(sc, BGE_HCC_MODE, val | BGE_HCCMODE_ENABLE); 1585 1586 /* Turn on RX BD completion state machine and enable attentions */ 1587 CSR_WRITE_4(sc, BGE_RBDC_MODE, 1588 BGE_RBDCMODE_ENABLE|BGE_RBDCMODE_ATTN); 1589 1590 /* Turn on RX list placement state machine */ 1591 CSR_WRITE_4(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE); 1592 1593 val = BGE_MACMODE_TXDMA_ENB | BGE_MACMODE_RXDMA_ENB | 1594 BGE_MACMODE_RX_STATS_CLEAR | BGE_MACMODE_TX_STATS_CLEAR | 1595 BGE_MACMODE_RX_STATS_ENB | BGE_MACMODE_TX_STATS_ENB | 1596 BGE_MACMODE_FRMHDR_DMA_ENB; 1597 1598 if (sc->bnx_flags & BNX_FLAG_TBI) 1599 val |= BGE_PORTMODE_TBI; 1600 else if (sc->bnx_flags & BNX_FLAG_MII_SERDES) 1601 val |= BGE_PORTMODE_GMII; 1602 else 1603 val |= BGE_PORTMODE_MII; 1604 1605 /* Allow APE to send/receive frames. */ 1606 if (sc->bnx_mfw_flags & BNX_MFW_ON_APE) 1607 val |= BGE_MACMODE_APE_RX_EN | BGE_MACMODE_APE_TX_EN; 1608 1609 /* Turn on DMA, clear stats */ 1610 CSR_WRITE_4(sc, BGE_MAC_MODE, val); 1611 DELAY(40); 1612 1613 /* Set misc. local control, enable interrupts on attentions */ 1614 BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_ONATTN); 1615 1616 #ifdef notdef 1617 /* Assert GPIO pins for PHY reset */ 1618 BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUT0| 1619 BGE_MLC_MISCIO_OUT1|BGE_MLC_MISCIO_OUT2); 1620 BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUTEN0| 1621 BGE_MLC_MISCIO_OUTEN1|BGE_MLC_MISCIO_OUTEN2); 1622 #endif 1623 1624 if (sc->bnx_intr_type == PCI_INTR_TYPE_MSIX) 1625 bnx_enable_msi(sc, TRUE); 1626 1627 /* Turn on write DMA state machine */ 1628 val = BGE_WDMAMODE_ENABLE|BGE_WDMAMODE_ALL_ATTNS; 1629 /* Enable host coalescing bug fix. */ 1630 val |= BGE_WDMAMODE_STATUS_TAG_FIX; 1631 if (sc->bnx_asicrev == BGE_ASICREV_BCM5785) { 1632 /* Request larger DMA burst size to get better performance. */ 1633 val |= BGE_WDMAMODE_BURST_ALL_DATA; 1634 } 1635 CSR_WRITE_4(sc, BGE_WDMA_MODE, val); 1636 DELAY(40); 1637 1638 if (BNX_IS_57765_PLUS(sc)) { 1639 uint32_t dmactl, dmactl_reg; 1640 1641 if (sc->bnx_asicrev == BGE_ASICREV_BCM5762) 1642 dmactl_reg = BGE_RDMA_RSRVCTRL2; 1643 else 1644 dmactl_reg = BGE_RDMA_RSRVCTRL; 1645 1646 dmactl = CSR_READ_4(sc, dmactl_reg); 1647 /* 1648 * Adjust tx margin to prevent TX data corruption and 1649 * fix internal FIFO overflow. 1650 */ 1651 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 || 1652 sc->bnx_asicrev == BGE_ASICREV_BCM5720 || 1653 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 1654 dmactl &= ~(BGE_RDMA_RSRVCTRL_FIFO_LWM_MASK | 1655 BGE_RDMA_RSRVCTRL_FIFO_HWM_MASK | 1656 BGE_RDMA_RSRVCTRL_TXMRGN_MASK); 1657 dmactl |= BGE_RDMA_RSRVCTRL_FIFO_LWM_1_5K | 1658 BGE_RDMA_RSRVCTRL_FIFO_HWM_1_5K | 1659 BGE_RDMA_RSRVCTRL_TXMRGN_320B; 1660 } 1661 /* 1662 * Enable fix for read DMA FIFO overruns. 1663 * The fix is to limit the number of RX BDs 1664 * the hardware would fetch at a fime. 1665 */ 1666 CSR_WRITE_4(sc, dmactl_reg, 1667 dmactl | BGE_RDMA_RSRVCTRL_FIFO_OFLW_FIX); 1668 } 1669 1670 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719) { 1671 CSR_WRITE_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL, 1672 CSR_READ_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL) | 1673 BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_4K | 1674 BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K); 1675 } else if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 || 1676 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 1677 uint32_t ctrl_reg; 1678 1679 if (sc->bnx_asicrev == BGE_ASICREV_BCM5762) 1680 ctrl_reg = BGE_RDMA_LSO_CRPTEN_CTRL2; 1681 else 1682 ctrl_reg = BGE_RDMA_LSO_CRPTEN_CTRL; 1683 1684 /* 1685 * Allow 4KB burst length reads for non-LSO frames. 1686 * Enable 512B burst length reads for buffer descriptors. 1687 */ 1688 CSR_WRITE_4(sc, ctrl_reg, 1689 CSR_READ_4(sc, ctrl_reg) | 1690 BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_512 | 1691 BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K); 1692 } 1693 1694 /* Turn on read DMA state machine */ 1695 val = BGE_RDMAMODE_ENABLE | BGE_RDMAMODE_ALL_ATTNS; 1696 if (sc->bnx_asicrev == BGE_ASICREV_BCM5717) 1697 val |= BGE_RDMAMODE_MULT_DMA_RD_DIS; 1698 if (sc->bnx_asicrev == BGE_ASICREV_BCM5784 || 1699 sc->bnx_asicrev == BGE_ASICREV_BCM5785 || 1700 sc->bnx_asicrev == BGE_ASICREV_BCM57780) { 1701 val |= BGE_RDMAMODE_BD_SBD_CRPT_ATTN | 1702 BGE_RDMAMODE_MBUF_RBD_CRPT_ATTN | 1703 BGE_RDMAMODE_MBUF_SBD_CRPT_ATTN; 1704 } 1705 if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 || 1706 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 1707 val |= CSR_READ_4(sc, BGE_RDMA_MODE) & 1708 BGE_RDMAMODE_H2BNC_VLAN_DET; 1709 /* 1710 * Allow multiple outstanding read requests from 1711 * non-LSO read DMA engine. 1712 */ 1713 val &= ~BGE_RDMAMODE_MULT_DMA_RD_DIS; 1714 } 1715 if (sc->bnx_asicrev == BGE_ASICREV_BCM57766) 1716 val |= BGE_RDMAMODE_JMB_2K_MMRR; 1717 if (sc->bnx_flags & BNX_FLAG_TSO) 1718 val |= BGE_RDMAMODE_TSO4_ENABLE; 1719 val |= BGE_RDMAMODE_FIFO_LONG_BURST; 1720 CSR_WRITE_4(sc, BGE_RDMA_MODE, val); 1721 DELAY(40); 1722 1723 /* Turn on RX data completion state machine */ 1724 CSR_WRITE_4(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE); 1725 1726 /* Turn on RX BD initiator state machine */ 1727 CSR_WRITE_4(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE); 1728 1729 /* Turn on RX data and RX BD initiator state machine */ 1730 CSR_WRITE_4(sc, BGE_RDBDI_MODE, BGE_RDBDIMODE_ENABLE); 1731 1732 /* Turn on send BD completion state machine */ 1733 CSR_WRITE_4(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE); 1734 1735 /* Turn on send data completion state machine */ 1736 val = BGE_SDCMODE_ENABLE; 1737 if (sc->bnx_asicrev == BGE_ASICREV_BCM5761) 1738 val |= BGE_SDCMODE_CDELAY; 1739 CSR_WRITE_4(sc, BGE_SDC_MODE, val); 1740 1741 /* Turn on send data initiator state machine */ 1742 if (sc->bnx_flags & BNX_FLAG_TSO) { 1743 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE | 1744 BGE_SDIMODE_HW_LSO_PRE_DMA); 1745 } else { 1746 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE); 1747 } 1748 1749 /* Turn on send BD initiator state machine */ 1750 val = BGE_SBDIMODE_ENABLE; 1751 if (sc->bnx_tx_ringcnt > 1) 1752 val |= BGE_SBDIMODE_MULTI_TXR; 1753 CSR_WRITE_4(sc, BGE_SBDI_MODE, val); 1754 1755 /* Turn on send BD selector state machine */ 1756 CSR_WRITE_4(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE); 1757 1758 CSR_WRITE_4(sc, BGE_SDI_STATS_ENABLE_MASK, 0x007FFFFF); 1759 CSR_WRITE_4(sc, BGE_SDI_STATS_CTL, 1760 BGE_SDISTATSCTL_ENABLE|BGE_SDISTATSCTL_FASTER); 1761 1762 /* ack/clear link change events */ 1763 CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED| 1764 BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE| 1765 BGE_MACSTAT_LINK_CHANGED); 1766 CSR_WRITE_4(sc, BGE_MI_STS, 0); 1767 1768 /* 1769 * Enable attention when the link has changed state for 1770 * devices that use auto polling. 1771 */ 1772 if (sc->bnx_flags & BNX_FLAG_TBI) { 1773 CSR_WRITE_4(sc, BGE_MI_STS, BGE_MISTS_LINK); 1774 } else { 1775 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) { 1776 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode); 1777 DELAY(80); 1778 } 1779 } 1780 1781 /* 1782 * Clear any pending link state attention. 1783 * Otherwise some link state change events may be lost until attention 1784 * is cleared by bnx_intr() -> bnx_softc.bnx_link_upd() sequence. 1785 * It's not necessary on newer BCM chips - perhaps enabling link 1786 * state change attentions implies clearing pending attention. 1787 */ 1788 CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED| 1789 BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE| 1790 BGE_MACSTAT_LINK_CHANGED); 1791 1792 /* Enable link state change attentions. */ 1793 BNX_SETBIT(sc, BGE_MAC_EVT_ENB, BGE_EVTENB_LINK_CHANGED); 1794 1795 return(0); 1796 } 1797 1798 /* 1799 * Probe for a Broadcom chip. Check the PCI vendor and device IDs 1800 * against our list and return its name if we find a match. Note 1801 * that since the Broadcom controller contains VPD support, we 1802 * can get the device name string from the controller itself instead 1803 * of the compiled-in string. This is a little slow, but it guarantees 1804 * we'll always announce the right product name. 1805 */ 1806 static int 1807 bnx_probe(device_t dev) 1808 { 1809 const struct bnx_type *t; 1810 uint16_t product, vendor; 1811 1812 if (!pci_is_pcie(dev)) 1813 return ENXIO; 1814 1815 product = pci_get_device(dev); 1816 vendor = pci_get_vendor(dev); 1817 1818 for (t = bnx_devs; t->bnx_name != NULL; t++) { 1819 if (vendor == t->bnx_vid && product == t->bnx_did) 1820 break; 1821 } 1822 if (t->bnx_name == NULL) 1823 return ENXIO; 1824 1825 device_set_desc(dev, t->bnx_name); 1826 return 0; 1827 } 1828 1829 static int 1830 bnx_attach(device_t dev) 1831 { 1832 struct ifnet *ifp; 1833 struct bnx_softc *sc; 1834 struct bnx_rx_std_ring *std; 1835 struct sysctl_ctx_list *ctx; 1836 struct sysctl_oid_list *tree; 1837 uint32_t hwcfg = 0; 1838 int error = 0, rid, capmask, i, std_cpuid, std_cpuid_def; 1839 uint8_t ether_addr[ETHER_ADDR_LEN]; 1840 uint16_t product; 1841 uintptr_t mii_priv = 0; 1842 #if defined(BNX_TSO_DEBUG) || defined(BNX_RSS_DEBUG) || defined(BNX_TSS_DEBUG) 1843 char desc[32]; 1844 #endif 1845 1846 sc = device_get_softc(dev); 1847 sc->bnx_dev = dev; 1848 callout_init_mp(&sc->bnx_tick_timer); 1849 lwkt_serialize_init(&sc->bnx_jslot_serializer); 1850 lwkt_serialize_init(&sc->bnx_main_serialize); 1851 1852 /* Always setup interrupt mailboxes */ 1853 for (i = 0; i < BNX_INTR_MAX; ++i) { 1854 callout_init_mp(&sc->bnx_intr_data[i].bnx_intr_timer); 1855 sc->bnx_intr_data[i].bnx_sc = sc; 1856 sc->bnx_intr_data[i].bnx_intr_mbx = BGE_MBX_IRQ0_LO + (i * 8); 1857 sc->bnx_intr_data[i].bnx_intr_rid = -1; 1858 sc->bnx_intr_data[i].bnx_intr_cpuid = -1; 1859 } 1860 1861 sc->bnx_func_addr = pci_get_function(dev); 1862 product = pci_get_device(dev); 1863 1864 #ifndef BURN_BRIDGES 1865 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) { 1866 uint32_t irq, mem; 1867 1868 irq = pci_read_config(dev, PCIR_INTLINE, 4); 1869 mem = pci_read_config(dev, BGE_PCI_BAR0, 4); 1870 1871 device_printf(dev, "chip is in D%d power mode " 1872 "-- setting to D0\n", pci_get_powerstate(dev)); 1873 1874 pci_set_powerstate(dev, PCI_POWERSTATE_D0); 1875 1876 pci_write_config(dev, PCIR_INTLINE, irq, 4); 1877 pci_write_config(dev, BGE_PCI_BAR0, mem, 4); 1878 } 1879 #endif /* !BURN_BRIDGE */ 1880 1881 /* 1882 * Map control/status registers. 1883 */ 1884 pci_enable_busmaster(dev); 1885 1886 rid = BGE_PCI_BAR0; 1887 sc->bnx_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 1888 RF_ACTIVE); 1889 1890 if (sc->bnx_res == NULL) { 1891 device_printf(dev, "couldn't map memory\n"); 1892 return ENXIO; 1893 } 1894 1895 sc->bnx_btag = rman_get_bustag(sc->bnx_res); 1896 sc->bnx_bhandle = rman_get_bushandle(sc->bnx_res); 1897 1898 /* Save various chip information */ 1899 sc->bnx_chipid = 1900 pci_read_config(dev, BGE_PCI_MISC_CTL, 4) >> 1901 BGE_PCIMISCCTL_ASICREV_SHIFT; 1902 if (BGE_ASICREV(sc->bnx_chipid) == BGE_ASICREV_USE_PRODID_REG) { 1903 /* All chips having dedicated ASICREV register have CPMU */ 1904 sc->bnx_flags |= BNX_FLAG_CPMU; 1905 1906 switch (product) { 1907 case PCI_PRODUCT_BROADCOM_BCM5717: 1908 case PCI_PRODUCT_BROADCOM_BCM5717C: 1909 case PCI_PRODUCT_BROADCOM_BCM5718: 1910 case PCI_PRODUCT_BROADCOM_BCM5719: 1911 case PCI_PRODUCT_BROADCOM_BCM5720_ALT: 1912 case PCI_PRODUCT_BROADCOM_BCM5725: 1913 case PCI_PRODUCT_BROADCOM_BCM5727: 1914 case PCI_PRODUCT_BROADCOM_BCM5762: 1915 sc->bnx_chipid = pci_read_config(dev, 1916 BGE_PCI_GEN2_PRODID_ASICREV, 4); 1917 break; 1918 1919 case PCI_PRODUCT_BROADCOM_BCM57761: 1920 case PCI_PRODUCT_BROADCOM_BCM57762: 1921 case PCI_PRODUCT_BROADCOM_BCM57765: 1922 case PCI_PRODUCT_BROADCOM_BCM57766: 1923 case PCI_PRODUCT_BROADCOM_BCM57781: 1924 case PCI_PRODUCT_BROADCOM_BCM57782: 1925 case PCI_PRODUCT_BROADCOM_BCM57785: 1926 case PCI_PRODUCT_BROADCOM_BCM57786: 1927 case PCI_PRODUCT_BROADCOM_BCM57791: 1928 case PCI_PRODUCT_BROADCOM_BCM57795: 1929 sc->bnx_chipid = pci_read_config(dev, 1930 BGE_PCI_GEN15_PRODID_ASICREV, 4); 1931 break; 1932 1933 default: 1934 sc->bnx_chipid = pci_read_config(dev, 1935 BGE_PCI_PRODID_ASICREV, 4); 1936 break; 1937 } 1938 } 1939 if (sc->bnx_chipid == BGE_CHIPID_BCM5717_C0) 1940 sc->bnx_chipid = BGE_CHIPID_BCM5720_A0; 1941 1942 sc->bnx_asicrev = BGE_ASICREV(sc->bnx_chipid); 1943 sc->bnx_chiprev = BGE_CHIPREV(sc->bnx_chipid); 1944 1945 switch (sc->bnx_asicrev) { 1946 case BGE_ASICREV_BCM5717: 1947 case BGE_ASICREV_BCM5719: 1948 case BGE_ASICREV_BCM5720: 1949 sc->bnx_flags |= BNX_FLAG_5717_PLUS | BNX_FLAG_57765_PLUS; 1950 break; 1951 1952 case BGE_ASICREV_BCM5762: 1953 sc->bnx_flags |= BNX_FLAG_57765_PLUS; 1954 break; 1955 1956 case BGE_ASICREV_BCM57765: 1957 case BGE_ASICREV_BCM57766: 1958 sc->bnx_flags |= BNX_FLAG_57765_FAMILY | BNX_FLAG_57765_PLUS; 1959 break; 1960 } 1961 1962 if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 || 1963 sc->bnx_asicrev == BGE_ASICREV_BCM5719 || 1964 sc->bnx_asicrev == BGE_ASICREV_BCM5720 || 1965 sc->bnx_asicrev == BGE_ASICREV_BCM5762) 1966 sc->bnx_flags |= BNX_FLAG_APE; 1967 1968 sc->bnx_flags |= BNX_FLAG_TSO; 1969 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 && 1970 sc->bnx_chipid == BGE_CHIPID_BCM5719_A0) 1971 sc->bnx_flags &= ~BNX_FLAG_TSO; 1972 1973 if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 || 1974 BNX_IS_57765_FAMILY(sc)) { 1975 /* 1976 * All BCM57785 and BCM5718 families chips have a bug that 1977 * under certain situation interrupt will not be enabled 1978 * even if status tag is written to interrupt mailbox. 1979 * 1980 * While BCM5719 and BCM5720 have a hardware workaround 1981 * which could fix the above bug. 1982 * See the comment near BGE_PCIDMARWCTL_TAGGED_STATUS_WA in 1983 * bnx_chipinit(). 1984 * 1985 * For the rest of the chips in these two families, we will 1986 * have to poll the status block at high rate (10ms currently) 1987 * to check whether the interrupt is hosed or not. 1988 * See bnx_check_intr_*() for details. 1989 */ 1990 sc->bnx_flags |= BNX_FLAG_STATUSTAG_BUG; 1991 } 1992 1993 sc->bnx_pciecap = pci_get_pciecap_ptr(sc->bnx_dev); 1994 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 || 1995 sc->bnx_asicrev == BGE_ASICREV_BCM5720) 1996 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_2048); 1997 else 1998 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_4096); 1999 device_printf(dev, "CHIP ID 0x%08x; " 2000 "ASIC REV 0x%02x; CHIP REV 0x%02x\n", 2001 sc->bnx_chipid, sc->bnx_asicrev, sc->bnx_chiprev); 2002 2003 /* 2004 * Set various PHY quirk flags. 2005 */ 2006 2007 capmask = MII_CAPMASK_DEFAULT; 2008 if (product == PCI_PRODUCT_BROADCOM_BCM57791 || 2009 product == PCI_PRODUCT_BROADCOM_BCM57795) { 2010 /* 10/100 only */ 2011 capmask &= ~BMSR_EXTSTAT; 2012 } 2013 2014 mii_priv |= BRGPHY_FLAG_WIRESPEED; 2015 if (sc->bnx_chipid == BGE_CHIPID_BCM5762_A0) 2016 mii_priv |= BRGPHY_FLAG_5762_A0; 2017 2018 /* 2019 * Chips with APE need BAR2 access for APE registers/memory. 2020 */ 2021 if (sc->bnx_flags & BNX_FLAG_APE) { 2022 uint32_t pcistate; 2023 2024 rid = PCIR_BAR(2); 2025 sc->bnx_res2 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 2026 RF_ACTIVE); 2027 if (sc->bnx_res2 == NULL) { 2028 device_printf(dev, "couldn't map BAR2 memory\n"); 2029 error = ENXIO; 2030 goto fail; 2031 } 2032 2033 /* Enable APE register/memory access by host driver. */ 2034 pcistate = pci_read_config(dev, BGE_PCI_PCISTATE, 4); 2035 pcistate |= BGE_PCISTATE_ALLOW_APE_CTLSPC_WR | 2036 BGE_PCISTATE_ALLOW_APE_SHMEM_WR | 2037 BGE_PCISTATE_ALLOW_APE_PSPACE_WR; 2038 pci_write_config(dev, BGE_PCI_PCISTATE, pcistate, 4); 2039 2040 bnx_ape_lock_init(sc); 2041 bnx_ape_read_fw_ver(sc); 2042 } 2043 2044 /* Initialize if_name earlier, so if_printf could be used */ 2045 ifp = &sc->arpcom.ac_if; 2046 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2047 2048 /* 2049 * Try to reset the chip. 2050 */ 2051 bnx_sig_pre_reset(sc, BNX_RESET_SHUTDOWN); 2052 bnx_reset(sc); 2053 bnx_sig_post_reset(sc, BNX_RESET_SHUTDOWN); 2054 2055 if (bnx_chipinit(sc)) { 2056 device_printf(dev, "chip initialization failed\n"); 2057 error = ENXIO; 2058 goto fail; 2059 } 2060 2061 /* 2062 * Get station address 2063 */ 2064 error = bnx_get_eaddr(sc, ether_addr); 2065 if (error) { 2066 device_printf(dev, "failed to read station address\n"); 2067 goto fail; 2068 } 2069 2070 /* Setup RX/TX and interrupt count */ 2071 bnx_setup_ring_cnt(sc); 2072 2073 if ((sc->bnx_rx_retcnt == 1 && sc->bnx_tx_ringcnt == 1) || 2074 (sc->bnx_rx_retcnt > 1 && sc->bnx_tx_ringcnt > 1)) { 2075 /* 2076 * The RX ring and the corresponding TX ring processing 2077 * should be on the same CPU, since they share the same 2078 * status block. 2079 */ 2080 sc->bnx_flags |= BNX_FLAG_RXTX_BUNDLE; 2081 if (bootverbose) 2082 device_printf(dev, "RX/TX bundle\n"); 2083 if (sc->bnx_tx_ringcnt > 1) { 2084 /* 2085 * Multiple TX rings do not share status block 2086 * with link status, so link status will have 2087 * to save its own status_tag. 2088 */ 2089 sc->bnx_flags |= BNX_FLAG_STATUS_HASTAG; 2090 if (bootverbose) 2091 device_printf(dev, "status needs tag\n"); 2092 } 2093 } else { 2094 KKASSERT(sc->bnx_rx_retcnt > 1 && sc->bnx_tx_ringcnt == 1); 2095 if (bootverbose) 2096 device_printf(dev, "RX/TX not bundled\n"); 2097 } 2098 2099 error = bnx_dma_alloc(dev); 2100 if (error) 2101 goto fail; 2102 2103 /* 2104 * Allocate interrupt 2105 */ 2106 error = bnx_alloc_intr(sc); 2107 if (error) 2108 goto fail; 2109 2110 /* Setup serializers */ 2111 bnx_setup_serialize(sc); 2112 2113 /* Set default tuneable values. */ 2114 sc->bnx_rx_coal_ticks = BNX_RX_COAL_TICKS_DEF; 2115 sc->bnx_tx_coal_ticks = BNX_TX_COAL_TICKS_DEF; 2116 sc->bnx_rx_coal_bds = BNX_RX_COAL_BDS_DEF; 2117 sc->bnx_rx_coal_bds_poll = sc->bnx_rx_ret_ring[0].bnx_rx_cntmax; 2118 sc->bnx_tx_coal_bds = BNX_TX_COAL_BDS_DEF; 2119 sc->bnx_tx_coal_bds_poll = BNX_TX_COAL_BDS_POLL_DEF; 2120 sc->bnx_rx_coal_bds_int = BNX_RX_COAL_BDS_INT_DEF; 2121 sc->bnx_tx_coal_bds_int = BNX_TX_COAL_BDS_INT_DEF; 2122 2123 /* Set up ifnet structure */ 2124 ifp->if_softc = sc; 2125 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2126 ifp->if_ioctl = bnx_ioctl; 2127 ifp->if_start = bnx_start; 2128 #ifdef IFPOLL_ENABLE 2129 ifp->if_npoll = bnx_npoll; 2130 #endif 2131 ifp->if_init = bnx_init; 2132 ifp->if_serialize = bnx_serialize; 2133 ifp->if_deserialize = bnx_deserialize; 2134 ifp->if_tryserialize = bnx_tryserialize; 2135 #ifdef INVARIANTS 2136 ifp->if_serialize_assert = bnx_serialize_assert; 2137 #endif 2138 ifp->if_mtu = ETHERMTU; 2139 ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2140 2141 ifp->if_capabilities |= IFCAP_HWCSUM; 2142 ifp->if_hwassist = BNX_CSUM_FEATURES; 2143 if (sc->bnx_flags & BNX_FLAG_TSO) { 2144 ifp->if_capabilities |= IFCAP_TSO; 2145 ifp->if_hwassist |= CSUM_TSO; 2146 } 2147 if (BNX_RSS_ENABLED(sc)) 2148 ifp->if_capabilities |= IFCAP_RSS; 2149 ifp->if_capenable = ifp->if_capabilities; 2150 2151 ifp->if_nmbclusters = BGE_STD_RX_RING_CNT; 2152 2153 ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1); 2154 ifq_set_ready(&ifp->if_snd); 2155 ifq_set_subq_cnt(&ifp->if_snd, sc->bnx_tx_ringcnt); 2156 2157 if (sc->bnx_tx_ringcnt > 1) { 2158 ifp->if_mapsubq = ifq_mapsubq_modulo; 2159 ifq_set_subq_divisor(&ifp->if_snd, sc->bnx_tx_ringcnt); 2160 } 2161 2162 /* 2163 * Figure out what sort of media we have by checking the 2164 * hardware config word in the first 32k of NIC internal memory, 2165 * or fall back to examining the EEPROM if necessary. 2166 * Note: on some BCM5700 cards, this value appears to be unset. 2167 * If that's the case, we have to rely on identifying the NIC 2168 * by its PCI subsystem ID, as we do below for the SysKonnect 2169 * SK-9D41. 2170 */ 2171 if (bnx_readmem_ind(sc, BGE_SRAM_DATA_SIG) == BGE_SRAM_DATA_SIG_MAGIC) { 2172 hwcfg = bnx_readmem_ind(sc, BGE_SRAM_DATA_CFG); 2173 } else { 2174 if (bnx_read_eeprom(sc, (caddr_t)&hwcfg, BGE_EE_HWCFG_OFFSET, 2175 sizeof(hwcfg))) { 2176 device_printf(dev, "failed to read EEPROM\n"); 2177 error = ENXIO; 2178 goto fail; 2179 } 2180 hwcfg = ntohl(hwcfg); 2181 } 2182 2183 /* The SysKonnect SK-9D41 is a 1000baseSX card. */ 2184 if (pci_get_subvendor(dev) == PCI_PRODUCT_SCHNEIDERKOCH_SK_9D41 || 2185 (hwcfg & BGE_HWCFG_MEDIA) == BGE_MEDIA_FIBER) 2186 sc->bnx_flags |= BNX_FLAG_TBI; 2187 2188 /* Setup MI MODE */ 2189 if (sc->bnx_flags & BNX_FLAG_CPMU) 2190 sc->bnx_mi_mode = BGE_MIMODE_500KHZ_CONST; 2191 else 2192 sc->bnx_mi_mode = BGE_MIMODE_BASE; 2193 2194 /* Setup link status update stuffs */ 2195 if (sc->bnx_flags & BNX_FLAG_TBI) { 2196 sc->bnx_link_upd = bnx_tbi_link_upd; 2197 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED; 2198 } else if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) { 2199 sc->bnx_link_upd = bnx_autopoll_link_upd; 2200 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED; 2201 } else { 2202 sc->bnx_link_upd = bnx_copper_link_upd; 2203 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED; 2204 } 2205 2206 /* Set default PHY address */ 2207 sc->bnx_phyno = 1; 2208 2209 /* 2210 * PHY address mapping for various devices. 2211 * 2212 * | F0 Cu | F0 Sr | F1 Cu | F1 Sr | 2213 * ---------+-------+-------+-------+-------+ 2214 * BCM57XX | 1 | X | X | X | 2215 * BCM5717 | 1 | 8 | 2 | 9 | 2216 * BCM5719 | 1 | 8 | 2 | 9 | 2217 * BCM5720 | 1 | 8 | 2 | 9 | 2218 * 2219 * | F2 Cu | F2 Sr | F3 Cu | F3 Sr | 2220 * ---------+-------+-------+-------+-------+ 2221 * BCM57XX | X | X | X | X | 2222 * BCM5717 | X | X | X | X | 2223 * BCM5719 | 3 | 10 | 4 | 11 | 2224 * BCM5720 | X | X | X | X | 2225 * 2226 * Other addresses may respond but they are not 2227 * IEEE compliant PHYs and should be ignored. 2228 */ 2229 if (BNX_IS_5717_PLUS(sc)) { 2230 if (sc->bnx_chipid == BGE_CHIPID_BCM5717_A0) { 2231 if (CSR_READ_4(sc, BGE_SGDIG_STS) & 2232 BGE_SGDIGSTS_IS_SERDES) 2233 sc->bnx_phyno = sc->bnx_func_addr + 8; 2234 else 2235 sc->bnx_phyno = sc->bnx_func_addr + 1; 2236 } else { 2237 if (CSR_READ_4(sc, BGE_CPMU_PHY_STRAP) & 2238 BGE_CPMU_PHY_STRAP_IS_SERDES) 2239 sc->bnx_phyno = sc->bnx_func_addr + 8; 2240 else 2241 sc->bnx_phyno = sc->bnx_func_addr + 1; 2242 } 2243 } 2244 2245 if (sc->bnx_flags & BNX_FLAG_TBI) { 2246 ifmedia_init(&sc->bnx_ifmedia, IFM_IMASK, 2247 bnx_ifmedia_upd, bnx_ifmedia_sts); 2248 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_1000_SX, 0, NULL); 2249 ifmedia_add(&sc->bnx_ifmedia, 2250 IFM_ETHER|IFM_1000_SX|IFM_FDX, 0, NULL); 2251 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO, 0, NULL); 2252 ifmedia_set(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO); 2253 sc->bnx_ifmedia.ifm_media = sc->bnx_ifmedia.ifm_cur->ifm_media; 2254 } else { 2255 struct mii_probe_args mii_args; 2256 2257 mii_probe_args_init(&mii_args, bnx_ifmedia_upd, bnx_ifmedia_sts); 2258 mii_args.mii_probemask = 1 << sc->bnx_phyno; 2259 mii_args.mii_capmask = capmask; 2260 mii_args.mii_privtag = MII_PRIVTAG_BRGPHY; 2261 mii_args.mii_priv = mii_priv; 2262 2263 error = mii_probe(dev, &sc->bnx_miibus, &mii_args); 2264 if (error) { 2265 device_printf(dev, "MII without any PHY!\n"); 2266 goto fail; 2267 } 2268 } 2269 2270 ctx = device_get_sysctl_ctx(sc->bnx_dev); 2271 tree = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->bnx_dev)); 2272 2273 SYSCTL_ADD_INT(ctx, tree, OID_AUTO, 2274 "rx_rings", CTLFLAG_RD, &sc->bnx_rx_retcnt, 0, "# of RX rings"); 2275 SYSCTL_ADD_INT(ctx, tree, OID_AUTO, 2276 "tx_rings", CTLFLAG_RD, &sc->bnx_tx_ringcnt, 0, "# of TX rings"); 2277 2278 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "rx_coal_ticks", 2279 CTLTYPE_INT | CTLFLAG_RW, 2280 sc, 0, bnx_sysctl_rx_coal_ticks, "I", 2281 "Receive coalescing ticks (usec)."); 2282 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "tx_coal_ticks", 2283 CTLTYPE_INT | CTLFLAG_RW, 2284 sc, 0, bnx_sysctl_tx_coal_ticks, "I", 2285 "Transmit coalescing ticks (usec)."); 2286 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "rx_coal_bds", 2287 CTLTYPE_INT | CTLFLAG_RW, 2288 sc, 0, bnx_sysctl_rx_coal_bds, "I", 2289 "Receive max coalesced BD count."); 2290 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "rx_coal_bds_poll", 2291 CTLTYPE_INT | CTLFLAG_RW, 2292 sc, 0, bnx_sysctl_rx_coal_bds_poll, "I", 2293 "Receive max coalesced BD count in polling."); 2294 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "tx_coal_bds", 2295 CTLTYPE_INT | CTLFLAG_RW, 2296 sc, 0, bnx_sysctl_tx_coal_bds, "I", 2297 "Transmit max coalesced BD count."); 2298 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "tx_coal_bds_poll", 2299 CTLTYPE_INT | CTLFLAG_RW, 2300 sc, 0, bnx_sysctl_tx_coal_bds_poll, "I", 2301 "Transmit max coalesced BD count in polling."); 2302 /* 2303 * A common design characteristic for many Broadcom 2304 * client controllers is that they only support a 2305 * single outstanding DMA read operation on the PCIe 2306 * bus. This means that it will take twice as long to 2307 * fetch a TX frame that is split into header and 2308 * payload buffers as it does to fetch a single, 2309 * contiguous TX frame (2 reads vs. 1 read). For these 2310 * controllers, coalescing buffers to reduce the number 2311 * of memory reads is effective way to get maximum 2312 * performance(about 940Mbps). Without collapsing TX 2313 * buffers the maximum TCP bulk transfer performance 2314 * is about 850Mbps. However forcing coalescing mbufs 2315 * consumes a lot of CPU cycles, so leave it off by 2316 * default. 2317 */ 2318 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, 2319 "force_defrag", CTLTYPE_INT | CTLFLAG_RW, 2320 sc, 0, bnx_sysctl_force_defrag, "I", 2321 "Force defragment on TX path"); 2322 2323 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, 2324 "tx_wreg", CTLTYPE_INT | CTLFLAG_RW, 2325 sc, 0, bnx_sysctl_tx_wreg, "I", 2326 "# of segments before writing to hardware register"); 2327 2328 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, 2329 "std_refill", CTLTYPE_INT | CTLFLAG_RW, 2330 sc, 0, bnx_sysctl_std_refill, "I", 2331 "# of packets received before scheduling standard refilling"); 2332 2333 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, 2334 "rx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW, 2335 sc, 0, bnx_sysctl_rx_coal_bds_int, "I", 2336 "Receive max coalesced BD count during interrupt."); 2337 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, 2338 "tx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW, 2339 sc, 0, bnx_sysctl_tx_coal_bds_int, "I", 2340 "Transmit max coalesced BD count during interrupt."); 2341 2342 if (sc->bnx_intr_type == PCI_INTR_TYPE_MSIX) { 2343 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "tx_cpumap", 2344 CTLTYPE_OPAQUE | CTLFLAG_RD, 2345 sc->bnx_tx_rmap, 0, if_ringmap_cpumap_sysctl, "I", 2346 "TX ring CPU map"); 2347 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "rx_cpumap", 2348 CTLTYPE_OPAQUE | CTLFLAG_RD, 2349 sc->bnx_rx_rmap, 0, if_ringmap_cpumap_sysctl, "I", 2350 "RX ring CPU map"); 2351 } else { 2352 #ifdef IFPOLL_ENABLE 2353 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "tx_poll_cpumap", 2354 CTLTYPE_OPAQUE | CTLFLAG_RD, 2355 sc->bnx_tx_rmap, 0, if_ringmap_cpumap_sysctl, "I", 2356 "TX poll CPU map"); 2357 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "rx_poll_cpumap", 2358 CTLTYPE_OPAQUE | CTLFLAG_RD, 2359 sc->bnx_rx_rmap, 0, if_ringmap_cpumap_sysctl, "I", 2360 "RX poll CPU map"); 2361 #endif 2362 } 2363 2364 #ifdef BNX_RSS_DEBUG 2365 SYSCTL_ADD_INT(ctx, tree, OID_AUTO, 2366 "std_refill_mask", CTLFLAG_RD, 2367 &sc->bnx_rx_std_ring.bnx_rx_std_refill, 0, ""); 2368 SYSCTL_ADD_INT(ctx, tree, OID_AUTO, 2369 "std_used", CTLFLAG_RD, 2370 &sc->bnx_rx_std_ring.bnx_rx_std_used, 0, ""); 2371 SYSCTL_ADD_INT(ctx, tree, OID_AUTO, 2372 "rss_debug", CTLFLAG_RW, &sc->bnx_rss_debug, 0, ""); 2373 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 2374 ksnprintf(desc, sizeof(desc), "rx_pkt%d", i); 2375 SYSCTL_ADD_ULONG(ctx, tree, OID_AUTO, 2376 desc, CTLFLAG_RW, &sc->bnx_rx_ret_ring[i].bnx_rx_pkt, ""); 2377 2378 ksnprintf(desc, sizeof(desc), "rx_force_sched%d", i); 2379 SYSCTL_ADD_ULONG(ctx, tree, OID_AUTO, 2380 desc, CTLFLAG_RW, 2381 &sc->bnx_rx_ret_ring[i].bnx_rx_force_sched, ""); 2382 } 2383 #endif 2384 #ifdef BNX_TSS_DEBUG 2385 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 2386 ksnprintf(desc, sizeof(desc), "tx_pkt%d", i); 2387 SYSCTL_ADD_ULONG(ctx, tree, OID_AUTO, 2388 desc, CTLFLAG_RW, &sc->bnx_tx_ring[i].bnx_tx_pkt, ""); 2389 } 2390 #endif 2391 2392 SYSCTL_ADD_ULONG(ctx, tree, OID_AUTO, 2393 "norxbds", CTLFLAG_RW, &sc->bnx_norxbds, ""); 2394 2395 SYSCTL_ADD_ULONG(ctx, tree, OID_AUTO, 2396 "errors", CTLFLAG_RW, &sc->bnx_errors, ""); 2397 2398 #ifdef BNX_TSO_DEBUG 2399 for (i = 0; i < BNX_TSO_NSTATS; ++i) { 2400 ksnprintf(desc, sizeof(desc), "tso%d", i + 1); 2401 SYSCTL_ADD_ULONG(ctx, tree, OID_AUTO, 2402 desc, CTLFLAG_RW, &sc->bnx_tsosegs[i], ""); 2403 } 2404 #endif 2405 2406 /* 2407 * Call MI attach routine. 2408 */ 2409 ether_ifattach(ifp, ether_addr, NULL); 2410 2411 /* Setup TX rings and subqueues */ 2412 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 2413 struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i); 2414 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 2415 2416 ifsq_set_cpuid(ifsq, txr->bnx_tx_cpuid); 2417 ifsq_set_hw_serialize(ifsq, &txr->bnx_tx_serialize); 2418 ifsq_set_priv(ifsq, txr); 2419 txr->bnx_ifsq = ifsq; 2420 2421 ifsq_watchdog_init(&txr->bnx_tx_watchdog, ifsq, bnx_watchdog); 2422 2423 if (bootverbose) { 2424 device_printf(dev, "txr %d -> cpu%d\n", i, 2425 txr->bnx_tx_cpuid); 2426 } 2427 } 2428 2429 error = bnx_setup_intr(sc); 2430 if (error) { 2431 ether_ifdetach(ifp); 2432 goto fail; 2433 } 2434 bnx_set_tick_cpuid(sc, FALSE); 2435 2436 /* 2437 * Create RX standard ring refilling thread 2438 */ 2439 std_cpuid_def = if_ringmap_cpumap(sc->bnx_rx_rmap, 0); 2440 std_cpuid = device_getenv_int(dev, "std.cpuid", std_cpuid_def); 2441 if (std_cpuid < 0 || std_cpuid >= ncpus) { 2442 device_printf(dev, "invalid std.cpuid %d, use %d\n", 2443 std_cpuid, std_cpuid_def); 2444 std_cpuid = std_cpuid_def; 2445 } 2446 2447 std = &sc->bnx_rx_std_ring; 2448 lwkt_create(bnx_rx_std_refill_ithread, std, &std->bnx_rx_std_ithread, 2449 NULL, TDF_NOSTART | TDF_INTTHREAD, std_cpuid, 2450 "%s std", device_get_nameunit(dev)); 2451 lwkt_setpri(std->bnx_rx_std_ithread, TDPRI_INT_MED); 2452 std->bnx_rx_std_ithread->td_preemptable = lwkt_preempt; 2453 2454 return(0); 2455 fail: 2456 bnx_detach(dev); 2457 return(error); 2458 } 2459 2460 static int 2461 bnx_detach(device_t dev) 2462 { 2463 struct bnx_softc *sc = device_get_softc(dev); 2464 struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring; 2465 2466 if (device_is_attached(dev)) { 2467 struct ifnet *ifp = &sc->arpcom.ac_if; 2468 2469 ifnet_serialize_all(ifp); 2470 bnx_stop(sc); 2471 bnx_teardown_intr(sc, sc->bnx_intr_cnt); 2472 ifnet_deserialize_all(ifp); 2473 2474 ether_ifdetach(ifp); 2475 } 2476 2477 if (std->bnx_rx_std_ithread != NULL) { 2478 tsleep_interlock(std, 0); 2479 2480 if (std->bnx_rx_std_ithread->td_gd == mycpu) { 2481 bnx_rx_std_refill_stop(std); 2482 } else { 2483 lwkt_send_ipiq(std->bnx_rx_std_ithread->td_gd, 2484 bnx_rx_std_refill_stop, std); 2485 } 2486 2487 tsleep(std, PINTERLOCKED, "bnx_detach", 0); 2488 if (bootverbose) 2489 device_printf(dev, "RX std ithread exited\n"); 2490 2491 lwkt_synchronize_ipiqs("bnx_detach_ipiq"); 2492 } 2493 2494 if (sc->bnx_flags & BNX_FLAG_TBI) 2495 ifmedia_removeall(&sc->bnx_ifmedia); 2496 if (sc->bnx_miibus) 2497 device_delete_child(dev, sc->bnx_miibus); 2498 bus_generic_detach(dev); 2499 2500 bnx_free_intr(sc); 2501 2502 if (sc->bnx_msix_mem_res != NULL) { 2503 bus_release_resource(dev, SYS_RES_MEMORY, sc->bnx_msix_mem_rid, 2504 sc->bnx_msix_mem_res); 2505 } 2506 if (sc->bnx_res != NULL) { 2507 bus_release_resource(dev, SYS_RES_MEMORY, 2508 BGE_PCI_BAR0, sc->bnx_res); 2509 } 2510 if (sc->bnx_res2 != NULL) { 2511 bus_release_resource(dev, SYS_RES_MEMORY, 2512 PCIR_BAR(2), sc->bnx_res2); 2513 } 2514 2515 bnx_dma_free(sc); 2516 2517 if (sc->bnx_serialize != NULL) 2518 kfree(sc->bnx_serialize, M_DEVBUF); 2519 2520 if (sc->bnx_rx_rmap != NULL) 2521 if_ringmap_free(sc->bnx_rx_rmap); 2522 if (sc->bnx_tx_rmap != NULL) 2523 if_ringmap_free(sc->bnx_tx_rmap); 2524 2525 return 0; 2526 } 2527 2528 static void 2529 bnx_reset(struct bnx_softc *sc) 2530 { 2531 device_t dev = sc->bnx_dev; 2532 uint32_t cachesize, command, reset, mac_mode, mac_mode_mask; 2533 void (*write_op)(struct bnx_softc *, uint32_t, uint32_t); 2534 int i, val = 0; 2535 uint16_t devctl; 2536 2537 mac_mode_mask = BGE_MACMODE_HALF_DUPLEX | BGE_MACMODE_PORTMODE; 2538 if (sc->bnx_mfw_flags & BNX_MFW_ON_APE) 2539 mac_mode_mask |= BGE_MACMODE_APE_RX_EN | BGE_MACMODE_APE_TX_EN; 2540 mac_mode = CSR_READ_4(sc, BGE_MAC_MODE) & mac_mode_mask; 2541 2542 write_op = bnx_writemem_direct; 2543 2544 CSR_WRITE_4(sc, BGE_NVRAM_SWARB, BGE_NVRAMSWARB_SET1); 2545 for (i = 0; i < 8000; i++) { 2546 if (CSR_READ_4(sc, BGE_NVRAM_SWARB) & BGE_NVRAMSWARB_GNT1) 2547 break; 2548 DELAY(20); 2549 } 2550 if (i == 8000) 2551 if_printf(&sc->arpcom.ac_if, "NVRAM lock timedout!\n"); 2552 2553 /* Take APE lock when performing reset. */ 2554 bnx_ape_lock(sc, BGE_APE_LOCK_GRC); 2555 2556 /* Save some important PCI state. */ 2557 cachesize = pci_read_config(dev, BGE_PCI_CACHESZ, 4); 2558 command = pci_read_config(dev, BGE_PCI_CMD, 4); 2559 2560 pci_write_config(dev, BGE_PCI_MISC_CTL, 2561 BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR| 2562 BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW| 2563 BGE_PCIMISCCTL_TAGGED_STATUS, 4); 2564 2565 /* Disable fastboot on controllers that support it. */ 2566 if (bootverbose) 2567 if_printf(&sc->arpcom.ac_if, "Disabling fastboot\n"); 2568 CSR_WRITE_4(sc, BGE_FASTBOOT_PC, 0x0); 2569 2570 /* 2571 * Write the magic number to SRAM at offset 0xB50. 2572 * When firmware finishes its initialization it will 2573 * write ~BGE_SRAM_FW_MB_MAGIC to the same location. 2574 */ 2575 bnx_writemem_ind(sc, BGE_SRAM_FW_MB, BGE_SRAM_FW_MB_MAGIC); 2576 2577 reset = BGE_MISCCFG_RESET_CORE_CLOCKS|(65<<1); 2578 2579 /* XXX: Broadcom Linux driver. */ 2580 /* Force PCI-E 1.0a mode */ 2581 if (!BNX_IS_57765_PLUS(sc) && 2582 CSR_READ_4(sc, BGE_PCIE_PHY_TSTCTL) == 2583 (BGE_PCIE_PHY_TSTCTL_PSCRAM | 2584 BGE_PCIE_PHY_TSTCTL_PCIE10)) { 2585 CSR_WRITE_4(sc, BGE_PCIE_PHY_TSTCTL, 2586 BGE_PCIE_PHY_TSTCTL_PSCRAM); 2587 } 2588 if (sc->bnx_chipid != BGE_CHIPID_BCM5750_A0) { 2589 /* Prevent PCIE link training during global reset */ 2590 CSR_WRITE_4(sc, BGE_MISC_CFG, (1<<29)); 2591 reset |= (1<<29); 2592 } 2593 2594 /* 2595 * Set GPHY Power Down Override to leave GPHY 2596 * powered up in D0 uninitialized. 2597 */ 2598 if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0) 2599 reset |= BGE_MISCCFG_GPHY_PD_OVERRIDE; 2600 2601 /* Issue global reset */ 2602 write_op(sc, BGE_MISC_CFG, reset); 2603 2604 DELAY(100 * 1000); 2605 2606 /* XXX: Broadcom Linux driver. */ 2607 if (sc->bnx_chipid == BGE_CHIPID_BCM5750_A0) { 2608 uint32_t v; 2609 2610 DELAY(500000); /* wait for link training to complete */ 2611 v = pci_read_config(dev, 0xc4, 4); 2612 pci_write_config(dev, 0xc4, v | (1<<15), 4); 2613 } 2614 2615 devctl = pci_read_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL, 2); 2616 2617 /* Disable no snoop and disable relaxed ordering. */ 2618 devctl &= ~(PCIEM_DEVCTL_RELAX_ORDER | PCIEM_DEVCTL_NOSNOOP); 2619 2620 /* Old PCI-E chips only support 128 bytes Max PayLoad Size. */ 2621 if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0) { 2622 devctl &= ~PCIEM_DEVCTL_MAX_PAYLOAD_MASK; 2623 devctl |= PCIEM_DEVCTL_MAX_PAYLOAD_128; 2624 } 2625 2626 pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL, 2627 devctl, 2); 2628 2629 /* Clear error status. */ 2630 pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVSTS, 2631 PCIEM_DEVSTS_CORR_ERR | 2632 PCIEM_DEVSTS_NFATAL_ERR | 2633 PCIEM_DEVSTS_FATAL_ERR | 2634 PCIEM_DEVSTS_UNSUPP_REQ, 2); 2635 2636 /* Reset some of the PCI state that got zapped by reset */ 2637 pci_write_config(dev, BGE_PCI_MISC_CTL, 2638 BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR| 2639 BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW| 2640 BGE_PCIMISCCTL_TAGGED_STATUS, 4); 2641 val = BGE_PCISTATE_ROM_ENABLE | BGE_PCISTATE_ROM_RETRY_ENABLE; 2642 if (sc->bnx_mfw_flags & BNX_MFW_ON_APE) { 2643 val |= BGE_PCISTATE_ALLOW_APE_CTLSPC_WR | 2644 BGE_PCISTATE_ALLOW_APE_SHMEM_WR | 2645 BGE_PCISTATE_ALLOW_APE_PSPACE_WR; 2646 } 2647 pci_write_config(dev, BGE_PCI_PCISTATE, val, 4); 2648 pci_write_config(dev, BGE_PCI_CACHESZ, cachesize, 4); 2649 pci_write_config(dev, BGE_PCI_CMD, command, 4); 2650 2651 /* Enable memory arbiter */ 2652 CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE); 2653 2654 /* Fix up byte swapping */ 2655 CSR_WRITE_4(sc, BGE_MODE_CTL, bnx_dma_swap_options(sc)); 2656 2657 val = CSR_READ_4(sc, BGE_MAC_MODE); 2658 val = (val & ~mac_mode_mask) | mac_mode; 2659 CSR_WRITE_4(sc, BGE_MAC_MODE, val); 2660 DELAY(40); 2661 2662 bnx_ape_unlock(sc, BGE_APE_LOCK_GRC); 2663 2664 /* 2665 * Poll until we see the 1's complement of the magic number. 2666 * This indicates that the firmware initialization is complete. 2667 */ 2668 for (i = 0; i < BNX_FIRMWARE_TIMEOUT; i++) { 2669 val = bnx_readmem_ind(sc, BGE_SRAM_FW_MB); 2670 if (val == ~BGE_SRAM_FW_MB_MAGIC) 2671 break; 2672 DELAY(10); 2673 } 2674 if (i == BNX_FIRMWARE_TIMEOUT) { 2675 if_printf(&sc->arpcom.ac_if, "firmware handshake " 2676 "timed out, found 0x%08x\n", val); 2677 } 2678 2679 /* BCM57765 A0 needs additional time before accessing. */ 2680 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0) 2681 DELAY(10 * 1000); 2682 2683 /* 2684 * The 5704 in TBI mode apparently needs some special 2685 * adjustment to insure the SERDES drive level is set 2686 * to 1.2V. 2687 */ 2688 if (sc->bnx_asicrev == BGE_ASICREV_BCM5704 && 2689 (sc->bnx_flags & BNX_FLAG_TBI)) { 2690 uint32_t serdescfg; 2691 2692 serdescfg = CSR_READ_4(sc, BGE_SERDES_CFG); 2693 serdescfg = (serdescfg & ~0xFFF) | 0x880; 2694 CSR_WRITE_4(sc, BGE_SERDES_CFG, serdescfg); 2695 } 2696 2697 CSR_WRITE_4(sc, BGE_MI_MODE, 2698 sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL); 2699 DELAY(80); 2700 2701 /* XXX: Broadcom Linux driver. */ 2702 if (!BNX_IS_57765_PLUS(sc)) { 2703 uint32_t v; 2704 2705 /* Enable Data FIFO protection. */ 2706 v = CSR_READ_4(sc, BGE_PCIE_TLDLPL_PORT); 2707 CSR_WRITE_4(sc, BGE_PCIE_TLDLPL_PORT, v | (1 << 25)); 2708 } 2709 2710 DELAY(10000); 2711 2712 if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) { 2713 BNX_CLRBIT(sc, BGE_CPMU_CLCK_ORIDE, 2714 CPMU_CLCK_ORIDE_MAC_ORIDE_EN); 2715 } 2716 } 2717 2718 /* 2719 * Frame reception handling. This is called if there's a frame 2720 * on the receive return list. 2721 * 2722 * Note: we have to be able to handle two possibilities here: 2723 * 1) the frame is from the jumbo recieve ring 2724 * 2) the frame is from the standard receive ring 2725 */ 2726 2727 static void 2728 bnx_rxeof(struct bnx_rx_ret_ring *ret, uint16_t rx_prod, int count) 2729 { 2730 struct bnx_softc *sc = ret->bnx_sc; 2731 struct bnx_rx_std_ring *std = ret->bnx_std; 2732 struct ifnet *ifp = &sc->arpcom.ac_if; 2733 int std_used = 0, cpuid = mycpuid; 2734 2735 while (ret->bnx_rx_saved_considx != rx_prod && count != 0) { 2736 struct pktinfo pi0, *pi = NULL; 2737 struct bge_rx_bd *cur_rx; 2738 struct bnx_rx_buf *rb; 2739 uint32_t rxidx; 2740 struct mbuf *m = NULL; 2741 uint16_t vlan_tag = 0; 2742 int have_tag = 0; 2743 2744 --count; 2745 2746 cur_rx = &ret->bnx_rx_ret_ring[ret->bnx_rx_saved_considx]; 2747 2748 rxidx = cur_rx->bge_idx; 2749 KKASSERT(rxidx < BGE_STD_RX_RING_CNT); 2750 2751 BNX_INC(ret->bnx_rx_saved_considx, BNX_RETURN_RING_CNT); 2752 #ifdef BNX_RSS_DEBUG 2753 ret->bnx_rx_pkt++; 2754 #endif 2755 2756 if (cur_rx->bge_flags & BGE_RXBDFLAG_VLAN_TAG) { 2757 have_tag = 1; 2758 vlan_tag = cur_rx->bge_vlan_tag; 2759 } 2760 2761 if (ret->bnx_rx_cnt >= ret->bnx_rx_cntmax) { 2762 atomic_add_int(&std->bnx_rx_std_used, std_used); 2763 std_used = 0; 2764 2765 bnx_rx_std_refill_sched(ret, std); 2766 } 2767 ret->bnx_rx_cnt++; 2768 ++std_used; 2769 2770 rb = &std->bnx_rx_std_buf[rxidx]; 2771 m = rb->bnx_rx_mbuf; 2772 if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) { 2773 IFNET_STAT_INC(ifp, ierrors, 1); 2774 cpu_sfence(); 2775 rb->bnx_rx_refilled = 1; 2776 continue; 2777 } 2778 if (bnx_newbuf_std(ret, rxidx, 0)) { 2779 IFNET_STAT_INC(ifp, ierrors, 1); 2780 continue; 2781 } 2782 2783 IFNET_STAT_INC(ifp, ipackets, 1); 2784 m->m_pkthdr.len = m->m_len = cur_rx->bge_len - ETHER_CRC_LEN; 2785 m->m_pkthdr.rcvif = ifp; 2786 2787 if ((ifp->if_capenable & IFCAP_RXCSUM) && 2788 (cur_rx->bge_flags & BGE_RXBDFLAG_IPV6) == 0) { 2789 if (cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) { 2790 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 2791 if ((cur_rx->bge_error_flag & 2792 BGE_RXERRFLAG_IP_CSUM_NOK) == 0) 2793 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2794 } 2795 if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) { 2796 m->m_pkthdr.csum_data = 2797 cur_rx->bge_tcp_udp_csum; 2798 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 2799 CSUM_PSEUDO_HDR; 2800 } 2801 } 2802 if (ifp->if_capenable & IFCAP_RSS) { 2803 pi = bnx_rss_info(&pi0, cur_rx); 2804 if (pi != NULL && 2805 (cur_rx->bge_flags & BGE_RXBDFLAG_RSS_HASH)) 2806 m_sethash(m, toeplitz_hash(cur_rx->bge_hash)); 2807 } 2808 2809 /* 2810 * If we received a packet with a vlan tag, pass it 2811 * to vlan_input() instead of ether_input(). 2812 */ 2813 if (have_tag) { 2814 m->m_flags |= M_VLANTAG; 2815 m->m_pkthdr.ether_vlantag = vlan_tag; 2816 } 2817 ifp->if_input(ifp, m, pi, cpuid); 2818 } 2819 bnx_writembx(sc, ret->bnx_rx_mbx, ret->bnx_rx_saved_considx); 2820 2821 if (std_used > 0) { 2822 int cur_std_used; 2823 2824 cur_std_used = atomic_fetchadd_int(&std->bnx_rx_std_used, 2825 std_used); 2826 if (cur_std_used + std_used >= (BGE_STD_RX_RING_CNT / 2)) { 2827 #ifdef BNX_RSS_DEBUG 2828 ret->bnx_rx_force_sched++; 2829 #endif 2830 bnx_rx_std_refill_sched(ret, std); 2831 } 2832 } 2833 } 2834 2835 static void 2836 bnx_txeof(struct bnx_tx_ring *txr, uint16_t tx_cons) 2837 { 2838 struct ifnet *ifp = &txr->bnx_sc->arpcom.ac_if; 2839 2840 /* 2841 * Go through our tx ring and free mbufs for those 2842 * frames that have been sent. 2843 */ 2844 while (txr->bnx_tx_saved_considx != tx_cons) { 2845 struct bnx_tx_buf *buf; 2846 uint32_t idx = 0; 2847 2848 idx = txr->bnx_tx_saved_considx; 2849 buf = &txr->bnx_tx_buf[idx]; 2850 if (buf->bnx_tx_mbuf != NULL) { 2851 IFNET_STAT_INC(ifp, opackets, 1); 2852 #ifdef BNX_TSS_DEBUG 2853 txr->bnx_tx_pkt++; 2854 #endif 2855 bus_dmamap_unload(txr->bnx_tx_mtag, 2856 buf->bnx_tx_dmamap); 2857 m_freem(buf->bnx_tx_mbuf); 2858 buf->bnx_tx_mbuf = NULL; 2859 } 2860 txr->bnx_tx_cnt--; 2861 BNX_INC(txr->bnx_tx_saved_considx, BGE_TX_RING_CNT); 2862 } 2863 2864 if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) >= 2865 (BNX_NSEG_RSVD + BNX_NSEG_SPARE)) 2866 ifsq_clr_oactive(txr->bnx_ifsq); 2867 2868 if (txr->bnx_tx_cnt == 0) 2869 txr->bnx_tx_watchdog.wd_timer = 0; 2870 2871 if (!ifsq_is_empty(txr->bnx_ifsq)) 2872 ifsq_devstart(txr->bnx_ifsq); 2873 } 2874 2875 static int 2876 bnx_handle_status(struct bnx_softc *sc) 2877 { 2878 uint32_t status; 2879 int handle = 0; 2880 2881 status = *sc->bnx_hw_status; 2882 2883 if (status & BGE_STATFLAG_ERROR) { 2884 uint32_t val; 2885 int reset = 0; 2886 2887 sc->bnx_errors++; 2888 2889 val = CSR_READ_4(sc, BGE_FLOW_ATTN); 2890 if (val & ~BGE_FLOWATTN_MB_LOWAT) { 2891 if_printf(&sc->arpcom.ac_if, 2892 "flow attn 0x%08x\n", val); 2893 reset = 1; 2894 } 2895 2896 val = CSR_READ_4(sc, BGE_MSI_STATUS); 2897 if (val & ~BGE_MSISTAT_MSI_PCI_REQ) { 2898 if_printf(&sc->arpcom.ac_if, 2899 "msi status 0x%08x\n", val); 2900 reset = 1; 2901 } 2902 2903 val = CSR_READ_4(sc, BGE_RDMA_STATUS); 2904 if (val) { 2905 if_printf(&sc->arpcom.ac_if, 2906 "rmda status 0x%08x\n", val); 2907 reset = 1; 2908 } 2909 2910 val = CSR_READ_4(sc, BGE_WDMA_STATUS); 2911 if (val) { 2912 if_printf(&sc->arpcom.ac_if, 2913 "wdma status 0x%08x\n", val); 2914 reset = 1; 2915 } 2916 2917 if (reset) { 2918 bnx_serialize_skipmain(sc); 2919 bnx_init(sc); 2920 bnx_deserialize_skipmain(sc); 2921 } 2922 handle = 1; 2923 } 2924 2925 if ((status & BGE_STATFLAG_LINKSTATE_CHANGED) || sc->bnx_link_evt) { 2926 if (bootverbose) { 2927 if_printf(&sc->arpcom.ac_if, "link change, " 2928 "link_evt %d\n", sc->bnx_link_evt); 2929 } 2930 bnx_link_poll(sc); 2931 handle = 1; 2932 } 2933 2934 return handle; 2935 } 2936 2937 #ifdef IFPOLL_ENABLE 2938 2939 static void 2940 bnx_npoll_rx(struct ifnet *ifp __unused, void *xret, int cycle) 2941 { 2942 struct bnx_rx_ret_ring *ret = xret; 2943 uint16_t rx_prod; 2944 2945 ASSERT_SERIALIZED(&ret->bnx_rx_ret_serialize); 2946 2947 ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag; 2948 cpu_lfence(); 2949 2950 rx_prod = *ret->bnx_rx_considx; 2951 if (ret->bnx_rx_saved_considx != rx_prod) 2952 bnx_rxeof(ret, rx_prod, cycle); 2953 } 2954 2955 static void 2956 bnx_npoll_tx_notag(struct ifnet *ifp __unused, void *xtxr, int cycle __unused) 2957 { 2958 struct bnx_tx_ring *txr = xtxr; 2959 uint16_t tx_cons; 2960 2961 ASSERT_SERIALIZED(&txr->bnx_tx_serialize); 2962 2963 tx_cons = *txr->bnx_tx_considx; 2964 if (txr->bnx_tx_saved_considx != tx_cons) 2965 bnx_txeof(txr, tx_cons); 2966 } 2967 2968 static void 2969 bnx_npoll_tx(struct ifnet *ifp, void *xtxr, int cycle) 2970 { 2971 struct bnx_tx_ring *txr = xtxr; 2972 2973 ASSERT_SERIALIZED(&txr->bnx_tx_serialize); 2974 2975 txr->bnx_saved_status_tag = *txr->bnx_hw_status_tag; 2976 cpu_lfence(); 2977 bnx_npoll_tx_notag(ifp, txr, cycle); 2978 } 2979 2980 static void 2981 bnx_npoll_status_notag(struct ifnet *ifp) 2982 { 2983 struct bnx_softc *sc = ifp->if_softc; 2984 2985 ASSERT_SERIALIZED(&sc->bnx_main_serialize); 2986 2987 if (bnx_handle_status(sc)) { 2988 /* 2989 * Status changes are handled; force the chip to 2990 * update the status block to reflect whether there 2991 * are more status changes or not, else staled status 2992 * changes are always seen. 2993 */ 2994 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW); 2995 } 2996 } 2997 2998 static void 2999 bnx_npoll_status(struct ifnet *ifp) 3000 { 3001 struct bnx_softc *sc = ifp->if_softc; 3002 3003 ASSERT_SERIALIZED(&sc->bnx_main_serialize); 3004 3005 sc->bnx_saved_status_tag = *sc->bnx_hw_status_tag; 3006 cpu_lfence(); 3007 bnx_npoll_status_notag(ifp); 3008 } 3009 3010 static void 3011 bnx_npoll(struct ifnet *ifp, struct ifpoll_info *info) 3012 { 3013 struct bnx_softc *sc = ifp->if_softc; 3014 int i; 3015 3016 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3017 3018 if (info != NULL) { 3019 if (sc->bnx_flags & BNX_FLAG_STATUS_HASTAG) 3020 info->ifpi_status.status_func = bnx_npoll_status; 3021 else 3022 info->ifpi_status.status_func = bnx_npoll_status_notag; 3023 info->ifpi_status.serializer = &sc->bnx_main_serialize; 3024 3025 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 3026 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 3027 int cpu = if_ringmap_cpumap(sc->bnx_tx_rmap, i); 3028 3029 KKASSERT(cpu < netisr_ncpus); 3030 if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) { 3031 info->ifpi_tx[cpu].poll_func = 3032 bnx_npoll_tx_notag; 3033 } else { 3034 info->ifpi_tx[cpu].poll_func = bnx_npoll_tx; 3035 } 3036 info->ifpi_tx[cpu].arg = txr; 3037 info->ifpi_tx[cpu].serializer = &txr->bnx_tx_serialize; 3038 ifsq_set_cpuid(txr->bnx_ifsq, cpu); 3039 } 3040 3041 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 3042 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i]; 3043 int cpu = if_ringmap_cpumap(sc->bnx_rx_rmap, i); 3044 3045 KKASSERT(cpu < netisr_ncpus); 3046 info->ifpi_rx[cpu].poll_func = bnx_npoll_rx; 3047 info->ifpi_rx[cpu].arg = ret; 3048 info->ifpi_rx[cpu].serializer = 3049 &ret->bnx_rx_ret_serialize; 3050 } 3051 3052 if (ifp->if_flags & IFF_RUNNING) { 3053 bnx_disable_intr(sc); 3054 bnx_set_tick_cpuid(sc, TRUE); 3055 3056 sc->bnx_coal_chg = BNX_TX_COAL_BDS_CHG | 3057 BNX_RX_COAL_BDS_CHG; 3058 bnx_coal_change(sc); 3059 } 3060 } else { 3061 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 3062 ifsq_set_cpuid(sc->bnx_tx_ring[i].bnx_ifsq, 3063 sc->bnx_tx_ring[i].bnx_tx_cpuid); 3064 } 3065 if (ifp->if_flags & IFF_RUNNING) { 3066 sc->bnx_coal_chg = BNX_TX_COAL_BDS_CHG | 3067 BNX_RX_COAL_BDS_CHG; 3068 bnx_coal_change(sc); 3069 3070 bnx_enable_intr(sc); 3071 bnx_set_tick_cpuid(sc, FALSE); 3072 } 3073 } 3074 } 3075 3076 #endif /* IFPOLL_ENABLE */ 3077 3078 static void 3079 bnx_intr_legacy(void *xsc) 3080 { 3081 struct bnx_softc *sc = xsc; 3082 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0]; 3083 3084 if (ret->bnx_saved_status_tag == *ret->bnx_hw_status_tag) { 3085 uint32_t val; 3086 3087 val = pci_read_config(sc->bnx_dev, BGE_PCI_PCISTATE, 4); 3088 if (val & BGE_PCISTAT_INTR_NOTACT) 3089 return; 3090 } 3091 3092 /* 3093 * NOTE: 3094 * Interrupt will have to be disabled if tagged status 3095 * is used, else interrupt will always be asserted on 3096 * certain chips (at least on BCM5750 AX/BX). 3097 */ 3098 bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1); 3099 3100 bnx_intr(sc); 3101 } 3102 3103 static void 3104 bnx_msi(void *xsc) 3105 { 3106 bnx_intr(xsc); 3107 } 3108 3109 static void 3110 bnx_intr(struct bnx_softc *sc) 3111 { 3112 struct ifnet *ifp = &sc->arpcom.ac_if; 3113 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0]; 3114 3115 ASSERT_SERIALIZED(&sc->bnx_main_serialize); 3116 3117 ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag; 3118 /* 3119 * Use a load fence to ensure that status_tag is saved 3120 * before rx_prod, tx_cons and status. 3121 */ 3122 cpu_lfence(); 3123 3124 bnx_handle_status(sc); 3125 3126 if (ifp->if_flags & IFF_RUNNING) { 3127 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; 3128 uint16_t rx_prod, tx_cons; 3129 3130 lwkt_serialize_enter(&ret->bnx_rx_ret_serialize); 3131 rx_prod = *ret->bnx_rx_considx; 3132 if (ret->bnx_rx_saved_considx != rx_prod) 3133 bnx_rxeof(ret, rx_prod, -1); 3134 lwkt_serialize_exit(&ret->bnx_rx_ret_serialize); 3135 3136 lwkt_serialize_enter(&txr->bnx_tx_serialize); 3137 tx_cons = *txr->bnx_tx_considx; 3138 if (txr->bnx_tx_saved_considx != tx_cons) 3139 bnx_txeof(txr, tx_cons); 3140 lwkt_serialize_exit(&txr->bnx_tx_serialize); 3141 } 3142 3143 bnx_writembx(sc, BGE_MBX_IRQ0_LO, ret->bnx_saved_status_tag << 24); 3144 } 3145 3146 static void 3147 bnx_msix_tx_status(void *xtxr) 3148 { 3149 struct bnx_tx_ring *txr = xtxr; 3150 struct bnx_softc *sc = txr->bnx_sc; 3151 struct ifnet *ifp = &sc->arpcom.ac_if; 3152 3153 ASSERT_SERIALIZED(&sc->bnx_main_serialize); 3154 3155 txr->bnx_saved_status_tag = *txr->bnx_hw_status_tag; 3156 /* 3157 * Use a load fence to ensure that status_tag is saved 3158 * before tx_cons and status. 3159 */ 3160 cpu_lfence(); 3161 3162 bnx_handle_status(sc); 3163 3164 if (ifp->if_flags & IFF_RUNNING) { 3165 uint16_t tx_cons; 3166 3167 lwkt_serialize_enter(&txr->bnx_tx_serialize); 3168 tx_cons = *txr->bnx_tx_considx; 3169 if (txr->bnx_tx_saved_considx != tx_cons) 3170 bnx_txeof(txr, tx_cons); 3171 lwkt_serialize_exit(&txr->bnx_tx_serialize); 3172 } 3173 3174 bnx_writembx(sc, BGE_MBX_IRQ0_LO, txr->bnx_saved_status_tag << 24); 3175 } 3176 3177 static void 3178 bnx_msix_rx(void *xret) 3179 { 3180 struct bnx_rx_ret_ring *ret = xret; 3181 uint16_t rx_prod; 3182 3183 ASSERT_SERIALIZED(&ret->bnx_rx_ret_serialize); 3184 3185 ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag; 3186 /* 3187 * Use a load fence to ensure that status_tag is saved 3188 * before rx_prod. 3189 */ 3190 cpu_lfence(); 3191 3192 rx_prod = *ret->bnx_rx_considx; 3193 if (ret->bnx_rx_saved_considx != rx_prod) 3194 bnx_rxeof(ret, rx_prod, -1); 3195 3196 bnx_writembx(ret->bnx_sc, ret->bnx_msix_mbx, 3197 ret->bnx_saved_status_tag << 24); 3198 } 3199 3200 static void 3201 bnx_msix_rxtx(void *xret) 3202 { 3203 struct bnx_rx_ret_ring *ret = xret; 3204 struct bnx_tx_ring *txr = ret->bnx_txr; 3205 uint16_t rx_prod, tx_cons; 3206 3207 ASSERT_SERIALIZED(&ret->bnx_rx_ret_serialize); 3208 3209 ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag; 3210 /* 3211 * Use a load fence to ensure that status_tag is saved 3212 * before rx_prod and tx_cons. 3213 */ 3214 cpu_lfence(); 3215 3216 rx_prod = *ret->bnx_rx_considx; 3217 if (ret->bnx_rx_saved_considx != rx_prod) 3218 bnx_rxeof(ret, rx_prod, -1); 3219 3220 lwkt_serialize_enter(&txr->bnx_tx_serialize); 3221 tx_cons = *txr->bnx_tx_considx; 3222 if (txr->bnx_tx_saved_considx != tx_cons) 3223 bnx_txeof(txr, tx_cons); 3224 lwkt_serialize_exit(&txr->bnx_tx_serialize); 3225 3226 bnx_writembx(ret->bnx_sc, ret->bnx_msix_mbx, 3227 ret->bnx_saved_status_tag << 24); 3228 } 3229 3230 static void 3231 bnx_msix_status(void *xsc) 3232 { 3233 struct bnx_softc *sc = xsc; 3234 3235 ASSERT_SERIALIZED(&sc->bnx_main_serialize); 3236 3237 sc->bnx_saved_status_tag = *sc->bnx_hw_status_tag; 3238 /* 3239 * Use a load fence to ensure that status_tag is saved 3240 * before status. 3241 */ 3242 cpu_lfence(); 3243 3244 bnx_handle_status(sc); 3245 3246 bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_saved_status_tag << 24); 3247 } 3248 3249 static void 3250 bnx_tick(void *xsc) 3251 { 3252 struct bnx_softc *sc = xsc; 3253 3254 lwkt_serialize_enter(&sc->bnx_main_serialize); 3255 3256 bnx_stats_update_regs(sc); 3257 3258 if (sc->bnx_flags & BNX_FLAG_TBI) { 3259 /* 3260 * Since in TBI mode auto-polling can't be used we should poll 3261 * link status manually. Here we register pending link event 3262 * and trigger interrupt. 3263 */ 3264 sc->bnx_link_evt++; 3265 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW); 3266 } else if (!sc->bnx_link) { 3267 mii_tick(device_get_softc(sc->bnx_miibus)); 3268 } 3269 3270 callout_reset_bycpu(&sc->bnx_tick_timer, hz, bnx_tick, sc, 3271 sc->bnx_tick_cpuid); 3272 3273 lwkt_serialize_exit(&sc->bnx_main_serialize); 3274 } 3275 3276 static void 3277 bnx_stats_update_regs(struct bnx_softc *sc) 3278 { 3279 struct ifnet *ifp = &sc->arpcom.ac_if; 3280 struct bge_mac_stats_regs stats; 3281 uint32_t *s, val; 3282 int i; 3283 3284 s = (uint32_t *)&stats; 3285 for (i = 0; i < sizeof(struct bge_mac_stats_regs); i += 4) { 3286 *s = CSR_READ_4(sc, BGE_RX_STATS + i); 3287 s++; 3288 } 3289 3290 IFNET_STAT_SET(ifp, collisions, 3291 (stats.dot3StatsSingleCollisionFrames + 3292 stats.dot3StatsMultipleCollisionFrames + 3293 stats.dot3StatsExcessiveCollisions + 3294 stats.dot3StatsLateCollisions)); 3295 3296 val = CSR_READ_4(sc, BGE_RXLP_LOCSTAT_OUT_OF_BDS); 3297 sc->bnx_norxbds += val; 3298 } 3299 3300 /* 3301 * Encapsulate an mbuf chain in the tx ring by coupling the mbuf data 3302 * pointers to descriptors. 3303 */ 3304 static int 3305 bnx_encap(struct bnx_tx_ring *txr, struct mbuf **m_head0, uint32_t *txidx, 3306 int *segs_used) 3307 { 3308 struct bge_tx_bd *d = NULL; 3309 uint16_t csum_flags = 0, vlan_tag = 0, mss = 0; 3310 bus_dma_segment_t segs[BNX_NSEG_NEW]; 3311 bus_dmamap_t map; 3312 int error, maxsegs, nsegs, idx, i; 3313 struct mbuf *m_head = *m_head0, *m_new; 3314 3315 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 3316 #ifdef BNX_TSO_DEBUG 3317 int tso_nsegs; 3318 #endif 3319 3320 error = bnx_setup_tso(txr, m_head0, &mss, &csum_flags); 3321 if (error) 3322 return error; 3323 m_head = *m_head0; 3324 3325 #ifdef BNX_TSO_DEBUG 3326 tso_nsegs = (m_head->m_pkthdr.len / 3327 m_head->m_pkthdr.tso_segsz) - 1; 3328 if (tso_nsegs > (BNX_TSO_NSTATS - 1)) 3329 tso_nsegs = BNX_TSO_NSTATS - 1; 3330 else if (tso_nsegs < 0) 3331 tso_nsegs = 0; 3332 txr->bnx_sc->bnx_tsosegs[tso_nsegs]++; 3333 #endif 3334 } else if (m_head->m_pkthdr.csum_flags & BNX_CSUM_FEATURES) { 3335 if (m_head->m_pkthdr.csum_flags & CSUM_IP) 3336 csum_flags |= BGE_TXBDFLAG_IP_CSUM; 3337 if (m_head->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)) 3338 csum_flags |= BGE_TXBDFLAG_TCP_UDP_CSUM; 3339 if (m_head->m_flags & M_LASTFRAG) 3340 csum_flags |= BGE_TXBDFLAG_IP_FRAG_END; 3341 else if (m_head->m_flags & M_FRAG) 3342 csum_flags |= BGE_TXBDFLAG_IP_FRAG; 3343 } 3344 if (m_head->m_flags & M_VLANTAG) { 3345 csum_flags |= BGE_TXBDFLAG_VLAN_TAG; 3346 vlan_tag = m_head->m_pkthdr.ether_vlantag; 3347 } 3348 3349 idx = *txidx; 3350 map = txr->bnx_tx_buf[idx].bnx_tx_dmamap; 3351 3352 maxsegs = (BGE_TX_RING_CNT - txr->bnx_tx_cnt) - BNX_NSEG_RSVD; 3353 KASSERT(maxsegs >= BNX_NSEG_SPARE, 3354 ("not enough segments %d", maxsegs)); 3355 3356 if (maxsegs > BNX_NSEG_NEW) 3357 maxsegs = BNX_NSEG_NEW; 3358 3359 /* 3360 * Pad outbound frame to BGE_MIN_FRAMELEN for an unusual reason. 3361 * The bge hardware will pad out Tx runts to BGE_MIN_FRAMELEN, 3362 * but when such padded frames employ the bge IP/TCP checksum 3363 * offload, the hardware checksum assist gives incorrect results 3364 * (possibly from incorporating its own padding into the UDP/TCP 3365 * checksum; who knows). If we pad such runts with zeros, the 3366 * onboard checksum comes out correct. 3367 */ 3368 if ((csum_flags & BGE_TXBDFLAG_TCP_UDP_CSUM) && 3369 m_head->m_pkthdr.len < BNX_MIN_FRAMELEN) { 3370 error = m_devpad(m_head, BNX_MIN_FRAMELEN); 3371 if (error) 3372 goto back; 3373 } 3374 3375 if ((txr->bnx_tx_flags & BNX_TX_FLAG_SHORTDMA) && 3376 m_head->m_next != NULL) { 3377 m_new = bnx_defrag_shortdma(m_head); 3378 if (m_new == NULL) { 3379 error = ENOBUFS; 3380 goto back; 3381 } 3382 *m_head0 = m_head = m_new; 3383 } 3384 if ((m_head->m_pkthdr.csum_flags & CSUM_TSO) == 0 && 3385 (txr->bnx_tx_flags & BNX_TX_FLAG_FORCE_DEFRAG) && 3386 m_head->m_next != NULL) { 3387 /* 3388 * Forcefully defragment mbuf chain to overcome hardware 3389 * limitation which only support a single outstanding 3390 * DMA read operation. If it fails, keep moving on using 3391 * the original mbuf chain. 3392 */ 3393 m_new = m_defrag(m_head, M_NOWAIT); 3394 if (m_new != NULL) 3395 *m_head0 = m_head = m_new; 3396 } 3397 3398 error = bus_dmamap_load_mbuf_defrag(txr->bnx_tx_mtag, map, 3399 m_head0, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 3400 if (error) 3401 goto back; 3402 *segs_used += nsegs; 3403 3404 m_head = *m_head0; 3405 bus_dmamap_sync(txr->bnx_tx_mtag, map, BUS_DMASYNC_PREWRITE); 3406 3407 for (i = 0; ; i++) { 3408 d = &txr->bnx_tx_ring[idx]; 3409 3410 d->bge_addr.bge_addr_lo = BGE_ADDR_LO(segs[i].ds_addr); 3411 d->bge_addr.bge_addr_hi = BGE_ADDR_HI(segs[i].ds_addr); 3412 d->bge_len = segs[i].ds_len; 3413 d->bge_flags = csum_flags; 3414 d->bge_vlan_tag = vlan_tag; 3415 d->bge_mss = mss; 3416 3417 if (i == nsegs - 1) 3418 break; 3419 BNX_INC(idx, BGE_TX_RING_CNT); 3420 } 3421 /* Mark the last segment as end of packet... */ 3422 d->bge_flags |= BGE_TXBDFLAG_END; 3423 3424 /* 3425 * Insure that the map for this transmission is placed at 3426 * the array index of the last descriptor in this chain. 3427 */ 3428 txr->bnx_tx_buf[*txidx].bnx_tx_dmamap = txr->bnx_tx_buf[idx].bnx_tx_dmamap; 3429 txr->bnx_tx_buf[idx].bnx_tx_dmamap = map; 3430 txr->bnx_tx_buf[idx].bnx_tx_mbuf = m_head; 3431 txr->bnx_tx_cnt += nsegs; 3432 3433 BNX_INC(idx, BGE_TX_RING_CNT); 3434 *txidx = idx; 3435 back: 3436 if (error) { 3437 m_freem(*m_head0); 3438 *m_head0 = NULL; 3439 } 3440 return error; 3441 } 3442 3443 /* 3444 * Main transmit routine. To avoid having to do mbuf copies, we put pointers 3445 * to the mbuf data regions directly in the transmit descriptors. 3446 */ 3447 static void 3448 bnx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 3449 { 3450 struct bnx_tx_ring *txr = ifsq_get_priv(ifsq); 3451 struct mbuf *m_head = NULL; 3452 uint32_t prodidx; 3453 int nsegs = 0; 3454 3455 KKASSERT(txr->bnx_ifsq == ifsq); 3456 ASSERT_SERIALIZED(&txr->bnx_tx_serialize); 3457 3458 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq)) 3459 return; 3460 3461 prodidx = txr->bnx_tx_prodidx; 3462 3463 while (txr->bnx_tx_buf[prodidx].bnx_tx_mbuf == NULL) { 3464 /* 3465 * Sanity check: avoid coming within BGE_NSEG_RSVD 3466 * descriptors of the end of the ring. Also make 3467 * sure there are BGE_NSEG_SPARE descriptors for 3468 * jumbo buffers' or TSO segments' defragmentation. 3469 */ 3470 if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) < 3471 (BNX_NSEG_RSVD + BNX_NSEG_SPARE)) { 3472 ifsq_set_oactive(ifsq); 3473 break; 3474 } 3475 3476 m_head = ifsq_dequeue(ifsq); 3477 if (m_head == NULL) 3478 break; 3479 3480 /* 3481 * Pack the data into the transmit ring. If we 3482 * don't have room, set the OACTIVE flag and wait 3483 * for the NIC to drain the ring. 3484 */ 3485 if (bnx_encap(txr, &m_head, &prodidx, &nsegs)) { 3486 ifsq_set_oactive(ifsq); 3487 IFNET_STAT_INC(ifp, oerrors, 1); 3488 break; 3489 } 3490 3491 if (nsegs >= txr->bnx_tx_wreg) { 3492 /* Transmit */ 3493 bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, prodidx); 3494 nsegs = 0; 3495 } 3496 3497 ETHER_BPF_MTAP(ifp, m_head); 3498 3499 /* 3500 * Set a timeout in case the chip goes out to lunch. 3501 */ 3502 txr->bnx_tx_watchdog.wd_timer = 5; 3503 } 3504 3505 if (nsegs > 0) { 3506 /* Transmit */ 3507 bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, prodidx); 3508 } 3509 txr->bnx_tx_prodidx = prodidx; 3510 } 3511 3512 static void 3513 bnx_init(void *xsc) 3514 { 3515 struct bnx_softc *sc = xsc; 3516 struct ifnet *ifp = &sc->arpcom.ac_if; 3517 uint16_t *m; 3518 uint32_t mode; 3519 int i; 3520 boolean_t polling; 3521 3522 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3523 3524 /* Cancel pending I/O and flush buffers. */ 3525 bnx_stop(sc); 3526 3527 bnx_sig_pre_reset(sc, BNX_RESET_START); 3528 bnx_reset(sc); 3529 bnx_sig_post_reset(sc, BNX_RESET_START); 3530 3531 bnx_chipinit(sc); 3532 3533 /* 3534 * Init the various state machines, ring 3535 * control blocks and firmware. 3536 */ 3537 if (bnx_blockinit(sc)) { 3538 if_printf(ifp, "initialization failure\n"); 3539 bnx_stop(sc); 3540 return; 3541 } 3542 3543 /* Specify MTU. */ 3544 CSR_WRITE_4(sc, BGE_RX_MTU, ifp->if_mtu + 3545 ETHER_HDR_LEN + ETHER_CRC_LEN + EVL_ENCAPLEN); 3546 3547 /* Load our MAC address. */ 3548 m = (uint16_t *)&sc->arpcom.ac_enaddr[0]; 3549 CSR_WRITE_4(sc, BGE_MAC_ADDR1_LO, htons(m[0])); 3550 CSR_WRITE_4(sc, BGE_MAC_ADDR1_HI, (htons(m[1]) << 16) | htons(m[2])); 3551 3552 /* Enable or disable promiscuous mode as needed. */ 3553 bnx_setpromisc(sc); 3554 3555 /* Program multicast filter. */ 3556 bnx_setmulti(sc); 3557 3558 /* Init RX ring. */ 3559 if (bnx_init_rx_ring_std(&sc->bnx_rx_std_ring)) { 3560 if_printf(ifp, "RX ring initialization failed\n"); 3561 bnx_stop(sc); 3562 return; 3563 } 3564 3565 /* Init jumbo RX ring. */ 3566 if (ifp->if_mtu > (ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN)) { 3567 if (bnx_init_rx_ring_jumbo(sc)) { 3568 if_printf(ifp, "Jumbo RX ring initialization failed\n"); 3569 bnx_stop(sc); 3570 return; 3571 } 3572 } 3573 3574 /* Init our RX return ring index */ 3575 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 3576 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i]; 3577 3578 ret->bnx_rx_saved_considx = 0; 3579 ret->bnx_rx_cnt = 0; 3580 } 3581 3582 /* Init TX ring. */ 3583 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) 3584 bnx_init_tx_ring(&sc->bnx_tx_ring[i]); 3585 3586 /* Enable TX MAC state machine lockup fix. */ 3587 mode = CSR_READ_4(sc, BGE_TX_MODE); 3588 mode |= BGE_TXMODE_MBUF_LOCKUP_FIX; 3589 if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 || 3590 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 3591 mode &= ~(BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE); 3592 mode |= CSR_READ_4(sc, BGE_TX_MODE) & 3593 (BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE); 3594 } 3595 /* Turn on transmitter */ 3596 CSR_WRITE_4(sc, BGE_TX_MODE, mode | BGE_TXMODE_ENABLE); 3597 DELAY(100); 3598 3599 /* Initialize RSS */ 3600 mode = BGE_RXMODE_ENABLE | BGE_RXMODE_IPV6_ENABLE; 3601 if (BNX_RSS_ENABLED(sc)) { 3602 bnx_init_rss(sc); 3603 mode |= BGE_RXMODE_RSS_ENABLE | 3604 BGE_RXMODE_RSS_HASH_MASK_BITS | 3605 BGE_RXMODE_RSS_IPV4_HASH | 3606 BGE_RXMODE_RSS_TCP_IPV4_HASH; 3607 } 3608 /* Turn on receiver */ 3609 BNX_SETBIT(sc, BGE_RX_MODE, mode); 3610 DELAY(10); 3611 3612 /* 3613 * Set the number of good frames to receive after RX MBUF 3614 * Low Watermark has been reached. After the RX MAC receives 3615 * this number of frames, it will drop subsequent incoming 3616 * frames until the MBUF High Watermark is reached. 3617 */ 3618 if (BNX_IS_57765_FAMILY(sc)) 3619 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 1); 3620 else 3621 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 2); 3622 3623 if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI || 3624 sc->bnx_intr_type == PCI_INTR_TYPE_MSIX) { 3625 if (bootverbose) { 3626 if_printf(ifp, "MSI_MODE: %#x\n", 3627 CSR_READ_4(sc, BGE_MSI_MODE)); 3628 } 3629 } 3630 3631 /* Tell firmware we're alive. */ 3632 BNX_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP); 3633 3634 /* Enable host interrupts if polling(4) is not enabled. */ 3635 PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_CLEAR_INTA, 4); 3636 3637 polling = FALSE; 3638 #ifdef IFPOLL_ENABLE 3639 if (ifp->if_flags & IFF_NPOLLING) 3640 polling = TRUE; 3641 #endif 3642 if (polling) 3643 bnx_disable_intr(sc); 3644 else 3645 bnx_enable_intr(sc); 3646 bnx_set_tick_cpuid(sc, polling); 3647 3648 ifp->if_flags |= IFF_RUNNING; 3649 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 3650 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 3651 3652 ifsq_clr_oactive(txr->bnx_ifsq); 3653 ifsq_watchdog_start(&txr->bnx_tx_watchdog); 3654 } 3655 3656 bnx_ifmedia_upd(ifp); 3657 3658 callout_reset_bycpu(&sc->bnx_tick_timer, hz, bnx_tick, sc, 3659 sc->bnx_tick_cpuid); 3660 } 3661 3662 /* 3663 * Set media options. 3664 */ 3665 static int 3666 bnx_ifmedia_upd(struct ifnet *ifp) 3667 { 3668 struct bnx_softc *sc = ifp->if_softc; 3669 3670 /* If this is a 1000baseX NIC, enable the TBI port. */ 3671 if (sc->bnx_flags & BNX_FLAG_TBI) { 3672 struct ifmedia *ifm = &sc->bnx_ifmedia; 3673 3674 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3675 return(EINVAL); 3676 3677 switch(IFM_SUBTYPE(ifm->ifm_media)) { 3678 case IFM_AUTO: 3679 break; 3680 3681 case IFM_1000_SX: 3682 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) { 3683 BNX_CLRBIT(sc, BGE_MAC_MODE, 3684 BGE_MACMODE_HALF_DUPLEX); 3685 } else { 3686 BNX_SETBIT(sc, BGE_MAC_MODE, 3687 BGE_MACMODE_HALF_DUPLEX); 3688 } 3689 DELAY(40); 3690 break; 3691 default: 3692 return(EINVAL); 3693 } 3694 } else { 3695 struct mii_data *mii = device_get_softc(sc->bnx_miibus); 3696 3697 sc->bnx_link_evt++; 3698 sc->bnx_link = 0; 3699 if (mii->mii_instance) { 3700 struct mii_softc *miisc; 3701 3702 LIST_FOREACH(miisc, &mii->mii_phys, mii_list) 3703 mii_phy_reset(miisc); 3704 } 3705 mii_mediachg(mii); 3706 3707 /* 3708 * Force an interrupt so that we will call bnx_link_upd 3709 * if needed and clear any pending link state attention. 3710 * Without this we are not getting any further interrupts 3711 * for link state changes and thus will not UP the link and 3712 * not be able to send in bnx_start. The only way to get 3713 * things working was to receive a packet and get an RX 3714 * intr. 3715 * 3716 * bnx_tick should help for fiber cards and we might not 3717 * need to do this here if BNX_FLAG_TBI is set but as 3718 * we poll for fiber anyway it should not harm. 3719 */ 3720 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW); 3721 } 3722 return(0); 3723 } 3724 3725 /* 3726 * Report current media status. 3727 */ 3728 static void 3729 bnx_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) 3730 { 3731 struct bnx_softc *sc = ifp->if_softc; 3732 3733 if ((ifp->if_flags & IFF_RUNNING) == 0) 3734 return; 3735 3736 if (sc->bnx_flags & BNX_FLAG_TBI) { 3737 ifmr->ifm_status = IFM_AVALID; 3738 ifmr->ifm_active = IFM_ETHER; 3739 if (CSR_READ_4(sc, BGE_MAC_STS) & 3740 BGE_MACSTAT_TBI_PCS_SYNCHED) { 3741 ifmr->ifm_status |= IFM_ACTIVE; 3742 } else { 3743 ifmr->ifm_active |= IFM_NONE; 3744 return; 3745 } 3746 3747 ifmr->ifm_active |= IFM_1000_SX; 3748 if (CSR_READ_4(sc, BGE_MAC_MODE) & BGE_MACMODE_HALF_DUPLEX) 3749 ifmr->ifm_active |= IFM_HDX; 3750 else 3751 ifmr->ifm_active |= IFM_FDX; 3752 } else { 3753 struct mii_data *mii = device_get_softc(sc->bnx_miibus); 3754 3755 mii_pollstat(mii); 3756 ifmr->ifm_active = mii->mii_media_active; 3757 ifmr->ifm_status = mii->mii_media_status; 3758 } 3759 } 3760 3761 static int 3762 bnx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) 3763 { 3764 struct bnx_softc *sc = ifp->if_softc; 3765 struct ifreq *ifr = (struct ifreq *)data; 3766 int mask, error = 0; 3767 3768 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3769 3770 switch (command) { 3771 case SIOCSIFMTU: 3772 if ((!BNX_IS_JUMBO_CAPABLE(sc) && ifr->ifr_mtu > ETHERMTU) || 3773 (BNX_IS_JUMBO_CAPABLE(sc) && 3774 ifr->ifr_mtu > BNX_JUMBO_MTU)) { 3775 error = EINVAL; 3776 } else if (ifp->if_mtu != ifr->ifr_mtu) { 3777 ifp->if_mtu = ifr->ifr_mtu; 3778 if (ifp->if_flags & IFF_RUNNING) 3779 bnx_init(sc); 3780 } 3781 break; 3782 case SIOCSIFFLAGS: 3783 if (ifp->if_flags & IFF_UP) { 3784 if (ifp->if_flags & IFF_RUNNING) { 3785 mask = ifp->if_flags ^ sc->bnx_if_flags; 3786 3787 /* 3788 * If only the state of the PROMISC flag 3789 * changed, then just use the 'set promisc 3790 * mode' command instead of reinitializing 3791 * the entire NIC. Doing a full re-init 3792 * means reloading the firmware and waiting 3793 * for it to start up, which may take a 3794 * second or two. Similarly for ALLMULTI. 3795 */ 3796 if (mask & IFF_PROMISC) 3797 bnx_setpromisc(sc); 3798 if (mask & IFF_ALLMULTI) 3799 bnx_setmulti(sc); 3800 } else { 3801 bnx_init(sc); 3802 } 3803 } else if (ifp->if_flags & IFF_RUNNING) { 3804 bnx_stop(sc); 3805 } 3806 sc->bnx_if_flags = ifp->if_flags; 3807 break; 3808 case SIOCADDMULTI: 3809 case SIOCDELMULTI: 3810 if (ifp->if_flags & IFF_RUNNING) 3811 bnx_setmulti(sc); 3812 break; 3813 case SIOCSIFMEDIA: 3814 case SIOCGIFMEDIA: 3815 if (sc->bnx_flags & BNX_FLAG_TBI) { 3816 error = ifmedia_ioctl(ifp, ifr, 3817 &sc->bnx_ifmedia, command); 3818 } else { 3819 struct mii_data *mii; 3820 3821 mii = device_get_softc(sc->bnx_miibus); 3822 error = ifmedia_ioctl(ifp, ifr, 3823 &mii->mii_media, command); 3824 } 3825 break; 3826 case SIOCSIFCAP: 3827 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 3828 if (mask & IFCAP_HWCSUM) { 3829 ifp->if_capenable ^= (mask & IFCAP_HWCSUM); 3830 if (ifp->if_capenable & IFCAP_TXCSUM) 3831 ifp->if_hwassist |= BNX_CSUM_FEATURES; 3832 else 3833 ifp->if_hwassist &= ~BNX_CSUM_FEATURES; 3834 } 3835 if (mask & IFCAP_TSO) { 3836 ifp->if_capenable ^= (mask & IFCAP_TSO); 3837 if (ifp->if_capenable & IFCAP_TSO) 3838 ifp->if_hwassist |= CSUM_TSO; 3839 else 3840 ifp->if_hwassist &= ~CSUM_TSO; 3841 } 3842 if (mask & IFCAP_RSS) 3843 ifp->if_capenable ^= IFCAP_RSS; 3844 break; 3845 default: 3846 error = ether_ioctl(ifp, command, data); 3847 break; 3848 } 3849 return error; 3850 } 3851 3852 static void 3853 bnx_watchdog(struct ifaltq_subque *ifsq) 3854 { 3855 struct ifnet *ifp = ifsq_get_ifp(ifsq); 3856 struct bnx_softc *sc = ifp->if_softc; 3857 int i; 3858 3859 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3860 3861 if_printf(ifp, "watchdog timeout -- resetting\n"); 3862 3863 bnx_init(sc); 3864 3865 IFNET_STAT_INC(ifp, oerrors, 1); 3866 3867 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) 3868 ifsq_devstart_sched(sc->bnx_tx_ring[i].bnx_ifsq); 3869 } 3870 3871 /* 3872 * Stop the adapter and free any mbufs allocated to the 3873 * RX and TX lists. 3874 */ 3875 static void 3876 bnx_stop(struct bnx_softc *sc) 3877 { 3878 struct ifnet *ifp = &sc->arpcom.ac_if; 3879 int i; 3880 3881 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3882 3883 callout_stop(&sc->bnx_tick_timer); 3884 3885 /* Disable host interrupts. */ 3886 bnx_disable_intr(sc); 3887 3888 /* 3889 * Tell firmware we're shutting down. 3890 */ 3891 bnx_sig_pre_reset(sc, BNX_RESET_SHUTDOWN); 3892 3893 /* 3894 * Disable all of the receiver blocks 3895 */ 3896 bnx_stop_block(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE); 3897 bnx_stop_block(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE); 3898 bnx_stop_block(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE); 3899 bnx_stop_block(sc, BGE_RDBDI_MODE, BGE_RBDIMODE_ENABLE); 3900 bnx_stop_block(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE); 3901 bnx_stop_block(sc, BGE_RBDC_MODE, BGE_RBDCMODE_ENABLE); 3902 3903 /* 3904 * Disable all of the transmit blocks 3905 */ 3906 bnx_stop_block(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE); 3907 bnx_stop_block(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE); 3908 bnx_stop_block(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE); 3909 bnx_stop_block(sc, BGE_RDMA_MODE, BGE_RDMAMODE_ENABLE); 3910 bnx_stop_block(sc, BGE_SDC_MODE, BGE_SDCMODE_ENABLE); 3911 bnx_stop_block(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE); 3912 3913 /* 3914 * Shut down all of the memory managers and related 3915 * state machines. 3916 */ 3917 bnx_stop_block(sc, BGE_HCC_MODE, BGE_HCCMODE_ENABLE); 3918 bnx_stop_block(sc, BGE_WDMA_MODE, BGE_WDMAMODE_ENABLE); 3919 CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF); 3920 CSR_WRITE_4(sc, BGE_FTQ_RESET, 0); 3921 3922 bnx_reset(sc); 3923 bnx_sig_post_reset(sc, BNX_RESET_SHUTDOWN); 3924 3925 /* 3926 * Tell firmware we're shutting down. 3927 */ 3928 BNX_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP); 3929 3930 /* Free the RX lists. */ 3931 bnx_free_rx_ring_std(&sc->bnx_rx_std_ring); 3932 3933 /* Free jumbo RX list. */ 3934 if (BNX_IS_JUMBO_CAPABLE(sc)) 3935 bnx_free_rx_ring_jumbo(sc); 3936 3937 /* Free TX buffers. */ 3938 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 3939 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 3940 3941 txr->bnx_saved_status_tag = 0; 3942 bnx_free_tx_ring(txr); 3943 } 3944 3945 /* Clear saved status tag */ 3946 for (i = 0; i < sc->bnx_rx_retcnt; ++i) 3947 sc->bnx_rx_ret_ring[i].bnx_saved_status_tag = 0; 3948 3949 sc->bnx_link = 0; 3950 sc->bnx_coal_chg = 0; 3951 3952 ifp->if_flags &= ~IFF_RUNNING; 3953 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 3954 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 3955 3956 ifsq_clr_oactive(txr->bnx_ifsq); 3957 ifsq_watchdog_stop(&txr->bnx_tx_watchdog); 3958 } 3959 } 3960 3961 /* 3962 * Stop all chip I/O so that the kernel's probe routines don't 3963 * get confused by errant DMAs when rebooting. 3964 */ 3965 static void 3966 bnx_shutdown(device_t dev) 3967 { 3968 struct bnx_softc *sc = device_get_softc(dev); 3969 struct ifnet *ifp = &sc->arpcom.ac_if; 3970 3971 ifnet_serialize_all(ifp); 3972 bnx_stop(sc); 3973 ifnet_deserialize_all(ifp); 3974 } 3975 3976 static int 3977 bnx_suspend(device_t dev) 3978 { 3979 struct bnx_softc *sc = device_get_softc(dev); 3980 struct ifnet *ifp = &sc->arpcom.ac_if; 3981 3982 ifnet_serialize_all(ifp); 3983 bnx_stop(sc); 3984 ifnet_deserialize_all(ifp); 3985 3986 return 0; 3987 } 3988 3989 static int 3990 bnx_resume(device_t dev) 3991 { 3992 struct bnx_softc *sc = device_get_softc(dev); 3993 struct ifnet *ifp = &sc->arpcom.ac_if; 3994 3995 ifnet_serialize_all(ifp); 3996 3997 if (ifp->if_flags & IFF_UP) { 3998 int i; 3999 4000 bnx_init(sc); 4001 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) 4002 ifsq_devstart_sched(sc->bnx_tx_ring[i].bnx_ifsq); 4003 } 4004 4005 ifnet_deserialize_all(ifp); 4006 4007 return 0; 4008 } 4009 4010 static void 4011 bnx_setpromisc(struct bnx_softc *sc) 4012 { 4013 struct ifnet *ifp = &sc->arpcom.ac_if; 4014 4015 if (ifp->if_flags & IFF_PROMISC) 4016 BNX_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC); 4017 else 4018 BNX_CLRBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC); 4019 } 4020 4021 static void 4022 bnx_dma_free(struct bnx_softc *sc) 4023 { 4024 struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring; 4025 int i; 4026 4027 /* Destroy RX return rings */ 4028 if (sc->bnx_rx_ret_ring != NULL) { 4029 for (i = 0; i < sc->bnx_rx_retcnt; ++i) 4030 bnx_destroy_rx_ret_ring(&sc->bnx_rx_ret_ring[i]); 4031 kfree(sc->bnx_rx_ret_ring, M_DEVBUF); 4032 } 4033 4034 /* Destroy RX mbuf DMA stuffs. */ 4035 if (std->bnx_rx_mtag != NULL) { 4036 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) { 4037 KKASSERT(std->bnx_rx_std_buf[i].bnx_rx_mbuf == NULL); 4038 bus_dmamap_destroy(std->bnx_rx_mtag, 4039 std->bnx_rx_std_buf[i].bnx_rx_dmamap); 4040 } 4041 bus_dma_tag_destroy(std->bnx_rx_mtag); 4042 } 4043 4044 /* Destroy standard RX ring */ 4045 bnx_dma_block_free(std->bnx_rx_std_ring_tag, 4046 std->bnx_rx_std_ring_map, std->bnx_rx_std_ring); 4047 4048 /* Destroy TX rings */ 4049 if (sc->bnx_tx_ring != NULL) { 4050 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) 4051 bnx_destroy_tx_ring(&sc->bnx_tx_ring[i]); 4052 kfree(sc->bnx_tx_ring, M_DEVBUF); 4053 } 4054 4055 if (BNX_IS_JUMBO_CAPABLE(sc)) 4056 bnx_free_jumbo_mem(sc); 4057 4058 /* Destroy status blocks */ 4059 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4060 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 4061 4062 bnx_dma_block_free(intr->bnx_status_tag, 4063 intr->bnx_status_map, intr->bnx_status_block); 4064 } 4065 4066 /* Destroy the parent tag */ 4067 if (sc->bnx_cdata.bnx_parent_tag != NULL) 4068 bus_dma_tag_destroy(sc->bnx_cdata.bnx_parent_tag); 4069 } 4070 4071 static int 4072 bnx_dma_alloc(device_t dev) 4073 { 4074 struct bnx_softc *sc = device_get_softc(dev); 4075 struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring; 4076 int i, error, mbx; 4077 4078 /* 4079 * Allocate the parent bus DMA tag appropriate for PCI. 4080 * 4081 * All of the NetExtreme/NetLink controllers have 4GB boundary 4082 * DMA bug. 4083 * Whenever an address crosses a multiple of the 4GB boundary 4084 * (including 4GB, 8Gb, 12Gb, etc.) and makes the transition 4085 * from 0xX_FFFF_FFFF to 0x(X+1)_0000_0000 an internal DMA 4086 * state machine will lockup and cause the device to hang. 4087 */ 4088 error = bus_dma_tag_create(NULL, 1, BGE_DMA_BOUNDARY_4G, 4089 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 4090 BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 4091 0, &sc->bnx_cdata.bnx_parent_tag); 4092 if (error) { 4093 device_printf(dev, "could not create parent DMA tag\n"); 4094 return error; 4095 } 4096 4097 /* 4098 * Create DMA stuffs for status blocks. 4099 */ 4100 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4101 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 4102 4103 error = bnx_dma_block_alloc(sc, 4104 __VM_CACHELINE_ALIGN(BGE_STATUS_BLK_SZ), 4105 &intr->bnx_status_tag, &intr->bnx_status_map, 4106 (void *)&intr->bnx_status_block, 4107 &intr->bnx_status_block_paddr); 4108 if (error) { 4109 device_printf(dev, 4110 "could not create %dth status block\n", i); 4111 return error; 4112 } 4113 } 4114 sc->bnx_hw_status = &sc->bnx_intr_data[0].bnx_status_block->bge_status; 4115 if (sc->bnx_flags & BNX_FLAG_STATUS_HASTAG) { 4116 sc->bnx_hw_status_tag = 4117 &sc->bnx_intr_data[0].bnx_status_block->bge_status_tag; 4118 } 4119 4120 /* 4121 * Create DMA tag and maps for RX mbufs. 4122 */ 4123 std->bnx_sc = sc; 4124 lwkt_serialize_init(&std->bnx_rx_std_serialize); 4125 error = bus_dma_tag_create(sc->bnx_cdata.bnx_parent_tag, 1, 0, 4126 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 4127 NULL, NULL, MCLBYTES, 1, MCLBYTES, 4128 BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK, &std->bnx_rx_mtag); 4129 if (error) { 4130 device_printf(dev, "could not create RX mbuf DMA tag\n"); 4131 return error; 4132 } 4133 4134 for (i = 0; i < BGE_STD_RX_RING_CNT; ++i) { 4135 error = bus_dmamap_create(std->bnx_rx_mtag, BUS_DMA_WAITOK, 4136 &std->bnx_rx_std_buf[i].bnx_rx_dmamap); 4137 if (error) { 4138 int j; 4139 4140 for (j = 0; j < i; ++j) { 4141 bus_dmamap_destroy(std->bnx_rx_mtag, 4142 std->bnx_rx_std_buf[j].bnx_rx_dmamap); 4143 } 4144 bus_dma_tag_destroy(std->bnx_rx_mtag); 4145 std->bnx_rx_mtag = NULL; 4146 4147 device_printf(dev, 4148 "could not create %dth RX mbuf DMA map\n", i); 4149 return error; 4150 } 4151 } 4152 4153 /* 4154 * Create DMA stuffs for standard RX ring. 4155 */ 4156 error = bnx_dma_block_alloc(sc, BGE_STD_RX_RING_SZ, 4157 &std->bnx_rx_std_ring_tag, 4158 &std->bnx_rx_std_ring_map, 4159 (void *)&std->bnx_rx_std_ring, 4160 &std->bnx_rx_std_ring_paddr); 4161 if (error) { 4162 device_printf(dev, "could not create std RX ring\n"); 4163 return error; 4164 } 4165 4166 /* 4167 * Create RX return rings 4168 */ 4169 mbx = BGE_MBX_RX_CONS0_LO; 4170 sc->bnx_rx_ret_ring = kmalloc_cachealign( 4171 sizeof(struct bnx_rx_ret_ring) * sc->bnx_rx_retcnt, M_DEVBUF, 4172 M_WAITOK | M_ZERO); 4173 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 4174 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i]; 4175 struct bnx_intr_data *intr; 4176 4177 ret->bnx_sc = sc; 4178 ret->bnx_std = std; 4179 ret->bnx_rx_mbx = mbx; 4180 ret->bnx_rx_cntmax = (BGE_STD_RX_RING_CNT / 4) / 4181 sc->bnx_rx_retcnt; 4182 ret->bnx_rx_mask = 1 << i; 4183 4184 if (!BNX_RSS_ENABLED(sc)) { 4185 intr = &sc->bnx_intr_data[0]; 4186 } else { 4187 KKASSERT(i + 1 < sc->bnx_intr_cnt); 4188 intr = &sc->bnx_intr_data[i + 1]; 4189 } 4190 4191 if (i == 0) { 4192 ret->bnx_rx_considx = 4193 &intr->bnx_status_block->bge_idx[0].bge_rx_prod_idx; 4194 } else if (i == 1) { 4195 ret->bnx_rx_considx = 4196 &intr->bnx_status_block->bge_rx_jumbo_cons_idx; 4197 } else if (i == 2) { 4198 ret->bnx_rx_considx = 4199 &intr->bnx_status_block->bge_rsvd1; 4200 } else if (i == 3) { 4201 ret->bnx_rx_considx = 4202 &intr->bnx_status_block->bge_rx_mini_cons_idx; 4203 } else { 4204 panic("unknown RX return ring %d\n", i); 4205 } 4206 ret->bnx_hw_status_tag = 4207 &intr->bnx_status_block->bge_status_tag; 4208 4209 error = bnx_create_rx_ret_ring(ret); 4210 if (error) { 4211 device_printf(dev, 4212 "could not create %dth RX ret ring\n", i); 4213 return error; 4214 } 4215 mbx += 8; 4216 } 4217 4218 /* 4219 * Create TX rings 4220 */ 4221 sc->bnx_tx_ring = kmalloc_cachealign( 4222 sizeof(struct bnx_tx_ring) * sc->bnx_tx_ringcnt, M_DEVBUF, 4223 M_WAITOK | M_ZERO); 4224 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 4225 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 4226 struct bnx_intr_data *intr; 4227 4228 txr->bnx_sc = sc; 4229 txr->bnx_tx_mbx = bnx_tx_mailbox[i]; 4230 4231 if (sc->bnx_tx_ringcnt == 1) { 4232 intr = &sc->bnx_intr_data[0]; 4233 } else { 4234 KKASSERT(i + 1 < sc->bnx_intr_cnt); 4235 intr = &sc->bnx_intr_data[i + 1]; 4236 } 4237 4238 if ((sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) == 0) { 4239 txr->bnx_hw_status_tag = 4240 &intr->bnx_status_block->bge_status_tag; 4241 } 4242 txr->bnx_tx_considx = 4243 &intr->bnx_status_block->bge_idx[0].bge_tx_cons_idx; 4244 4245 error = bnx_create_tx_ring(txr); 4246 if (error) { 4247 device_printf(dev, 4248 "could not create %dth TX ring\n", i); 4249 return error; 4250 } 4251 } 4252 4253 /* 4254 * Create jumbo buffer pool. 4255 */ 4256 if (BNX_IS_JUMBO_CAPABLE(sc)) { 4257 error = bnx_alloc_jumbo_mem(sc); 4258 if (error) { 4259 device_printf(dev, 4260 "could not create jumbo buffer pool\n"); 4261 return error; 4262 } 4263 } 4264 4265 return 0; 4266 } 4267 4268 static int 4269 bnx_dma_block_alloc(struct bnx_softc *sc, bus_size_t size, bus_dma_tag_t *tag, 4270 bus_dmamap_t *map, void **addr, bus_addr_t *paddr) 4271 { 4272 bus_dmamem_t dmem; 4273 int error; 4274 4275 error = bus_dmamem_coherent(sc->bnx_cdata.bnx_parent_tag, PAGE_SIZE, 0, 4276 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 4277 size, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem); 4278 if (error) 4279 return error; 4280 4281 *tag = dmem.dmem_tag; 4282 *map = dmem.dmem_map; 4283 *addr = dmem.dmem_addr; 4284 *paddr = dmem.dmem_busaddr; 4285 4286 return 0; 4287 } 4288 4289 static void 4290 bnx_dma_block_free(bus_dma_tag_t tag, bus_dmamap_t map, void *addr) 4291 { 4292 if (tag != NULL) { 4293 bus_dmamap_unload(tag, map); 4294 bus_dmamem_free(tag, addr, map); 4295 bus_dma_tag_destroy(tag); 4296 } 4297 } 4298 4299 static void 4300 bnx_tbi_link_upd(struct bnx_softc *sc, uint32_t status) 4301 { 4302 struct ifnet *ifp = &sc->arpcom.ac_if; 4303 4304 #define PCS_ENCODE_ERR (BGE_MACSTAT_PORT_DECODE_ERROR|BGE_MACSTAT_MI_COMPLETE) 4305 4306 /* 4307 * Sometimes PCS encoding errors are detected in 4308 * TBI mode (on fiber NICs), and for some reason 4309 * the chip will signal them as link changes. 4310 * If we get a link change event, but the 'PCS 4311 * encoding error' bit in the MAC status register 4312 * is set, don't bother doing a link check. 4313 * This avoids spurious "gigabit link up" messages 4314 * that sometimes appear on fiber NICs during 4315 * periods of heavy traffic. 4316 */ 4317 if (status & BGE_MACSTAT_TBI_PCS_SYNCHED) { 4318 if (!sc->bnx_link) { 4319 sc->bnx_link++; 4320 if (sc->bnx_asicrev == BGE_ASICREV_BCM5704) { 4321 BNX_CLRBIT(sc, BGE_MAC_MODE, 4322 BGE_MACMODE_TBI_SEND_CFGS); 4323 DELAY(40); 4324 } 4325 CSR_WRITE_4(sc, BGE_MAC_STS, 0xFFFFFFFF); 4326 4327 if (bootverbose) 4328 if_printf(ifp, "link UP\n"); 4329 4330 ifp->if_link_state = LINK_STATE_UP; 4331 if_link_state_change(ifp); 4332 } 4333 } else if ((status & PCS_ENCODE_ERR) != PCS_ENCODE_ERR) { 4334 if (sc->bnx_link) { 4335 sc->bnx_link = 0; 4336 4337 if (bootverbose) 4338 if_printf(ifp, "link DOWN\n"); 4339 4340 ifp->if_link_state = LINK_STATE_DOWN; 4341 if_link_state_change(ifp); 4342 } 4343 } 4344 4345 #undef PCS_ENCODE_ERR 4346 4347 /* Clear the attention. */ 4348 CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED | 4349 BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE | 4350 BGE_MACSTAT_LINK_CHANGED); 4351 } 4352 4353 static void 4354 bnx_copper_link_upd(struct bnx_softc *sc, uint32_t status __unused) 4355 { 4356 struct ifnet *ifp = &sc->arpcom.ac_if; 4357 struct mii_data *mii = device_get_softc(sc->bnx_miibus); 4358 4359 mii_pollstat(mii); 4360 bnx_miibus_statchg(sc->bnx_dev); 4361 4362 if (bootverbose) { 4363 if (sc->bnx_link) 4364 if_printf(ifp, "link UP\n"); 4365 else 4366 if_printf(ifp, "link DOWN\n"); 4367 } 4368 4369 /* Clear the attention. */ 4370 CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED | 4371 BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE | 4372 BGE_MACSTAT_LINK_CHANGED); 4373 } 4374 4375 static void 4376 bnx_autopoll_link_upd(struct bnx_softc *sc, uint32_t status __unused) 4377 { 4378 struct ifnet *ifp = &sc->arpcom.ac_if; 4379 struct mii_data *mii = device_get_softc(sc->bnx_miibus); 4380 4381 mii_pollstat(mii); 4382 4383 if (!sc->bnx_link && 4384 (mii->mii_media_status & IFM_ACTIVE) && 4385 IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) { 4386 sc->bnx_link++; 4387 if (bootverbose) 4388 if_printf(ifp, "link UP\n"); 4389 } else if (sc->bnx_link && 4390 (!(mii->mii_media_status & IFM_ACTIVE) || 4391 IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) { 4392 sc->bnx_link = 0; 4393 if (bootverbose) 4394 if_printf(ifp, "link DOWN\n"); 4395 } 4396 4397 /* Clear the attention. */ 4398 CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED | 4399 BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE | 4400 BGE_MACSTAT_LINK_CHANGED); 4401 } 4402 4403 static int 4404 bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS) 4405 { 4406 struct bnx_softc *sc = arg1; 4407 4408 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4409 &sc->bnx_rx_coal_ticks, 4410 BNX_RX_COAL_TICKS_MIN, BNX_RX_COAL_TICKS_MAX, 4411 BNX_RX_COAL_TICKS_CHG); 4412 } 4413 4414 static int 4415 bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS) 4416 { 4417 struct bnx_softc *sc = arg1; 4418 4419 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4420 &sc->bnx_tx_coal_ticks, 4421 BNX_TX_COAL_TICKS_MIN, BNX_TX_COAL_TICKS_MAX, 4422 BNX_TX_COAL_TICKS_CHG); 4423 } 4424 4425 static int 4426 bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS) 4427 { 4428 struct bnx_softc *sc = arg1; 4429 4430 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4431 &sc->bnx_rx_coal_bds, 4432 BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX, 4433 BNX_RX_COAL_BDS_CHG); 4434 } 4435 4436 static int 4437 bnx_sysctl_rx_coal_bds_poll(SYSCTL_HANDLER_ARGS) 4438 { 4439 struct bnx_softc *sc = arg1; 4440 4441 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4442 &sc->bnx_rx_coal_bds_poll, 4443 BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX, 4444 BNX_RX_COAL_BDS_CHG); 4445 } 4446 4447 static int 4448 bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS) 4449 { 4450 struct bnx_softc *sc = arg1; 4451 4452 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4453 &sc->bnx_tx_coal_bds, 4454 BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX, 4455 BNX_TX_COAL_BDS_CHG); 4456 } 4457 4458 static int 4459 bnx_sysctl_tx_coal_bds_poll(SYSCTL_HANDLER_ARGS) 4460 { 4461 struct bnx_softc *sc = arg1; 4462 4463 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4464 &sc->bnx_tx_coal_bds_poll, 4465 BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX, 4466 BNX_TX_COAL_BDS_CHG); 4467 } 4468 4469 static int 4470 bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS) 4471 { 4472 struct bnx_softc *sc = arg1; 4473 4474 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4475 &sc->bnx_rx_coal_bds_int, 4476 BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX, 4477 BNX_RX_COAL_BDS_INT_CHG); 4478 } 4479 4480 static int 4481 bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS) 4482 { 4483 struct bnx_softc *sc = arg1; 4484 4485 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4486 &sc->bnx_tx_coal_bds_int, 4487 BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX, 4488 BNX_TX_COAL_BDS_INT_CHG); 4489 } 4490 4491 static int 4492 bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *coal, 4493 int coal_min, int coal_max, uint32_t coal_chg_mask) 4494 { 4495 struct bnx_softc *sc = arg1; 4496 struct ifnet *ifp = &sc->arpcom.ac_if; 4497 int error = 0, v; 4498 4499 ifnet_serialize_all(ifp); 4500 4501 v = *coal; 4502 error = sysctl_handle_int(oidp, &v, 0, req); 4503 if (!error && req->newptr != NULL) { 4504 if (v < coal_min || v > coal_max) { 4505 error = EINVAL; 4506 } else { 4507 *coal = v; 4508 sc->bnx_coal_chg |= coal_chg_mask; 4509 4510 /* Commit changes */ 4511 bnx_coal_change(sc); 4512 } 4513 } 4514 4515 ifnet_deserialize_all(ifp); 4516 return error; 4517 } 4518 4519 static void 4520 bnx_coal_change(struct bnx_softc *sc) 4521 { 4522 struct ifnet *ifp = &sc->arpcom.ac_if; 4523 int i; 4524 4525 ASSERT_IFNET_SERIALIZED_ALL(ifp); 4526 4527 if (sc->bnx_coal_chg & BNX_RX_COAL_TICKS_CHG) { 4528 if (sc->bnx_rx_retcnt == 1) { 4529 CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS, 4530 sc->bnx_rx_coal_ticks); 4531 i = 0; 4532 } else { 4533 CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS, 0); 4534 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 4535 CSR_WRITE_4(sc, BGE_VEC1_RX_COAL_TICKS + 4536 (i * BGE_VEC_COALSET_SIZE), 4537 sc->bnx_rx_coal_ticks); 4538 } 4539 } 4540 for (; i < BNX_INTR_MAX - 1; ++i) { 4541 CSR_WRITE_4(sc, BGE_VEC1_RX_COAL_TICKS + 4542 (i * BGE_VEC_COALSET_SIZE), 0); 4543 } 4544 if (bootverbose) { 4545 if_printf(ifp, "rx_coal_ticks -> %u\n", 4546 sc->bnx_rx_coal_ticks); 4547 } 4548 } 4549 4550 if (sc->bnx_coal_chg & BNX_TX_COAL_TICKS_CHG) { 4551 if (sc->bnx_tx_ringcnt == 1) { 4552 CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS, 4553 sc->bnx_tx_coal_ticks); 4554 i = 0; 4555 } else { 4556 CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS, 0); 4557 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 4558 CSR_WRITE_4(sc, BGE_VEC1_TX_COAL_TICKS + 4559 (i * BGE_VEC_COALSET_SIZE), 4560 sc->bnx_tx_coal_ticks); 4561 } 4562 } 4563 for (; i < BNX_INTR_MAX - 1; ++i) { 4564 CSR_WRITE_4(sc, BGE_VEC1_TX_COAL_TICKS + 4565 (i * BGE_VEC_COALSET_SIZE), 0); 4566 } 4567 if (bootverbose) { 4568 if_printf(ifp, "tx_coal_ticks -> %u\n", 4569 sc->bnx_tx_coal_ticks); 4570 } 4571 } 4572 4573 if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_CHG) { 4574 uint32_t rx_coal_bds; 4575 4576 if (ifp->if_flags & IFF_NPOLLING) 4577 rx_coal_bds = sc->bnx_rx_coal_bds_poll; 4578 else 4579 rx_coal_bds = sc->bnx_rx_coal_bds; 4580 4581 if (sc->bnx_rx_retcnt == 1) { 4582 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, rx_coal_bds); 4583 i = 0; 4584 } else { 4585 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, 0); 4586 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 4587 CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS + 4588 (i * BGE_VEC_COALSET_SIZE), rx_coal_bds); 4589 } 4590 } 4591 for (; i < BNX_INTR_MAX - 1; ++i) { 4592 CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS + 4593 (i * BGE_VEC_COALSET_SIZE), 0); 4594 } 4595 if (bootverbose) { 4596 if_printf(ifp, "%srx_coal_bds -> %u\n", 4597 (ifp->if_flags & IFF_NPOLLING) ? "polling " : "", 4598 rx_coal_bds); 4599 } 4600 } 4601 4602 if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_CHG) { 4603 uint32_t tx_coal_bds; 4604 4605 if (ifp->if_flags & IFF_NPOLLING) 4606 tx_coal_bds = sc->bnx_tx_coal_bds_poll; 4607 else 4608 tx_coal_bds = sc->bnx_tx_coal_bds; 4609 4610 if (sc->bnx_tx_ringcnt == 1) { 4611 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, tx_coal_bds); 4612 i = 0; 4613 } else { 4614 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, 0); 4615 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 4616 CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS + 4617 (i * BGE_VEC_COALSET_SIZE), tx_coal_bds); 4618 } 4619 } 4620 for (; i < BNX_INTR_MAX - 1; ++i) { 4621 CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS + 4622 (i * BGE_VEC_COALSET_SIZE), 0); 4623 } 4624 if (bootverbose) { 4625 if_printf(ifp, "%stx_coal_bds -> %u\n", 4626 (ifp->if_flags & IFF_NPOLLING) ? "polling " : "", 4627 tx_coal_bds); 4628 } 4629 } 4630 4631 if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_INT_CHG) { 4632 if (sc->bnx_rx_retcnt == 1) { 4633 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT, 4634 sc->bnx_rx_coal_bds_int); 4635 i = 0; 4636 } else { 4637 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT, 0); 4638 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 4639 CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS_INT + 4640 (i * BGE_VEC_COALSET_SIZE), 4641 sc->bnx_rx_coal_bds_int); 4642 } 4643 } 4644 for (; i < BNX_INTR_MAX - 1; ++i) { 4645 CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS_INT + 4646 (i * BGE_VEC_COALSET_SIZE), 0); 4647 } 4648 if (bootverbose) { 4649 if_printf(ifp, "rx_coal_bds_int -> %u\n", 4650 sc->bnx_rx_coal_bds_int); 4651 } 4652 } 4653 4654 if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_INT_CHG) { 4655 if (sc->bnx_tx_ringcnt == 1) { 4656 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT, 4657 sc->bnx_tx_coal_bds_int); 4658 i = 0; 4659 } else { 4660 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT, 0); 4661 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 4662 CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS_INT + 4663 (i * BGE_VEC_COALSET_SIZE), 4664 sc->bnx_tx_coal_bds_int); 4665 } 4666 } 4667 for (; i < BNX_INTR_MAX - 1; ++i) { 4668 CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS_INT + 4669 (i * BGE_VEC_COALSET_SIZE), 0); 4670 } 4671 if (bootverbose) { 4672 if_printf(ifp, "tx_coal_bds_int -> %u\n", 4673 sc->bnx_tx_coal_bds_int); 4674 } 4675 } 4676 4677 sc->bnx_coal_chg = 0; 4678 } 4679 4680 static void 4681 bnx_check_intr_rxtx(void *xintr) 4682 { 4683 struct bnx_intr_data *intr = xintr; 4684 struct bnx_rx_ret_ring *ret; 4685 struct bnx_tx_ring *txr; 4686 struct ifnet *ifp; 4687 4688 lwkt_serialize_enter(intr->bnx_intr_serialize); 4689 4690 KKASSERT(mycpuid == intr->bnx_intr_cpuid); 4691 4692 ifp = &intr->bnx_sc->arpcom.ac_if; 4693 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) { 4694 lwkt_serialize_exit(intr->bnx_intr_serialize); 4695 return; 4696 } 4697 4698 txr = intr->bnx_txr; 4699 ret = intr->bnx_ret; 4700 4701 if (*ret->bnx_rx_considx != ret->bnx_rx_saved_considx || 4702 *txr->bnx_tx_considx != txr->bnx_tx_saved_considx) { 4703 if (intr->bnx_rx_check_considx == ret->bnx_rx_saved_considx && 4704 intr->bnx_tx_check_considx == txr->bnx_tx_saved_considx) { 4705 if (!intr->bnx_intr_maylose) { 4706 intr->bnx_intr_maylose = TRUE; 4707 goto done; 4708 } 4709 if (bootverbose) 4710 if_printf(ifp, "lost interrupt\n"); 4711 intr->bnx_intr_func(intr->bnx_intr_arg); 4712 } 4713 } 4714 intr->bnx_intr_maylose = FALSE; 4715 intr->bnx_rx_check_considx = ret->bnx_rx_saved_considx; 4716 intr->bnx_tx_check_considx = txr->bnx_tx_saved_considx; 4717 4718 done: 4719 callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL, 4720 intr->bnx_intr_check, intr); 4721 lwkt_serialize_exit(intr->bnx_intr_serialize); 4722 } 4723 4724 static void 4725 bnx_check_intr_tx(void *xintr) 4726 { 4727 struct bnx_intr_data *intr = xintr; 4728 struct bnx_tx_ring *txr; 4729 struct ifnet *ifp; 4730 4731 lwkt_serialize_enter(intr->bnx_intr_serialize); 4732 4733 KKASSERT(mycpuid == intr->bnx_intr_cpuid); 4734 4735 ifp = &intr->bnx_sc->arpcom.ac_if; 4736 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) { 4737 lwkt_serialize_exit(intr->bnx_intr_serialize); 4738 return; 4739 } 4740 4741 txr = intr->bnx_txr; 4742 4743 if (*txr->bnx_tx_considx != txr->bnx_tx_saved_considx) { 4744 if (intr->bnx_tx_check_considx == txr->bnx_tx_saved_considx) { 4745 if (!intr->bnx_intr_maylose) { 4746 intr->bnx_intr_maylose = TRUE; 4747 goto done; 4748 } 4749 if (bootverbose) 4750 if_printf(ifp, "lost interrupt\n"); 4751 intr->bnx_intr_func(intr->bnx_intr_arg); 4752 } 4753 } 4754 intr->bnx_intr_maylose = FALSE; 4755 intr->bnx_tx_check_considx = txr->bnx_tx_saved_considx; 4756 4757 done: 4758 callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL, 4759 intr->bnx_intr_check, intr); 4760 lwkt_serialize_exit(intr->bnx_intr_serialize); 4761 } 4762 4763 static void 4764 bnx_check_intr_rx(void *xintr) 4765 { 4766 struct bnx_intr_data *intr = xintr; 4767 struct bnx_rx_ret_ring *ret; 4768 struct ifnet *ifp; 4769 4770 lwkt_serialize_enter(intr->bnx_intr_serialize); 4771 4772 KKASSERT(mycpuid == intr->bnx_intr_cpuid); 4773 4774 ifp = &intr->bnx_sc->arpcom.ac_if; 4775 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) { 4776 lwkt_serialize_exit(intr->bnx_intr_serialize); 4777 return; 4778 } 4779 4780 ret = intr->bnx_ret; 4781 4782 if (*ret->bnx_rx_considx != ret->bnx_rx_saved_considx) { 4783 if (intr->bnx_rx_check_considx == ret->bnx_rx_saved_considx) { 4784 if (!intr->bnx_intr_maylose) { 4785 intr->bnx_intr_maylose = TRUE; 4786 goto done; 4787 } 4788 if (bootverbose) 4789 if_printf(ifp, "lost interrupt\n"); 4790 intr->bnx_intr_func(intr->bnx_intr_arg); 4791 } 4792 } 4793 intr->bnx_intr_maylose = FALSE; 4794 intr->bnx_rx_check_considx = ret->bnx_rx_saved_considx; 4795 4796 done: 4797 callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL, 4798 intr->bnx_intr_check, intr); 4799 lwkt_serialize_exit(intr->bnx_intr_serialize); 4800 } 4801 4802 static void 4803 bnx_enable_intr(struct bnx_softc *sc) 4804 { 4805 struct ifnet *ifp = &sc->arpcom.ac_if; 4806 int i; 4807 4808 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4809 lwkt_serialize_handler_enable( 4810 sc->bnx_intr_data[i].bnx_intr_serialize); 4811 } 4812 4813 /* 4814 * Enable interrupt. 4815 */ 4816 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4817 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 4818 4819 bnx_writembx(sc, intr->bnx_intr_mbx, 4820 (*intr->bnx_saved_status_tag) << 24); 4821 /* XXX Linux driver */ 4822 bnx_writembx(sc, intr->bnx_intr_mbx, 4823 (*intr->bnx_saved_status_tag) << 24); 4824 } 4825 4826 /* 4827 * Unmask the interrupt when we stop polling. 4828 */ 4829 PCI_CLRBIT(sc->bnx_dev, BGE_PCI_MISC_CTL, 4830 BGE_PCIMISCCTL_MASK_PCI_INTR, 4); 4831 4832 /* 4833 * Trigger another interrupt, since above writing 4834 * to interrupt mailbox0 may acknowledge pending 4835 * interrupt. 4836 */ 4837 BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET); 4838 4839 if (sc->bnx_flags & BNX_FLAG_STATUSTAG_BUG) { 4840 if (bootverbose) 4841 if_printf(ifp, "status tag bug workaround\n"); 4842 4843 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4844 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 4845 4846 if (intr->bnx_intr_check == NULL) 4847 continue; 4848 intr->bnx_intr_maylose = FALSE; 4849 intr->bnx_rx_check_considx = 0; 4850 intr->bnx_tx_check_considx = 0; 4851 callout_reset_bycpu(&intr->bnx_intr_timer, 4852 BNX_INTR_CKINTVL, intr->bnx_intr_check, intr, 4853 intr->bnx_intr_cpuid); 4854 } 4855 } 4856 } 4857 4858 static void 4859 bnx_disable_intr(struct bnx_softc *sc) 4860 { 4861 int i; 4862 4863 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4864 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 4865 4866 callout_stop(&intr->bnx_intr_timer); 4867 intr->bnx_intr_maylose = FALSE; 4868 intr->bnx_rx_check_considx = 0; 4869 intr->bnx_tx_check_considx = 0; 4870 } 4871 4872 /* 4873 * Mask the interrupt when we start polling. 4874 */ 4875 PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL, 4876 BGE_PCIMISCCTL_MASK_PCI_INTR, 4); 4877 4878 /* 4879 * Acknowledge possible asserted interrupt. 4880 */ 4881 for (i = 0; i < BNX_INTR_MAX; ++i) 4882 bnx_writembx(sc, sc->bnx_intr_data[i].bnx_intr_mbx, 1); 4883 4884 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4885 lwkt_serialize_handler_disable( 4886 sc->bnx_intr_data[i].bnx_intr_serialize); 4887 } 4888 } 4889 4890 static int 4891 bnx_get_eaddr_mem(struct bnx_softc *sc, uint8_t ether_addr[]) 4892 { 4893 uint32_t mac_addr; 4894 int ret = 1; 4895 4896 mac_addr = bnx_readmem_ind(sc, 0x0c14); 4897 if ((mac_addr >> 16) == 0x484b) { 4898 ether_addr[0] = (uint8_t)(mac_addr >> 8); 4899 ether_addr[1] = (uint8_t)mac_addr; 4900 mac_addr = bnx_readmem_ind(sc, 0x0c18); 4901 ether_addr[2] = (uint8_t)(mac_addr >> 24); 4902 ether_addr[3] = (uint8_t)(mac_addr >> 16); 4903 ether_addr[4] = (uint8_t)(mac_addr >> 8); 4904 ether_addr[5] = (uint8_t)mac_addr; 4905 ret = 0; 4906 } 4907 return ret; 4908 } 4909 4910 static int 4911 bnx_get_eaddr_nvram(struct bnx_softc *sc, uint8_t ether_addr[]) 4912 { 4913 int mac_offset = BGE_EE_MAC_OFFSET; 4914 4915 if (BNX_IS_5717_PLUS(sc)) { 4916 int f; 4917 4918 f = pci_get_function(sc->bnx_dev); 4919 if (f & 1) 4920 mac_offset = BGE_EE_MAC_OFFSET_5717; 4921 if (f > 1) 4922 mac_offset += BGE_EE_MAC_OFFSET_5717_OFF; 4923 } 4924 4925 return bnx_read_nvram(sc, ether_addr, mac_offset + 2, ETHER_ADDR_LEN); 4926 } 4927 4928 static int 4929 bnx_get_eaddr_eeprom(struct bnx_softc *sc, uint8_t ether_addr[]) 4930 { 4931 if (sc->bnx_flags & BNX_FLAG_NO_EEPROM) 4932 return 1; 4933 4934 return bnx_read_eeprom(sc, ether_addr, BGE_EE_MAC_OFFSET + 2, 4935 ETHER_ADDR_LEN); 4936 } 4937 4938 static int 4939 bnx_get_eaddr(struct bnx_softc *sc, uint8_t eaddr[]) 4940 { 4941 static const bnx_eaddr_fcn_t bnx_eaddr_funcs[] = { 4942 /* NOTE: Order is critical */ 4943 bnx_get_eaddr_mem, 4944 bnx_get_eaddr_nvram, 4945 bnx_get_eaddr_eeprom, 4946 NULL 4947 }; 4948 const bnx_eaddr_fcn_t *func; 4949 4950 for (func = bnx_eaddr_funcs; *func != NULL; ++func) { 4951 if ((*func)(sc, eaddr) == 0) 4952 break; 4953 } 4954 return (*func == NULL ? ENXIO : 0); 4955 } 4956 4957 /* 4958 * NOTE: 'm' is not freed upon failure 4959 */ 4960 struct mbuf * 4961 bnx_defrag_shortdma(struct mbuf *m) 4962 { 4963 struct mbuf *n; 4964 int found; 4965 4966 /* 4967 * If device receive two back-to-back send BDs with less than 4968 * or equal to 8 total bytes then the device may hang. The two 4969 * back-to-back send BDs must in the same frame for this failure 4970 * to occur. Scan mbuf chains and see whether two back-to-back 4971 * send BDs are there. If this is the case, allocate new mbuf 4972 * and copy the frame to workaround the silicon bug. 4973 */ 4974 for (n = m, found = 0; n != NULL; n = n->m_next) { 4975 if (n->m_len < 8) { 4976 found++; 4977 if (found > 1) 4978 break; 4979 continue; 4980 } 4981 found = 0; 4982 } 4983 4984 if (found > 1) 4985 n = m_defrag(m, M_NOWAIT); 4986 else 4987 n = m; 4988 return n; 4989 } 4990 4991 static void 4992 bnx_stop_block(struct bnx_softc *sc, bus_size_t reg, uint32_t bit) 4993 { 4994 int i; 4995 4996 BNX_CLRBIT(sc, reg, bit); 4997 for (i = 0; i < BNX_TIMEOUT; i++) { 4998 if ((CSR_READ_4(sc, reg) & bit) == 0) 4999 return; 5000 DELAY(100); 5001 } 5002 } 5003 5004 static void 5005 bnx_link_poll(struct bnx_softc *sc) 5006 { 5007 uint32_t status; 5008 5009 status = CSR_READ_4(sc, BGE_MAC_STS); 5010 if ((status & sc->bnx_link_chg) || sc->bnx_link_evt) { 5011 sc->bnx_link_evt = 0; 5012 sc->bnx_link_upd(sc, status); 5013 } 5014 } 5015 5016 static void 5017 bnx_enable_msi(struct bnx_softc *sc, boolean_t is_msix) 5018 { 5019 uint32_t msi_mode; 5020 5021 msi_mode = CSR_READ_4(sc, BGE_MSI_MODE); 5022 msi_mode |= BGE_MSIMODE_ENABLE; 5023 /* 5024 * NOTE: 5025 * 5718-PG105-R says that "one shot" mode does not work 5026 * if MSI is used, however, it obviously works. 5027 */ 5028 msi_mode &= ~BGE_MSIMODE_ONESHOT_DISABLE; 5029 if (is_msix) 5030 msi_mode |= BGE_MSIMODE_MSIX_MULTIMODE; 5031 else 5032 msi_mode &= ~BGE_MSIMODE_MSIX_MULTIMODE; 5033 CSR_WRITE_4(sc, BGE_MSI_MODE, msi_mode); 5034 } 5035 5036 static uint32_t 5037 bnx_dma_swap_options(struct bnx_softc *sc) 5038 { 5039 uint32_t dma_options; 5040 5041 dma_options = BGE_MODECTL_WORDSWAP_NONFRAME | 5042 BGE_MODECTL_BYTESWAP_DATA | BGE_MODECTL_WORDSWAP_DATA; 5043 #if BYTE_ORDER == BIG_ENDIAN 5044 dma_options |= BGE_MODECTL_BYTESWAP_NONFRAME; 5045 #endif 5046 return dma_options; 5047 } 5048 5049 static int 5050 bnx_setup_tso(struct bnx_tx_ring *txr, struct mbuf **mp, 5051 uint16_t *mss0, uint16_t *flags0) 5052 { 5053 struct mbuf *m; 5054 struct ip *ip; 5055 struct tcphdr *th; 5056 int thoff, iphlen, hoff, hlen; 5057 uint16_t flags, mss; 5058 5059 m = *mp; 5060 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable")); 5061 5062 hoff = m->m_pkthdr.csum_lhlen; 5063 iphlen = m->m_pkthdr.csum_iphlen; 5064 thoff = m->m_pkthdr.csum_thlen; 5065 5066 KASSERT(hoff > 0, ("invalid ether header len")); 5067 KASSERT(iphlen > 0, ("invalid ip header len")); 5068 KASSERT(thoff > 0, ("invalid tcp header len")); 5069 5070 if (__predict_false(m->m_len < hoff + iphlen + thoff)) { 5071 m = m_pullup(m, hoff + iphlen + thoff); 5072 if (m == NULL) { 5073 *mp = NULL; 5074 return ENOBUFS; 5075 } 5076 *mp = m; 5077 } 5078 ip = mtodoff(m, struct ip *, hoff); 5079 th = mtodoff(m, struct tcphdr *, hoff + iphlen); 5080 5081 mss = m->m_pkthdr.tso_segsz; 5082 flags = BGE_TXBDFLAG_CPU_PRE_DMA | BGE_TXBDFLAG_CPU_POST_DMA; 5083 5084 ip->ip_len = htons(mss + iphlen + thoff); 5085 th->th_sum = 0; 5086 5087 hlen = (iphlen + thoff) >> 2; 5088 mss |= ((hlen & 0x3) << 14); 5089 flags |= ((hlen & 0xf8) << 7) | ((hlen & 0x4) << 2); 5090 5091 *mss0 = mss; 5092 *flags0 = flags; 5093 5094 return 0; 5095 } 5096 5097 static int 5098 bnx_create_tx_ring(struct bnx_tx_ring *txr) 5099 { 5100 bus_size_t txmaxsz, txmaxsegsz; 5101 int i, error; 5102 5103 lwkt_serialize_init(&txr->bnx_tx_serialize); 5104 5105 /* 5106 * Create DMA tag and maps for TX mbufs. 5107 */ 5108 if (txr->bnx_sc->bnx_flags & BNX_FLAG_TSO) 5109 txmaxsz = IP_MAXPACKET + sizeof(struct ether_vlan_header); 5110 else 5111 txmaxsz = BNX_JUMBO_FRAMELEN; 5112 if (txr->bnx_sc->bnx_asicrev == BGE_ASICREV_BCM57766) 5113 txmaxsegsz = MCLBYTES; 5114 else 5115 txmaxsegsz = PAGE_SIZE; 5116 error = bus_dma_tag_create(txr->bnx_sc->bnx_cdata.bnx_parent_tag, 5117 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 5118 txmaxsz, BNX_NSEG_NEW, txmaxsegsz, 5119 BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 5120 &txr->bnx_tx_mtag); 5121 if (error) { 5122 device_printf(txr->bnx_sc->bnx_dev, 5123 "could not create TX mbuf DMA tag\n"); 5124 return error; 5125 } 5126 5127 for (i = 0; i < BGE_TX_RING_CNT; i++) { 5128 error = bus_dmamap_create(txr->bnx_tx_mtag, 5129 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 5130 &txr->bnx_tx_buf[i].bnx_tx_dmamap); 5131 if (error) { 5132 int j; 5133 5134 for (j = 0; j < i; ++j) { 5135 bus_dmamap_destroy(txr->bnx_tx_mtag, 5136 txr->bnx_tx_buf[j].bnx_tx_dmamap); 5137 } 5138 bus_dma_tag_destroy(txr->bnx_tx_mtag); 5139 txr->bnx_tx_mtag = NULL; 5140 5141 device_printf(txr->bnx_sc->bnx_dev, 5142 "could not create TX mbuf DMA map\n"); 5143 return error; 5144 } 5145 } 5146 5147 /* 5148 * Create DMA stuffs for TX ring. 5149 */ 5150 error = bnx_dma_block_alloc(txr->bnx_sc, BGE_TX_RING_SZ, 5151 &txr->bnx_tx_ring_tag, 5152 &txr->bnx_tx_ring_map, 5153 (void *)&txr->bnx_tx_ring, 5154 &txr->bnx_tx_ring_paddr); 5155 if (error) { 5156 device_printf(txr->bnx_sc->bnx_dev, 5157 "could not create TX ring\n"); 5158 return error; 5159 } 5160 5161 txr->bnx_tx_flags |= BNX_TX_FLAG_SHORTDMA; 5162 txr->bnx_tx_wreg = BNX_TX_WREG_NSEGS; 5163 5164 return 0; 5165 } 5166 5167 static void 5168 bnx_destroy_tx_ring(struct bnx_tx_ring *txr) 5169 { 5170 /* Destroy TX mbuf DMA stuffs. */ 5171 if (txr->bnx_tx_mtag != NULL) { 5172 int i; 5173 5174 for (i = 0; i < BGE_TX_RING_CNT; i++) { 5175 KKASSERT(txr->bnx_tx_buf[i].bnx_tx_mbuf == NULL); 5176 bus_dmamap_destroy(txr->bnx_tx_mtag, 5177 txr->bnx_tx_buf[i].bnx_tx_dmamap); 5178 } 5179 bus_dma_tag_destroy(txr->bnx_tx_mtag); 5180 } 5181 5182 /* Destroy TX ring */ 5183 bnx_dma_block_free(txr->bnx_tx_ring_tag, 5184 txr->bnx_tx_ring_map, txr->bnx_tx_ring); 5185 } 5186 5187 static int 5188 bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS) 5189 { 5190 struct bnx_softc *sc = (void *)arg1; 5191 struct ifnet *ifp = &sc->arpcom.ac_if; 5192 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; 5193 int error, defrag, i; 5194 5195 if (txr->bnx_tx_flags & BNX_TX_FLAG_FORCE_DEFRAG) 5196 defrag = 1; 5197 else 5198 defrag = 0; 5199 5200 error = sysctl_handle_int(oidp, &defrag, 0, req); 5201 if (error || req->newptr == NULL) 5202 return error; 5203 5204 ifnet_serialize_all(ifp); 5205 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 5206 txr = &sc->bnx_tx_ring[i]; 5207 if (defrag) 5208 txr->bnx_tx_flags |= BNX_TX_FLAG_FORCE_DEFRAG; 5209 else 5210 txr->bnx_tx_flags &= ~BNX_TX_FLAG_FORCE_DEFRAG; 5211 } 5212 ifnet_deserialize_all(ifp); 5213 5214 return 0; 5215 } 5216 5217 static int 5218 bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS) 5219 { 5220 struct bnx_softc *sc = (void *)arg1; 5221 struct ifnet *ifp = &sc->arpcom.ac_if; 5222 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; 5223 int error, tx_wreg, i; 5224 5225 tx_wreg = txr->bnx_tx_wreg; 5226 error = sysctl_handle_int(oidp, &tx_wreg, 0, req); 5227 if (error || req->newptr == NULL) 5228 return error; 5229 5230 ifnet_serialize_all(ifp); 5231 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) 5232 sc->bnx_tx_ring[i].bnx_tx_wreg = tx_wreg; 5233 ifnet_deserialize_all(ifp); 5234 5235 return 0; 5236 } 5237 5238 static int 5239 bnx_create_rx_ret_ring(struct bnx_rx_ret_ring *ret) 5240 { 5241 int error; 5242 5243 lwkt_serialize_init(&ret->bnx_rx_ret_serialize); 5244 5245 /* 5246 * Create DMA stuffs for RX return ring. 5247 */ 5248 error = bnx_dma_block_alloc(ret->bnx_sc, 5249 BGE_RX_RTN_RING_SZ(BNX_RETURN_RING_CNT), 5250 &ret->bnx_rx_ret_ring_tag, 5251 &ret->bnx_rx_ret_ring_map, 5252 (void *)&ret->bnx_rx_ret_ring, 5253 &ret->bnx_rx_ret_ring_paddr); 5254 if (error) { 5255 device_printf(ret->bnx_sc->bnx_dev, 5256 "could not create RX ret ring\n"); 5257 return error; 5258 } 5259 5260 /* Shadow standard ring's RX mbuf DMA tag */ 5261 ret->bnx_rx_mtag = ret->bnx_std->bnx_rx_mtag; 5262 5263 /* 5264 * Create tmp DMA map for RX mbufs. 5265 */ 5266 error = bus_dmamap_create(ret->bnx_rx_mtag, BUS_DMA_WAITOK, 5267 &ret->bnx_rx_tmpmap); 5268 if (error) { 5269 device_printf(ret->bnx_sc->bnx_dev, 5270 "could not create tmp RX mbuf DMA map\n"); 5271 ret->bnx_rx_mtag = NULL; 5272 return error; 5273 } 5274 return 0; 5275 } 5276 5277 static void 5278 bnx_destroy_rx_ret_ring(struct bnx_rx_ret_ring *ret) 5279 { 5280 /* Destroy tmp RX mbuf DMA map */ 5281 if (ret->bnx_rx_mtag != NULL) 5282 bus_dmamap_destroy(ret->bnx_rx_mtag, ret->bnx_rx_tmpmap); 5283 5284 /* Destroy RX return ring */ 5285 bnx_dma_block_free(ret->bnx_rx_ret_ring_tag, 5286 ret->bnx_rx_ret_ring_map, ret->bnx_rx_ret_ring); 5287 } 5288 5289 static int 5290 bnx_alloc_intr(struct bnx_softc *sc) 5291 { 5292 struct bnx_intr_data *intr; 5293 u_int intr_flags; 5294 int error; 5295 5296 if (sc->bnx_intr_cnt > 1) { 5297 error = bnx_alloc_msix(sc); 5298 if (error) 5299 return error; 5300 KKASSERT(sc->bnx_intr_type == PCI_INTR_TYPE_MSIX); 5301 return 0; 5302 } 5303 5304 KKASSERT(sc->bnx_intr_cnt == 1); 5305 5306 intr = &sc->bnx_intr_data[0]; 5307 intr->bnx_ret = &sc->bnx_rx_ret_ring[0]; 5308 intr->bnx_txr = &sc->bnx_tx_ring[0]; 5309 intr->bnx_intr_serialize = &sc->bnx_main_serialize; 5310 intr->bnx_intr_check = bnx_check_intr_rxtx; 5311 intr->bnx_saved_status_tag = &intr->bnx_ret->bnx_saved_status_tag; 5312 5313 sc->bnx_intr_type = pci_alloc_1intr(sc->bnx_dev, bnx_msi_enable, 5314 &intr->bnx_intr_rid, &intr_flags); 5315 5316 intr->bnx_intr_res = bus_alloc_resource_any(sc->bnx_dev, SYS_RES_IRQ, 5317 &intr->bnx_intr_rid, intr_flags); 5318 if (intr->bnx_intr_res == NULL) { 5319 device_printf(sc->bnx_dev, "could not alloc interrupt\n"); 5320 return ENXIO; 5321 } 5322 5323 if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI) { 5324 bnx_enable_msi(sc, FALSE); 5325 intr->bnx_intr_func = bnx_msi; 5326 if (bootverbose) 5327 device_printf(sc->bnx_dev, "oneshot MSI\n"); 5328 } else { 5329 intr->bnx_intr_func = bnx_intr_legacy; 5330 } 5331 intr->bnx_intr_arg = sc; 5332 intr->bnx_intr_cpuid = rman_get_cpuid(intr->bnx_intr_res); 5333 5334 intr->bnx_txr->bnx_tx_cpuid = intr->bnx_intr_cpuid; 5335 5336 return 0; 5337 } 5338 5339 static int 5340 bnx_setup_intr(struct bnx_softc *sc) 5341 { 5342 int error, i; 5343 5344 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 5345 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 5346 5347 error = bus_setup_intr_descr(sc->bnx_dev, intr->bnx_intr_res, 5348 INTR_MPSAFE, intr->bnx_intr_func, intr->bnx_intr_arg, 5349 &intr->bnx_intr_hand, intr->bnx_intr_serialize, 5350 intr->bnx_intr_desc); 5351 if (error) { 5352 device_printf(sc->bnx_dev, 5353 "could not set up %dth intr\n", i); 5354 bnx_teardown_intr(sc, i); 5355 return error; 5356 } 5357 } 5358 return 0; 5359 } 5360 5361 static void 5362 bnx_teardown_intr(struct bnx_softc *sc, int cnt) 5363 { 5364 int i; 5365 5366 for (i = 0; i < cnt; ++i) { 5367 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 5368 5369 bus_teardown_intr(sc->bnx_dev, intr->bnx_intr_res, 5370 intr->bnx_intr_hand); 5371 } 5372 } 5373 5374 static void 5375 bnx_free_intr(struct bnx_softc *sc) 5376 { 5377 if (sc->bnx_intr_type != PCI_INTR_TYPE_MSIX) { 5378 struct bnx_intr_data *intr; 5379 5380 KKASSERT(sc->bnx_intr_cnt <= 1); 5381 intr = &sc->bnx_intr_data[0]; 5382 5383 if (intr->bnx_intr_res != NULL) { 5384 bus_release_resource(sc->bnx_dev, SYS_RES_IRQ, 5385 intr->bnx_intr_rid, intr->bnx_intr_res); 5386 } 5387 if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI) 5388 pci_release_msi(sc->bnx_dev); 5389 } else { 5390 bnx_free_msix(sc, TRUE); 5391 } 5392 } 5393 5394 static void 5395 bnx_setup_serialize(struct bnx_softc *sc) 5396 { 5397 int i, j; 5398 5399 /* 5400 * Allocate serializer array 5401 */ 5402 5403 /* Main + RX STD + TX + RX RET */ 5404 sc->bnx_serialize_cnt = 1 + 1 + sc->bnx_tx_ringcnt + sc->bnx_rx_retcnt; 5405 5406 sc->bnx_serialize = 5407 kmalloc(sc->bnx_serialize_cnt * sizeof(struct lwkt_serialize *), 5408 M_DEVBUF, M_WAITOK | M_ZERO); 5409 5410 /* 5411 * Setup serializers 5412 * 5413 * NOTE: Order is critical 5414 */ 5415 5416 i = 0; 5417 5418 KKASSERT(i < sc->bnx_serialize_cnt); 5419 sc->bnx_serialize[i++] = &sc->bnx_main_serialize; 5420 5421 KKASSERT(i < sc->bnx_serialize_cnt); 5422 sc->bnx_serialize[i++] = &sc->bnx_rx_std_ring.bnx_rx_std_serialize; 5423 5424 for (j = 0; j < sc->bnx_rx_retcnt; ++j) { 5425 KKASSERT(i < sc->bnx_serialize_cnt); 5426 sc->bnx_serialize[i++] = 5427 &sc->bnx_rx_ret_ring[j].bnx_rx_ret_serialize; 5428 } 5429 5430 for (j = 0; j < sc->bnx_tx_ringcnt; ++j) { 5431 KKASSERT(i < sc->bnx_serialize_cnt); 5432 sc->bnx_serialize[i++] = 5433 &sc->bnx_tx_ring[j].bnx_tx_serialize; 5434 } 5435 5436 KKASSERT(i == sc->bnx_serialize_cnt); 5437 } 5438 5439 static void 5440 bnx_serialize(struct ifnet *ifp, enum ifnet_serialize slz) 5441 { 5442 struct bnx_softc *sc = ifp->if_softc; 5443 5444 ifnet_serialize_array_enter(sc->bnx_serialize, 5445 sc->bnx_serialize_cnt, slz); 5446 } 5447 5448 static void 5449 bnx_deserialize(struct ifnet *ifp, enum ifnet_serialize slz) 5450 { 5451 struct bnx_softc *sc = ifp->if_softc; 5452 5453 ifnet_serialize_array_exit(sc->bnx_serialize, 5454 sc->bnx_serialize_cnt, slz); 5455 } 5456 5457 static int 5458 bnx_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz) 5459 { 5460 struct bnx_softc *sc = ifp->if_softc; 5461 5462 return ifnet_serialize_array_try(sc->bnx_serialize, 5463 sc->bnx_serialize_cnt, slz); 5464 } 5465 5466 #ifdef INVARIANTS 5467 5468 static void 5469 bnx_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz, 5470 boolean_t serialized) 5471 { 5472 struct bnx_softc *sc = ifp->if_softc; 5473 5474 ifnet_serialize_array_assert(sc->bnx_serialize, sc->bnx_serialize_cnt, 5475 slz, serialized); 5476 } 5477 5478 #endif /* INVARIANTS */ 5479 5480 static void 5481 bnx_set_tick_cpuid(struct bnx_softc *sc, boolean_t polling) 5482 { 5483 if (polling) 5484 sc->bnx_tick_cpuid = 0; /* XXX */ 5485 else 5486 sc->bnx_tick_cpuid = sc->bnx_intr_data[0].bnx_intr_cpuid; 5487 } 5488 5489 static void 5490 bnx_rx_std_refill_ithread(void *xstd) 5491 { 5492 struct bnx_rx_std_ring *std = xstd; 5493 struct globaldata *gd = mycpu; 5494 5495 crit_enter_gd(gd); 5496 5497 while (!std->bnx_rx_std_stop) { 5498 if (std->bnx_rx_std_refill) { 5499 lwkt_serialize_handler_call( 5500 &std->bnx_rx_std_serialize, 5501 bnx_rx_std_refill, std, NULL); 5502 } 5503 5504 crit_exit_gd(gd); 5505 crit_enter_gd(gd); 5506 5507 atomic_poll_release_int(&std->bnx_rx_std_running); 5508 cpu_mfence(); 5509 5510 if (!std->bnx_rx_std_refill && !std->bnx_rx_std_stop) { 5511 lwkt_deschedule_self(gd->gd_curthread); 5512 lwkt_switch(); 5513 } 5514 } 5515 5516 crit_exit_gd(gd); 5517 5518 wakeup(std); 5519 5520 lwkt_exit(); 5521 } 5522 5523 static void 5524 bnx_rx_std_refill(void *xstd, void *frame __unused) 5525 { 5526 struct bnx_rx_std_ring *std = xstd; 5527 int cnt, refill_mask; 5528 5529 again: 5530 cnt = 0; 5531 5532 cpu_lfence(); 5533 refill_mask = std->bnx_rx_std_refill; 5534 atomic_clear_int(&std->bnx_rx_std_refill, refill_mask); 5535 5536 while (refill_mask) { 5537 uint16_t check_idx = std->bnx_rx_std; 5538 int ret_idx; 5539 5540 ret_idx = bsfl(refill_mask); 5541 for (;;) { 5542 struct bnx_rx_buf *rb; 5543 int refilled; 5544 5545 BNX_INC(check_idx, BGE_STD_RX_RING_CNT); 5546 rb = &std->bnx_rx_std_buf[check_idx]; 5547 refilled = rb->bnx_rx_refilled; 5548 cpu_lfence(); 5549 if (refilled) { 5550 bnx_setup_rxdesc_std(std, check_idx); 5551 std->bnx_rx_std = check_idx; 5552 ++cnt; 5553 if (cnt >= 8) { 5554 atomic_subtract_int( 5555 &std->bnx_rx_std_used, cnt); 5556 bnx_writembx(std->bnx_sc, 5557 BGE_MBX_RX_STD_PROD_LO, 5558 std->bnx_rx_std); 5559 cnt = 0; 5560 } 5561 } else { 5562 break; 5563 } 5564 } 5565 refill_mask &= ~(1 << ret_idx); 5566 } 5567 5568 if (cnt) { 5569 atomic_subtract_int(&std->bnx_rx_std_used, cnt); 5570 bnx_writembx(std->bnx_sc, BGE_MBX_RX_STD_PROD_LO, 5571 std->bnx_rx_std); 5572 } 5573 5574 if (std->bnx_rx_std_refill) 5575 goto again; 5576 5577 atomic_poll_release_int(&std->bnx_rx_std_running); 5578 cpu_mfence(); 5579 5580 if (std->bnx_rx_std_refill) 5581 goto again; 5582 } 5583 5584 static int 5585 bnx_sysctl_std_refill(SYSCTL_HANDLER_ARGS) 5586 { 5587 struct bnx_softc *sc = (void *)arg1; 5588 struct ifnet *ifp = &sc->arpcom.ac_if; 5589 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0]; 5590 int error, cntmax, i; 5591 5592 cntmax = ret->bnx_rx_cntmax; 5593 error = sysctl_handle_int(oidp, &cntmax, 0, req); 5594 if (error || req->newptr == NULL) 5595 return error; 5596 5597 ifnet_serialize_all(ifp); 5598 5599 if ((cntmax * sc->bnx_rx_retcnt) >= BGE_STD_RX_RING_CNT / 2) { 5600 error = EINVAL; 5601 goto back; 5602 } 5603 5604 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) 5605 sc->bnx_rx_ret_ring[i].bnx_rx_cntmax = cntmax; 5606 error = 0; 5607 5608 back: 5609 ifnet_deserialize_all(ifp); 5610 5611 return error; 5612 } 5613 5614 static void 5615 bnx_init_rss(struct bnx_softc *sc) 5616 { 5617 uint8_t key[BGE_RSS_KEYREG_CNT * BGE_RSS_KEYREG_SIZE]; 5618 int i, j, r; 5619 5620 KKASSERT(BNX_RSS_ENABLED(sc)); 5621 5622 /* 5623 * Configure RSS redirect table. 5624 */ 5625 if_ringmap_rdrtable(sc->bnx_rx_rmap, sc->bnx_rdr_table, 5626 BNX_RDRTABLE_SIZE); 5627 r = 0; 5628 for (j = 0; j < BGE_RSS_INDIR_TBL_CNT; ++j) { 5629 uint32_t tbl = 0; 5630 5631 for (i = 0; i < BGE_RSS_INDIR_TBLENT_CNT; ++i) { 5632 uint32_t q; 5633 5634 q = sc->bnx_rdr_table[r]; 5635 tbl |= q << (BGE_RSS_INDIR_TBLENT_SHIFT * 5636 (BGE_RSS_INDIR_TBLENT_CNT - i - 1)); 5637 ++r; 5638 } 5639 5640 BNX_RSS_DPRINTF(sc, 1, "tbl%d %08x\n", j, tbl); 5641 CSR_WRITE_4(sc, BGE_RSS_INDIR_TBL(j), tbl); 5642 } 5643 5644 toeplitz_get_key(key, sizeof(key)); 5645 for (i = 0; i < BGE_RSS_KEYREG_CNT; ++i) { 5646 uint32_t keyreg; 5647 5648 keyreg = BGE_RSS_KEYREG_VAL(key, i); 5649 5650 BNX_RSS_DPRINTF(sc, 1, "key%d %08x\n", i, keyreg); 5651 CSR_WRITE_4(sc, BGE_RSS_KEYREG(i), keyreg); 5652 } 5653 } 5654 5655 static void 5656 bnx_setup_ring_cnt(struct bnx_softc *sc) 5657 { 5658 int msix_enable, msix_cnt, msix_ring, ring_max, ring_cnt; 5659 5660 /* One RX ring. */ 5661 sc->bnx_rx_rmap = if_ringmap_alloc(sc->bnx_dev, 1, 1); 5662 5663 if (netisr_ncpus == 1) 5664 goto skip_rx; 5665 5666 msix_enable = device_getenv_int(sc->bnx_dev, "msix.enable", 5667 bnx_msix_enable); 5668 if (!msix_enable) 5669 goto skip_rx; 5670 5671 /* 5672 * One MSI-X vector is dedicated to status or single TX queue, 5673 * so make sure that there are enough MSI-X vectors. 5674 */ 5675 msix_cnt = pci_msix_count(sc->bnx_dev); 5676 if (msix_cnt <= 1) 5677 goto skip_rx; 5678 if (bootverbose) 5679 device_printf(sc->bnx_dev, "MSI-X count %d\n", msix_cnt); 5680 msix_ring = msix_cnt - 1; 5681 5682 /* 5683 * Setup RX ring count 5684 */ 5685 ring_max = BNX_RX_RING_MAX; 5686 if (ring_max > msix_ring) 5687 ring_max = msix_ring; 5688 ring_cnt = device_getenv_int(sc->bnx_dev, "rx_rings", bnx_rx_rings); 5689 5690 if_ringmap_free(sc->bnx_rx_rmap); 5691 sc->bnx_rx_rmap = if_ringmap_alloc(sc->bnx_dev, ring_cnt, ring_max); 5692 5693 skip_rx: 5694 sc->bnx_rx_retcnt = if_ringmap_count(sc->bnx_rx_rmap); 5695 5696 /* 5697 * Setup TX ring count 5698 * 5699 * Currently only BCM5719 and BCM5720 support multiple TX rings 5700 * and the TX ring count must be less than the RX ring count. 5701 */ 5702 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 || 5703 sc->bnx_asicrev == BGE_ASICREV_BCM5720) { 5704 ring_max = BNX_TX_RING_MAX; 5705 if (ring_max > sc->bnx_rx_retcnt) 5706 ring_max = sc->bnx_rx_retcnt; 5707 ring_cnt = device_getenv_int(sc->bnx_dev, "tx_rings", 5708 bnx_tx_rings); 5709 } else { 5710 ring_max = 1; 5711 ring_cnt = 1; 5712 } 5713 sc->bnx_tx_rmap = if_ringmap_alloc(sc->bnx_dev, ring_cnt, ring_max); 5714 if_ringmap_align(sc->bnx_dev, sc->bnx_rx_rmap, sc->bnx_tx_rmap); 5715 5716 sc->bnx_tx_ringcnt = if_ringmap_count(sc->bnx_tx_rmap); 5717 KASSERT(sc->bnx_tx_ringcnt <= sc->bnx_rx_retcnt, 5718 ("invalid TX ring count %d and RX ring count %d", 5719 sc->bnx_tx_ringcnt, sc->bnx_rx_retcnt)); 5720 5721 /* 5722 * Setup interrupt count. 5723 */ 5724 if (sc->bnx_rx_retcnt == 1) { 5725 sc->bnx_intr_cnt = 1; 5726 } else { 5727 /* 5728 * We need one extra MSI-X vector for link status or 5729 * TX ring (if only one TX ring is enabled). 5730 */ 5731 sc->bnx_intr_cnt = sc->bnx_rx_retcnt + 1; 5732 } 5733 KKASSERT(sc->bnx_intr_cnt <= BNX_INTR_MAX); 5734 5735 if (bootverbose) { 5736 device_printf(sc->bnx_dev, "intr count %d, " 5737 "RX ring %d, TX ring %d\n", sc->bnx_intr_cnt, 5738 sc->bnx_rx_retcnt, sc->bnx_tx_ringcnt); 5739 } 5740 } 5741 5742 static int 5743 bnx_alloc_msix(struct bnx_softc *sc) 5744 { 5745 struct bnx_intr_data *intr; 5746 boolean_t setup = FALSE; 5747 int error, i; 5748 5749 KKASSERT(sc->bnx_intr_cnt > 1); 5750 KKASSERT(sc->bnx_intr_cnt == sc->bnx_rx_retcnt + 1); 5751 5752 if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) { 5753 /* 5754 * Link status 5755 */ 5756 intr = &sc->bnx_intr_data[0]; 5757 5758 intr->bnx_intr_serialize = &sc->bnx_main_serialize; 5759 intr->bnx_saved_status_tag = &sc->bnx_saved_status_tag; 5760 5761 intr->bnx_intr_func = bnx_msix_status; 5762 intr->bnx_intr_arg = sc; 5763 intr->bnx_intr_cpuid = 0; /* XXX */ 5764 5765 ksnprintf(intr->bnx_intr_desc0, sizeof(intr->bnx_intr_desc0), 5766 "%s sts", device_get_nameunit(sc->bnx_dev)); 5767 intr->bnx_intr_desc = intr->bnx_intr_desc0; 5768 5769 /* 5770 * RX/TX rings 5771 */ 5772 for (i = 1; i < sc->bnx_intr_cnt; ++i) { 5773 int idx = i - 1; 5774 5775 intr = &sc->bnx_intr_data[i]; 5776 5777 KKASSERT(idx < sc->bnx_rx_retcnt); 5778 intr->bnx_ret = &sc->bnx_rx_ret_ring[idx]; 5779 if (idx < sc->bnx_tx_ringcnt) { 5780 intr->bnx_txr = &sc->bnx_tx_ring[idx]; 5781 intr->bnx_ret->bnx_txr = intr->bnx_txr; 5782 } 5783 5784 intr->bnx_intr_serialize = 5785 &intr->bnx_ret->bnx_rx_ret_serialize; 5786 intr->bnx_saved_status_tag = 5787 &intr->bnx_ret->bnx_saved_status_tag; 5788 5789 intr->bnx_intr_arg = intr->bnx_ret; 5790 intr->bnx_intr_cpuid = 5791 if_ringmap_cpumap(sc->bnx_rx_rmap, idx); 5792 KKASSERT(intr->bnx_intr_cpuid < netisr_ncpus); 5793 5794 if (intr->bnx_txr == NULL) { 5795 intr->bnx_intr_check = bnx_check_intr_rx; 5796 intr->bnx_intr_func = bnx_msix_rx; 5797 ksnprintf(intr->bnx_intr_desc0, 5798 sizeof(intr->bnx_intr_desc0), "%s rx%d", 5799 device_get_nameunit(sc->bnx_dev), idx); 5800 } else { 5801 #ifdef INVARIANTS 5802 int tx_cpuid; 5803 #endif 5804 5805 intr->bnx_intr_check = bnx_check_intr_rxtx; 5806 intr->bnx_intr_func = bnx_msix_rxtx; 5807 ksnprintf(intr->bnx_intr_desc0, 5808 sizeof(intr->bnx_intr_desc0), "%s rxtx%d", 5809 device_get_nameunit(sc->bnx_dev), idx); 5810 5811 #ifdef INVARIANTS 5812 tx_cpuid = if_ringmap_cpumap(sc->bnx_tx_rmap, 5813 idx); 5814 KASSERT(intr->bnx_intr_cpuid == tx_cpuid, 5815 ("RX intr cpu%d, TX intr cpu%d, mismatch", 5816 intr->bnx_intr_cpuid, tx_cpuid)); 5817 #endif 5818 intr->bnx_txr->bnx_tx_cpuid = 5819 intr->bnx_intr_cpuid; 5820 } 5821 intr->bnx_intr_desc = intr->bnx_intr_desc0; 5822 5823 intr->bnx_ret->bnx_msix_mbx = intr->bnx_intr_mbx; 5824 } 5825 } else { 5826 /* 5827 * TX ring0 and link status 5828 */ 5829 intr = &sc->bnx_intr_data[0]; 5830 5831 intr->bnx_txr = &sc->bnx_tx_ring[0]; 5832 intr->bnx_intr_serialize = &sc->bnx_main_serialize; 5833 intr->bnx_intr_check = bnx_check_intr_tx; 5834 intr->bnx_saved_status_tag = 5835 &intr->bnx_txr->bnx_saved_status_tag; 5836 5837 intr->bnx_intr_func = bnx_msix_tx_status; 5838 intr->bnx_intr_arg = intr->bnx_txr; 5839 intr->bnx_intr_cpuid = if_ringmap_cpumap(sc->bnx_tx_rmap, 0); 5840 KKASSERT(intr->bnx_intr_cpuid < netisr_ncpus); 5841 5842 ksnprintf(intr->bnx_intr_desc0, sizeof(intr->bnx_intr_desc0), 5843 "%s ststx", device_get_nameunit(sc->bnx_dev)); 5844 intr->bnx_intr_desc = intr->bnx_intr_desc0; 5845 5846 intr->bnx_txr->bnx_tx_cpuid = intr->bnx_intr_cpuid; 5847 5848 /* 5849 * RX rings 5850 */ 5851 for (i = 1; i < sc->bnx_intr_cnt; ++i) { 5852 int idx = i - 1; 5853 5854 intr = &sc->bnx_intr_data[i]; 5855 5856 KKASSERT(idx < sc->bnx_rx_retcnt); 5857 intr->bnx_ret = &sc->bnx_rx_ret_ring[idx]; 5858 intr->bnx_intr_serialize = 5859 &intr->bnx_ret->bnx_rx_ret_serialize; 5860 intr->bnx_intr_check = bnx_check_intr_rx; 5861 intr->bnx_saved_status_tag = 5862 &intr->bnx_ret->bnx_saved_status_tag; 5863 5864 intr->bnx_intr_func = bnx_msix_rx; 5865 intr->bnx_intr_arg = intr->bnx_ret; 5866 intr->bnx_intr_cpuid = 5867 if_ringmap_cpumap(sc->bnx_rx_rmap, idx); 5868 KKASSERT(intr->bnx_intr_cpuid < netisr_ncpus); 5869 5870 ksnprintf(intr->bnx_intr_desc0, 5871 sizeof(intr->bnx_intr_desc0), "%s rx%d", 5872 device_get_nameunit(sc->bnx_dev), idx); 5873 intr->bnx_intr_desc = intr->bnx_intr_desc0; 5874 5875 intr->bnx_ret->bnx_msix_mbx = intr->bnx_intr_mbx; 5876 } 5877 } 5878 5879 if (BNX_IS_5717_PLUS(sc)) { 5880 sc->bnx_msix_mem_rid = PCIR_BAR(4); 5881 } else { 5882 if (sc->bnx_res2 == NULL) 5883 sc->bnx_msix_mem_rid = PCIR_BAR(2); 5884 } 5885 if (sc->bnx_msix_mem_rid != 0) { 5886 sc->bnx_msix_mem_res = bus_alloc_resource_any(sc->bnx_dev, 5887 SYS_RES_MEMORY, &sc->bnx_msix_mem_rid, RF_ACTIVE); 5888 if (sc->bnx_msix_mem_res == NULL) { 5889 device_printf(sc->bnx_dev, 5890 "could not alloc MSI-X table\n"); 5891 return ENXIO; 5892 } 5893 } 5894 5895 bnx_enable_msi(sc, TRUE); 5896 5897 error = pci_setup_msix(sc->bnx_dev); 5898 if (error) { 5899 device_printf(sc->bnx_dev, "could not setup MSI-X\n"); 5900 goto back; 5901 } 5902 setup = TRUE; 5903 5904 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 5905 intr = &sc->bnx_intr_data[i]; 5906 5907 error = pci_alloc_msix_vector(sc->bnx_dev, i, 5908 &intr->bnx_intr_rid, intr->bnx_intr_cpuid); 5909 if (error) { 5910 device_printf(sc->bnx_dev, 5911 "could not alloc MSI-X %d on cpu%d\n", 5912 i, intr->bnx_intr_cpuid); 5913 goto back; 5914 } 5915 5916 intr->bnx_intr_res = bus_alloc_resource_any(sc->bnx_dev, 5917 SYS_RES_IRQ, &intr->bnx_intr_rid, RF_ACTIVE); 5918 if (intr->bnx_intr_res == NULL) { 5919 device_printf(sc->bnx_dev, 5920 "could not alloc MSI-X %d resource\n", i); 5921 error = ENXIO; 5922 goto back; 5923 } 5924 } 5925 5926 pci_enable_msix(sc->bnx_dev); 5927 sc->bnx_intr_type = PCI_INTR_TYPE_MSIX; 5928 back: 5929 if (error) 5930 bnx_free_msix(sc, setup); 5931 return error; 5932 } 5933 5934 static void 5935 bnx_free_msix(struct bnx_softc *sc, boolean_t setup) 5936 { 5937 int i; 5938 5939 KKASSERT(sc->bnx_intr_cnt > 1); 5940 5941 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 5942 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 5943 5944 if (intr->bnx_intr_res != NULL) { 5945 bus_release_resource(sc->bnx_dev, SYS_RES_IRQ, 5946 intr->bnx_intr_rid, intr->bnx_intr_res); 5947 } 5948 if (intr->bnx_intr_rid >= 0) { 5949 pci_release_msix_vector(sc->bnx_dev, 5950 intr->bnx_intr_rid); 5951 } 5952 } 5953 if (setup) 5954 pci_teardown_msix(sc->bnx_dev); 5955 } 5956 5957 static void 5958 bnx_rx_std_refill_sched_ipi(void *xret) 5959 { 5960 struct bnx_rx_ret_ring *ret = xret; 5961 struct bnx_rx_std_ring *std = ret->bnx_std; 5962 struct globaldata *gd = mycpu; 5963 5964 crit_enter_gd(gd); 5965 5966 atomic_set_int(&std->bnx_rx_std_refill, ret->bnx_rx_mask); 5967 cpu_sfence(); 5968 5969 KKASSERT(std->bnx_rx_std_ithread->td_gd == gd); 5970 lwkt_schedule(std->bnx_rx_std_ithread); 5971 5972 crit_exit_gd(gd); 5973 } 5974 5975 static void 5976 bnx_rx_std_refill_stop(void *xstd) 5977 { 5978 struct bnx_rx_std_ring *std = xstd; 5979 struct globaldata *gd = mycpu; 5980 5981 crit_enter_gd(gd); 5982 5983 std->bnx_rx_std_stop = 1; 5984 cpu_sfence(); 5985 5986 KKASSERT(std->bnx_rx_std_ithread->td_gd == gd); 5987 lwkt_schedule(std->bnx_rx_std_ithread); 5988 5989 crit_exit_gd(gd); 5990 } 5991 5992 static void 5993 bnx_serialize_skipmain(struct bnx_softc *sc) 5994 { 5995 lwkt_serialize_array_enter(sc->bnx_serialize, 5996 sc->bnx_serialize_cnt, 1); 5997 } 5998 5999 static void 6000 bnx_deserialize_skipmain(struct bnx_softc *sc) 6001 { 6002 lwkt_serialize_array_exit(sc->bnx_serialize, 6003 sc->bnx_serialize_cnt, 1); 6004 } 6005 6006 static void 6007 bnx_rx_std_refill_sched(struct bnx_rx_ret_ring *ret, 6008 struct bnx_rx_std_ring *std) 6009 { 6010 struct globaldata *gd = mycpu; 6011 6012 ret->bnx_rx_cnt = 0; 6013 cpu_sfence(); 6014 6015 crit_enter_gd(gd); 6016 6017 atomic_set_int(&std->bnx_rx_std_refill, ret->bnx_rx_mask); 6018 cpu_sfence(); 6019 if (atomic_poll_acquire_int(&std->bnx_rx_std_running)) { 6020 if (std->bnx_rx_std_ithread->td_gd == gd) { 6021 lwkt_schedule(std->bnx_rx_std_ithread); 6022 } else { 6023 lwkt_send_ipiq(std->bnx_rx_std_ithread->td_gd, 6024 bnx_rx_std_refill_sched_ipi, ret); 6025 } 6026 } 6027 6028 crit_exit_gd(gd); 6029 } 6030 6031 static struct pktinfo * 6032 bnx_rss_info(struct pktinfo *pi, const struct bge_rx_bd *cur_rx) 6033 { 6034 /* Don't pick up IPv6 packet */ 6035 if (cur_rx->bge_flags & BGE_RXBDFLAG_IPV6) 6036 return NULL; 6037 6038 /* Don't pick up IP packet w/o IP checksum */ 6039 if ((cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) == 0 || 6040 (cur_rx->bge_error_flag & BGE_RXERRFLAG_IP_CSUM_NOK)) 6041 return NULL; 6042 6043 /* Don't pick up IP packet w/o TCP/UDP checksum */ 6044 if ((cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) == 0) 6045 return NULL; 6046 6047 /* May be IP fragment */ 6048 if (cur_rx->bge_tcp_udp_csum != 0xffff) 6049 return NULL; 6050 6051 if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_IS_TCP) 6052 pi->pi_l3proto = IPPROTO_TCP; 6053 else 6054 pi->pi_l3proto = IPPROTO_UDP; 6055 pi->pi_netisr = NETISR_IP; 6056 pi->pi_flags = 0; 6057 6058 return pi; 6059 } 6060 6061 static void 6062 bnx_sig_pre_reset(struct bnx_softc *sc, int type) 6063 { 6064 if (type == BNX_RESET_START || type == BNX_RESET_SUSPEND) 6065 bnx_ape_driver_state_change(sc, type); 6066 } 6067 6068 static void 6069 bnx_sig_post_reset(struct bnx_softc *sc, int type) 6070 { 6071 if (type == BNX_RESET_SHUTDOWN) 6072 bnx_ape_driver_state_change(sc, type); 6073 } 6074 6075 /* 6076 * Clear all stale locks and select the lock for this driver instance. 6077 */ 6078 static void 6079 bnx_ape_lock_init(struct bnx_softc *sc) 6080 { 6081 uint32_t bit, regbase; 6082 int i; 6083 6084 regbase = BGE_APE_PER_LOCK_GRANT; 6085 6086 /* Clear any stale locks. */ 6087 for (i = BGE_APE_LOCK_PHY0; i <= BGE_APE_LOCK_GPIO; i++) { 6088 switch (i) { 6089 case BGE_APE_LOCK_PHY0: 6090 case BGE_APE_LOCK_PHY1: 6091 case BGE_APE_LOCK_PHY2: 6092 case BGE_APE_LOCK_PHY3: 6093 bit = BGE_APE_LOCK_GRANT_DRIVER0; 6094 break; 6095 6096 default: 6097 if (sc->bnx_func_addr == 0) 6098 bit = BGE_APE_LOCK_GRANT_DRIVER0; 6099 else 6100 bit = 1 << sc->bnx_func_addr; 6101 break; 6102 } 6103 APE_WRITE_4(sc, regbase + 4 * i, bit); 6104 } 6105 6106 /* Select the PHY lock based on the device's function number. */ 6107 switch (sc->bnx_func_addr) { 6108 case 0: 6109 sc->bnx_phy_ape_lock = BGE_APE_LOCK_PHY0; 6110 break; 6111 6112 case 1: 6113 sc->bnx_phy_ape_lock = BGE_APE_LOCK_PHY1; 6114 break; 6115 6116 case 2: 6117 sc->bnx_phy_ape_lock = BGE_APE_LOCK_PHY2; 6118 break; 6119 6120 case 3: 6121 sc->bnx_phy_ape_lock = BGE_APE_LOCK_PHY3; 6122 break; 6123 6124 default: 6125 device_printf(sc->bnx_dev, 6126 "PHY lock not supported on this function\n"); 6127 break; 6128 } 6129 } 6130 6131 /* 6132 * Check for APE firmware, set flags, and print version info. 6133 */ 6134 static void 6135 bnx_ape_read_fw_ver(struct bnx_softc *sc) 6136 { 6137 const char *fwtype; 6138 uint32_t apedata, features; 6139 6140 /* Check for a valid APE signature in shared memory. */ 6141 apedata = APE_READ_4(sc, BGE_APE_SEG_SIG); 6142 if (apedata != BGE_APE_SEG_SIG_MAGIC) { 6143 device_printf(sc->bnx_dev, "no APE signature\n"); 6144 sc->bnx_mfw_flags &= ~BNX_MFW_ON_APE; 6145 return; 6146 } 6147 6148 /* Check if APE firmware is running. */ 6149 apedata = APE_READ_4(sc, BGE_APE_FW_STATUS); 6150 if ((apedata & BGE_APE_FW_STATUS_READY) == 0) { 6151 device_printf(sc->bnx_dev, "APE signature found " 6152 "but FW status not ready! 0x%08x\n", apedata); 6153 return; 6154 } 6155 6156 sc->bnx_mfw_flags |= BNX_MFW_ON_APE; 6157 6158 /* Fetch the APE firwmare type and version. */ 6159 apedata = APE_READ_4(sc, BGE_APE_FW_VERSION); 6160 features = APE_READ_4(sc, BGE_APE_FW_FEATURES); 6161 if (features & BGE_APE_FW_FEATURE_NCSI) { 6162 sc->bnx_mfw_flags |= BNX_MFW_TYPE_NCSI; 6163 fwtype = "NCSI"; 6164 } else if (features & BGE_APE_FW_FEATURE_DASH) { 6165 sc->bnx_mfw_flags |= BNX_MFW_TYPE_DASH; 6166 fwtype = "DASH"; 6167 } else { 6168 fwtype = "UNKN"; 6169 } 6170 6171 /* Print the APE firmware version. */ 6172 device_printf(sc->bnx_dev, "APE FW version: %s v%d.%d.%d.%d\n", 6173 fwtype, 6174 (apedata & BGE_APE_FW_VERSION_MAJMSK) >> BGE_APE_FW_VERSION_MAJSFT, 6175 (apedata & BGE_APE_FW_VERSION_MINMSK) >> BGE_APE_FW_VERSION_MINSFT, 6176 (apedata & BGE_APE_FW_VERSION_REVMSK) >> BGE_APE_FW_VERSION_REVSFT, 6177 (apedata & BGE_APE_FW_VERSION_BLDMSK)); 6178 } 6179 6180 static int 6181 bnx_ape_lock(struct bnx_softc *sc, int locknum) 6182 { 6183 uint32_t bit, gnt, req, status; 6184 int i, off; 6185 6186 if ((sc->bnx_mfw_flags & BNX_MFW_ON_APE) == 0) 6187 return 0; 6188 6189 /* Lock request/grant registers have different bases. */ 6190 req = BGE_APE_PER_LOCK_REQ; 6191 gnt = BGE_APE_PER_LOCK_GRANT; 6192 6193 off = 4 * locknum; 6194 6195 switch (locknum) { 6196 case BGE_APE_LOCK_GPIO: 6197 /* Lock required when using GPIO. */ 6198 if (sc->bnx_func_addr == 0) 6199 bit = BGE_APE_LOCK_REQ_DRIVER0; 6200 else 6201 bit = 1 << sc->bnx_func_addr; 6202 break; 6203 6204 case BGE_APE_LOCK_GRC: 6205 /* Lock required to reset the device. */ 6206 if (sc->bnx_func_addr == 0) 6207 bit = BGE_APE_LOCK_REQ_DRIVER0; 6208 else 6209 bit = 1 << sc->bnx_func_addr; 6210 break; 6211 6212 case BGE_APE_LOCK_MEM: 6213 /* Lock required when accessing certain APE memory. */ 6214 if (sc->bnx_func_addr == 0) 6215 bit = BGE_APE_LOCK_REQ_DRIVER0; 6216 else 6217 bit = 1 << sc->bnx_func_addr; 6218 break; 6219 6220 case BGE_APE_LOCK_PHY0: 6221 case BGE_APE_LOCK_PHY1: 6222 case BGE_APE_LOCK_PHY2: 6223 case BGE_APE_LOCK_PHY3: 6224 /* Lock required when accessing PHYs. */ 6225 bit = BGE_APE_LOCK_REQ_DRIVER0; 6226 break; 6227 6228 default: 6229 return EINVAL; 6230 } 6231 6232 /* Request a lock. */ 6233 APE_WRITE_4(sc, req + off, bit); 6234 6235 /* Wait up to 1 second to acquire lock. */ 6236 for (i = 0; i < 20000; i++) { 6237 status = APE_READ_4(sc, gnt + off); 6238 if (status == bit) 6239 break; 6240 DELAY(50); 6241 } 6242 6243 /* Handle any errors. */ 6244 if (status != bit) { 6245 if_printf(&sc->arpcom.ac_if, "APE lock %d request failed! " 6246 "request = 0x%04x[0x%04x], status = 0x%04x[0x%04x]\n", 6247 locknum, req + off, bit & 0xFFFF, gnt + off, 6248 status & 0xFFFF); 6249 /* Revoke the lock request. */ 6250 APE_WRITE_4(sc, gnt + off, bit); 6251 return EBUSY; 6252 } 6253 6254 return 0; 6255 } 6256 6257 static void 6258 bnx_ape_unlock(struct bnx_softc *sc, int locknum) 6259 { 6260 uint32_t bit, gnt; 6261 int off; 6262 6263 if ((sc->bnx_mfw_flags & BNX_MFW_ON_APE) == 0) 6264 return; 6265 6266 gnt = BGE_APE_PER_LOCK_GRANT; 6267 6268 off = 4 * locknum; 6269 6270 switch (locknum) { 6271 case BGE_APE_LOCK_GPIO: 6272 if (sc->bnx_func_addr == 0) 6273 bit = BGE_APE_LOCK_GRANT_DRIVER0; 6274 else 6275 bit = 1 << sc->bnx_func_addr; 6276 break; 6277 6278 case BGE_APE_LOCK_GRC: 6279 if (sc->bnx_func_addr == 0) 6280 bit = BGE_APE_LOCK_GRANT_DRIVER0; 6281 else 6282 bit = 1 << sc->bnx_func_addr; 6283 break; 6284 6285 case BGE_APE_LOCK_MEM: 6286 if (sc->bnx_func_addr == 0) 6287 bit = BGE_APE_LOCK_GRANT_DRIVER0; 6288 else 6289 bit = 1 << sc->bnx_func_addr; 6290 break; 6291 6292 case BGE_APE_LOCK_PHY0: 6293 case BGE_APE_LOCK_PHY1: 6294 case BGE_APE_LOCK_PHY2: 6295 case BGE_APE_LOCK_PHY3: 6296 bit = BGE_APE_LOCK_GRANT_DRIVER0; 6297 break; 6298 6299 default: 6300 return; 6301 } 6302 6303 APE_WRITE_4(sc, gnt + off, bit); 6304 } 6305 6306 /* 6307 * Send an event to the APE firmware. 6308 */ 6309 static void 6310 bnx_ape_send_event(struct bnx_softc *sc, uint32_t event) 6311 { 6312 uint32_t apedata; 6313 int i; 6314 6315 /* NCSI does not support APE events. */ 6316 if ((sc->bnx_mfw_flags & BNX_MFW_ON_APE) == 0) 6317 return; 6318 6319 /* Wait up to 1ms for APE to service previous event. */ 6320 for (i = 10; i > 0; i--) { 6321 if (bnx_ape_lock(sc, BGE_APE_LOCK_MEM) != 0) 6322 break; 6323 apedata = APE_READ_4(sc, BGE_APE_EVENT_STATUS); 6324 if ((apedata & BGE_APE_EVENT_STATUS_EVENT_PENDING) == 0) { 6325 APE_WRITE_4(sc, BGE_APE_EVENT_STATUS, event | 6326 BGE_APE_EVENT_STATUS_EVENT_PENDING); 6327 bnx_ape_unlock(sc, BGE_APE_LOCK_MEM); 6328 APE_WRITE_4(sc, BGE_APE_EVENT, BGE_APE_EVENT_1); 6329 break; 6330 } 6331 bnx_ape_unlock(sc, BGE_APE_LOCK_MEM); 6332 DELAY(100); 6333 } 6334 if (i == 0) { 6335 if_printf(&sc->arpcom.ac_if, 6336 "APE event 0x%08x send timed out\n", event); 6337 } 6338 } 6339 6340 static void 6341 bnx_ape_driver_state_change(struct bnx_softc *sc, int kind) 6342 { 6343 uint32_t apedata, event; 6344 6345 if ((sc->bnx_mfw_flags & BNX_MFW_ON_APE) == 0) 6346 return; 6347 6348 switch (kind) { 6349 case BNX_RESET_START: 6350 /* If this is the first load, clear the load counter. */ 6351 apedata = APE_READ_4(sc, BGE_APE_HOST_SEG_SIG); 6352 if (apedata != BGE_APE_HOST_SEG_SIG_MAGIC) { 6353 APE_WRITE_4(sc, BGE_APE_HOST_INIT_COUNT, 0); 6354 } else { 6355 apedata = APE_READ_4(sc, BGE_APE_HOST_INIT_COUNT); 6356 APE_WRITE_4(sc, BGE_APE_HOST_INIT_COUNT, ++apedata); 6357 } 6358 APE_WRITE_4(sc, BGE_APE_HOST_SEG_SIG, 6359 BGE_APE_HOST_SEG_SIG_MAGIC); 6360 APE_WRITE_4(sc, BGE_APE_HOST_SEG_LEN, 6361 BGE_APE_HOST_SEG_LEN_MAGIC); 6362 6363 /* Add some version info if bnx(4) supports it. */ 6364 APE_WRITE_4(sc, BGE_APE_HOST_DRIVER_ID, 6365 BGE_APE_HOST_DRIVER_ID_MAGIC(1, 0)); 6366 APE_WRITE_4(sc, BGE_APE_HOST_BEHAVIOR, 6367 BGE_APE_HOST_BEHAV_NO_PHYLOCK); 6368 APE_WRITE_4(sc, BGE_APE_HOST_HEARTBEAT_INT_MS, 6369 BGE_APE_HOST_HEARTBEAT_INT_DISABLE); 6370 APE_WRITE_4(sc, BGE_APE_HOST_DRVR_STATE, 6371 BGE_APE_HOST_DRVR_STATE_START); 6372 event = BGE_APE_EVENT_STATUS_STATE_START; 6373 break; 6374 6375 case BNX_RESET_SHUTDOWN: 6376 APE_WRITE_4(sc, BGE_APE_HOST_DRVR_STATE, 6377 BGE_APE_HOST_DRVR_STATE_UNLOAD); 6378 event = BGE_APE_EVENT_STATUS_STATE_UNLOAD; 6379 break; 6380 6381 case BNX_RESET_SUSPEND: 6382 event = BGE_APE_EVENT_STATUS_STATE_SUSPEND; 6383 break; 6384 6385 default: 6386 return; 6387 } 6388 6389 bnx_ape_send_event(sc, event | BGE_APE_EVENT_STATUS_DRIVER_EVNT | 6390 BGE_APE_EVENT_STATUS_STATE_CHNGE); 6391 } 6392