1 /* 2 * Copyright (c) 2001 Wind River Systems 3 * Copyright (c) 1997, 1998, 1999, 2001 4 * Bill Paul <wpaul@windriver.com>. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. All advertising materials mentioning features or use of this software 15 * must display the following acknowledgement: 16 * This product includes software developed by Bill Paul. 17 * 4. Neither the name of the author nor the names of any co-contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD 25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 31 * THE POSSIBILITY OF SUCH DAMAGE. 32 * 33 * $FreeBSD: src/sys/dev/bge/if_bge.c,v 1.3.2.39 2005/07/03 03:41:18 silby Exp $ 34 */ 35 36 #include "opt_bnx.h" 37 #include "opt_ifpoll.h" 38 39 #include <sys/param.h> 40 #include <sys/bus.h> 41 #include <sys/endian.h> 42 #include <sys/kernel.h> 43 #include <sys/interrupt.h> 44 #include <sys/mbuf.h> 45 #include <sys/malloc.h> 46 #include <sys/queue.h> 47 #include <sys/rman.h> 48 #include <sys/serialize.h> 49 #include <sys/socket.h> 50 #include <sys/sockio.h> 51 #include <sys/sysctl.h> 52 53 #include <netinet/ip.h> 54 #include <netinet/tcp.h> 55 56 #include <net/bpf.h> 57 #include <net/ethernet.h> 58 #include <net/if.h> 59 #include <net/if_arp.h> 60 #include <net/if_dl.h> 61 #include <net/if_media.h> 62 #include <net/if_poll.h> 63 #include <net/if_types.h> 64 #include <net/ifq_var.h> 65 #include <net/toeplitz.h> 66 #include <net/toeplitz2.h> 67 #include <net/vlan/if_vlan_var.h> 68 #include <net/vlan/if_vlan_ether.h> 69 70 #include <dev/netif/mii_layer/mii.h> 71 #include <dev/netif/mii_layer/miivar.h> 72 #include <dev/netif/mii_layer/brgphyreg.h> 73 74 #include <bus/pci/pcidevs.h> 75 #include <bus/pci/pcireg.h> 76 #include <bus/pci/pcivar.h> 77 78 #include <dev/netif/bge/if_bgereg.h> 79 #include <dev/netif/bnx/if_bnxvar.h> 80 81 /* "device miibus" required. See GENERIC if you get errors here. */ 82 #include "miibus_if.h" 83 84 #define BNX_CSUM_FEATURES (CSUM_IP | CSUM_TCP | CSUM_UDP) 85 86 #define BNX_INTR_CKINTVL ((10 * hz) / 1000) /* 10ms */ 87 88 #ifdef BNX_RSS_DEBUG 89 #define BNX_RSS_DPRINTF(sc, lvl, fmt, ...) \ 90 do { \ 91 if (sc->bnx_rss_debug >= lvl) \ 92 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \ 93 } while (0) 94 #else /* !BNX_RSS_DEBUG */ 95 #define BNX_RSS_DPRINTF(sc, lvl, fmt, ...) ((void)0) 96 #endif /* BNX_RSS_DEBUG */ 97 98 static const struct bnx_type { 99 uint16_t bnx_vid; 100 uint16_t bnx_did; 101 char *bnx_name; 102 } bnx_devs[] = { 103 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717, 104 "Broadcom BCM5717 Gigabit Ethernet" }, 105 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717C, 106 "Broadcom BCM5717C Gigabit Ethernet" }, 107 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5718, 108 "Broadcom BCM5718 Gigabit Ethernet" }, 109 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5719, 110 "Broadcom BCM5719 Gigabit Ethernet" }, 111 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5720_ALT, 112 "Broadcom BCM5720 Gigabit Ethernet" }, 113 114 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5725, 115 "Broadcom BCM5725 Gigabit Ethernet" }, 116 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5727, 117 "Broadcom BCM5727 Gigabit Ethernet" }, 118 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5762, 119 "Broadcom BCM5762 Gigabit Ethernet" }, 120 121 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57761, 122 "Broadcom BCM57761 Gigabit Ethernet" }, 123 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57762, 124 "Broadcom BCM57762 Gigabit Ethernet" }, 125 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57765, 126 "Broadcom BCM57765 Gigabit Ethernet" }, 127 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57766, 128 "Broadcom BCM57766 Gigabit Ethernet" }, 129 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57781, 130 "Broadcom BCM57781 Gigabit Ethernet" }, 131 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57782, 132 "Broadcom BCM57782 Gigabit Ethernet" }, 133 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57785, 134 "Broadcom BCM57785 Gigabit Ethernet" }, 135 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57786, 136 "Broadcom BCM57786 Gigabit Ethernet" }, 137 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57791, 138 "Broadcom BCM57791 Fast Ethernet" }, 139 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57795, 140 "Broadcom BCM57795 Fast Ethernet" }, 141 142 { 0, 0, NULL } 143 }; 144 145 static const int bnx_tx_mailbox[BNX_TX_RING_MAX] = { 146 BGE_MBX_TX_HOST_PROD0_LO, 147 BGE_MBX_TX_HOST_PROD0_HI, 148 BGE_MBX_TX_HOST_PROD1_LO, 149 BGE_MBX_TX_HOST_PROD1_HI 150 }; 151 152 #define BNX_IS_JUMBO_CAPABLE(sc) ((sc)->bnx_flags & BNX_FLAG_JUMBO) 153 #define BNX_IS_5717_PLUS(sc) ((sc)->bnx_flags & BNX_FLAG_5717_PLUS) 154 #define BNX_IS_57765_PLUS(sc) ((sc)->bnx_flags & BNX_FLAG_57765_PLUS) 155 #define BNX_IS_57765_FAMILY(sc) \ 156 ((sc)->bnx_flags & BNX_FLAG_57765_FAMILY) 157 158 typedef int (*bnx_eaddr_fcn_t)(struct bnx_softc *, uint8_t[]); 159 160 static int bnx_probe(device_t); 161 static int bnx_attach(device_t); 162 static int bnx_detach(device_t); 163 static void bnx_shutdown(device_t); 164 static int bnx_suspend(device_t); 165 static int bnx_resume(device_t); 166 static int bnx_miibus_readreg(device_t, int, int); 167 static int bnx_miibus_writereg(device_t, int, int, int); 168 static void bnx_miibus_statchg(device_t); 169 170 static int bnx_handle_status(struct bnx_softc *); 171 #ifdef IFPOLL_ENABLE 172 static void bnx_npoll(struct ifnet *, struct ifpoll_info *); 173 static void bnx_npoll_rx(struct ifnet *, void *, int); 174 static void bnx_npoll_tx(struct ifnet *, void *, int); 175 static void bnx_npoll_tx_notag(struct ifnet *, void *, int); 176 static void bnx_npoll_status(struct ifnet *); 177 static void bnx_npoll_status_notag(struct ifnet *); 178 #endif 179 static void bnx_intr_legacy(void *); 180 static void bnx_msi(void *); 181 static void bnx_intr(struct bnx_softc *); 182 static void bnx_msix_status(void *); 183 static void bnx_msix_tx_status(void *); 184 static void bnx_msix_rx(void *); 185 static void bnx_msix_rxtx(void *); 186 static void bnx_enable_intr(struct bnx_softc *); 187 static void bnx_disable_intr(struct bnx_softc *); 188 static void bnx_txeof(struct bnx_tx_ring *, uint16_t); 189 static void bnx_rxeof(struct bnx_rx_ret_ring *, uint16_t, int); 190 static int bnx_alloc_intr(struct bnx_softc *); 191 static int bnx_setup_intr(struct bnx_softc *); 192 static void bnx_free_intr(struct bnx_softc *); 193 static void bnx_teardown_intr(struct bnx_softc *, int); 194 static int bnx_alloc_msix(struct bnx_softc *); 195 static void bnx_free_msix(struct bnx_softc *, boolean_t); 196 static void bnx_check_intr_rxtx(void *); 197 static void bnx_check_intr_rx(void *); 198 static void bnx_check_intr_tx(void *); 199 static void bnx_rx_std_refill_ithread(void *); 200 static void bnx_rx_std_refill(void *, void *); 201 static void bnx_rx_std_refill_sched_ipi(void *); 202 static void bnx_rx_std_refill_stop(void *); 203 static void bnx_rx_std_refill_sched(struct bnx_rx_ret_ring *, 204 struct bnx_rx_std_ring *); 205 206 static void bnx_start(struct ifnet *, struct ifaltq_subque *); 207 static int bnx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 208 static void bnx_init(void *); 209 static void bnx_stop(struct bnx_softc *); 210 static void bnx_watchdog(struct ifaltq_subque *); 211 static int bnx_ifmedia_upd(struct ifnet *); 212 static void bnx_ifmedia_sts(struct ifnet *, struct ifmediareq *); 213 static void bnx_tick(void *); 214 static void bnx_serialize(struct ifnet *, enum ifnet_serialize); 215 static void bnx_deserialize(struct ifnet *, enum ifnet_serialize); 216 static int bnx_tryserialize(struct ifnet *, enum ifnet_serialize); 217 #ifdef INVARIANTS 218 static void bnx_serialize_assert(struct ifnet *, enum ifnet_serialize, 219 boolean_t); 220 #endif 221 static void bnx_serialize_skipmain(struct bnx_softc *); 222 static void bnx_deserialize_skipmain(struct bnx_softc *sc); 223 224 static int bnx_alloc_jumbo_mem(struct bnx_softc *); 225 static void bnx_free_jumbo_mem(struct bnx_softc *); 226 static struct bnx_jslot 227 *bnx_jalloc(struct bnx_softc *); 228 static void bnx_jfree(void *); 229 static void bnx_jref(void *); 230 static int bnx_newbuf_std(struct bnx_rx_ret_ring *, int, int); 231 static int bnx_newbuf_jumbo(struct bnx_softc *, int, int); 232 static void bnx_setup_rxdesc_std(struct bnx_rx_std_ring *, int); 233 static void bnx_setup_rxdesc_jumbo(struct bnx_softc *, int); 234 static int bnx_init_rx_ring_std(struct bnx_rx_std_ring *); 235 static void bnx_free_rx_ring_std(struct bnx_rx_std_ring *); 236 static int bnx_init_rx_ring_jumbo(struct bnx_softc *); 237 static void bnx_free_rx_ring_jumbo(struct bnx_softc *); 238 static void bnx_free_tx_ring(struct bnx_tx_ring *); 239 static int bnx_init_tx_ring(struct bnx_tx_ring *); 240 static int bnx_create_tx_ring(struct bnx_tx_ring *); 241 static void bnx_destroy_tx_ring(struct bnx_tx_ring *); 242 static int bnx_create_rx_ret_ring(struct bnx_rx_ret_ring *); 243 static void bnx_destroy_rx_ret_ring(struct bnx_rx_ret_ring *); 244 static int bnx_dma_alloc(device_t); 245 static void bnx_dma_free(struct bnx_softc *); 246 static int bnx_dma_block_alloc(struct bnx_softc *, bus_size_t, 247 bus_dma_tag_t *, bus_dmamap_t *, void **, bus_addr_t *); 248 static void bnx_dma_block_free(bus_dma_tag_t, bus_dmamap_t, void *); 249 static struct mbuf * 250 bnx_defrag_shortdma(struct mbuf *); 251 static int bnx_encap(struct bnx_tx_ring *, struct mbuf **, 252 uint32_t *, int *); 253 static int bnx_setup_tso(struct bnx_tx_ring *, struct mbuf **, 254 uint16_t *, uint16_t *); 255 static void bnx_setup_serialize(struct bnx_softc *); 256 static void bnx_set_tick_cpuid(struct bnx_softc *, boolean_t); 257 static void bnx_setup_ring_cnt(struct bnx_softc *); 258 259 static struct pktinfo *bnx_rss_info(struct pktinfo *, 260 const struct bge_rx_bd *); 261 static void bnx_init_rss(struct bnx_softc *); 262 static void bnx_reset(struct bnx_softc *); 263 static int bnx_chipinit(struct bnx_softc *); 264 static int bnx_blockinit(struct bnx_softc *); 265 static void bnx_stop_block(struct bnx_softc *, bus_size_t, uint32_t); 266 static void bnx_enable_msi(struct bnx_softc *, boolean_t); 267 static void bnx_setmulti(struct bnx_softc *); 268 static void bnx_setpromisc(struct bnx_softc *); 269 static void bnx_stats_update_regs(struct bnx_softc *); 270 static uint32_t bnx_dma_swap_options(struct bnx_softc *); 271 272 static uint32_t bnx_readmem_ind(struct bnx_softc *, uint32_t); 273 static void bnx_writemem_ind(struct bnx_softc *, uint32_t, uint32_t); 274 #ifdef notdef 275 static uint32_t bnx_readreg_ind(struct bnx_softc *, uint32_t); 276 #endif 277 static void bnx_writemem_direct(struct bnx_softc *, uint32_t, uint32_t); 278 static void bnx_writembx(struct bnx_softc *, int, int); 279 static int bnx_read_nvram(struct bnx_softc *, caddr_t, int, int); 280 static uint8_t bnx_eeprom_getbyte(struct bnx_softc *, uint32_t, uint8_t *); 281 static int bnx_read_eeprom(struct bnx_softc *, caddr_t, uint32_t, size_t); 282 283 static void bnx_tbi_link_upd(struct bnx_softc *, uint32_t); 284 static void bnx_copper_link_upd(struct bnx_softc *, uint32_t); 285 static void bnx_autopoll_link_upd(struct bnx_softc *, uint32_t); 286 static void bnx_link_poll(struct bnx_softc *); 287 288 static int bnx_get_eaddr_mem(struct bnx_softc *, uint8_t[]); 289 static int bnx_get_eaddr_nvram(struct bnx_softc *, uint8_t[]); 290 static int bnx_get_eaddr_eeprom(struct bnx_softc *, uint8_t[]); 291 static int bnx_get_eaddr(struct bnx_softc *, uint8_t[]); 292 293 static void bnx_coal_change(struct bnx_softc *); 294 static int bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS); 295 static int bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS); 296 static int bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS); 297 static int bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS); 298 static int bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS); 299 static int bnx_sysctl_rx_coal_bds_poll(SYSCTL_HANDLER_ARGS); 300 static int bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS); 301 static int bnx_sysctl_tx_coal_bds_poll(SYSCTL_HANDLER_ARGS); 302 static int bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS); 303 static int bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS); 304 static int bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *, 305 int, int, uint32_t); 306 #ifdef IFPOLL_ENABLE 307 static int bnx_sysctl_npoll_offset(SYSCTL_HANDLER_ARGS); 308 static int bnx_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS); 309 static int bnx_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS); 310 #endif 311 static int bnx_sysctl_std_refill(SYSCTL_HANDLER_ARGS); 312 313 static int bnx_msi_enable = 1; 314 static int bnx_msix_enable = 1; 315 316 static int bnx_rx_rings = 0; /* auto */ 317 static int bnx_tx_rings = 0; /* auto */ 318 319 TUNABLE_INT("hw.bnx.msi.enable", &bnx_msi_enable); 320 TUNABLE_INT("hw.bnx.msix.enable", &bnx_msix_enable); 321 TUNABLE_INT("hw.bnx.rx_rings", &bnx_rx_rings); 322 TUNABLE_INT("hw.bnx.tx_rings", &bnx_tx_rings); 323 324 static device_method_t bnx_methods[] = { 325 /* Device interface */ 326 DEVMETHOD(device_probe, bnx_probe), 327 DEVMETHOD(device_attach, bnx_attach), 328 DEVMETHOD(device_detach, bnx_detach), 329 DEVMETHOD(device_shutdown, bnx_shutdown), 330 DEVMETHOD(device_suspend, bnx_suspend), 331 DEVMETHOD(device_resume, bnx_resume), 332 333 /* bus interface */ 334 DEVMETHOD(bus_print_child, bus_generic_print_child), 335 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 336 337 /* MII interface */ 338 DEVMETHOD(miibus_readreg, bnx_miibus_readreg), 339 DEVMETHOD(miibus_writereg, bnx_miibus_writereg), 340 DEVMETHOD(miibus_statchg, bnx_miibus_statchg), 341 342 DEVMETHOD_END 343 }; 344 345 static DEFINE_CLASS_0(bnx, bnx_driver, bnx_methods, sizeof(struct bnx_softc)); 346 static devclass_t bnx_devclass; 347 348 DECLARE_DUMMY_MODULE(if_bnx); 349 DRIVER_MODULE(if_bnx, pci, bnx_driver, bnx_devclass, NULL, NULL); 350 DRIVER_MODULE(miibus, bnx, miibus_driver, miibus_devclass, NULL, NULL); 351 352 static uint32_t 353 bnx_readmem_ind(struct bnx_softc *sc, uint32_t off) 354 { 355 device_t dev = sc->bnx_dev; 356 uint32_t val; 357 358 pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4); 359 val = pci_read_config(dev, BGE_PCI_MEMWIN_DATA, 4); 360 pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4); 361 return (val); 362 } 363 364 static void 365 bnx_writemem_ind(struct bnx_softc *sc, uint32_t off, uint32_t val) 366 { 367 device_t dev = sc->bnx_dev; 368 369 pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4); 370 pci_write_config(dev, BGE_PCI_MEMWIN_DATA, val, 4); 371 pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4); 372 } 373 374 static void 375 bnx_writemem_direct(struct bnx_softc *sc, uint32_t off, uint32_t val) 376 { 377 CSR_WRITE_4(sc, off, val); 378 } 379 380 static void 381 bnx_writembx(struct bnx_softc *sc, int off, int val) 382 { 383 CSR_WRITE_4(sc, off, val); 384 } 385 386 /* 387 * Read a sequence of bytes from NVRAM. 388 */ 389 static int 390 bnx_read_nvram(struct bnx_softc *sc, caddr_t dest, int off, int cnt) 391 { 392 return (1); 393 } 394 395 /* 396 * Read a byte of data stored in the EEPROM at address 'addr.' The 397 * BCM570x supports both the traditional bitbang interface and an 398 * auto access interface for reading the EEPROM. We use the auto 399 * access method. 400 */ 401 static uint8_t 402 bnx_eeprom_getbyte(struct bnx_softc *sc, uint32_t addr, uint8_t *dest) 403 { 404 int i; 405 uint32_t byte = 0; 406 407 /* 408 * Enable use of auto EEPROM access so we can avoid 409 * having to use the bitbang method. 410 */ 411 BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_AUTO_EEPROM); 412 413 /* Reset the EEPROM, load the clock period. */ 414 CSR_WRITE_4(sc, BGE_EE_ADDR, 415 BGE_EEADDR_RESET|BGE_EEHALFCLK(BGE_HALFCLK_384SCL)); 416 DELAY(20); 417 418 /* Issue the read EEPROM command. */ 419 CSR_WRITE_4(sc, BGE_EE_ADDR, BGE_EE_READCMD | addr); 420 421 /* Wait for completion */ 422 for(i = 0; i < BNX_TIMEOUT * 10; i++) { 423 DELAY(10); 424 if (CSR_READ_4(sc, BGE_EE_ADDR) & BGE_EEADDR_DONE) 425 break; 426 } 427 428 if (i == BNX_TIMEOUT) { 429 if_printf(&sc->arpcom.ac_if, "eeprom read timed out\n"); 430 return(1); 431 } 432 433 /* Get result. */ 434 byte = CSR_READ_4(sc, BGE_EE_DATA); 435 436 *dest = (byte >> ((addr % 4) * 8)) & 0xFF; 437 438 return(0); 439 } 440 441 /* 442 * Read a sequence of bytes from the EEPROM. 443 */ 444 static int 445 bnx_read_eeprom(struct bnx_softc *sc, caddr_t dest, uint32_t off, size_t len) 446 { 447 size_t i; 448 int err; 449 uint8_t byte; 450 451 for (byte = 0, err = 0, i = 0; i < len; i++) { 452 err = bnx_eeprom_getbyte(sc, off + i, &byte); 453 if (err) 454 break; 455 *(dest + i) = byte; 456 } 457 458 return(err ? 1 : 0); 459 } 460 461 static int 462 bnx_miibus_readreg(device_t dev, int phy, int reg) 463 { 464 struct bnx_softc *sc = device_get_softc(dev); 465 uint32_t val; 466 int i; 467 468 KASSERT(phy == sc->bnx_phyno, 469 ("invalid phyno %d, should be %d", phy, sc->bnx_phyno)); 470 471 /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */ 472 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) { 473 CSR_WRITE_4(sc, BGE_MI_MODE, 474 sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL); 475 DELAY(80); 476 } 477 478 CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_READ | BGE_MICOMM_BUSY | 479 BGE_MIPHY(phy) | BGE_MIREG(reg)); 480 481 /* Poll for the PHY register access to complete. */ 482 for (i = 0; i < BNX_TIMEOUT; i++) { 483 DELAY(10); 484 val = CSR_READ_4(sc, BGE_MI_COMM); 485 if ((val & BGE_MICOMM_BUSY) == 0) { 486 DELAY(5); 487 val = CSR_READ_4(sc, BGE_MI_COMM); 488 break; 489 } 490 } 491 if (i == BNX_TIMEOUT) { 492 if_printf(&sc->arpcom.ac_if, "PHY read timed out " 493 "(phy %d, reg %d, val 0x%08x)\n", phy, reg, val); 494 val = 0; 495 } 496 497 /* Restore the autopoll bit if necessary. */ 498 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) { 499 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode); 500 DELAY(80); 501 } 502 503 if (val & BGE_MICOMM_READFAIL) 504 return 0; 505 506 return (val & 0xFFFF); 507 } 508 509 static int 510 bnx_miibus_writereg(device_t dev, int phy, int reg, int val) 511 { 512 struct bnx_softc *sc = device_get_softc(dev); 513 int i; 514 515 KASSERT(phy == sc->bnx_phyno, 516 ("invalid phyno %d, should be %d", phy, sc->bnx_phyno)); 517 518 /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */ 519 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) { 520 CSR_WRITE_4(sc, BGE_MI_MODE, 521 sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL); 522 DELAY(80); 523 } 524 525 CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_WRITE | BGE_MICOMM_BUSY | 526 BGE_MIPHY(phy) | BGE_MIREG(reg) | val); 527 528 for (i = 0; i < BNX_TIMEOUT; i++) { 529 DELAY(10); 530 if (!(CSR_READ_4(sc, BGE_MI_COMM) & BGE_MICOMM_BUSY)) { 531 DELAY(5); 532 CSR_READ_4(sc, BGE_MI_COMM); /* dummy read */ 533 break; 534 } 535 } 536 if (i == BNX_TIMEOUT) { 537 if_printf(&sc->arpcom.ac_if, "PHY write timed out " 538 "(phy %d, reg %d, val %d)\n", phy, reg, val); 539 } 540 541 /* Restore the autopoll bit if necessary. */ 542 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) { 543 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode); 544 DELAY(80); 545 } 546 547 return 0; 548 } 549 550 static void 551 bnx_miibus_statchg(device_t dev) 552 { 553 struct bnx_softc *sc; 554 struct mii_data *mii; 555 556 sc = device_get_softc(dev); 557 mii = device_get_softc(sc->bnx_miibus); 558 559 if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) == 560 (IFM_ACTIVE | IFM_AVALID)) { 561 switch (IFM_SUBTYPE(mii->mii_media_active)) { 562 case IFM_10_T: 563 case IFM_100_TX: 564 sc->bnx_link = 1; 565 break; 566 case IFM_1000_T: 567 case IFM_1000_SX: 568 case IFM_2500_SX: 569 sc->bnx_link = 1; 570 break; 571 default: 572 sc->bnx_link = 0; 573 break; 574 } 575 } else { 576 sc->bnx_link = 0; 577 } 578 if (sc->bnx_link == 0) 579 return; 580 581 BNX_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_PORTMODE); 582 if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T || 583 IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX) { 584 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_GMII); 585 } else { 586 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_MII); 587 } 588 589 if ((mii->mii_media_active & IFM_GMASK) == IFM_FDX) { 590 BNX_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX); 591 } else { 592 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX); 593 } 594 } 595 596 /* 597 * Memory management for jumbo frames. 598 */ 599 static int 600 bnx_alloc_jumbo_mem(struct bnx_softc *sc) 601 { 602 struct ifnet *ifp = &sc->arpcom.ac_if; 603 struct bnx_jslot *entry; 604 uint8_t *ptr; 605 bus_addr_t paddr; 606 int i, error; 607 608 /* 609 * Create tag for jumbo mbufs. 610 * This is really a bit of a kludge. We allocate a special 611 * jumbo buffer pool which (thanks to the way our DMA 612 * memory allocation works) will consist of contiguous 613 * pages. This means that even though a jumbo buffer might 614 * be larger than a page size, we don't really need to 615 * map it into more than one DMA segment. However, the 616 * default mbuf tag will result in multi-segment mappings, 617 * so we have to create a special jumbo mbuf tag that 618 * lets us get away with mapping the jumbo buffers as 619 * a single segment. I think eventually the driver should 620 * be changed so that it uses ordinary mbufs and cluster 621 * buffers, i.e. jumbo frames can span multiple DMA 622 * descriptors. But that's a project for another day. 623 */ 624 625 /* 626 * Create DMA stuffs for jumbo RX ring. 627 */ 628 error = bnx_dma_block_alloc(sc, BGE_JUMBO_RX_RING_SZ, 629 &sc->bnx_cdata.bnx_rx_jumbo_ring_tag, 630 &sc->bnx_cdata.bnx_rx_jumbo_ring_map, 631 (void *)&sc->bnx_ldata.bnx_rx_jumbo_ring, 632 &sc->bnx_ldata.bnx_rx_jumbo_ring_paddr); 633 if (error) { 634 if_printf(ifp, "could not create jumbo RX ring\n"); 635 return error; 636 } 637 638 /* 639 * Create DMA stuffs for jumbo buffer block. 640 */ 641 error = bnx_dma_block_alloc(sc, BNX_JMEM, 642 &sc->bnx_cdata.bnx_jumbo_tag, 643 &sc->bnx_cdata.bnx_jumbo_map, 644 (void **)&sc->bnx_ldata.bnx_jumbo_buf, 645 &paddr); 646 if (error) { 647 if_printf(ifp, "could not create jumbo buffer\n"); 648 return error; 649 } 650 651 SLIST_INIT(&sc->bnx_jfree_listhead); 652 653 /* 654 * Now divide it up into 9K pieces and save the addresses 655 * in an array. Note that we play an evil trick here by using 656 * the first few bytes in the buffer to hold the the address 657 * of the softc structure for this interface. This is because 658 * bnx_jfree() needs it, but it is called by the mbuf management 659 * code which will not pass it to us explicitly. 660 */ 661 for (i = 0, ptr = sc->bnx_ldata.bnx_jumbo_buf; i < BNX_JSLOTS; i++) { 662 entry = &sc->bnx_cdata.bnx_jslots[i]; 663 entry->bnx_sc = sc; 664 entry->bnx_buf = ptr; 665 entry->bnx_paddr = paddr; 666 entry->bnx_inuse = 0; 667 entry->bnx_slot = i; 668 SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead, entry, jslot_link); 669 670 ptr += BNX_JLEN; 671 paddr += BNX_JLEN; 672 } 673 return 0; 674 } 675 676 static void 677 bnx_free_jumbo_mem(struct bnx_softc *sc) 678 { 679 /* Destroy jumbo RX ring. */ 680 bnx_dma_block_free(sc->bnx_cdata.bnx_rx_jumbo_ring_tag, 681 sc->bnx_cdata.bnx_rx_jumbo_ring_map, 682 sc->bnx_ldata.bnx_rx_jumbo_ring); 683 684 /* Destroy jumbo buffer block. */ 685 bnx_dma_block_free(sc->bnx_cdata.bnx_jumbo_tag, 686 sc->bnx_cdata.bnx_jumbo_map, 687 sc->bnx_ldata.bnx_jumbo_buf); 688 } 689 690 /* 691 * Allocate a jumbo buffer. 692 */ 693 static struct bnx_jslot * 694 bnx_jalloc(struct bnx_softc *sc) 695 { 696 struct bnx_jslot *entry; 697 698 lwkt_serialize_enter(&sc->bnx_jslot_serializer); 699 entry = SLIST_FIRST(&sc->bnx_jfree_listhead); 700 if (entry) { 701 SLIST_REMOVE_HEAD(&sc->bnx_jfree_listhead, jslot_link); 702 entry->bnx_inuse = 1; 703 } else { 704 if_printf(&sc->arpcom.ac_if, "no free jumbo buffers\n"); 705 } 706 lwkt_serialize_exit(&sc->bnx_jslot_serializer); 707 return(entry); 708 } 709 710 /* 711 * Adjust usage count on a jumbo buffer. 712 */ 713 static void 714 bnx_jref(void *arg) 715 { 716 struct bnx_jslot *entry = (struct bnx_jslot *)arg; 717 struct bnx_softc *sc = entry->bnx_sc; 718 719 if (sc == NULL) 720 panic("bnx_jref: can't find softc pointer!"); 721 722 if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) { 723 panic("bnx_jref: asked to reference buffer " 724 "that we don't manage!"); 725 } else if (entry->bnx_inuse == 0) { 726 panic("bnx_jref: buffer already free!"); 727 } else { 728 atomic_add_int(&entry->bnx_inuse, 1); 729 } 730 } 731 732 /* 733 * Release a jumbo buffer. 734 */ 735 static void 736 bnx_jfree(void *arg) 737 { 738 struct bnx_jslot *entry = (struct bnx_jslot *)arg; 739 struct bnx_softc *sc = entry->bnx_sc; 740 741 if (sc == NULL) 742 panic("bnx_jfree: can't find softc pointer!"); 743 744 if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) { 745 panic("bnx_jfree: asked to free buffer that we don't manage!"); 746 } else if (entry->bnx_inuse == 0) { 747 panic("bnx_jfree: buffer already free!"); 748 } else { 749 /* 750 * Possible MP race to 0, use the serializer. The atomic insn 751 * is still needed for races against bnx_jref(). 752 */ 753 lwkt_serialize_enter(&sc->bnx_jslot_serializer); 754 atomic_subtract_int(&entry->bnx_inuse, 1); 755 if (entry->bnx_inuse == 0) { 756 SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead, 757 entry, jslot_link); 758 } 759 lwkt_serialize_exit(&sc->bnx_jslot_serializer); 760 } 761 } 762 763 764 /* 765 * Intialize a standard receive ring descriptor. 766 */ 767 static int 768 bnx_newbuf_std(struct bnx_rx_ret_ring *ret, int i, int init) 769 { 770 struct mbuf *m_new = NULL; 771 bus_dma_segment_t seg; 772 bus_dmamap_t map; 773 int error, nsegs; 774 struct bnx_rx_buf *rb; 775 776 rb = &ret->bnx_std->bnx_rx_std_buf[i]; 777 KASSERT(!rb->bnx_rx_refilled, ("RX buf %dth has been refilled", i)); 778 779 m_new = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR); 780 if (m_new == NULL) { 781 error = ENOBUFS; 782 goto back; 783 } 784 m_new->m_len = m_new->m_pkthdr.len = MCLBYTES; 785 m_adj(m_new, ETHER_ALIGN); 786 787 error = bus_dmamap_load_mbuf_segment(ret->bnx_rx_mtag, 788 ret->bnx_rx_tmpmap, m_new, &seg, 1, &nsegs, BUS_DMA_NOWAIT); 789 if (error) { 790 m_freem(m_new); 791 goto back; 792 } 793 794 if (!init) { 795 bus_dmamap_sync(ret->bnx_rx_mtag, rb->bnx_rx_dmamap, 796 BUS_DMASYNC_POSTREAD); 797 bus_dmamap_unload(ret->bnx_rx_mtag, rb->bnx_rx_dmamap); 798 } 799 800 map = ret->bnx_rx_tmpmap; 801 ret->bnx_rx_tmpmap = rb->bnx_rx_dmamap; 802 803 rb->bnx_rx_dmamap = map; 804 rb->bnx_rx_mbuf = m_new; 805 rb->bnx_rx_paddr = seg.ds_addr; 806 rb->bnx_rx_len = m_new->m_len; 807 back: 808 cpu_sfence(); 809 rb->bnx_rx_refilled = 1; 810 return error; 811 } 812 813 static void 814 bnx_setup_rxdesc_std(struct bnx_rx_std_ring *std, int i) 815 { 816 struct bnx_rx_buf *rb; 817 struct bge_rx_bd *r; 818 bus_addr_t paddr; 819 int len; 820 821 rb = &std->bnx_rx_std_buf[i]; 822 KASSERT(rb->bnx_rx_refilled, ("RX buf %dth is not refilled", i)); 823 824 paddr = rb->bnx_rx_paddr; 825 len = rb->bnx_rx_len; 826 827 cpu_mfence(); 828 829 rb->bnx_rx_refilled = 0; 830 831 r = &std->bnx_rx_std_ring[i]; 832 r->bge_addr.bge_addr_lo = BGE_ADDR_LO(paddr); 833 r->bge_addr.bge_addr_hi = BGE_ADDR_HI(paddr); 834 r->bge_len = len; 835 r->bge_idx = i; 836 r->bge_flags = BGE_RXBDFLAG_END; 837 } 838 839 /* 840 * Initialize a jumbo receive ring descriptor. This allocates 841 * a jumbo buffer from the pool managed internally by the driver. 842 */ 843 static int 844 bnx_newbuf_jumbo(struct bnx_softc *sc, int i, int init) 845 { 846 struct mbuf *m_new = NULL; 847 struct bnx_jslot *buf; 848 bus_addr_t paddr; 849 850 /* Allocate the mbuf. */ 851 MGETHDR(m_new, init ? MB_WAIT : MB_DONTWAIT, MT_DATA); 852 if (m_new == NULL) 853 return ENOBUFS; 854 855 /* Allocate the jumbo buffer */ 856 buf = bnx_jalloc(sc); 857 if (buf == NULL) { 858 m_freem(m_new); 859 return ENOBUFS; 860 } 861 862 /* Attach the buffer to the mbuf. */ 863 m_new->m_ext.ext_arg = buf; 864 m_new->m_ext.ext_buf = buf->bnx_buf; 865 m_new->m_ext.ext_free = bnx_jfree; 866 m_new->m_ext.ext_ref = bnx_jref; 867 m_new->m_ext.ext_size = BNX_JUMBO_FRAMELEN; 868 869 m_new->m_flags |= M_EXT; 870 871 m_new->m_data = m_new->m_ext.ext_buf; 872 m_new->m_len = m_new->m_pkthdr.len = m_new->m_ext.ext_size; 873 874 paddr = buf->bnx_paddr; 875 m_adj(m_new, ETHER_ALIGN); 876 paddr += ETHER_ALIGN; 877 878 /* Save necessary information */ 879 sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_rx_mbuf = m_new; 880 sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_rx_paddr = paddr; 881 882 /* Set up the descriptor. */ 883 bnx_setup_rxdesc_jumbo(sc, i); 884 return 0; 885 } 886 887 static void 888 bnx_setup_rxdesc_jumbo(struct bnx_softc *sc, int i) 889 { 890 struct bge_rx_bd *r; 891 struct bnx_rx_buf *rc; 892 893 r = &sc->bnx_ldata.bnx_rx_jumbo_ring[i]; 894 rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i]; 895 896 r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_rx_paddr); 897 r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_rx_paddr); 898 r->bge_len = rc->bnx_rx_mbuf->m_len; 899 r->bge_idx = i; 900 r->bge_flags = BGE_RXBDFLAG_END|BGE_RXBDFLAG_JUMBO_RING; 901 } 902 903 static int 904 bnx_init_rx_ring_std(struct bnx_rx_std_ring *std) 905 { 906 int i, error; 907 908 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) { 909 /* Use the first RX return ring's tmp RX mbuf DMA map */ 910 error = bnx_newbuf_std(&std->bnx_sc->bnx_rx_ret_ring[0], i, 1); 911 if (error) 912 return error; 913 bnx_setup_rxdesc_std(std, i); 914 } 915 916 std->bnx_rx_std_used = 0; 917 std->bnx_rx_std_refill = 0; 918 std->bnx_rx_std_running = 0; 919 cpu_sfence(); 920 lwkt_serialize_handler_enable(&std->bnx_rx_std_serialize); 921 922 std->bnx_rx_std = BGE_STD_RX_RING_CNT - 1; 923 bnx_writembx(std->bnx_sc, BGE_MBX_RX_STD_PROD_LO, std->bnx_rx_std); 924 925 return(0); 926 } 927 928 static void 929 bnx_free_rx_ring_std(struct bnx_rx_std_ring *std) 930 { 931 int i; 932 933 lwkt_serialize_handler_disable(&std->bnx_rx_std_serialize); 934 935 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) { 936 struct bnx_rx_buf *rb = &std->bnx_rx_std_buf[i]; 937 938 rb->bnx_rx_refilled = 0; 939 if (rb->bnx_rx_mbuf != NULL) { 940 bus_dmamap_unload(std->bnx_rx_mtag, rb->bnx_rx_dmamap); 941 m_freem(rb->bnx_rx_mbuf); 942 rb->bnx_rx_mbuf = NULL; 943 } 944 bzero(&std->bnx_rx_std_ring[i], sizeof(struct bge_rx_bd)); 945 } 946 } 947 948 static int 949 bnx_init_rx_ring_jumbo(struct bnx_softc *sc) 950 { 951 struct bge_rcb *rcb; 952 int i, error; 953 954 for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) { 955 error = bnx_newbuf_jumbo(sc, i, 1); 956 if (error) 957 return error; 958 } 959 960 sc->bnx_jumbo = BGE_JUMBO_RX_RING_CNT - 1; 961 962 rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb; 963 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0, 0); 964 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags); 965 966 bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bnx_jumbo); 967 968 return(0); 969 } 970 971 static void 972 bnx_free_rx_ring_jumbo(struct bnx_softc *sc) 973 { 974 int i; 975 976 for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) { 977 struct bnx_rx_buf *rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i]; 978 979 if (rc->bnx_rx_mbuf != NULL) { 980 m_freem(rc->bnx_rx_mbuf); 981 rc->bnx_rx_mbuf = NULL; 982 } 983 bzero(&sc->bnx_ldata.bnx_rx_jumbo_ring[i], 984 sizeof(struct bge_rx_bd)); 985 } 986 } 987 988 static void 989 bnx_free_tx_ring(struct bnx_tx_ring *txr) 990 { 991 int i; 992 993 for (i = 0; i < BGE_TX_RING_CNT; i++) { 994 struct bnx_tx_buf *buf = &txr->bnx_tx_buf[i]; 995 996 if (buf->bnx_tx_mbuf != NULL) { 997 bus_dmamap_unload(txr->bnx_tx_mtag, 998 buf->bnx_tx_dmamap); 999 m_freem(buf->bnx_tx_mbuf); 1000 buf->bnx_tx_mbuf = NULL; 1001 } 1002 bzero(&txr->bnx_tx_ring[i], sizeof(struct bge_tx_bd)); 1003 } 1004 txr->bnx_tx_saved_considx = BNX_TXCONS_UNSET; 1005 } 1006 1007 static int 1008 bnx_init_tx_ring(struct bnx_tx_ring *txr) 1009 { 1010 txr->bnx_tx_cnt = 0; 1011 txr->bnx_tx_saved_considx = 0; 1012 txr->bnx_tx_prodidx = 0; 1013 1014 /* Initialize transmit producer index for host-memory send ring. */ 1015 bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, txr->bnx_tx_prodidx); 1016 1017 return(0); 1018 } 1019 1020 static void 1021 bnx_setmulti(struct bnx_softc *sc) 1022 { 1023 struct ifnet *ifp; 1024 struct ifmultiaddr *ifma; 1025 uint32_t hashes[4] = { 0, 0, 0, 0 }; 1026 int h, i; 1027 1028 ifp = &sc->arpcom.ac_if; 1029 1030 if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) { 1031 for (i = 0; i < 4; i++) 1032 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0xFFFFFFFF); 1033 return; 1034 } 1035 1036 /* First, zot all the existing filters. */ 1037 for (i = 0; i < 4; i++) 1038 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0); 1039 1040 /* Now program new ones. */ 1041 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1042 if (ifma->ifma_addr->sa_family != AF_LINK) 1043 continue; 1044 h = ether_crc32_le( 1045 LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1046 ETHER_ADDR_LEN) & 0x7f; 1047 hashes[(h & 0x60) >> 5] |= 1 << (h & 0x1F); 1048 } 1049 1050 for (i = 0; i < 4; i++) 1051 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), hashes[i]); 1052 } 1053 1054 /* 1055 * Do endian, PCI and DMA initialization. Also check the on-board ROM 1056 * self-test results. 1057 */ 1058 static int 1059 bnx_chipinit(struct bnx_softc *sc) 1060 { 1061 uint32_t dma_rw_ctl, mode_ctl; 1062 int i; 1063 1064 /* Set endian type before we access any non-PCI registers. */ 1065 pci_write_config(sc->bnx_dev, BGE_PCI_MISC_CTL, 1066 BGE_INIT | BGE_PCIMISCCTL_TAGGED_STATUS, 4); 1067 1068 /* Clear the MAC control register */ 1069 CSR_WRITE_4(sc, BGE_MAC_MODE, 0); 1070 1071 /* 1072 * Clear the MAC statistics block in the NIC's 1073 * internal memory. 1074 */ 1075 for (i = BGE_STATS_BLOCK; 1076 i < BGE_STATS_BLOCK_END + 1; i += sizeof(uint32_t)) 1077 BNX_MEMWIN_WRITE(sc, i, 0); 1078 1079 for (i = BGE_STATUS_BLOCK; 1080 i < BGE_STATUS_BLOCK_END + 1; i += sizeof(uint32_t)) 1081 BNX_MEMWIN_WRITE(sc, i, 0); 1082 1083 if (BNX_IS_57765_FAMILY(sc)) { 1084 uint32_t val; 1085 1086 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0) { 1087 mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL); 1088 val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS; 1089 1090 /* Access the lower 1K of PL PCI-E block registers. */ 1091 CSR_WRITE_4(sc, BGE_MODE_CTL, 1092 val | BGE_MODECTL_PCIE_PL_SEL); 1093 1094 val = CSR_READ_4(sc, BGE_PCIE_PL_LO_PHYCTL5); 1095 val |= BGE_PCIE_PL_LO_PHYCTL5_DIS_L2CLKREQ; 1096 CSR_WRITE_4(sc, BGE_PCIE_PL_LO_PHYCTL5, val); 1097 1098 CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl); 1099 } 1100 if (sc->bnx_chiprev != BGE_CHIPREV_57765_AX) { 1101 /* Fix transmit hangs */ 1102 val = CSR_READ_4(sc, BGE_CPMU_PADRNG_CTL); 1103 val |= BGE_CPMU_PADRNG_CTL_RDIV2; 1104 CSR_WRITE_4(sc, BGE_CPMU_PADRNG_CTL, val); 1105 1106 mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL); 1107 val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS; 1108 1109 /* Access the lower 1K of DL PCI-E block registers. */ 1110 CSR_WRITE_4(sc, BGE_MODE_CTL, 1111 val | BGE_MODECTL_PCIE_DL_SEL); 1112 1113 val = CSR_READ_4(sc, BGE_PCIE_DL_LO_FTSMAX); 1114 val &= ~BGE_PCIE_DL_LO_FTSMAX_MASK; 1115 val |= BGE_PCIE_DL_LO_FTSMAX_VAL; 1116 CSR_WRITE_4(sc, BGE_PCIE_DL_LO_FTSMAX, val); 1117 1118 CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl); 1119 } 1120 1121 val = CSR_READ_4(sc, BGE_CPMU_LSPD_10MB_CLK); 1122 val &= ~BGE_CPMU_LSPD_10MB_MACCLK_MASK; 1123 val |= BGE_CPMU_LSPD_10MB_MACCLK_6_25; 1124 CSR_WRITE_4(sc, BGE_CPMU_LSPD_10MB_CLK, val); 1125 } 1126 1127 /* 1128 * Set up the PCI DMA control register. 1129 */ 1130 dma_rw_ctl = pci_read_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, 4); 1131 /* 1132 * Disable 32bytes cache alignment for DMA write to host memory 1133 * 1134 * NOTE: 1135 * 64bytes cache alignment for DMA write to host memory is still 1136 * enabled. 1137 */ 1138 dma_rw_ctl |= BGE_PCIDMARWCTL_DIS_CACHE_ALIGNMENT; 1139 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0) 1140 dma_rw_ctl &= ~BGE_PCIDMARWCTL_CRDRDR_RDMA_MRRS_MSK; 1141 /* 1142 * Enable HW workaround for controllers that misinterpret 1143 * a status tag update and leave interrupts permanently 1144 * disabled. 1145 */ 1146 if (sc->bnx_asicrev != BGE_ASICREV_BCM5717 && 1147 sc->bnx_asicrev != BGE_ASICREV_BCM5762 && 1148 !BNX_IS_57765_FAMILY(sc)) 1149 dma_rw_ctl |= BGE_PCIDMARWCTL_TAGGED_STATUS_WA; 1150 if (bootverbose) { 1151 if_printf(&sc->arpcom.ac_if, "DMA read/write %#x\n", 1152 dma_rw_ctl); 1153 } 1154 pci_write_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, dma_rw_ctl, 4); 1155 1156 /* 1157 * Set up general mode register. 1158 */ 1159 mode_ctl = bnx_dma_swap_options(sc) | BGE_MODECTL_MAC_ATTN_INTR | 1160 BGE_MODECTL_HOST_SEND_BDS | BGE_MODECTL_TX_NO_PHDR_CSUM; 1161 CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl); 1162 1163 /* 1164 * Disable memory write invalidate. Apparently it is not supported 1165 * properly by these devices. Also ensure that INTx isn't disabled, 1166 * as these chips need it even when using MSI. 1167 */ 1168 PCI_CLRBIT(sc->bnx_dev, BGE_PCI_CMD, 1169 (PCIM_CMD_MWRICEN | PCIM_CMD_INTxDIS), 4); 1170 1171 /* Set the timer prescaler (always 66Mhz) */ 1172 CSR_WRITE_4(sc, BGE_MISC_CFG, 65 << 1/*BGE_32BITTIME_66MHZ*/); 1173 1174 return(0); 1175 } 1176 1177 static int 1178 bnx_blockinit(struct bnx_softc *sc) 1179 { 1180 struct bnx_intr_data *intr; 1181 struct bge_rcb *rcb; 1182 bus_size_t vrcb; 1183 bge_hostaddr taddr; 1184 uint32_t val; 1185 int i, limit; 1186 1187 /* 1188 * Initialize the memory window pointer register so that 1189 * we can access the first 32K of internal NIC RAM. This will 1190 * allow us to set up the TX send ring RCBs and the RX return 1191 * ring RCBs, plus other things which live in NIC memory. 1192 */ 1193 CSR_WRITE_4(sc, BGE_PCI_MEMWIN_BASEADDR, 0); 1194 1195 /* Configure mbuf pool watermarks */ 1196 if (BNX_IS_57765_PLUS(sc)) { 1197 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0); 1198 if (sc->arpcom.ac_if.if_mtu > ETHERMTU) { 1199 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x7e); 1200 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xea); 1201 } else { 1202 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x2a); 1203 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xa0); 1204 } 1205 } else { 1206 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0); 1207 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x10); 1208 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x60); 1209 } 1210 1211 /* Configure DMA resource watermarks */ 1212 CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LOWAT, 5); 1213 CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_HIWAT, 10); 1214 1215 /* Enable buffer manager */ 1216 val = BGE_BMANMODE_ENABLE | BGE_BMANMODE_LOMBUF_ATTN; 1217 /* 1218 * Change the arbitration algorithm of TXMBUF read request to 1219 * round-robin instead of priority based for BCM5719. When 1220 * TXFIFO is almost empty, RDMA will hold its request until 1221 * TXFIFO is not almost empty. 1222 */ 1223 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719) 1224 val |= BGE_BMANMODE_NO_TX_UNDERRUN; 1225 if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 || 1226 sc->bnx_chipid == BGE_CHIPID_BCM5719_A0 || 1227 sc->bnx_chipid == BGE_CHIPID_BCM5720_A0) 1228 val |= BGE_BMANMODE_LOMBUF_ATTN; 1229 CSR_WRITE_4(sc, BGE_BMAN_MODE, val); 1230 1231 /* Poll for buffer manager start indication */ 1232 for (i = 0; i < BNX_TIMEOUT; i++) { 1233 if (CSR_READ_4(sc, BGE_BMAN_MODE) & BGE_BMANMODE_ENABLE) 1234 break; 1235 DELAY(10); 1236 } 1237 1238 if (i == BNX_TIMEOUT) { 1239 if_printf(&sc->arpcom.ac_if, 1240 "buffer manager failed to start\n"); 1241 return(ENXIO); 1242 } 1243 1244 /* Enable flow-through queues */ 1245 CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF); 1246 CSR_WRITE_4(sc, BGE_FTQ_RESET, 0); 1247 1248 /* Wait until queue initialization is complete */ 1249 for (i = 0; i < BNX_TIMEOUT; i++) { 1250 if (CSR_READ_4(sc, BGE_FTQ_RESET) == 0) 1251 break; 1252 DELAY(10); 1253 } 1254 1255 if (i == BNX_TIMEOUT) { 1256 if_printf(&sc->arpcom.ac_if, 1257 "flow-through queue init failed\n"); 1258 return(ENXIO); 1259 } 1260 1261 /* 1262 * Summary of rings supported by the controller: 1263 * 1264 * Standard Receive Producer Ring 1265 * - This ring is used to feed receive buffers for "standard" 1266 * sized frames (typically 1536 bytes) to the controller. 1267 * 1268 * Jumbo Receive Producer Ring 1269 * - This ring is used to feed receive buffers for jumbo sized 1270 * frames (i.e. anything bigger than the "standard" frames) 1271 * to the controller. 1272 * 1273 * Mini Receive Producer Ring 1274 * - This ring is used to feed receive buffers for "mini" 1275 * sized frames to the controller. 1276 * - This feature required external memory for the controller 1277 * but was never used in a production system. Should always 1278 * be disabled. 1279 * 1280 * Receive Return Ring 1281 * - After the controller has placed an incoming frame into a 1282 * receive buffer that buffer is moved into a receive return 1283 * ring. The driver is then responsible to passing the 1284 * buffer up to the stack. BCM5718/BCM57785 families support 1285 * multiple receive return rings. 1286 * 1287 * Send Ring 1288 * - This ring is used for outgoing frames. BCM5719/BCM5720 1289 * support multiple send rings. 1290 */ 1291 1292 /* Initialize the standard receive producer ring control block. */ 1293 rcb = &sc->bnx_ldata.bnx_info.bnx_std_rx_rcb; 1294 rcb->bge_hostaddr.bge_addr_lo = 1295 BGE_ADDR_LO(sc->bnx_rx_std_ring.bnx_rx_std_ring_paddr); 1296 rcb->bge_hostaddr.bge_addr_hi = 1297 BGE_ADDR_HI(sc->bnx_rx_std_ring.bnx_rx_std_ring_paddr); 1298 if (BNX_IS_57765_PLUS(sc)) { 1299 /* 1300 * Bits 31-16: Programmable ring size (2048, 1024, 512, .., 32) 1301 * Bits 15-2 : Maximum RX frame size 1302 * Bit 1 : 1 = Ring Disabled, 0 = Ring ENabled 1303 * Bit 0 : Reserved 1304 */ 1305 rcb->bge_maxlen_flags = 1306 BGE_RCB_MAXLEN_FLAGS(512, BNX_MAX_FRAMELEN << 2); 1307 } else { 1308 /* 1309 * Bits 31-16: Programmable ring size (512, 256, 128, 64, 32) 1310 * Bits 15-2 : Reserved (should be 0) 1311 * Bit 1 : 1 = Ring Disabled, 0 = Ring Enabled 1312 * Bit 0 : Reserved 1313 */ 1314 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(512, 0); 1315 } 1316 if (BNX_IS_5717_PLUS(sc)) 1317 rcb->bge_nicaddr = BGE_STD_RX_RINGS_5717; 1318 else 1319 rcb->bge_nicaddr = BGE_STD_RX_RINGS; 1320 /* Write the standard receive producer ring control block. */ 1321 CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_HI, rcb->bge_hostaddr.bge_addr_hi); 1322 CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_LO, rcb->bge_hostaddr.bge_addr_lo); 1323 CSR_WRITE_4(sc, BGE_RX_STD_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags); 1324 if (!BNX_IS_5717_PLUS(sc)) 1325 CSR_WRITE_4(sc, BGE_RX_STD_RCB_NICADDR, rcb->bge_nicaddr); 1326 /* Reset the standard receive producer ring producer index. */ 1327 bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, 0); 1328 1329 /* 1330 * Initialize the jumbo RX producer ring control 1331 * block. We set the 'ring disabled' bit in the 1332 * flags field until we're actually ready to start 1333 * using this ring (i.e. once we set the MTU 1334 * high enough to require it). 1335 */ 1336 if (BNX_IS_JUMBO_CAPABLE(sc)) { 1337 rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb; 1338 /* Get the jumbo receive producer ring RCB parameters. */ 1339 rcb->bge_hostaddr.bge_addr_lo = 1340 BGE_ADDR_LO(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr); 1341 rcb->bge_hostaddr.bge_addr_hi = 1342 BGE_ADDR_HI(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr); 1343 rcb->bge_maxlen_flags = 1344 BGE_RCB_MAXLEN_FLAGS(BNX_MAX_FRAMELEN, 1345 BGE_RCB_FLAG_RING_DISABLED); 1346 if (BNX_IS_5717_PLUS(sc)) 1347 rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS_5717; 1348 else 1349 rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS; 1350 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_HI, 1351 rcb->bge_hostaddr.bge_addr_hi); 1352 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_LO, 1353 rcb->bge_hostaddr.bge_addr_lo); 1354 /* Program the jumbo receive producer ring RCB parameters. */ 1355 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, 1356 rcb->bge_maxlen_flags); 1357 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_NICADDR, rcb->bge_nicaddr); 1358 /* Reset the jumbo receive producer ring producer index. */ 1359 bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, 0); 1360 } 1361 1362 /* 1363 * The BD ring replenish thresholds control how often the 1364 * hardware fetches new BD's from the producer rings in host 1365 * memory. Setting the value too low on a busy system can 1366 * starve the hardware and recue the throughpout. 1367 * 1368 * Set the BD ring replentish thresholds. The recommended 1369 * values are 1/8th the number of descriptors allocated to 1370 * each ring. 1371 */ 1372 val = 8; 1373 CSR_WRITE_4(sc, BGE_RBDI_STD_REPL_THRESH, val); 1374 if (BNX_IS_JUMBO_CAPABLE(sc)) { 1375 CSR_WRITE_4(sc, BGE_RBDI_JUMBO_REPL_THRESH, 1376 BGE_JUMBO_RX_RING_CNT/8); 1377 } 1378 if (BNX_IS_57765_PLUS(sc)) { 1379 CSR_WRITE_4(sc, BGE_STD_REPLENISH_LWM, 32); 1380 CSR_WRITE_4(sc, BGE_JMB_REPLENISH_LWM, 16); 1381 } 1382 1383 /* 1384 * Disable all send rings by setting the 'ring disabled' bit 1385 * in the flags field of all the TX send ring control blocks, 1386 * located in NIC memory. 1387 */ 1388 if (BNX_IS_5717_PLUS(sc)) 1389 limit = 4; 1390 else if (BNX_IS_57765_FAMILY(sc) || 1391 sc->bnx_asicrev == BGE_ASICREV_BCM5762) 1392 limit = 2; 1393 else 1394 limit = 1; 1395 vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB; 1396 for (i = 0; i < limit; i++) { 1397 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags, 1398 BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED)); 1399 vrcb += sizeof(struct bge_rcb); 1400 } 1401 1402 /* 1403 * Configure send ring RCBs 1404 */ 1405 vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB; 1406 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 1407 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 1408 1409 BGE_HOSTADDR(taddr, txr->bnx_tx_ring_paddr); 1410 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 1411 taddr.bge_addr_hi); 1412 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 1413 taddr.bge_addr_lo); 1414 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags, 1415 BGE_RCB_MAXLEN_FLAGS(BGE_TX_RING_CNT, 0)); 1416 vrcb += sizeof(struct bge_rcb); 1417 } 1418 1419 /* 1420 * Disable all receive return rings by setting the 1421 * 'ring disabled' bit in the flags field of all the receive 1422 * return ring control blocks, located in NIC memory. 1423 */ 1424 if (BNX_IS_5717_PLUS(sc)) { 1425 /* Should be 17, use 16 until we get an SRAM map. */ 1426 limit = 16; 1427 } else if (BNX_IS_57765_FAMILY(sc) || 1428 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 1429 limit = 4; 1430 } else { 1431 limit = 1; 1432 } 1433 /* Disable all receive return rings. */ 1434 vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB; 1435 for (i = 0; i < limit; i++) { 1436 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 0); 1437 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 0); 1438 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags, 1439 BGE_RCB_FLAG_RING_DISABLED); 1440 bnx_writembx(sc, BGE_MBX_RX_CONS0_LO + 1441 (i * (sizeof(uint64_t))), 0); 1442 vrcb += sizeof(struct bge_rcb); 1443 } 1444 1445 /* 1446 * Set up receive return rings. 1447 */ 1448 vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB; 1449 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 1450 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i]; 1451 1452 BGE_HOSTADDR(taddr, ret->bnx_rx_ret_ring_paddr); 1453 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 1454 taddr.bge_addr_hi); 1455 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 1456 taddr.bge_addr_lo); 1457 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags, 1458 BGE_RCB_MAXLEN_FLAGS(BNX_RETURN_RING_CNT, 0)); 1459 vrcb += sizeof(struct bge_rcb); 1460 } 1461 1462 /* Set random backoff seed for TX */ 1463 CSR_WRITE_4(sc, BGE_TX_RANDOM_BACKOFF, 1464 sc->arpcom.ac_enaddr[0] + sc->arpcom.ac_enaddr[1] + 1465 sc->arpcom.ac_enaddr[2] + sc->arpcom.ac_enaddr[3] + 1466 sc->arpcom.ac_enaddr[4] + sc->arpcom.ac_enaddr[5] + 1467 BGE_TX_BACKOFF_SEED_MASK); 1468 1469 /* Set inter-packet gap */ 1470 val = 0x2620; 1471 if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 || 1472 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 1473 val |= CSR_READ_4(sc, BGE_TX_LENGTHS) & 1474 (BGE_TXLEN_JMB_FRM_LEN_MSK | BGE_TXLEN_CNT_DN_VAL_MSK); 1475 } 1476 CSR_WRITE_4(sc, BGE_TX_LENGTHS, val); 1477 1478 /* 1479 * Specify which ring to use for packets that don't match 1480 * any RX rules. 1481 */ 1482 CSR_WRITE_4(sc, BGE_RX_RULES_CFG, 0x08); 1483 1484 /* 1485 * Configure number of RX lists. One interrupt distribution 1486 * list, sixteen active lists, one bad frames class. 1487 */ 1488 CSR_WRITE_4(sc, BGE_RXLP_CFG, 0x181); 1489 1490 /* Inialize RX list placement stats mask. */ 1491 CSR_WRITE_4(sc, BGE_RXLP_STATS_ENABLE_MASK, 0x007FFFFF); 1492 CSR_WRITE_4(sc, BGE_RXLP_STATS_CTL, 0x1); 1493 1494 /* Disable host coalescing until we get it set up */ 1495 CSR_WRITE_4(sc, BGE_HCC_MODE, 0x00000000); 1496 1497 /* Poll to make sure it's shut down. */ 1498 for (i = 0; i < BNX_TIMEOUT; i++) { 1499 if (!(CSR_READ_4(sc, BGE_HCC_MODE) & BGE_HCCMODE_ENABLE)) 1500 break; 1501 DELAY(10); 1502 } 1503 1504 if (i == BNX_TIMEOUT) { 1505 if_printf(&sc->arpcom.ac_if, 1506 "host coalescing engine failed to idle\n"); 1507 return(ENXIO); 1508 } 1509 1510 /* Set up host coalescing defaults */ 1511 sc->bnx_coal_chg = BNX_RX_COAL_TICKS_CHG | 1512 BNX_TX_COAL_TICKS_CHG | 1513 BNX_RX_COAL_BDS_CHG | 1514 BNX_TX_COAL_BDS_CHG | 1515 BNX_RX_COAL_BDS_INT_CHG | 1516 BNX_TX_COAL_BDS_INT_CHG; 1517 bnx_coal_change(sc); 1518 1519 /* 1520 * Set up addresses of status blocks 1521 */ 1522 intr = &sc->bnx_intr_data[0]; 1523 bzero(intr->bnx_status_block, BGE_STATUS_BLK_SZ); 1524 CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_HI, 1525 BGE_ADDR_HI(intr->bnx_status_block_paddr)); 1526 CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_LO, 1527 BGE_ADDR_LO(intr->bnx_status_block_paddr)); 1528 for (i = 1; i < sc->bnx_intr_cnt; ++i) { 1529 intr = &sc->bnx_intr_data[i]; 1530 bzero(intr->bnx_status_block, BGE_STATUS_BLK_SZ); 1531 CSR_WRITE_4(sc, BGE_VEC1_STATUSBLK_ADDR_HI + ((i - 1) * 8), 1532 BGE_ADDR_HI(intr->bnx_status_block_paddr)); 1533 CSR_WRITE_4(sc, BGE_VEC1_STATUSBLK_ADDR_LO + ((i - 1) * 8), 1534 BGE_ADDR_LO(intr->bnx_status_block_paddr)); 1535 } 1536 1537 /* Set up status block partail update size. */ 1538 val = BGE_STATBLKSZ_32BYTE; 1539 #if 0 1540 /* 1541 * Does not seem to have visible effect in both 1542 * bulk data (1472B UDP datagram) and tiny data 1543 * (18B UDP datagram) TX tests. 1544 */ 1545 val |= BGE_HCCMODE_CLRTICK_TX; 1546 #endif 1547 /* Turn on host coalescing state machine */ 1548 CSR_WRITE_4(sc, BGE_HCC_MODE, val | BGE_HCCMODE_ENABLE); 1549 1550 /* Turn on RX BD completion state machine and enable attentions */ 1551 CSR_WRITE_4(sc, BGE_RBDC_MODE, 1552 BGE_RBDCMODE_ENABLE|BGE_RBDCMODE_ATTN); 1553 1554 /* Turn on RX list placement state machine */ 1555 CSR_WRITE_4(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE); 1556 1557 val = BGE_MACMODE_TXDMA_ENB | BGE_MACMODE_RXDMA_ENB | 1558 BGE_MACMODE_RX_STATS_CLEAR | BGE_MACMODE_TX_STATS_CLEAR | 1559 BGE_MACMODE_RX_STATS_ENB | BGE_MACMODE_TX_STATS_ENB | 1560 BGE_MACMODE_FRMHDR_DMA_ENB; 1561 1562 if (sc->bnx_flags & BNX_FLAG_TBI) 1563 val |= BGE_PORTMODE_TBI; 1564 else if (sc->bnx_flags & BNX_FLAG_MII_SERDES) 1565 val |= BGE_PORTMODE_GMII; 1566 else 1567 val |= BGE_PORTMODE_MII; 1568 1569 /* Turn on DMA, clear stats */ 1570 CSR_WRITE_4(sc, BGE_MAC_MODE, val); 1571 1572 /* Set misc. local control, enable interrupts on attentions */ 1573 CSR_WRITE_4(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_ONATTN); 1574 1575 #ifdef notdef 1576 /* Assert GPIO pins for PHY reset */ 1577 BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUT0| 1578 BGE_MLC_MISCIO_OUT1|BGE_MLC_MISCIO_OUT2); 1579 BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUTEN0| 1580 BGE_MLC_MISCIO_OUTEN1|BGE_MLC_MISCIO_OUTEN2); 1581 #endif 1582 1583 if (sc->bnx_intr_type == PCI_INTR_TYPE_MSIX) 1584 bnx_enable_msi(sc, TRUE); 1585 1586 /* Turn on write DMA state machine */ 1587 val = BGE_WDMAMODE_ENABLE|BGE_WDMAMODE_ALL_ATTNS; 1588 /* Enable host coalescing bug fix. */ 1589 val |= BGE_WDMAMODE_STATUS_TAG_FIX; 1590 if (sc->bnx_asicrev == BGE_ASICREV_BCM5785) { 1591 /* Request larger DMA burst size to get better performance. */ 1592 val |= BGE_WDMAMODE_BURST_ALL_DATA; 1593 } 1594 CSR_WRITE_4(sc, BGE_WDMA_MODE, val); 1595 DELAY(40); 1596 1597 if (BNX_IS_57765_PLUS(sc)) { 1598 uint32_t dmactl, dmactl_reg; 1599 1600 if (sc->bnx_asicrev == BGE_ASICREV_BCM5762) 1601 dmactl_reg = BGE_RDMA_RSRVCTRL2; 1602 else 1603 dmactl_reg = BGE_RDMA_RSRVCTRL; 1604 1605 dmactl = CSR_READ_4(sc, dmactl_reg); 1606 /* 1607 * Adjust tx margin to prevent TX data corruption and 1608 * fix internal FIFO overflow. 1609 */ 1610 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 || 1611 sc->bnx_asicrev == BGE_ASICREV_BCM5720 || 1612 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 1613 dmactl &= ~(BGE_RDMA_RSRVCTRL_FIFO_LWM_MASK | 1614 BGE_RDMA_RSRVCTRL_FIFO_HWM_MASK | 1615 BGE_RDMA_RSRVCTRL_TXMRGN_MASK); 1616 dmactl |= BGE_RDMA_RSRVCTRL_FIFO_LWM_1_5K | 1617 BGE_RDMA_RSRVCTRL_FIFO_HWM_1_5K | 1618 BGE_RDMA_RSRVCTRL_TXMRGN_320B; 1619 } 1620 /* 1621 * Enable fix for read DMA FIFO overruns. 1622 * The fix is to limit the number of RX BDs 1623 * the hardware would fetch at a fime. 1624 */ 1625 CSR_WRITE_4(sc, dmactl_reg, 1626 dmactl | BGE_RDMA_RSRVCTRL_FIFO_OFLW_FIX); 1627 } 1628 1629 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719) { 1630 CSR_WRITE_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL, 1631 CSR_READ_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL) | 1632 BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_4K | 1633 BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K); 1634 } else if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 || 1635 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 1636 uint32_t ctrl_reg; 1637 1638 if (sc->bnx_asicrev == BGE_ASICREV_BCM5762) 1639 ctrl_reg = BGE_RDMA_LSO_CRPTEN_CTRL2; 1640 else 1641 ctrl_reg = BGE_RDMA_LSO_CRPTEN_CTRL; 1642 1643 /* 1644 * Allow 4KB burst length reads for non-LSO frames. 1645 * Enable 512B burst length reads for buffer descriptors. 1646 */ 1647 CSR_WRITE_4(sc, ctrl_reg, 1648 CSR_READ_4(sc, ctrl_reg) | 1649 BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_512 | 1650 BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K); 1651 } 1652 1653 /* Turn on read DMA state machine */ 1654 val = BGE_RDMAMODE_ENABLE | BGE_RDMAMODE_ALL_ATTNS; 1655 if (sc->bnx_asicrev == BGE_ASICREV_BCM5717) 1656 val |= BGE_RDMAMODE_MULT_DMA_RD_DIS; 1657 if (sc->bnx_asicrev == BGE_ASICREV_BCM5784 || 1658 sc->bnx_asicrev == BGE_ASICREV_BCM5785 || 1659 sc->bnx_asicrev == BGE_ASICREV_BCM57780) { 1660 val |= BGE_RDMAMODE_BD_SBD_CRPT_ATTN | 1661 BGE_RDMAMODE_MBUF_RBD_CRPT_ATTN | 1662 BGE_RDMAMODE_MBUF_SBD_CRPT_ATTN; 1663 } 1664 if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 || 1665 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 1666 val |= CSR_READ_4(sc, BGE_RDMA_MODE) & 1667 BGE_RDMAMODE_H2BNC_VLAN_DET; 1668 /* 1669 * Allow multiple outstanding read requests from 1670 * non-LSO read DMA engine. 1671 */ 1672 val &= ~BGE_RDMAMODE_MULT_DMA_RD_DIS; 1673 } 1674 if (sc->bnx_asicrev == BGE_ASICREV_BCM57766) 1675 val |= BGE_RDMAMODE_JMB_2K_MMRR; 1676 if (sc->bnx_flags & BNX_FLAG_TSO) 1677 val |= BGE_RDMAMODE_TSO4_ENABLE; 1678 val |= BGE_RDMAMODE_FIFO_LONG_BURST; 1679 CSR_WRITE_4(sc, BGE_RDMA_MODE, val); 1680 DELAY(40); 1681 1682 /* Turn on RX data completion state machine */ 1683 CSR_WRITE_4(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE); 1684 1685 /* Turn on RX BD initiator state machine */ 1686 CSR_WRITE_4(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE); 1687 1688 /* Turn on RX data and RX BD initiator state machine */ 1689 CSR_WRITE_4(sc, BGE_RDBDI_MODE, BGE_RDBDIMODE_ENABLE); 1690 1691 /* Turn on send BD completion state machine */ 1692 CSR_WRITE_4(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE); 1693 1694 /* Turn on send data completion state machine */ 1695 val = BGE_SDCMODE_ENABLE; 1696 if (sc->bnx_asicrev == BGE_ASICREV_BCM5761) 1697 val |= BGE_SDCMODE_CDELAY; 1698 CSR_WRITE_4(sc, BGE_SDC_MODE, val); 1699 1700 /* Turn on send data initiator state machine */ 1701 if (sc->bnx_flags & BNX_FLAG_TSO) { 1702 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE | 1703 BGE_SDIMODE_HW_LSO_PRE_DMA); 1704 } else { 1705 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE); 1706 } 1707 1708 /* Turn on send BD initiator state machine */ 1709 val = BGE_SBDIMODE_ENABLE; 1710 if (sc->bnx_tx_ringcnt > 1) 1711 val |= BGE_SBDIMODE_MULTI_TXR; 1712 CSR_WRITE_4(sc, BGE_SBDI_MODE, val); 1713 1714 /* Turn on send BD selector state machine */ 1715 CSR_WRITE_4(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE); 1716 1717 CSR_WRITE_4(sc, BGE_SDI_STATS_ENABLE_MASK, 0x007FFFFF); 1718 CSR_WRITE_4(sc, BGE_SDI_STATS_CTL, 1719 BGE_SDISTATSCTL_ENABLE|BGE_SDISTATSCTL_FASTER); 1720 1721 /* ack/clear link change events */ 1722 CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED| 1723 BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE| 1724 BGE_MACSTAT_LINK_CHANGED); 1725 CSR_WRITE_4(sc, BGE_MI_STS, 0); 1726 1727 /* 1728 * Enable attention when the link has changed state for 1729 * devices that use auto polling. 1730 */ 1731 if (sc->bnx_flags & BNX_FLAG_TBI) { 1732 CSR_WRITE_4(sc, BGE_MI_STS, BGE_MISTS_LINK); 1733 } else { 1734 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) { 1735 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode); 1736 DELAY(80); 1737 } 1738 } 1739 1740 /* 1741 * Clear any pending link state attention. 1742 * Otherwise some link state change events may be lost until attention 1743 * is cleared by bnx_intr() -> bnx_softc.bnx_link_upd() sequence. 1744 * It's not necessary on newer BCM chips - perhaps enabling link 1745 * state change attentions implies clearing pending attention. 1746 */ 1747 CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED| 1748 BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE| 1749 BGE_MACSTAT_LINK_CHANGED); 1750 1751 /* Enable link state change attentions. */ 1752 BNX_SETBIT(sc, BGE_MAC_EVT_ENB, BGE_EVTENB_LINK_CHANGED); 1753 1754 return(0); 1755 } 1756 1757 /* 1758 * Probe for a Broadcom chip. Check the PCI vendor and device IDs 1759 * against our list and return its name if we find a match. Note 1760 * that since the Broadcom controller contains VPD support, we 1761 * can get the device name string from the controller itself instead 1762 * of the compiled-in string. This is a little slow, but it guarantees 1763 * we'll always announce the right product name. 1764 */ 1765 static int 1766 bnx_probe(device_t dev) 1767 { 1768 const struct bnx_type *t; 1769 uint16_t product, vendor; 1770 1771 if (!pci_is_pcie(dev)) 1772 return ENXIO; 1773 1774 product = pci_get_device(dev); 1775 vendor = pci_get_vendor(dev); 1776 1777 for (t = bnx_devs; t->bnx_name != NULL; t++) { 1778 if (vendor == t->bnx_vid && product == t->bnx_did) 1779 break; 1780 } 1781 if (t->bnx_name == NULL) 1782 return ENXIO; 1783 1784 device_set_desc(dev, t->bnx_name); 1785 return 0; 1786 } 1787 1788 static int 1789 bnx_attach(device_t dev) 1790 { 1791 struct ifnet *ifp; 1792 struct bnx_softc *sc; 1793 struct bnx_rx_std_ring *std; 1794 uint32_t hwcfg = 0; 1795 int error = 0, rid, capmask, i, std_cpuid, std_cpuid_def; 1796 uint8_t ether_addr[ETHER_ADDR_LEN]; 1797 uint16_t product; 1798 uintptr_t mii_priv = 0; 1799 #if defined(BNX_TSO_DEBUG) || defined(BNX_RSS_DEBUG) || defined(BNX_TSS_DEBUG) 1800 char desc[32]; 1801 #endif 1802 #ifdef IFPOLL_ENABLE 1803 int offset, offset_def; 1804 #endif 1805 1806 sc = device_get_softc(dev); 1807 sc->bnx_dev = dev; 1808 callout_init_mp(&sc->bnx_tick_timer); 1809 lwkt_serialize_init(&sc->bnx_jslot_serializer); 1810 lwkt_serialize_init(&sc->bnx_main_serialize); 1811 1812 /* Always setup interrupt mailboxes */ 1813 for (i = 0; i < BNX_INTR_MAX; ++i) { 1814 callout_init_mp(&sc->bnx_intr_data[i].bnx_intr_timer); 1815 sc->bnx_intr_data[i].bnx_sc = sc; 1816 sc->bnx_intr_data[i].bnx_intr_mbx = BGE_MBX_IRQ0_LO + (i * 8); 1817 sc->bnx_intr_data[i].bnx_intr_rid = -1; 1818 sc->bnx_intr_data[i].bnx_intr_cpuid = -1; 1819 } 1820 1821 product = pci_get_device(dev); 1822 1823 #ifndef BURN_BRIDGES 1824 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) { 1825 uint32_t irq, mem; 1826 1827 irq = pci_read_config(dev, PCIR_INTLINE, 4); 1828 mem = pci_read_config(dev, BGE_PCI_BAR0, 4); 1829 1830 device_printf(dev, "chip is in D%d power mode " 1831 "-- setting to D0\n", pci_get_powerstate(dev)); 1832 1833 pci_set_powerstate(dev, PCI_POWERSTATE_D0); 1834 1835 pci_write_config(dev, PCIR_INTLINE, irq, 4); 1836 pci_write_config(dev, BGE_PCI_BAR0, mem, 4); 1837 } 1838 #endif /* !BURN_BRIDGE */ 1839 1840 /* 1841 * Map control/status registers. 1842 */ 1843 pci_enable_busmaster(dev); 1844 1845 rid = BGE_PCI_BAR0; 1846 sc->bnx_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 1847 RF_ACTIVE); 1848 1849 if (sc->bnx_res == NULL) { 1850 device_printf(dev, "couldn't map memory\n"); 1851 return ENXIO; 1852 } 1853 1854 sc->bnx_btag = rman_get_bustag(sc->bnx_res); 1855 sc->bnx_bhandle = rman_get_bushandle(sc->bnx_res); 1856 1857 /* Save various chip information */ 1858 sc->bnx_chipid = 1859 pci_read_config(dev, BGE_PCI_MISC_CTL, 4) >> 1860 BGE_PCIMISCCTL_ASICREV_SHIFT; 1861 if (BGE_ASICREV(sc->bnx_chipid) == BGE_ASICREV_USE_PRODID_REG) { 1862 /* All chips having dedicated ASICREV register have CPMU */ 1863 sc->bnx_flags |= BNX_FLAG_CPMU; 1864 1865 switch (product) { 1866 case PCI_PRODUCT_BROADCOM_BCM5717: 1867 case PCI_PRODUCT_BROADCOM_BCM5717C: 1868 case PCI_PRODUCT_BROADCOM_BCM5718: 1869 case PCI_PRODUCT_BROADCOM_BCM5719: 1870 case PCI_PRODUCT_BROADCOM_BCM5720_ALT: 1871 case PCI_PRODUCT_BROADCOM_BCM5725: 1872 case PCI_PRODUCT_BROADCOM_BCM5727: 1873 case PCI_PRODUCT_BROADCOM_BCM5762: 1874 sc->bnx_chipid = pci_read_config(dev, 1875 BGE_PCI_GEN2_PRODID_ASICREV, 4); 1876 break; 1877 1878 case PCI_PRODUCT_BROADCOM_BCM57761: 1879 case PCI_PRODUCT_BROADCOM_BCM57762: 1880 case PCI_PRODUCT_BROADCOM_BCM57765: 1881 case PCI_PRODUCT_BROADCOM_BCM57766: 1882 case PCI_PRODUCT_BROADCOM_BCM57781: 1883 case PCI_PRODUCT_BROADCOM_BCM57782: 1884 case PCI_PRODUCT_BROADCOM_BCM57785: 1885 case PCI_PRODUCT_BROADCOM_BCM57786: 1886 case PCI_PRODUCT_BROADCOM_BCM57791: 1887 case PCI_PRODUCT_BROADCOM_BCM57795: 1888 sc->bnx_chipid = pci_read_config(dev, 1889 BGE_PCI_GEN15_PRODID_ASICREV, 4); 1890 break; 1891 1892 default: 1893 sc->bnx_chipid = pci_read_config(dev, 1894 BGE_PCI_PRODID_ASICREV, 4); 1895 break; 1896 } 1897 } 1898 if (sc->bnx_chipid == BGE_CHIPID_BCM5717_C0) 1899 sc->bnx_chipid = BGE_CHIPID_BCM5720_A0; 1900 1901 sc->bnx_asicrev = BGE_ASICREV(sc->bnx_chipid); 1902 sc->bnx_chiprev = BGE_CHIPREV(sc->bnx_chipid); 1903 1904 switch (sc->bnx_asicrev) { 1905 case BGE_ASICREV_BCM5717: 1906 case BGE_ASICREV_BCM5719: 1907 case BGE_ASICREV_BCM5720: 1908 sc->bnx_flags |= BNX_FLAG_5717_PLUS | BNX_FLAG_57765_PLUS; 1909 break; 1910 1911 case BGE_ASICREV_BCM5762: 1912 sc->bnx_flags |= BNX_FLAG_57765_PLUS; 1913 break; 1914 1915 case BGE_ASICREV_BCM57765: 1916 case BGE_ASICREV_BCM57766: 1917 sc->bnx_flags |= BNX_FLAG_57765_FAMILY | BNX_FLAG_57765_PLUS; 1918 break; 1919 } 1920 1921 sc->bnx_flags |= BNX_FLAG_TSO; 1922 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 && 1923 sc->bnx_chipid == BGE_CHIPID_BCM5719_A0) 1924 sc->bnx_flags &= ~BNX_FLAG_TSO; 1925 1926 if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 || 1927 BNX_IS_57765_FAMILY(sc)) { 1928 /* 1929 * All BCM57785 and BCM5718 families chips have a bug that 1930 * under certain situation interrupt will not be enabled 1931 * even if status tag is written to interrupt mailbox. 1932 * 1933 * While BCM5719 and BCM5720 have a hardware workaround 1934 * which could fix the above bug. 1935 * See the comment near BGE_PCIDMARWCTL_TAGGED_STATUS_WA in 1936 * bnx_chipinit(). 1937 * 1938 * For the rest of the chips in these two families, we will 1939 * have to poll the status block at high rate (10ms currently) 1940 * to check whether the interrupt is hosed or not. 1941 * See bnx_check_intr_*() for details. 1942 */ 1943 sc->bnx_flags |= BNX_FLAG_STATUSTAG_BUG; 1944 } 1945 1946 sc->bnx_pciecap = pci_get_pciecap_ptr(sc->bnx_dev); 1947 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 || 1948 sc->bnx_asicrev == BGE_ASICREV_BCM5720) 1949 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_2048); 1950 else 1951 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_4096); 1952 device_printf(dev, "CHIP ID 0x%08x; " 1953 "ASIC REV 0x%02x; CHIP REV 0x%02x\n", 1954 sc->bnx_chipid, sc->bnx_asicrev, sc->bnx_chiprev); 1955 1956 /* 1957 * Set various PHY quirk flags. 1958 */ 1959 1960 capmask = MII_CAPMASK_DEFAULT; 1961 if (product == PCI_PRODUCT_BROADCOM_BCM57791 || 1962 product == PCI_PRODUCT_BROADCOM_BCM57795) { 1963 /* 10/100 only */ 1964 capmask &= ~BMSR_EXTSTAT; 1965 } 1966 1967 mii_priv |= BRGPHY_FLAG_WIRESPEED; 1968 if (sc->bnx_chipid == BGE_CHIPID_BCM5762_A0) 1969 mii_priv |= BRGPHY_FLAG_5762_A0; 1970 1971 /* Initialize if_name earlier, so if_printf could be used */ 1972 ifp = &sc->arpcom.ac_if; 1973 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1974 1975 /* Try to reset the chip. */ 1976 bnx_reset(sc); 1977 1978 if (bnx_chipinit(sc)) { 1979 device_printf(dev, "chip initialization failed\n"); 1980 error = ENXIO; 1981 goto fail; 1982 } 1983 1984 /* 1985 * Get station address 1986 */ 1987 error = bnx_get_eaddr(sc, ether_addr); 1988 if (error) { 1989 device_printf(dev, "failed to read station address\n"); 1990 goto fail; 1991 } 1992 1993 /* Setup RX/TX and interrupt count */ 1994 bnx_setup_ring_cnt(sc); 1995 1996 if ((sc->bnx_rx_retcnt == 1 && sc->bnx_tx_ringcnt == 1) || 1997 (sc->bnx_rx_retcnt > 1 && sc->bnx_tx_ringcnt > 1)) { 1998 /* 1999 * The RX ring and the corresponding TX ring processing 2000 * should be on the same CPU, since they share the same 2001 * status block. 2002 */ 2003 sc->bnx_flags |= BNX_FLAG_RXTX_BUNDLE; 2004 if (bootverbose) 2005 device_printf(dev, "RX/TX bundle\n"); 2006 if (sc->bnx_tx_ringcnt > 1) { 2007 /* 2008 * Multiple TX rings do not share status block 2009 * with link status, so link status will have 2010 * to save its own status_tag. 2011 */ 2012 sc->bnx_flags |= BNX_FLAG_STATUS_HASTAG; 2013 if (bootverbose) 2014 device_printf(dev, "status needs tag\n"); 2015 } 2016 } else { 2017 KKASSERT(sc->bnx_rx_retcnt > 1 && sc->bnx_tx_ringcnt == 1); 2018 if (bootverbose) 2019 device_printf(dev, "RX/TX not bundled\n"); 2020 } 2021 2022 error = bnx_dma_alloc(dev); 2023 if (error) 2024 goto fail; 2025 2026 #ifdef IFPOLL_ENABLE 2027 if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) { 2028 /* 2029 * NPOLLING RX/TX CPU offset 2030 */ 2031 if (sc->bnx_rx_retcnt == ncpus2) { 2032 offset = 0; 2033 } else { 2034 offset_def = 2035 (sc->bnx_rx_retcnt * device_get_unit(dev)) % ncpus2; 2036 offset = device_getenv_int(dev, "npoll.offset", 2037 offset_def); 2038 if (offset >= ncpus2 || 2039 offset % sc->bnx_rx_retcnt != 0) { 2040 device_printf(dev, "invalid npoll.offset %d, " 2041 "use %d\n", offset, offset_def); 2042 offset = offset_def; 2043 } 2044 } 2045 sc->bnx_npoll_rxoff = offset; 2046 sc->bnx_npoll_txoff = offset; 2047 } else { 2048 /* 2049 * NPOLLING RX CPU offset 2050 */ 2051 if (sc->bnx_rx_retcnt == ncpus2) { 2052 offset = 0; 2053 } else { 2054 offset_def = 2055 (sc->bnx_rx_retcnt * device_get_unit(dev)) % ncpus2; 2056 offset = device_getenv_int(dev, "npoll.rxoff", 2057 offset_def); 2058 if (offset >= ncpus2 || 2059 offset % sc->bnx_rx_retcnt != 0) { 2060 device_printf(dev, "invalid npoll.rxoff %d, " 2061 "use %d\n", offset, offset_def); 2062 offset = offset_def; 2063 } 2064 } 2065 sc->bnx_npoll_rxoff = offset; 2066 2067 /* 2068 * NPOLLING TX CPU offset 2069 */ 2070 offset_def = device_get_unit(dev) % ncpus2; 2071 offset = device_getenv_int(dev, "npoll.txoff", offset_def); 2072 if (offset >= ncpus2) { 2073 device_printf(dev, "invalid npoll.txoff %d, use %d\n", 2074 offset, offset_def); 2075 offset = offset_def; 2076 } 2077 sc->bnx_npoll_txoff = offset; 2078 } 2079 #endif /* IFPOLL_ENABLE */ 2080 2081 /* 2082 * Allocate interrupt 2083 */ 2084 error = bnx_alloc_intr(sc); 2085 if (error) 2086 goto fail; 2087 2088 /* Setup serializers */ 2089 bnx_setup_serialize(sc); 2090 2091 /* Set default tuneable values. */ 2092 sc->bnx_rx_coal_ticks = BNX_RX_COAL_TICKS_DEF; 2093 sc->bnx_tx_coal_ticks = BNX_TX_COAL_TICKS_DEF; 2094 sc->bnx_rx_coal_bds = BNX_RX_COAL_BDS_DEF; 2095 sc->bnx_rx_coal_bds_poll = sc->bnx_rx_ret_ring[0].bnx_rx_cntmax; 2096 sc->bnx_tx_coal_bds = BNX_TX_COAL_BDS_DEF; 2097 sc->bnx_tx_coal_bds_poll = BNX_TX_COAL_BDS_POLL_DEF; 2098 sc->bnx_rx_coal_bds_int = BNX_RX_COAL_BDS_INT_DEF; 2099 sc->bnx_tx_coal_bds_int = BNX_TX_COAL_BDS_INT_DEF; 2100 2101 /* Set up ifnet structure */ 2102 ifp->if_softc = sc; 2103 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2104 ifp->if_ioctl = bnx_ioctl; 2105 ifp->if_start = bnx_start; 2106 #ifdef IFPOLL_ENABLE 2107 ifp->if_npoll = bnx_npoll; 2108 #endif 2109 ifp->if_init = bnx_init; 2110 ifp->if_serialize = bnx_serialize; 2111 ifp->if_deserialize = bnx_deserialize; 2112 ifp->if_tryserialize = bnx_tryserialize; 2113 #ifdef INVARIANTS 2114 ifp->if_serialize_assert = bnx_serialize_assert; 2115 #endif 2116 ifp->if_mtu = ETHERMTU; 2117 ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2118 2119 ifp->if_capabilities |= IFCAP_HWCSUM; 2120 ifp->if_hwassist = BNX_CSUM_FEATURES; 2121 if (sc->bnx_flags & BNX_FLAG_TSO) { 2122 ifp->if_capabilities |= IFCAP_TSO; 2123 ifp->if_hwassist |= CSUM_TSO; 2124 } 2125 if (BNX_RSS_ENABLED(sc)) 2126 ifp->if_capabilities |= IFCAP_RSS; 2127 ifp->if_capenable = ifp->if_capabilities; 2128 2129 ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1); 2130 ifq_set_ready(&ifp->if_snd); 2131 ifq_set_subq_cnt(&ifp->if_snd, sc->bnx_tx_ringcnt); 2132 2133 if (sc->bnx_tx_ringcnt > 1) { 2134 ifp->if_mapsubq = ifq_mapsubq_mask; 2135 ifq_set_subq_mask(&ifp->if_snd, sc->bnx_tx_ringcnt - 1); 2136 } 2137 2138 /* 2139 * Figure out what sort of media we have by checking the 2140 * hardware config word in the first 32k of NIC internal memory, 2141 * or fall back to examining the EEPROM if necessary. 2142 * Note: on some BCM5700 cards, this value appears to be unset. 2143 * If that's the case, we have to rely on identifying the NIC 2144 * by its PCI subsystem ID, as we do below for the SysKonnect 2145 * SK-9D41. 2146 */ 2147 if (bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_SIG) == BGE_MAGIC_NUMBER) { 2148 hwcfg = bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_NICCFG); 2149 } else { 2150 if (bnx_read_eeprom(sc, (caddr_t)&hwcfg, BGE_EE_HWCFG_OFFSET, 2151 sizeof(hwcfg))) { 2152 device_printf(dev, "failed to read EEPROM\n"); 2153 error = ENXIO; 2154 goto fail; 2155 } 2156 hwcfg = ntohl(hwcfg); 2157 } 2158 2159 /* The SysKonnect SK-9D41 is a 1000baseSX card. */ 2160 if (pci_get_subvendor(dev) == PCI_PRODUCT_SCHNEIDERKOCH_SK_9D41 || 2161 (hwcfg & BGE_HWCFG_MEDIA) == BGE_MEDIA_FIBER) 2162 sc->bnx_flags |= BNX_FLAG_TBI; 2163 2164 /* Setup MI MODE */ 2165 if (sc->bnx_flags & BNX_FLAG_CPMU) 2166 sc->bnx_mi_mode = BGE_MIMODE_500KHZ_CONST; 2167 else 2168 sc->bnx_mi_mode = BGE_MIMODE_BASE; 2169 2170 /* Setup link status update stuffs */ 2171 if (sc->bnx_flags & BNX_FLAG_TBI) { 2172 sc->bnx_link_upd = bnx_tbi_link_upd; 2173 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED; 2174 } else if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) { 2175 sc->bnx_link_upd = bnx_autopoll_link_upd; 2176 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED; 2177 } else { 2178 sc->bnx_link_upd = bnx_copper_link_upd; 2179 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED; 2180 } 2181 2182 /* Set default PHY address */ 2183 sc->bnx_phyno = 1; 2184 2185 /* 2186 * PHY address mapping for various devices. 2187 * 2188 * | F0 Cu | F0 Sr | F1 Cu | F1 Sr | 2189 * ---------+-------+-------+-------+-------+ 2190 * BCM57XX | 1 | X | X | X | 2191 * BCM5704 | 1 | X | 1 | X | 2192 * BCM5717 | 1 | 8 | 2 | 9 | 2193 * BCM5719 | 1 | 8 | 2 | 9 | 2194 * BCM5720 | 1 | 8 | 2 | 9 | 2195 * 2196 * Other addresses may respond but they are not 2197 * IEEE compliant PHYs and should be ignored. 2198 */ 2199 if (BNX_IS_5717_PLUS(sc)) { 2200 int f; 2201 2202 f = pci_get_function(dev); 2203 if (sc->bnx_chipid == BGE_CHIPID_BCM5717_A0) { 2204 if (CSR_READ_4(sc, BGE_SGDIG_STS) & 2205 BGE_SGDIGSTS_IS_SERDES) 2206 sc->bnx_phyno = f + 8; 2207 else 2208 sc->bnx_phyno = f + 1; 2209 } else { 2210 if (CSR_READ_4(sc, BGE_CPMU_PHY_STRAP) & 2211 BGE_CPMU_PHY_STRAP_IS_SERDES) 2212 sc->bnx_phyno = f + 8; 2213 else 2214 sc->bnx_phyno = f + 1; 2215 } 2216 } 2217 2218 if (sc->bnx_flags & BNX_FLAG_TBI) { 2219 ifmedia_init(&sc->bnx_ifmedia, IFM_IMASK, 2220 bnx_ifmedia_upd, bnx_ifmedia_sts); 2221 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_1000_SX, 0, NULL); 2222 ifmedia_add(&sc->bnx_ifmedia, 2223 IFM_ETHER|IFM_1000_SX|IFM_FDX, 0, NULL); 2224 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO, 0, NULL); 2225 ifmedia_set(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO); 2226 sc->bnx_ifmedia.ifm_media = sc->bnx_ifmedia.ifm_cur->ifm_media; 2227 } else { 2228 struct mii_probe_args mii_args; 2229 2230 mii_probe_args_init(&mii_args, bnx_ifmedia_upd, bnx_ifmedia_sts); 2231 mii_args.mii_probemask = 1 << sc->bnx_phyno; 2232 mii_args.mii_capmask = capmask; 2233 mii_args.mii_privtag = MII_PRIVTAG_BRGPHY; 2234 mii_args.mii_priv = mii_priv; 2235 2236 error = mii_probe(dev, &sc->bnx_miibus, &mii_args); 2237 if (error) { 2238 device_printf(dev, "MII without any PHY!\n"); 2239 goto fail; 2240 } 2241 } 2242 2243 /* 2244 * Create sysctl nodes. 2245 */ 2246 sysctl_ctx_init(&sc->bnx_sysctl_ctx); 2247 sc->bnx_sysctl_tree = SYSCTL_ADD_NODE(&sc->bnx_sysctl_ctx, 2248 SYSCTL_STATIC_CHILDREN(_hw), 2249 OID_AUTO, 2250 device_get_nameunit(dev), 2251 CTLFLAG_RD, 0, ""); 2252 if (sc->bnx_sysctl_tree == NULL) { 2253 device_printf(dev, "can't add sysctl node\n"); 2254 error = ENXIO; 2255 goto fail; 2256 } 2257 2258 SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx, 2259 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2260 "rx_rings", CTLFLAG_RD, &sc->bnx_rx_retcnt, 0, "# of RX rings"); 2261 SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx, 2262 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2263 "tx_rings", CTLFLAG_RD, &sc->bnx_tx_ringcnt, 0, "# of TX rings"); 2264 2265 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2266 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), 2267 OID_AUTO, "rx_coal_ticks", 2268 CTLTYPE_INT | CTLFLAG_RW, 2269 sc, 0, bnx_sysctl_rx_coal_ticks, "I", 2270 "Receive coalescing ticks (usec)."); 2271 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2272 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), 2273 OID_AUTO, "tx_coal_ticks", 2274 CTLTYPE_INT | CTLFLAG_RW, 2275 sc, 0, bnx_sysctl_tx_coal_ticks, "I", 2276 "Transmit coalescing ticks (usec)."); 2277 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2278 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), 2279 OID_AUTO, "rx_coal_bds", 2280 CTLTYPE_INT | CTLFLAG_RW, 2281 sc, 0, bnx_sysctl_rx_coal_bds, "I", 2282 "Receive max coalesced BD count."); 2283 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2284 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), 2285 OID_AUTO, "rx_coal_bds_poll", 2286 CTLTYPE_INT | CTLFLAG_RW, 2287 sc, 0, bnx_sysctl_rx_coal_bds_poll, "I", 2288 "Receive max coalesced BD count in polling."); 2289 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2290 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), 2291 OID_AUTO, "tx_coal_bds", 2292 CTLTYPE_INT | CTLFLAG_RW, 2293 sc, 0, bnx_sysctl_tx_coal_bds, "I", 2294 "Transmit max coalesced BD count."); 2295 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2296 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), 2297 OID_AUTO, "tx_coal_bds_poll", 2298 CTLTYPE_INT | CTLFLAG_RW, 2299 sc, 0, bnx_sysctl_tx_coal_bds_poll, "I", 2300 "Transmit max coalesced BD count in polling."); 2301 /* 2302 * A common design characteristic for many Broadcom 2303 * client controllers is that they only support a 2304 * single outstanding DMA read operation on the PCIe 2305 * bus. This means that it will take twice as long to 2306 * fetch a TX frame that is split into header and 2307 * payload buffers as it does to fetch a single, 2308 * contiguous TX frame (2 reads vs. 1 read). For these 2309 * controllers, coalescing buffers to reduce the number 2310 * of memory reads is effective way to get maximum 2311 * performance(about 940Mbps). Without collapsing TX 2312 * buffers the maximum TCP bulk transfer performance 2313 * is about 850Mbps. However forcing coalescing mbufs 2314 * consumes a lot of CPU cycles, so leave it off by 2315 * default. 2316 */ 2317 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2318 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2319 "force_defrag", CTLTYPE_INT | CTLFLAG_RW, 2320 sc, 0, bnx_sysctl_force_defrag, "I", 2321 "Force defragment on TX path"); 2322 2323 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2324 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2325 "tx_wreg", CTLTYPE_INT | CTLFLAG_RW, 2326 sc, 0, bnx_sysctl_tx_wreg, "I", 2327 "# of segments before writing to hardware register"); 2328 2329 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2330 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2331 "std_refill", CTLTYPE_INT | CTLFLAG_RW, 2332 sc, 0, bnx_sysctl_std_refill, "I", 2333 "# of packets received before scheduling standard refilling"); 2334 2335 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2336 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2337 "rx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW, 2338 sc, 0, bnx_sysctl_rx_coal_bds_int, "I", 2339 "Receive max coalesced BD count during interrupt."); 2340 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2341 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2342 "tx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW, 2343 sc, 0, bnx_sysctl_tx_coal_bds_int, "I", 2344 "Transmit max coalesced BD count during interrupt."); 2345 2346 #ifdef IFPOLL_ENABLE 2347 if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) { 2348 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2349 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2350 "npoll_offset", CTLTYPE_INT | CTLFLAG_RW, 2351 sc, 0, bnx_sysctl_npoll_offset, "I", 2352 "NPOLLING cpu offset"); 2353 } else { 2354 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2355 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2356 "npoll_rxoff", CTLTYPE_INT | CTLFLAG_RW, 2357 sc, 0, bnx_sysctl_npoll_rxoff, "I", 2358 "NPOLLING RX cpu offset"); 2359 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx, 2360 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2361 "npoll_txoff", CTLTYPE_INT | CTLFLAG_RW, 2362 sc, 0, bnx_sysctl_npoll_txoff, "I", 2363 "NPOLLING TX cpu offset"); 2364 } 2365 #endif 2366 2367 #ifdef BNX_RSS_DEBUG 2368 SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx, 2369 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2370 "std_refill_mask", CTLFLAG_RD, 2371 &sc->bnx_rx_std_ring.bnx_rx_std_refill, 0, ""); 2372 SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx, 2373 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2374 "std_used", CTLFLAG_RD, 2375 &sc->bnx_rx_std_ring.bnx_rx_std_used, 0, ""); 2376 SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx, 2377 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2378 "rss_debug", CTLFLAG_RW, &sc->bnx_rss_debug, 0, ""); 2379 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 2380 ksnprintf(desc, sizeof(desc), "rx_pkt%d", i); 2381 SYSCTL_ADD_ULONG(&sc->bnx_sysctl_ctx, 2382 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2383 desc, CTLFLAG_RW, &sc->bnx_rx_ret_ring[i].bnx_rx_pkt, ""); 2384 2385 ksnprintf(desc, sizeof(desc), "rx_force_sched%d", i); 2386 SYSCTL_ADD_ULONG(&sc->bnx_sysctl_ctx, 2387 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2388 desc, CTLFLAG_RW, 2389 &sc->bnx_rx_ret_ring[i].bnx_rx_force_sched, ""); 2390 } 2391 #endif 2392 #ifdef BNX_TSS_DEBUG 2393 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 2394 ksnprintf(desc, sizeof(desc), "tx_pkt%d", i); 2395 SYSCTL_ADD_ULONG(&sc->bnx_sysctl_ctx, 2396 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2397 desc, CTLFLAG_RW, &sc->bnx_tx_ring[i].bnx_tx_pkt, ""); 2398 } 2399 #endif 2400 2401 SYSCTL_ADD_ULONG(&sc->bnx_sysctl_ctx, 2402 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2403 "norxbds", CTLFLAG_RW, &sc->bnx_norxbds, ""); 2404 2405 SYSCTL_ADD_ULONG(&sc->bnx_sysctl_ctx, 2406 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2407 "errors", CTLFLAG_RW, &sc->bnx_errors, ""); 2408 2409 #ifdef BNX_TSO_DEBUG 2410 for (i = 0; i < BNX_TSO_NSTATS; ++i) { 2411 ksnprintf(desc, sizeof(desc), "tso%d", i + 1); 2412 SYSCTL_ADD_ULONG(&sc->bnx_sysctl_ctx, 2413 SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO, 2414 desc, CTLFLAG_RW, &sc->bnx_tsosegs[i], ""); 2415 } 2416 #endif 2417 2418 /* 2419 * Call MI attach routine. 2420 */ 2421 ether_ifattach(ifp, ether_addr, NULL); 2422 2423 /* Setup TX rings and subqueues */ 2424 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 2425 struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i); 2426 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 2427 2428 ifsq_set_cpuid(ifsq, txr->bnx_tx_cpuid); 2429 ifsq_set_hw_serialize(ifsq, &txr->bnx_tx_serialize); 2430 ifsq_set_priv(ifsq, txr); 2431 txr->bnx_ifsq = ifsq; 2432 2433 ifsq_watchdog_init(&txr->bnx_tx_watchdog, ifsq, bnx_watchdog); 2434 2435 if (bootverbose) { 2436 device_printf(dev, "txr %d -> cpu%d\n", i, 2437 txr->bnx_tx_cpuid); 2438 } 2439 } 2440 2441 error = bnx_setup_intr(sc); 2442 if (error) { 2443 ether_ifdetach(ifp); 2444 goto fail; 2445 } 2446 bnx_set_tick_cpuid(sc, FALSE); 2447 2448 /* 2449 * Create RX standard ring refilling thread 2450 */ 2451 std_cpuid_def = device_get_unit(dev) % ncpus; 2452 std_cpuid = device_getenv_int(dev, "std.cpuid", std_cpuid_def); 2453 if (std_cpuid < 0 || std_cpuid >= ncpus) { 2454 device_printf(dev, "invalid std.cpuid %d, use %d\n", 2455 std_cpuid, std_cpuid_def); 2456 std_cpuid = std_cpuid_def; 2457 } 2458 2459 std = &sc->bnx_rx_std_ring; 2460 lwkt_create(bnx_rx_std_refill_ithread, std, NULL, 2461 &std->bnx_rx_std_ithread, TDF_NOSTART | TDF_INTTHREAD, std_cpuid, 2462 "%s std", device_get_nameunit(dev)); 2463 lwkt_setpri(&std->bnx_rx_std_ithread, TDPRI_INT_MED); 2464 std->bnx_rx_std_ithread.td_preemptable = lwkt_preempt; 2465 sc->bnx_flags |= BNX_FLAG_STD_THREAD; 2466 2467 return(0); 2468 fail: 2469 bnx_detach(dev); 2470 return(error); 2471 } 2472 2473 static int 2474 bnx_detach(device_t dev) 2475 { 2476 struct bnx_softc *sc = device_get_softc(dev); 2477 2478 if (device_is_attached(dev)) { 2479 struct ifnet *ifp = &sc->arpcom.ac_if; 2480 2481 ifnet_serialize_all(ifp); 2482 bnx_stop(sc); 2483 bnx_reset(sc); 2484 bnx_teardown_intr(sc, sc->bnx_intr_cnt); 2485 ifnet_deserialize_all(ifp); 2486 2487 ether_ifdetach(ifp); 2488 } 2489 2490 if (sc->bnx_flags & BNX_FLAG_STD_THREAD) { 2491 struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring; 2492 2493 tsleep_interlock(std, 0); 2494 2495 if (std->bnx_rx_std_ithread.td_gd == mycpu) { 2496 bnx_rx_std_refill_stop(std); 2497 } else { 2498 lwkt_send_ipiq(std->bnx_rx_std_ithread.td_gd, 2499 bnx_rx_std_refill_stop, std); 2500 } 2501 2502 tsleep(std, PINTERLOCKED, "bnx_detach", 0); 2503 if (bootverbose) 2504 device_printf(dev, "RX std ithread exited\n"); 2505 2506 lwkt_synchronize_ipiqs("bnx_detach_ipiq"); 2507 } 2508 2509 if (sc->bnx_flags & BNX_FLAG_TBI) 2510 ifmedia_removeall(&sc->bnx_ifmedia); 2511 if (sc->bnx_miibus) 2512 device_delete_child(dev, sc->bnx_miibus); 2513 bus_generic_detach(dev); 2514 2515 bnx_free_intr(sc); 2516 2517 if (sc->bnx_msix_mem_res != NULL) { 2518 bus_release_resource(dev, SYS_RES_MEMORY, sc->bnx_msix_mem_rid, 2519 sc->bnx_msix_mem_res); 2520 } 2521 if (sc->bnx_res != NULL) { 2522 bus_release_resource(dev, SYS_RES_MEMORY, 2523 BGE_PCI_BAR0, sc->bnx_res); 2524 } 2525 2526 if (sc->bnx_sysctl_tree != NULL) 2527 sysctl_ctx_free(&sc->bnx_sysctl_ctx); 2528 2529 bnx_dma_free(sc); 2530 2531 if (sc->bnx_serialize != NULL) 2532 kfree(sc->bnx_serialize, M_DEVBUF); 2533 2534 return 0; 2535 } 2536 2537 static void 2538 bnx_reset(struct bnx_softc *sc) 2539 { 2540 device_t dev; 2541 uint32_t cachesize, command, pcistate, reset; 2542 void (*write_op)(struct bnx_softc *, uint32_t, uint32_t); 2543 int i, val = 0; 2544 uint16_t devctl; 2545 2546 dev = sc->bnx_dev; 2547 2548 write_op = bnx_writemem_direct; 2549 2550 /* Save some important PCI state. */ 2551 cachesize = pci_read_config(dev, BGE_PCI_CACHESZ, 4); 2552 command = pci_read_config(dev, BGE_PCI_CMD, 4); 2553 pcistate = pci_read_config(dev, BGE_PCI_PCISTATE, 4); 2554 2555 pci_write_config(dev, BGE_PCI_MISC_CTL, 2556 BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR| 2557 BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW| 2558 BGE_PCIMISCCTL_TAGGED_STATUS, 4); 2559 2560 /* Disable fastboot on controllers that support it. */ 2561 if (bootverbose) 2562 if_printf(&sc->arpcom.ac_if, "Disabling fastboot\n"); 2563 CSR_WRITE_4(sc, BGE_FASTBOOT_PC, 0x0); 2564 2565 /* 2566 * Write the magic number to SRAM at offset 0xB50. 2567 * When firmware finishes its initialization it will 2568 * write ~BGE_MAGIC_NUMBER to the same location. 2569 */ 2570 bnx_writemem_ind(sc, BGE_SOFTWARE_GENCOMM, BGE_MAGIC_NUMBER); 2571 2572 reset = BGE_MISCCFG_RESET_CORE_CLOCKS|(65<<1); 2573 2574 /* XXX: Broadcom Linux driver. */ 2575 /* Force PCI-E 1.0a mode */ 2576 if (!BNX_IS_57765_PLUS(sc) && 2577 CSR_READ_4(sc, BGE_PCIE_PHY_TSTCTL) == 2578 (BGE_PCIE_PHY_TSTCTL_PSCRAM | 2579 BGE_PCIE_PHY_TSTCTL_PCIE10)) { 2580 CSR_WRITE_4(sc, BGE_PCIE_PHY_TSTCTL, 2581 BGE_PCIE_PHY_TSTCTL_PSCRAM); 2582 } 2583 if (sc->bnx_chipid != BGE_CHIPID_BCM5750_A0) { 2584 /* Prevent PCIE link training during global reset */ 2585 CSR_WRITE_4(sc, BGE_MISC_CFG, (1<<29)); 2586 reset |= (1<<29); 2587 } 2588 2589 /* 2590 * Set GPHY Power Down Override to leave GPHY 2591 * powered up in D0 uninitialized. 2592 */ 2593 if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0) 2594 reset |= BGE_MISCCFG_GPHY_PD_OVERRIDE; 2595 2596 /* Issue global reset */ 2597 write_op(sc, BGE_MISC_CFG, reset); 2598 2599 DELAY(1000); 2600 2601 /* XXX: Broadcom Linux driver. */ 2602 if (sc->bnx_chipid == BGE_CHIPID_BCM5750_A0) { 2603 uint32_t v; 2604 2605 DELAY(500000); /* wait for link training to complete */ 2606 v = pci_read_config(dev, 0xc4, 4); 2607 pci_write_config(dev, 0xc4, v | (1<<15), 4); 2608 } 2609 2610 devctl = pci_read_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL, 2); 2611 2612 /* Disable no snoop and disable relaxed ordering. */ 2613 devctl &= ~(PCIEM_DEVCTL_RELAX_ORDER | PCIEM_DEVCTL_NOSNOOP); 2614 2615 /* Old PCI-E chips only support 128 bytes Max PayLoad Size. */ 2616 if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0) { 2617 devctl &= ~PCIEM_DEVCTL_MAX_PAYLOAD_MASK; 2618 devctl |= PCIEM_DEVCTL_MAX_PAYLOAD_128; 2619 } 2620 2621 pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL, 2622 devctl, 2); 2623 2624 /* Clear error status. */ 2625 pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVSTS, 2626 PCIEM_DEVSTS_CORR_ERR | 2627 PCIEM_DEVSTS_NFATAL_ERR | 2628 PCIEM_DEVSTS_FATAL_ERR | 2629 PCIEM_DEVSTS_UNSUPP_REQ, 2); 2630 2631 /* Reset some of the PCI state that got zapped by reset */ 2632 pci_write_config(dev, BGE_PCI_MISC_CTL, 2633 BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR| 2634 BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW| 2635 BGE_PCIMISCCTL_TAGGED_STATUS, 4); 2636 pci_write_config(dev, BGE_PCI_CACHESZ, cachesize, 4); 2637 pci_write_config(dev, BGE_PCI_CMD, command, 4); 2638 write_op(sc, BGE_MISC_CFG, (65 << 1)); 2639 2640 /* Enable memory arbiter */ 2641 CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE); 2642 2643 /* 2644 * Poll until we see the 1's complement of the magic number. 2645 * This indicates that the firmware initialization is complete. 2646 */ 2647 for (i = 0; i < BNX_FIRMWARE_TIMEOUT; i++) { 2648 val = bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM); 2649 if (val == ~BGE_MAGIC_NUMBER) 2650 break; 2651 DELAY(10); 2652 } 2653 if (i == BNX_FIRMWARE_TIMEOUT) { 2654 if_printf(&sc->arpcom.ac_if, "firmware handshake " 2655 "timed out, found 0x%08x\n", val); 2656 } 2657 2658 /* BCM57765 A0 needs additional time before accessing. */ 2659 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0) 2660 DELAY(10 * 1000); 2661 2662 /* 2663 * XXX Wait for the value of the PCISTATE register to 2664 * return to its original pre-reset state. This is a 2665 * fairly good indicator of reset completion. If we don't 2666 * wait for the reset to fully complete, trying to read 2667 * from the device's non-PCI registers may yield garbage 2668 * results. 2669 */ 2670 for (i = 0; i < BNX_TIMEOUT; i++) { 2671 if (pci_read_config(dev, BGE_PCI_PCISTATE, 4) == pcistate) 2672 break; 2673 DELAY(10); 2674 } 2675 2676 /* Fix up byte swapping */ 2677 CSR_WRITE_4(sc, BGE_MODE_CTL, bnx_dma_swap_options(sc)); 2678 2679 CSR_WRITE_4(sc, BGE_MAC_MODE, 0); 2680 2681 /* 2682 * The 5704 in TBI mode apparently needs some special 2683 * adjustment to insure the SERDES drive level is set 2684 * to 1.2V. 2685 */ 2686 if (sc->bnx_asicrev == BGE_ASICREV_BCM5704 && 2687 (sc->bnx_flags & BNX_FLAG_TBI)) { 2688 uint32_t serdescfg; 2689 2690 serdescfg = CSR_READ_4(sc, BGE_SERDES_CFG); 2691 serdescfg = (serdescfg & ~0xFFF) | 0x880; 2692 CSR_WRITE_4(sc, BGE_SERDES_CFG, serdescfg); 2693 } 2694 2695 CSR_WRITE_4(sc, BGE_MI_MODE, 2696 sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL); 2697 DELAY(80); 2698 2699 /* XXX: Broadcom Linux driver. */ 2700 if (!BNX_IS_57765_PLUS(sc)) { 2701 uint32_t v; 2702 2703 /* Enable Data FIFO protection. */ 2704 v = CSR_READ_4(sc, BGE_PCIE_TLDLPL_PORT); 2705 CSR_WRITE_4(sc, BGE_PCIE_TLDLPL_PORT, v | (1 << 25)); 2706 } 2707 2708 DELAY(10000); 2709 2710 if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) { 2711 BNX_CLRBIT(sc, BGE_CPMU_CLCK_ORIDE, 2712 CPMU_CLCK_ORIDE_MAC_ORIDE_EN); 2713 } 2714 } 2715 2716 /* 2717 * Frame reception handling. This is called if there's a frame 2718 * on the receive return list. 2719 * 2720 * Note: we have to be able to handle two possibilities here: 2721 * 1) the frame is from the jumbo recieve ring 2722 * 2) the frame is from the standard receive ring 2723 */ 2724 2725 static void 2726 bnx_rxeof(struct bnx_rx_ret_ring *ret, uint16_t rx_prod, int count) 2727 { 2728 struct bnx_softc *sc = ret->bnx_sc; 2729 struct bnx_rx_std_ring *std = ret->bnx_std; 2730 struct ifnet *ifp = &sc->arpcom.ac_if; 2731 int std_used = 0; 2732 2733 while (ret->bnx_rx_saved_considx != rx_prod && count != 0) { 2734 struct pktinfo pi0, *pi = NULL; 2735 struct bge_rx_bd *cur_rx; 2736 struct bnx_rx_buf *rb; 2737 uint32_t rxidx; 2738 struct mbuf *m = NULL; 2739 uint16_t vlan_tag = 0; 2740 int have_tag = 0; 2741 2742 --count; 2743 2744 cur_rx = &ret->bnx_rx_ret_ring[ret->bnx_rx_saved_considx]; 2745 2746 rxidx = cur_rx->bge_idx; 2747 KKASSERT(rxidx < BGE_STD_RX_RING_CNT); 2748 2749 BNX_INC(ret->bnx_rx_saved_considx, BNX_RETURN_RING_CNT); 2750 #ifdef BNX_RSS_DEBUG 2751 ret->bnx_rx_pkt++; 2752 #endif 2753 2754 if (cur_rx->bge_flags & BGE_RXBDFLAG_VLAN_TAG) { 2755 have_tag = 1; 2756 vlan_tag = cur_rx->bge_vlan_tag; 2757 } 2758 2759 if (ret->bnx_rx_cnt >= ret->bnx_rx_cntmax) { 2760 atomic_add_int(&std->bnx_rx_std_used, std_used); 2761 std_used = 0; 2762 2763 bnx_rx_std_refill_sched(ret, std); 2764 } 2765 ret->bnx_rx_cnt++; 2766 ++std_used; 2767 2768 rb = &std->bnx_rx_std_buf[rxidx]; 2769 m = rb->bnx_rx_mbuf; 2770 if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) { 2771 IFNET_STAT_INC(ifp, ierrors, 1); 2772 cpu_sfence(); 2773 rb->bnx_rx_refilled = 1; 2774 continue; 2775 } 2776 if (bnx_newbuf_std(ret, rxidx, 0)) { 2777 IFNET_STAT_INC(ifp, ierrors, 1); 2778 continue; 2779 } 2780 2781 IFNET_STAT_INC(ifp, ipackets, 1); 2782 m->m_pkthdr.len = m->m_len = cur_rx->bge_len - ETHER_CRC_LEN; 2783 m->m_pkthdr.rcvif = ifp; 2784 2785 if ((ifp->if_capenable & IFCAP_RXCSUM) && 2786 (cur_rx->bge_flags & BGE_RXBDFLAG_IPV6) == 0) { 2787 if (cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) { 2788 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 2789 if ((cur_rx->bge_error_flag & 2790 BGE_RXERRFLAG_IP_CSUM_NOK) == 0) 2791 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2792 } 2793 if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) { 2794 m->m_pkthdr.csum_data = 2795 cur_rx->bge_tcp_udp_csum; 2796 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 2797 CSUM_PSEUDO_HDR; 2798 } 2799 } 2800 if (ifp->if_capenable & IFCAP_RSS) { 2801 pi = bnx_rss_info(&pi0, cur_rx); 2802 if (pi != NULL && 2803 (cur_rx->bge_flags & BGE_RXBDFLAG_RSS_HASH)) { 2804 m->m_flags |= M_HASH; 2805 m->m_pkthdr.hash = 2806 toeplitz_hash(cur_rx->bge_hash); 2807 } 2808 } 2809 2810 /* 2811 * If we received a packet with a vlan tag, pass it 2812 * to vlan_input() instead of ether_input(). 2813 */ 2814 if (have_tag) { 2815 m->m_flags |= M_VLANTAG; 2816 m->m_pkthdr.ether_vlantag = vlan_tag; 2817 } 2818 ether_input_pkt(ifp, m, pi); 2819 } 2820 bnx_writembx(sc, ret->bnx_rx_mbx, ret->bnx_rx_saved_considx); 2821 2822 if (std_used > 0) { 2823 int cur_std_used; 2824 2825 cur_std_used = atomic_fetchadd_int(&std->bnx_rx_std_used, 2826 std_used); 2827 if (cur_std_used + std_used >= (BGE_STD_RX_RING_CNT / 2)) { 2828 #ifdef BNX_RSS_DEBUG 2829 ret->bnx_rx_force_sched++; 2830 #endif 2831 bnx_rx_std_refill_sched(ret, std); 2832 } 2833 } 2834 } 2835 2836 static void 2837 bnx_txeof(struct bnx_tx_ring *txr, uint16_t tx_cons) 2838 { 2839 struct ifnet *ifp = &txr->bnx_sc->arpcom.ac_if; 2840 2841 /* 2842 * Go through our tx ring and free mbufs for those 2843 * frames that have been sent. 2844 */ 2845 while (txr->bnx_tx_saved_considx != tx_cons) { 2846 struct bnx_tx_buf *buf; 2847 uint32_t idx = 0; 2848 2849 idx = txr->bnx_tx_saved_considx; 2850 buf = &txr->bnx_tx_buf[idx]; 2851 if (buf->bnx_tx_mbuf != NULL) { 2852 IFNET_STAT_INC(ifp, opackets, 1); 2853 #ifdef BNX_TSS_DEBUG 2854 txr->bnx_tx_pkt++; 2855 #endif 2856 bus_dmamap_unload(txr->bnx_tx_mtag, 2857 buf->bnx_tx_dmamap); 2858 m_freem(buf->bnx_tx_mbuf); 2859 buf->bnx_tx_mbuf = NULL; 2860 } 2861 txr->bnx_tx_cnt--; 2862 BNX_INC(txr->bnx_tx_saved_considx, BGE_TX_RING_CNT); 2863 } 2864 2865 if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) >= 2866 (BNX_NSEG_RSVD + BNX_NSEG_SPARE)) 2867 ifsq_clr_oactive(txr->bnx_ifsq); 2868 2869 if (txr->bnx_tx_cnt == 0) 2870 txr->bnx_tx_watchdog.wd_timer = 0; 2871 2872 if (!ifsq_is_empty(txr->bnx_ifsq)) 2873 ifsq_devstart(txr->bnx_ifsq); 2874 } 2875 2876 static int 2877 bnx_handle_status(struct bnx_softc *sc) 2878 { 2879 uint32_t status; 2880 int handle = 0; 2881 2882 status = *sc->bnx_hw_status; 2883 2884 if (status & BGE_STATFLAG_ERROR) { 2885 uint32_t val; 2886 int reset = 0; 2887 2888 sc->bnx_errors++; 2889 2890 val = CSR_READ_4(sc, BGE_FLOW_ATTN); 2891 if (val & ~BGE_FLOWATTN_MB_LOWAT) { 2892 if_printf(&sc->arpcom.ac_if, 2893 "flow attn 0x%08x\n", val); 2894 reset = 1; 2895 } 2896 2897 val = CSR_READ_4(sc, BGE_MSI_STATUS); 2898 if (val & ~BGE_MSISTAT_MSI_PCI_REQ) { 2899 if_printf(&sc->arpcom.ac_if, 2900 "msi status 0x%08x\n", val); 2901 reset = 1; 2902 } 2903 2904 val = CSR_READ_4(sc, BGE_RDMA_STATUS); 2905 if (val) { 2906 if_printf(&sc->arpcom.ac_if, 2907 "rmda status 0x%08x\n", val); 2908 reset = 1; 2909 } 2910 2911 val = CSR_READ_4(sc, BGE_WDMA_STATUS); 2912 if (val) { 2913 if_printf(&sc->arpcom.ac_if, 2914 "wdma status 0x%08x\n", val); 2915 reset = 1; 2916 } 2917 2918 if (reset) { 2919 bnx_serialize_skipmain(sc); 2920 bnx_init(sc); 2921 bnx_deserialize_skipmain(sc); 2922 } 2923 handle = 1; 2924 } 2925 2926 if ((status & BGE_STATFLAG_LINKSTATE_CHANGED) || sc->bnx_link_evt) { 2927 if (bootverbose) { 2928 if_printf(&sc->arpcom.ac_if, "link change, " 2929 "link_evt %d\n", sc->bnx_link_evt); 2930 } 2931 bnx_link_poll(sc); 2932 handle = 1; 2933 } 2934 2935 return handle; 2936 } 2937 2938 #ifdef IFPOLL_ENABLE 2939 2940 static void 2941 bnx_npoll_rx(struct ifnet *ifp __unused, void *xret, int cycle) 2942 { 2943 struct bnx_rx_ret_ring *ret = xret; 2944 uint16_t rx_prod; 2945 2946 ASSERT_SERIALIZED(&ret->bnx_rx_ret_serialize); 2947 2948 ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag; 2949 cpu_lfence(); 2950 2951 rx_prod = *ret->bnx_rx_considx; 2952 if (ret->bnx_rx_saved_considx != rx_prod) 2953 bnx_rxeof(ret, rx_prod, cycle); 2954 } 2955 2956 static void 2957 bnx_npoll_tx_notag(struct ifnet *ifp __unused, void *xtxr, int cycle __unused) 2958 { 2959 struct bnx_tx_ring *txr = xtxr; 2960 uint16_t tx_cons; 2961 2962 ASSERT_SERIALIZED(&txr->bnx_tx_serialize); 2963 2964 tx_cons = *txr->bnx_tx_considx; 2965 if (txr->bnx_tx_saved_considx != tx_cons) 2966 bnx_txeof(txr, tx_cons); 2967 } 2968 2969 static void 2970 bnx_npoll_tx(struct ifnet *ifp, void *xtxr, int cycle) 2971 { 2972 struct bnx_tx_ring *txr = xtxr; 2973 2974 ASSERT_SERIALIZED(&txr->bnx_tx_serialize); 2975 2976 txr->bnx_saved_status_tag = *txr->bnx_hw_status_tag; 2977 cpu_lfence(); 2978 bnx_npoll_tx_notag(ifp, txr, cycle); 2979 } 2980 2981 static void 2982 bnx_npoll_status_notag(struct ifnet *ifp) 2983 { 2984 struct bnx_softc *sc = ifp->if_softc; 2985 2986 ASSERT_SERIALIZED(&sc->bnx_main_serialize); 2987 2988 if (bnx_handle_status(sc)) { 2989 /* 2990 * Status changes are handled; force the chip to 2991 * update the status block to reflect whether there 2992 * are more status changes or not, else staled status 2993 * changes are always seen. 2994 */ 2995 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW); 2996 } 2997 } 2998 2999 static void 3000 bnx_npoll_status(struct ifnet *ifp) 3001 { 3002 struct bnx_softc *sc = ifp->if_softc; 3003 3004 ASSERT_SERIALIZED(&sc->bnx_main_serialize); 3005 3006 sc->bnx_saved_status_tag = *sc->bnx_hw_status_tag; 3007 cpu_lfence(); 3008 bnx_npoll_status_notag(ifp); 3009 } 3010 3011 static void 3012 bnx_npoll(struct ifnet *ifp, struct ifpoll_info *info) 3013 { 3014 struct bnx_softc *sc = ifp->if_softc; 3015 int i; 3016 3017 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3018 3019 if (info != NULL) { 3020 if (sc->bnx_flags & BNX_FLAG_STATUS_HASTAG) 3021 info->ifpi_status.status_func = bnx_npoll_status; 3022 else 3023 info->ifpi_status.status_func = bnx_npoll_status_notag; 3024 info->ifpi_status.serializer = &sc->bnx_main_serialize; 3025 3026 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 3027 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 3028 int idx = i + sc->bnx_npoll_txoff; 3029 3030 KKASSERT(idx < ncpus2); 3031 if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) { 3032 info->ifpi_tx[idx].poll_func = 3033 bnx_npoll_tx_notag; 3034 } else { 3035 info->ifpi_tx[idx].poll_func = bnx_npoll_tx; 3036 } 3037 info->ifpi_tx[idx].arg = txr; 3038 info->ifpi_tx[idx].serializer = &txr->bnx_tx_serialize; 3039 ifsq_set_cpuid(txr->bnx_ifsq, idx); 3040 } 3041 3042 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 3043 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i]; 3044 int idx = i + sc->bnx_npoll_rxoff; 3045 3046 KKASSERT(idx < ncpus2); 3047 info->ifpi_rx[idx].poll_func = bnx_npoll_rx; 3048 info->ifpi_rx[idx].arg = ret; 3049 info->ifpi_rx[idx].serializer = 3050 &ret->bnx_rx_ret_serialize; 3051 } 3052 3053 if (ifp->if_flags & IFF_RUNNING) { 3054 bnx_disable_intr(sc); 3055 bnx_set_tick_cpuid(sc, TRUE); 3056 3057 sc->bnx_coal_chg = BNX_TX_COAL_BDS_CHG | 3058 BNX_RX_COAL_BDS_CHG; 3059 bnx_coal_change(sc); 3060 } 3061 } else { 3062 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 3063 ifsq_set_cpuid(sc->bnx_tx_ring[i].bnx_ifsq, 3064 sc->bnx_tx_ring[i].bnx_tx_cpuid); 3065 } 3066 if (ifp->if_flags & IFF_RUNNING) { 3067 sc->bnx_coal_chg = BNX_TX_COAL_BDS_CHG | 3068 BNX_RX_COAL_BDS_CHG; 3069 bnx_coal_change(sc); 3070 3071 bnx_enable_intr(sc); 3072 bnx_set_tick_cpuid(sc, FALSE); 3073 } 3074 } 3075 } 3076 3077 #endif /* IFPOLL_ENABLE */ 3078 3079 static void 3080 bnx_intr_legacy(void *xsc) 3081 { 3082 struct bnx_softc *sc = xsc; 3083 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0]; 3084 3085 if (ret->bnx_saved_status_tag == *ret->bnx_hw_status_tag) { 3086 uint32_t val; 3087 3088 val = pci_read_config(sc->bnx_dev, BGE_PCI_PCISTATE, 4); 3089 if (val & BGE_PCISTAT_INTR_NOTACT) 3090 return; 3091 } 3092 3093 /* 3094 * NOTE: 3095 * Interrupt will have to be disabled if tagged status 3096 * is used, else interrupt will always be asserted on 3097 * certain chips (at least on BCM5750 AX/BX). 3098 */ 3099 bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1); 3100 3101 bnx_intr(sc); 3102 } 3103 3104 static void 3105 bnx_msi(void *xsc) 3106 { 3107 bnx_intr(xsc); 3108 } 3109 3110 static void 3111 bnx_intr(struct bnx_softc *sc) 3112 { 3113 struct ifnet *ifp = &sc->arpcom.ac_if; 3114 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0]; 3115 3116 ASSERT_SERIALIZED(&sc->bnx_main_serialize); 3117 3118 ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag; 3119 /* 3120 * Use a load fence to ensure that status_tag is saved 3121 * before rx_prod, tx_cons and status. 3122 */ 3123 cpu_lfence(); 3124 3125 bnx_handle_status(sc); 3126 3127 if (ifp->if_flags & IFF_RUNNING) { 3128 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; 3129 uint16_t rx_prod, tx_cons; 3130 3131 lwkt_serialize_enter(&ret->bnx_rx_ret_serialize); 3132 rx_prod = *ret->bnx_rx_considx; 3133 if (ret->bnx_rx_saved_considx != rx_prod) 3134 bnx_rxeof(ret, rx_prod, -1); 3135 lwkt_serialize_exit(&ret->bnx_rx_ret_serialize); 3136 3137 lwkt_serialize_enter(&txr->bnx_tx_serialize); 3138 tx_cons = *txr->bnx_tx_considx; 3139 if (txr->bnx_tx_saved_considx != tx_cons) 3140 bnx_txeof(txr, tx_cons); 3141 lwkt_serialize_exit(&txr->bnx_tx_serialize); 3142 } 3143 3144 bnx_writembx(sc, BGE_MBX_IRQ0_LO, ret->bnx_saved_status_tag << 24); 3145 } 3146 3147 static void 3148 bnx_msix_tx_status(void *xtxr) 3149 { 3150 struct bnx_tx_ring *txr = xtxr; 3151 struct bnx_softc *sc = txr->bnx_sc; 3152 struct ifnet *ifp = &sc->arpcom.ac_if; 3153 3154 ASSERT_SERIALIZED(&sc->bnx_main_serialize); 3155 3156 txr->bnx_saved_status_tag = *txr->bnx_hw_status_tag; 3157 /* 3158 * Use a load fence to ensure that status_tag is saved 3159 * before tx_cons and status. 3160 */ 3161 cpu_lfence(); 3162 3163 bnx_handle_status(sc); 3164 3165 if (ifp->if_flags & IFF_RUNNING) { 3166 uint16_t tx_cons; 3167 3168 lwkt_serialize_enter(&txr->bnx_tx_serialize); 3169 tx_cons = *txr->bnx_tx_considx; 3170 if (txr->bnx_tx_saved_considx != tx_cons) 3171 bnx_txeof(txr, tx_cons); 3172 lwkt_serialize_exit(&txr->bnx_tx_serialize); 3173 } 3174 3175 bnx_writembx(sc, BGE_MBX_IRQ0_LO, txr->bnx_saved_status_tag << 24); 3176 } 3177 3178 static void 3179 bnx_msix_rx(void *xret) 3180 { 3181 struct bnx_rx_ret_ring *ret = xret; 3182 uint16_t rx_prod; 3183 3184 ASSERT_SERIALIZED(&ret->bnx_rx_ret_serialize); 3185 3186 ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag; 3187 /* 3188 * Use a load fence to ensure that status_tag is saved 3189 * before rx_prod. 3190 */ 3191 cpu_lfence(); 3192 3193 rx_prod = *ret->bnx_rx_considx; 3194 if (ret->bnx_rx_saved_considx != rx_prod) 3195 bnx_rxeof(ret, rx_prod, -1); 3196 3197 bnx_writembx(ret->bnx_sc, ret->bnx_msix_mbx, 3198 ret->bnx_saved_status_tag << 24); 3199 } 3200 3201 static void 3202 bnx_msix_rxtx(void *xret) 3203 { 3204 struct bnx_rx_ret_ring *ret = xret; 3205 struct bnx_tx_ring *txr = ret->bnx_txr; 3206 uint16_t rx_prod, tx_cons; 3207 3208 ASSERT_SERIALIZED(&ret->bnx_rx_ret_serialize); 3209 3210 ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag; 3211 /* 3212 * Use a load fence to ensure that status_tag is saved 3213 * before rx_prod and tx_cons. 3214 */ 3215 cpu_lfence(); 3216 3217 rx_prod = *ret->bnx_rx_considx; 3218 if (ret->bnx_rx_saved_considx != rx_prod) 3219 bnx_rxeof(ret, rx_prod, -1); 3220 3221 lwkt_serialize_enter(&txr->bnx_tx_serialize); 3222 tx_cons = *txr->bnx_tx_considx; 3223 if (txr->bnx_tx_saved_considx != tx_cons) 3224 bnx_txeof(txr, tx_cons); 3225 lwkt_serialize_exit(&txr->bnx_tx_serialize); 3226 3227 bnx_writembx(ret->bnx_sc, ret->bnx_msix_mbx, 3228 ret->bnx_saved_status_tag << 24); 3229 } 3230 3231 static void 3232 bnx_msix_status(void *xsc) 3233 { 3234 struct bnx_softc *sc = xsc; 3235 3236 ASSERT_SERIALIZED(&sc->bnx_main_serialize); 3237 3238 sc->bnx_saved_status_tag = *sc->bnx_hw_status_tag; 3239 /* 3240 * Use a load fence to ensure that status_tag is saved 3241 * before status. 3242 */ 3243 cpu_lfence(); 3244 3245 bnx_handle_status(sc); 3246 3247 bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_saved_status_tag << 24); 3248 } 3249 3250 static void 3251 bnx_tick(void *xsc) 3252 { 3253 struct bnx_softc *sc = xsc; 3254 3255 lwkt_serialize_enter(&sc->bnx_main_serialize); 3256 3257 bnx_stats_update_regs(sc); 3258 3259 if (sc->bnx_flags & BNX_FLAG_TBI) { 3260 /* 3261 * Since in TBI mode auto-polling can't be used we should poll 3262 * link status manually. Here we register pending link event 3263 * and trigger interrupt. 3264 */ 3265 sc->bnx_link_evt++; 3266 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW); 3267 } else if (!sc->bnx_link) { 3268 mii_tick(device_get_softc(sc->bnx_miibus)); 3269 } 3270 3271 callout_reset_bycpu(&sc->bnx_tick_timer, hz, bnx_tick, sc, 3272 sc->bnx_tick_cpuid); 3273 3274 lwkt_serialize_exit(&sc->bnx_main_serialize); 3275 } 3276 3277 static void 3278 bnx_stats_update_regs(struct bnx_softc *sc) 3279 { 3280 struct ifnet *ifp = &sc->arpcom.ac_if; 3281 struct bge_mac_stats_regs stats; 3282 uint32_t *s, val; 3283 int i; 3284 3285 s = (uint32_t *)&stats; 3286 for (i = 0; i < sizeof(struct bge_mac_stats_regs); i += 4) { 3287 *s = CSR_READ_4(sc, BGE_RX_STATS + i); 3288 s++; 3289 } 3290 3291 IFNET_STAT_SET(ifp, collisions, 3292 (stats.dot3StatsSingleCollisionFrames + 3293 stats.dot3StatsMultipleCollisionFrames + 3294 stats.dot3StatsExcessiveCollisions + 3295 stats.dot3StatsLateCollisions)); 3296 3297 val = CSR_READ_4(sc, BGE_RXLP_LOCSTAT_OUT_OF_BDS); 3298 sc->bnx_norxbds += val; 3299 } 3300 3301 /* 3302 * Encapsulate an mbuf chain in the tx ring by coupling the mbuf data 3303 * pointers to descriptors. 3304 */ 3305 static int 3306 bnx_encap(struct bnx_tx_ring *txr, struct mbuf **m_head0, uint32_t *txidx, 3307 int *segs_used) 3308 { 3309 struct bge_tx_bd *d = NULL; 3310 uint16_t csum_flags = 0, vlan_tag = 0, mss = 0; 3311 bus_dma_segment_t segs[BNX_NSEG_NEW]; 3312 bus_dmamap_t map; 3313 int error, maxsegs, nsegs, idx, i; 3314 struct mbuf *m_head = *m_head0, *m_new; 3315 3316 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 3317 #ifdef BNX_TSO_DEBUG 3318 int tso_nsegs; 3319 #endif 3320 3321 error = bnx_setup_tso(txr, m_head0, &mss, &csum_flags); 3322 if (error) 3323 return error; 3324 m_head = *m_head0; 3325 3326 #ifdef BNX_TSO_DEBUG 3327 tso_nsegs = (m_head->m_pkthdr.len / 3328 m_head->m_pkthdr.tso_segsz) - 1; 3329 if (tso_nsegs > (BNX_TSO_NSTATS - 1)) 3330 tso_nsegs = BNX_TSO_NSTATS - 1; 3331 else if (tso_nsegs < 0) 3332 tso_nsegs = 0; 3333 txr->bnx_sc->bnx_tsosegs[tso_nsegs]++; 3334 #endif 3335 } else if (m_head->m_pkthdr.csum_flags & BNX_CSUM_FEATURES) { 3336 if (m_head->m_pkthdr.csum_flags & CSUM_IP) 3337 csum_flags |= BGE_TXBDFLAG_IP_CSUM; 3338 if (m_head->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)) 3339 csum_flags |= BGE_TXBDFLAG_TCP_UDP_CSUM; 3340 if (m_head->m_flags & M_LASTFRAG) 3341 csum_flags |= BGE_TXBDFLAG_IP_FRAG_END; 3342 else if (m_head->m_flags & M_FRAG) 3343 csum_flags |= BGE_TXBDFLAG_IP_FRAG; 3344 } 3345 if (m_head->m_flags & M_VLANTAG) { 3346 csum_flags |= BGE_TXBDFLAG_VLAN_TAG; 3347 vlan_tag = m_head->m_pkthdr.ether_vlantag; 3348 } 3349 3350 idx = *txidx; 3351 map = txr->bnx_tx_buf[idx].bnx_tx_dmamap; 3352 3353 maxsegs = (BGE_TX_RING_CNT - txr->bnx_tx_cnt) - BNX_NSEG_RSVD; 3354 KASSERT(maxsegs >= BNX_NSEG_SPARE, 3355 ("not enough segments %d", maxsegs)); 3356 3357 if (maxsegs > BNX_NSEG_NEW) 3358 maxsegs = BNX_NSEG_NEW; 3359 3360 /* 3361 * Pad outbound frame to BGE_MIN_FRAMELEN for an unusual reason. 3362 * The bge hardware will pad out Tx runts to BGE_MIN_FRAMELEN, 3363 * but when such padded frames employ the bge IP/TCP checksum 3364 * offload, the hardware checksum assist gives incorrect results 3365 * (possibly from incorporating its own padding into the UDP/TCP 3366 * checksum; who knows). If we pad such runts with zeros, the 3367 * onboard checksum comes out correct. 3368 */ 3369 if ((csum_flags & BGE_TXBDFLAG_TCP_UDP_CSUM) && 3370 m_head->m_pkthdr.len < BNX_MIN_FRAMELEN) { 3371 error = m_devpad(m_head, BNX_MIN_FRAMELEN); 3372 if (error) 3373 goto back; 3374 } 3375 3376 if ((txr->bnx_tx_flags & BNX_TX_FLAG_SHORTDMA) && 3377 m_head->m_next != NULL) { 3378 m_new = bnx_defrag_shortdma(m_head); 3379 if (m_new == NULL) { 3380 error = ENOBUFS; 3381 goto back; 3382 } 3383 *m_head0 = m_head = m_new; 3384 } 3385 if ((m_head->m_pkthdr.csum_flags & CSUM_TSO) == 0 && 3386 (txr->bnx_tx_flags & BNX_TX_FLAG_FORCE_DEFRAG) && 3387 m_head->m_next != NULL) { 3388 /* 3389 * Forcefully defragment mbuf chain to overcome hardware 3390 * limitation which only support a single outstanding 3391 * DMA read operation. If it fails, keep moving on using 3392 * the original mbuf chain. 3393 */ 3394 m_new = m_defrag(m_head, MB_DONTWAIT); 3395 if (m_new != NULL) 3396 *m_head0 = m_head = m_new; 3397 } 3398 3399 error = bus_dmamap_load_mbuf_defrag(txr->bnx_tx_mtag, map, 3400 m_head0, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 3401 if (error) 3402 goto back; 3403 *segs_used += nsegs; 3404 3405 m_head = *m_head0; 3406 bus_dmamap_sync(txr->bnx_tx_mtag, map, BUS_DMASYNC_PREWRITE); 3407 3408 for (i = 0; ; i++) { 3409 d = &txr->bnx_tx_ring[idx]; 3410 3411 d->bge_addr.bge_addr_lo = BGE_ADDR_LO(segs[i].ds_addr); 3412 d->bge_addr.bge_addr_hi = BGE_ADDR_HI(segs[i].ds_addr); 3413 d->bge_len = segs[i].ds_len; 3414 d->bge_flags = csum_flags; 3415 d->bge_vlan_tag = vlan_tag; 3416 d->bge_mss = mss; 3417 3418 if (i == nsegs - 1) 3419 break; 3420 BNX_INC(idx, BGE_TX_RING_CNT); 3421 } 3422 /* Mark the last segment as end of packet... */ 3423 d->bge_flags |= BGE_TXBDFLAG_END; 3424 3425 /* 3426 * Insure that the map for this transmission is placed at 3427 * the array index of the last descriptor in this chain. 3428 */ 3429 txr->bnx_tx_buf[*txidx].bnx_tx_dmamap = txr->bnx_tx_buf[idx].bnx_tx_dmamap; 3430 txr->bnx_tx_buf[idx].bnx_tx_dmamap = map; 3431 txr->bnx_tx_buf[idx].bnx_tx_mbuf = m_head; 3432 txr->bnx_tx_cnt += nsegs; 3433 3434 BNX_INC(idx, BGE_TX_RING_CNT); 3435 *txidx = idx; 3436 back: 3437 if (error) { 3438 m_freem(*m_head0); 3439 *m_head0 = NULL; 3440 } 3441 return error; 3442 } 3443 3444 /* 3445 * Main transmit routine. To avoid having to do mbuf copies, we put pointers 3446 * to the mbuf data regions directly in the transmit descriptors. 3447 */ 3448 static void 3449 bnx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 3450 { 3451 struct bnx_tx_ring *txr = ifsq_get_priv(ifsq); 3452 struct mbuf *m_head = NULL; 3453 uint32_t prodidx; 3454 int nsegs = 0; 3455 3456 KKASSERT(txr->bnx_ifsq == ifsq); 3457 ASSERT_SERIALIZED(&txr->bnx_tx_serialize); 3458 3459 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq)) 3460 return; 3461 3462 prodidx = txr->bnx_tx_prodidx; 3463 3464 while (txr->bnx_tx_buf[prodidx].bnx_tx_mbuf == NULL) { 3465 /* 3466 * Sanity check: avoid coming within BGE_NSEG_RSVD 3467 * descriptors of the end of the ring. Also make 3468 * sure there are BGE_NSEG_SPARE descriptors for 3469 * jumbo buffers' or TSO segments' defragmentation. 3470 */ 3471 if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) < 3472 (BNX_NSEG_RSVD + BNX_NSEG_SPARE)) { 3473 ifsq_set_oactive(ifsq); 3474 break; 3475 } 3476 3477 m_head = ifsq_dequeue(ifsq); 3478 if (m_head == NULL) 3479 break; 3480 3481 /* 3482 * Pack the data into the transmit ring. If we 3483 * don't have room, set the OACTIVE flag and wait 3484 * for the NIC to drain the ring. 3485 */ 3486 if (bnx_encap(txr, &m_head, &prodidx, &nsegs)) { 3487 ifsq_set_oactive(ifsq); 3488 IFNET_STAT_INC(ifp, oerrors, 1); 3489 break; 3490 } 3491 3492 if (nsegs >= txr->bnx_tx_wreg) { 3493 /* Transmit */ 3494 bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, prodidx); 3495 nsegs = 0; 3496 } 3497 3498 ETHER_BPF_MTAP(ifp, m_head); 3499 3500 /* 3501 * Set a timeout in case the chip goes out to lunch. 3502 */ 3503 txr->bnx_tx_watchdog.wd_timer = 5; 3504 } 3505 3506 if (nsegs > 0) { 3507 /* Transmit */ 3508 bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, prodidx); 3509 } 3510 txr->bnx_tx_prodidx = prodidx; 3511 } 3512 3513 static void 3514 bnx_init(void *xsc) 3515 { 3516 struct bnx_softc *sc = xsc; 3517 struct ifnet *ifp = &sc->arpcom.ac_if; 3518 uint16_t *m; 3519 uint32_t mode; 3520 int i; 3521 boolean_t polling; 3522 3523 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3524 3525 /* Cancel pending I/O and flush buffers. */ 3526 bnx_stop(sc); 3527 bnx_reset(sc); 3528 bnx_chipinit(sc); 3529 3530 /* 3531 * Init the various state machines, ring 3532 * control blocks and firmware. 3533 */ 3534 if (bnx_blockinit(sc)) { 3535 if_printf(ifp, "initialization failure\n"); 3536 bnx_stop(sc); 3537 return; 3538 } 3539 3540 /* Specify MTU. */ 3541 CSR_WRITE_4(sc, BGE_RX_MTU, ifp->if_mtu + 3542 ETHER_HDR_LEN + ETHER_CRC_LEN + EVL_ENCAPLEN); 3543 3544 /* Load our MAC address. */ 3545 m = (uint16_t *)&sc->arpcom.ac_enaddr[0]; 3546 CSR_WRITE_4(sc, BGE_MAC_ADDR1_LO, htons(m[0])); 3547 CSR_WRITE_4(sc, BGE_MAC_ADDR1_HI, (htons(m[1]) << 16) | htons(m[2])); 3548 3549 /* Enable or disable promiscuous mode as needed. */ 3550 bnx_setpromisc(sc); 3551 3552 /* Program multicast filter. */ 3553 bnx_setmulti(sc); 3554 3555 /* Init RX ring. */ 3556 if (bnx_init_rx_ring_std(&sc->bnx_rx_std_ring)) { 3557 if_printf(ifp, "RX ring initialization failed\n"); 3558 bnx_stop(sc); 3559 return; 3560 } 3561 3562 /* Init jumbo RX ring. */ 3563 if (ifp->if_mtu > (ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN)) { 3564 if (bnx_init_rx_ring_jumbo(sc)) { 3565 if_printf(ifp, "Jumbo RX ring initialization failed\n"); 3566 bnx_stop(sc); 3567 return; 3568 } 3569 } 3570 3571 /* Init our RX return ring index */ 3572 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 3573 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i]; 3574 3575 ret->bnx_rx_saved_considx = 0; 3576 ret->bnx_rx_cnt = 0; 3577 } 3578 3579 /* Init TX ring. */ 3580 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) 3581 bnx_init_tx_ring(&sc->bnx_tx_ring[i]); 3582 3583 /* Enable TX MAC state machine lockup fix. */ 3584 mode = CSR_READ_4(sc, BGE_TX_MODE); 3585 mode |= BGE_TXMODE_MBUF_LOCKUP_FIX; 3586 if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 || 3587 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 3588 mode &= ~(BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE); 3589 mode |= CSR_READ_4(sc, BGE_TX_MODE) & 3590 (BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE); 3591 } 3592 /* Turn on transmitter */ 3593 CSR_WRITE_4(sc, BGE_TX_MODE, mode | BGE_TXMODE_ENABLE); 3594 3595 /* Initialize RSS */ 3596 mode = BGE_RXMODE_ENABLE; 3597 if (BNX_RSS_ENABLED(sc)) { 3598 bnx_init_rss(sc); 3599 mode |= BGE_RXMODE_RSS_ENABLE | 3600 BGE_RXMODE_RSS_HASH_MASK_BITS | 3601 BGE_RXMODE_RSS_IPV4_HASH | 3602 BGE_RXMODE_RSS_TCP_IPV4_HASH; 3603 } 3604 /* Turn on receiver */ 3605 BNX_SETBIT(sc, BGE_RX_MODE, mode); 3606 3607 /* 3608 * Set the number of good frames to receive after RX MBUF 3609 * Low Watermark has been reached. After the RX MAC receives 3610 * this number of frames, it will drop subsequent incoming 3611 * frames until the MBUF High Watermark is reached. 3612 */ 3613 if (BNX_IS_57765_FAMILY(sc)) 3614 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 1); 3615 else 3616 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 2); 3617 3618 if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI || 3619 sc->bnx_intr_type == PCI_INTR_TYPE_MSIX) { 3620 if (bootverbose) { 3621 if_printf(ifp, "MSI_MODE: %#x\n", 3622 CSR_READ_4(sc, BGE_MSI_MODE)); 3623 } 3624 } 3625 3626 /* Tell firmware we're alive. */ 3627 BNX_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP); 3628 3629 /* Enable host interrupts if polling(4) is not enabled. */ 3630 PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_CLEAR_INTA, 4); 3631 3632 polling = FALSE; 3633 #ifdef IFPOLL_ENABLE 3634 if (ifp->if_flags & IFF_NPOLLING) 3635 polling = TRUE; 3636 #endif 3637 if (polling) 3638 bnx_disable_intr(sc); 3639 else 3640 bnx_enable_intr(sc); 3641 bnx_set_tick_cpuid(sc, polling); 3642 3643 bnx_ifmedia_upd(ifp); 3644 3645 ifp->if_flags |= IFF_RUNNING; 3646 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 3647 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 3648 3649 ifsq_clr_oactive(txr->bnx_ifsq); 3650 ifsq_watchdog_start(&txr->bnx_tx_watchdog); 3651 } 3652 3653 callout_reset_bycpu(&sc->bnx_tick_timer, hz, bnx_tick, sc, 3654 sc->bnx_tick_cpuid); 3655 } 3656 3657 /* 3658 * Set media options. 3659 */ 3660 static int 3661 bnx_ifmedia_upd(struct ifnet *ifp) 3662 { 3663 struct bnx_softc *sc = ifp->if_softc; 3664 3665 /* If this is a 1000baseX NIC, enable the TBI port. */ 3666 if (sc->bnx_flags & BNX_FLAG_TBI) { 3667 struct ifmedia *ifm = &sc->bnx_ifmedia; 3668 3669 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3670 return(EINVAL); 3671 3672 switch(IFM_SUBTYPE(ifm->ifm_media)) { 3673 case IFM_AUTO: 3674 break; 3675 3676 case IFM_1000_SX: 3677 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) { 3678 BNX_CLRBIT(sc, BGE_MAC_MODE, 3679 BGE_MACMODE_HALF_DUPLEX); 3680 } else { 3681 BNX_SETBIT(sc, BGE_MAC_MODE, 3682 BGE_MACMODE_HALF_DUPLEX); 3683 } 3684 break; 3685 default: 3686 return(EINVAL); 3687 } 3688 } else { 3689 struct mii_data *mii = device_get_softc(sc->bnx_miibus); 3690 3691 sc->bnx_link_evt++; 3692 sc->bnx_link = 0; 3693 if (mii->mii_instance) { 3694 struct mii_softc *miisc; 3695 3696 LIST_FOREACH(miisc, &mii->mii_phys, mii_list) 3697 mii_phy_reset(miisc); 3698 } 3699 mii_mediachg(mii); 3700 3701 /* 3702 * Force an interrupt so that we will call bnx_link_upd 3703 * if needed and clear any pending link state attention. 3704 * Without this we are not getting any further interrupts 3705 * for link state changes and thus will not UP the link and 3706 * not be able to send in bnx_start. The only way to get 3707 * things working was to receive a packet and get an RX 3708 * intr. 3709 * 3710 * bnx_tick should help for fiber cards and we might not 3711 * need to do this here if BNX_FLAG_TBI is set but as 3712 * we poll for fiber anyway it should not harm. 3713 */ 3714 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW); 3715 } 3716 return(0); 3717 } 3718 3719 /* 3720 * Report current media status. 3721 */ 3722 static void 3723 bnx_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) 3724 { 3725 struct bnx_softc *sc = ifp->if_softc; 3726 3727 if (sc->bnx_flags & BNX_FLAG_TBI) { 3728 ifmr->ifm_status = IFM_AVALID; 3729 ifmr->ifm_active = IFM_ETHER; 3730 if (CSR_READ_4(sc, BGE_MAC_STS) & 3731 BGE_MACSTAT_TBI_PCS_SYNCHED) { 3732 ifmr->ifm_status |= IFM_ACTIVE; 3733 } else { 3734 ifmr->ifm_active |= IFM_NONE; 3735 return; 3736 } 3737 3738 ifmr->ifm_active |= IFM_1000_SX; 3739 if (CSR_READ_4(sc, BGE_MAC_MODE) & BGE_MACMODE_HALF_DUPLEX) 3740 ifmr->ifm_active |= IFM_HDX; 3741 else 3742 ifmr->ifm_active |= IFM_FDX; 3743 } else { 3744 struct mii_data *mii = device_get_softc(sc->bnx_miibus); 3745 3746 mii_pollstat(mii); 3747 ifmr->ifm_active = mii->mii_media_active; 3748 ifmr->ifm_status = mii->mii_media_status; 3749 } 3750 } 3751 3752 static int 3753 bnx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) 3754 { 3755 struct bnx_softc *sc = ifp->if_softc; 3756 struct ifreq *ifr = (struct ifreq *)data; 3757 int mask, error = 0; 3758 3759 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3760 3761 switch (command) { 3762 case SIOCSIFMTU: 3763 if ((!BNX_IS_JUMBO_CAPABLE(sc) && ifr->ifr_mtu > ETHERMTU) || 3764 (BNX_IS_JUMBO_CAPABLE(sc) && 3765 ifr->ifr_mtu > BNX_JUMBO_MTU)) { 3766 error = EINVAL; 3767 } else if (ifp->if_mtu != ifr->ifr_mtu) { 3768 ifp->if_mtu = ifr->ifr_mtu; 3769 if (ifp->if_flags & IFF_RUNNING) 3770 bnx_init(sc); 3771 } 3772 break; 3773 case SIOCSIFFLAGS: 3774 if (ifp->if_flags & IFF_UP) { 3775 if (ifp->if_flags & IFF_RUNNING) { 3776 mask = ifp->if_flags ^ sc->bnx_if_flags; 3777 3778 /* 3779 * If only the state of the PROMISC flag 3780 * changed, then just use the 'set promisc 3781 * mode' command instead of reinitializing 3782 * the entire NIC. Doing a full re-init 3783 * means reloading the firmware and waiting 3784 * for it to start up, which may take a 3785 * second or two. Similarly for ALLMULTI. 3786 */ 3787 if (mask & IFF_PROMISC) 3788 bnx_setpromisc(sc); 3789 if (mask & IFF_ALLMULTI) 3790 bnx_setmulti(sc); 3791 } else { 3792 bnx_init(sc); 3793 } 3794 } else if (ifp->if_flags & IFF_RUNNING) { 3795 bnx_stop(sc); 3796 } 3797 sc->bnx_if_flags = ifp->if_flags; 3798 break; 3799 case SIOCADDMULTI: 3800 case SIOCDELMULTI: 3801 if (ifp->if_flags & IFF_RUNNING) 3802 bnx_setmulti(sc); 3803 break; 3804 case SIOCSIFMEDIA: 3805 case SIOCGIFMEDIA: 3806 if (sc->bnx_flags & BNX_FLAG_TBI) { 3807 error = ifmedia_ioctl(ifp, ifr, 3808 &sc->bnx_ifmedia, command); 3809 } else { 3810 struct mii_data *mii; 3811 3812 mii = device_get_softc(sc->bnx_miibus); 3813 error = ifmedia_ioctl(ifp, ifr, 3814 &mii->mii_media, command); 3815 } 3816 break; 3817 case SIOCSIFCAP: 3818 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 3819 if (mask & IFCAP_HWCSUM) { 3820 ifp->if_capenable ^= (mask & IFCAP_HWCSUM); 3821 if (ifp->if_capenable & IFCAP_TXCSUM) 3822 ifp->if_hwassist |= BNX_CSUM_FEATURES; 3823 else 3824 ifp->if_hwassist &= ~BNX_CSUM_FEATURES; 3825 } 3826 if (mask & IFCAP_TSO) { 3827 ifp->if_capenable ^= (mask & IFCAP_TSO); 3828 if (ifp->if_capenable & IFCAP_TSO) 3829 ifp->if_hwassist |= CSUM_TSO; 3830 else 3831 ifp->if_hwassist &= ~CSUM_TSO; 3832 } 3833 if (mask & IFCAP_RSS) 3834 ifp->if_capenable ^= IFCAP_RSS; 3835 break; 3836 default: 3837 error = ether_ioctl(ifp, command, data); 3838 break; 3839 } 3840 return error; 3841 } 3842 3843 static void 3844 bnx_watchdog(struct ifaltq_subque *ifsq) 3845 { 3846 struct ifnet *ifp = ifsq_get_ifp(ifsq); 3847 struct bnx_softc *sc = ifp->if_softc; 3848 int i; 3849 3850 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3851 3852 if_printf(ifp, "watchdog timeout -- resetting\n"); 3853 3854 bnx_init(sc); 3855 3856 IFNET_STAT_INC(ifp, oerrors, 1); 3857 3858 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) 3859 ifsq_devstart_sched(sc->bnx_tx_ring[i].bnx_ifsq); 3860 } 3861 3862 /* 3863 * Stop the adapter and free any mbufs allocated to the 3864 * RX and TX lists. 3865 */ 3866 static void 3867 bnx_stop(struct bnx_softc *sc) 3868 { 3869 struct ifnet *ifp = &sc->arpcom.ac_if; 3870 int i; 3871 3872 ASSERT_IFNET_SERIALIZED_ALL(ifp); 3873 3874 callout_stop(&sc->bnx_tick_timer); 3875 3876 /* 3877 * Disable all of the receiver blocks 3878 */ 3879 bnx_stop_block(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE); 3880 bnx_stop_block(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE); 3881 bnx_stop_block(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE); 3882 bnx_stop_block(sc, BGE_RDBDI_MODE, BGE_RBDIMODE_ENABLE); 3883 bnx_stop_block(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE); 3884 bnx_stop_block(sc, BGE_RBDC_MODE, BGE_RBDCMODE_ENABLE); 3885 3886 /* 3887 * Disable all of the transmit blocks 3888 */ 3889 bnx_stop_block(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE); 3890 bnx_stop_block(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE); 3891 bnx_stop_block(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE); 3892 bnx_stop_block(sc, BGE_RDMA_MODE, BGE_RDMAMODE_ENABLE); 3893 bnx_stop_block(sc, BGE_SDC_MODE, BGE_SDCMODE_ENABLE); 3894 bnx_stop_block(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE); 3895 3896 /* 3897 * Shut down all of the memory managers and related 3898 * state machines. 3899 */ 3900 bnx_stop_block(sc, BGE_HCC_MODE, BGE_HCCMODE_ENABLE); 3901 bnx_stop_block(sc, BGE_WDMA_MODE, BGE_WDMAMODE_ENABLE); 3902 CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF); 3903 CSR_WRITE_4(sc, BGE_FTQ_RESET, 0); 3904 3905 /* Disable host interrupts. */ 3906 bnx_disable_intr(sc); 3907 3908 /* 3909 * Tell firmware we're shutting down. 3910 */ 3911 BNX_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP); 3912 3913 /* Free the RX lists. */ 3914 bnx_free_rx_ring_std(&sc->bnx_rx_std_ring); 3915 3916 /* Free jumbo RX list. */ 3917 if (BNX_IS_JUMBO_CAPABLE(sc)) 3918 bnx_free_rx_ring_jumbo(sc); 3919 3920 /* Free TX buffers. */ 3921 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 3922 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 3923 3924 txr->bnx_saved_status_tag = 0; 3925 bnx_free_tx_ring(txr); 3926 } 3927 3928 /* Clear saved status tag */ 3929 for (i = 0; i < sc->bnx_rx_retcnt; ++i) 3930 sc->bnx_rx_ret_ring[i].bnx_saved_status_tag = 0; 3931 3932 sc->bnx_link = 0; 3933 sc->bnx_coal_chg = 0; 3934 3935 ifp->if_flags &= ~IFF_RUNNING; 3936 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 3937 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 3938 3939 ifsq_clr_oactive(txr->bnx_ifsq); 3940 ifsq_watchdog_stop(&txr->bnx_tx_watchdog); 3941 } 3942 } 3943 3944 /* 3945 * Stop all chip I/O so that the kernel's probe routines don't 3946 * get confused by errant DMAs when rebooting. 3947 */ 3948 static void 3949 bnx_shutdown(device_t dev) 3950 { 3951 struct bnx_softc *sc = device_get_softc(dev); 3952 struct ifnet *ifp = &sc->arpcom.ac_if; 3953 3954 ifnet_serialize_all(ifp); 3955 bnx_stop(sc); 3956 bnx_reset(sc); 3957 ifnet_deserialize_all(ifp); 3958 } 3959 3960 static int 3961 bnx_suspend(device_t dev) 3962 { 3963 struct bnx_softc *sc = device_get_softc(dev); 3964 struct ifnet *ifp = &sc->arpcom.ac_if; 3965 3966 ifnet_serialize_all(ifp); 3967 bnx_stop(sc); 3968 ifnet_deserialize_all(ifp); 3969 3970 return 0; 3971 } 3972 3973 static int 3974 bnx_resume(device_t dev) 3975 { 3976 struct bnx_softc *sc = device_get_softc(dev); 3977 struct ifnet *ifp = &sc->arpcom.ac_if; 3978 3979 ifnet_serialize_all(ifp); 3980 3981 if (ifp->if_flags & IFF_UP) { 3982 int i; 3983 3984 bnx_init(sc); 3985 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) 3986 ifsq_devstart_sched(sc->bnx_tx_ring[i].bnx_ifsq); 3987 } 3988 3989 ifnet_deserialize_all(ifp); 3990 3991 return 0; 3992 } 3993 3994 static void 3995 bnx_setpromisc(struct bnx_softc *sc) 3996 { 3997 struct ifnet *ifp = &sc->arpcom.ac_if; 3998 3999 if (ifp->if_flags & IFF_PROMISC) 4000 BNX_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC); 4001 else 4002 BNX_CLRBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC); 4003 } 4004 4005 static void 4006 bnx_dma_free(struct bnx_softc *sc) 4007 { 4008 struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring; 4009 int i; 4010 4011 /* Destroy RX return rings */ 4012 if (sc->bnx_rx_ret_ring != NULL) { 4013 for (i = 0; i < sc->bnx_rx_retcnt; ++i) 4014 bnx_destroy_rx_ret_ring(&sc->bnx_rx_ret_ring[i]); 4015 kfree(sc->bnx_rx_ret_ring, M_DEVBUF); 4016 } 4017 4018 /* Destroy RX mbuf DMA stuffs. */ 4019 if (std->bnx_rx_mtag != NULL) { 4020 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) { 4021 KKASSERT(std->bnx_rx_std_buf[i].bnx_rx_mbuf == NULL); 4022 bus_dmamap_destroy(std->bnx_rx_mtag, 4023 std->bnx_rx_std_buf[i].bnx_rx_dmamap); 4024 } 4025 bus_dma_tag_destroy(std->bnx_rx_mtag); 4026 } 4027 4028 /* Destroy standard RX ring */ 4029 bnx_dma_block_free(std->bnx_rx_std_ring_tag, 4030 std->bnx_rx_std_ring_map, std->bnx_rx_std_ring); 4031 4032 /* Destroy TX rings */ 4033 if (sc->bnx_tx_ring != NULL) { 4034 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) 4035 bnx_destroy_tx_ring(&sc->bnx_tx_ring[i]); 4036 kfree(sc->bnx_tx_ring, M_DEVBUF); 4037 } 4038 4039 if (BNX_IS_JUMBO_CAPABLE(sc)) 4040 bnx_free_jumbo_mem(sc); 4041 4042 /* Destroy status blocks */ 4043 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4044 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 4045 4046 bnx_dma_block_free(intr->bnx_status_tag, 4047 intr->bnx_status_map, intr->bnx_status_block); 4048 } 4049 4050 /* Destroy the parent tag */ 4051 if (sc->bnx_cdata.bnx_parent_tag != NULL) 4052 bus_dma_tag_destroy(sc->bnx_cdata.bnx_parent_tag); 4053 } 4054 4055 static int 4056 bnx_dma_alloc(device_t dev) 4057 { 4058 struct bnx_softc *sc = device_get_softc(dev); 4059 struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring; 4060 int i, error, mbx; 4061 4062 /* 4063 * Allocate the parent bus DMA tag appropriate for PCI. 4064 * 4065 * All of the NetExtreme/NetLink controllers have 4GB boundary 4066 * DMA bug. 4067 * Whenever an address crosses a multiple of the 4GB boundary 4068 * (including 4GB, 8Gb, 12Gb, etc.) and makes the transition 4069 * from 0xX_FFFF_FFFF to 0x(X+1)_0000_0000 an internal DMA 4070 * state machine will lockup and cause the device to hang. 4071 */ 4072 error = bus_dma_tag_create(NULL, 1, BGE_DMA_BOUNDARY_4G, 4073 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 4074 BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 4075 0, &sc->bnx_cdata.bnx_parent_tag); 4076 if (error) { 4077 device_printf(dev, "could not create parent DMA tag\n"); 4078 return error; 4079 } 4080 4081 /* 4082 * Create DMA stuffs for status blocks. 4083 */ 4084 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4085 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 4086 4087 error = bnx_dma_block_alloc(sc, 4088 __VM_CACHELINE_ALIGN(BGE_STATUS_BLK_SZ), 4089 &intr->bnx_status_tag, &intr->bnx_status_map, 4090 (void *)&intr->bnx_status_block, 4091 &intr->bnx_status_block_paddr); 4092 if (error) { 4093 device_printf(dev, 4094 "could not create %dth status block\n", i); 4095 return error; 4096 } 4097 } 4098 sc->bnx_hw_status = &sc->bnx_intr_data[0].bnx_status_block->bge_status; 4099 if (sc->bnx_flags & BNX_FLAG_STATUS_HASTAG) { 4100 sc->bnx_hw_status_tag = 4101 &sc->bnx_intr_data[0].bnx_status_block->bge_status_tag; 4102 } 4103 4104 /* 4105 * Create DMA tag and maps for RX mbufs. 4106 */ 4107 std->bnx_sc = sc; 4108 lwkt_serialize_init(&std->bnx_rx_std_serialize); 4109 error = bus_dma_tag_create(sc->bnx_cdata.bnx_parent_tag, 1, 0, 4110 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 4111 NULL, NULL, MCLBYTES, 1, MCLBYTES, 4112 BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK, &std->bnx_rx_mtag); 4113 if (error) { 4114 device_printf(dev, "could not create RX mbuf DMA tag\n"); 4115 return error; 4116 } 4117 4118 for (i = 0; i < BGE_STD_RX_RING_CNT; ++i) { 4119 error = bus_dmamap_create(std->bnx_rx_mtag, BUS_DMA_WAITOK, 4120 &std->bnx_rx_std_buf[i].bnx_rx_dmamap); 4121 if (error) { 4122 int j; 4123 4124 for (j = 0; j < i; ++j) { 4125 bus_dmamap_destroy(std->bnx_rx_mtag, 4126 std->bnx_rx_std_buf[j].bnx_rx_dmamap); 4127 } 4128 bus_dma_tag_destroy(std->bnx_rx_mtag); 4129 std->bnx_rx_mtag = NULL; 4130 4131 device_printf(dev, 4132 "could not create %dth RX mbuf DMA map\n", i); 4133 return error; 4134 } 4135 } 4136 4137 /* 4138 * Create DMA stuffs for standard RX ring. 4139 */ 4140 error = bnx_dma_block_alloc(sc, BGE_STD_RX_RING_SZ, 4141 &std->bnx_rx_std_ring_tag, 4142 &std->bnx_rx_std_ring_map, 4143 (void *)&std->bnx_rx_std_ring, 4144 &std->bnx_rx_std_ring_paddr); 4145 if (error) { 4146 device_printf(dev, "could not create std RX ring\n"); 4147 return error; 4148 } 4149 4150 /* 4151 * Create RX return rings 4152 */ 4153 mbx = BGE_MBX_RX_CONS0_LO; 4154 sc->bnx_rx_ret_ring = kmalloc_cachealign( 4155 sizeof(struct bnx_rx_ret_ring) * sc->bnx_rx_retcnt, M_DEVBUF, 4156 M_WAITOK | M_ZERO); 4157 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 4158 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i]; 4159 struct bnx_intr_data *intr; 4160 4161 ret->bnx_sc = sc; 4162 ret->bnx_std = std; 4163 ret->bnx_rx_mbx = mbx; 4164 ret->bnx_rx_cntmax = (BGE_STD_RX_RING_CNT / 4) / 4165 sc->bnx_rx_retcnt; 4166 ret->bnx_rx_mask = 1 << i; 4167 4168 if (!BNX_RSS_ENABLED(sc)) { 4169 intr = &sc->bnx_intr_data[0]; 4170 } else { 4171 KKASSERT(i + 1 < sc->bnx_intr_cnt); 4172 intr = &sc->bnx_intr_data[i + 1]; 4173 } 4174 4175 if (i == 0) { 4176 ret->bnx_rx_considx = 4177 &intr->bnx_status_block->bge_idx[0].bge_rx_prod_idx; 4178 } else if (i == 1) { 4179 ret->bnx_rx_considx = 4180 &intr->bnx_status_block->bge_rx_jumbo_cons_idx; 4181 } else if (i == 2) { 4182 ret->bnx_rx_considx = 4183 &intr->bnx_status_block->bge_rsvd1; 4184 } else if (i == 3) { 4185 ret->bnx_rx_considx = 4186 &intr->bnx_status_block->bge_rx_mini_cons_idx; 4187 } else { 4188 panic("unknown RX return ring %d\n", i); 4189 } 4190 ret->bnx_hw_status_tag = 4191 &intr->bnx_status_block->bge_status_tag; 4192 4193 error = bnx_create_rx_ret_ring(ret); 4194 if (error) { 4195 device_printf(dev, 4196 "could not create %dth RX ret ring\n", i); 4197 return error; 4198 } 4199 mbx += 8; 4200 } 4201 4202 /* 4203 * Create TX rings 4204 */ 4205 sc->bnx_tx_ring = kmalloc_cachealign( 4206 sizeof(struct bnx_tx_ring) * sc->bnx_tx_ringcnt, M_DEVBUF, 4207 M_WAITOK | M_ZERO); 4208 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 4209 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i]; 4210 struct bnx_intr_data *intr; 4211 4212 txr->bnx_sc = sc; 4213 txr->bnx_tx_mbx = bnx_tx_mailbox[i]; 4214 4215 if (sc->bnx_tx_ringcnt == 1) { 4216 intr = &sc->bnx_intr_data[0]; 4217 } else { 4218 KKASSERT(i + 1 < sc->bnx_intr_cnt); 4219 intr = &sc->bnx_intr_data[i + 1]; 4220 } 4221 4222 if ((sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) == 0) { 4223 txr->bnx_hw_status_tag = 4224 &intr->bnx_status_block->bge_status_tag; 4225 } 4226 txr->bnx_tx_considx = 4227 &intr->bnx_status_block->bge_idx[0].bge_tx_cons_idx; 4228 4229 error = bnx_create_tx_ring(txr); 4230 if (error) { 4231 device_printf(dev, 4232 "could not create %dth TX ring\n", i); 4233 return error; 4234 } 4235 } 4236 4237 /* 4238 * Create jumbo buffer pool. 4239 */ 4240 if (BNX_IS_JUMBO_CAPABLE(sc)) { 4241 error = bnx_alloc_jumbo_mem(sc); 4242 if (error) { 4243 device_printf(dev, 4244 "could not create jumbo buffer pool\n"); 4245 return error; 4246 } 4247 } 4248 4249 return 0; 4250 } 4251 4252 static int 4253 bnx_dma_block_alloc(struct bnx_softc *sc, bus_size_t size, bus_dma_tag_t *tag, 4254 bus_dmamap_t *map, void **addr, bus_addr_t *paddr) 4255 { 4256 bus_dmamem_t dmem; 4257 int error; 4258 4259 error = bus_dmamem_coherent(sc->bnx_cdata.bnx_parent_tag, PAGE_SIZE, 0, 4260 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 4261 size, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem); 4262 if (error) 4263 return error; 4264 4265 *tag = dmem.dmem_tag; 4266 *map = dmem.dmem_map; 4267 *addr = dmem.dmem_addr; 4268 *paddr = dmem.dmem_busaddr; 4269 4270 return 0; 4271 } 4272 4273 static void 4274 bnx_dma_block_free(bus_dma_tag_t tag, bus_dmamap_t map, void *addr) 4275 { 4276 if (tag != NULL) { 4277 bus_dmamap_unload(tag, map); 4278 bus_dmamem_free(tag, addr, map); 4279 bus_dma_tag_destroy(tag); 4280 } 4281 } 4282 4283 static void 4284 bnx_tbi_link_upd(struct bnx_softc *sc, uint32_t status) 4285 { 4286 struct ifnet *ifp = &sc->arpcom.ac_if; 4287 4288 #define PCS_ENCODE_ERR (BGE_MACSTAT_PORT_DECODE_ERROR|BGE_MACSTAT_MI_COMPLETE) 4289 4290 /* 4291 * Sometimes PCS encoding errors are detected in 4292 * TBI mode (on fiber NICs), and for some reason 4293 * the chip will signal them as link changes. 4294 * If we get a link change event, but the 'PCS 4295 * encoding error' bit in the MAC status register 4296 * is set, don't bother doing a link check. 4297 * This avoids spurious "gigabit link up" messages 4298 * that sometimes appear on fiber NICs during 4299 * periods of heavy traffic. 4300 */ 4301 if (status & BGE_MACSTAT_TBI_PCS_SYNCHED) { 4302 if (!sc->bnx_link) { 4303 sc->bnx_link++; 4304 if (sc->bnx_asicrev == BGE_ASICREV_BCM5704) { 4305 BNX_CLRBIT(sc, BGE_MAC_MODE, 4306 BGE_MACMODE_TBI_SEND_CFGS); 4307 } 4308 CSR_WRITE_4(sc, BGE_MAC_STS, 0xFFFFFFFF); 4309 4310 if (bootverbose) 4311 if_printf(ifp, "link UP\n"); 4312 4313 ifp->if_link_state = LINK_STATE_UP; 4314 if_link_state_change(ifp); 4315 } 4316 } else if ((status & PCS_ENCODE_ERR) != PCS_ENCODE_ERR) { 4317 if (sc->bnx_link) { 4318 sc->bnx_link = 0; 4319 4320 if (bootverbose) 4321 if_printf(ifp, "link DOWN\n"); 4322 4323 ifp->if_link_state = LINK_STATE_DOWN; 4324 if_link_state_change(ifp); 4325 } 4326 } 4327 4328 #undef PCS_ENCODE_ERR 4329 4330 /* Clear the attention. */ 4331 CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED | 4332 BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE | 4333 BGE_MACSTAT_LINK_CHANGED); 4334 } 4335 4336 static void 4337 bnx_copper_link_upd(struct bnx_softc *sc, uint32_t status __unused) 4338 { 4339 struct ifnet *ifp = &sc->arpcom.ac_if; 4340 struct mii_data *mii = device_get_softc(sc->bnx_miibus); 4341 4342 mii_pollstat(mii); 4343 bnx_miibus_statchg(sc->bnx_dev); 4344 4345 if (bootverbose) { 4346 if (sc->bnx_link) 4347 if_printf(ifp, "link UP\n"); 4348 else 4349 if_printf(ifp, "link DOWN\n"); 4350 } 4351 4352 /* Clear the attention. */ 4353 CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED | 4354 BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE | 4355 BGE_MACSTAT_LINK_CHANGED); 4356 } 4357 4358 static void 4359 bnx_autopoll_link_upd(struct bnx_softc *sc, uint32_t status __unused) 4360 { 4361 struct ifnet *ifp = &sc->arpcom.ac_if; 4362 struct mii_data *mii = device_get_softc(sc->bnx_miibus); 4363 4364 mii_pollstat(mii); 4365 4366 if (!sc->bnx_link && 4367 (mii->mii_media_status & IFM_ACTIVE) && 4368 IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) { 4369 sc->bnx_link++; 4370 if (bootverbose) 4371 if_printf(ifp, "link UP\n"); 4372 } else if (sc->bnx_link && 4373 (!(mii->mii_media_status & IFM_ACTIVE) || 4374 IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) { 4375 sc->bnx_link = 0; 4376 if (bootverbose) 4377 if_printf(ifp, "link DOWN\n"); 4378 } 4379 4380 /* Clear the attention. */ 4381 CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED | 4382 BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE | 4383 BGE_MACSTAT_LINK_CHANGED); 4384 } 4385 4386 static int 4387 bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS) 4388 { 4389 struct bnx_softc *sc = arg1; 4390 4391 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4392 &sc->bnx_rx_coal_ticks, 4393 BNX_RX_COAL_TICKS_MIN, BNX_RX_COAL_TICKS_MAX, 4394 BNX_RX_COAL_TICKS_CHG); 4395 } 4396 4397 static int 4398 bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS) 4399 { 4400 struct bnx_softc *sc = arg1; 4401 4402 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4403 &sc->bnx_tx_coal_ticks, 4404 BNX_TX_COAL_TICKS_MIN, BNX_TX_COAL_TICKS_MAX, 4405 BNX_TX_COAL_TICKS_CHG); 4406 } 4407 4408 static int 4409 bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS) 4410 { 4411 struct bnx_softc *sc = arg1; 4412 4413 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4414 &sc->bnx_rx_coal_bds, 4415 BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX, 4416 BNX_RX_COAL_BDS_CHG); 4417 } 4418 4419 static int 4420 bnx_sysctl_rx_coal_bds_poll(SYSCTL_HANDLER_ARGS) 4421 { 4422 struct bnx_softc *sc = arg1; 4423 4424 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4425 &sc->bnx_rx_coal_bds_poll, 4426 BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX, 4427 BNX_RX_COAL_BDS_CHG); 4428 } 4429 4430 static int 4431 bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS) 4432 { 4433 struct bnx_softc *sc = arg1; 4434 4435 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4436 &sc->bnx_tx_coal_bds, 4437 BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX, 4438 BNX_TX_COAL_BDS_CHG); 4439 } 4440 4441 static int 4442 bnx_sysctl_tx_coal_bds_poll(SYSCTL_HANDLER_ARGS) 4443 { 4444 struct bnx_softc *sc = arg1; 4445 4446 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4447 &sc->bnx_tx_coal_bds_poll, 4448 BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX, 4449 BNX_TX_COAL_BDS_CHG); 4450 } 4451 4452 static int 4453 bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS) 4454 { 4455 struct bnx_softc *sc = arg1; 4456 4457 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4458 &sc->bnx_rx_coal_bds_int, 4459 BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX, 4460 BNX_RX_COAL_BDS_INT_CHG); 4461 } 4462 4463 static int 4464 bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS) 4465 { 4466 struct bnx_softc *sc = arg1; 4467 4468 return bnx_sysctl_coal_chg(oidp, arg1, arg2, req, 4469 &sc->bnx_tx_coal_bds_int, 4470 BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX, 4471 BNX_TX_COAL_BDS_INT_CHG); 4472 } 4473 4474 static int 4475 bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *coal, 4476 int coal_min, int coal_max, uint32_t coal_chg_mask) 4477 { 4478 struct bnx_softc *sc = arg1; 4479 struct ifnet *ifp = &sc->arpcom.ac_if; 4480 int error = 0, v; 4481 4482 ifnet_serialize_all(ifp); 4483 4484 v = *coal; 4485 error = sysctl_handle_int(oidp, &v, 0, req); 4486 if (!error && req->newptr != NULL) { 4487 if (v < coal_min || v > coal_max) { 4488 error = EINVAL; 4489 } else { 4490 *coal = v; 4491 sc->bnx_coal_chg |= coal_chg_mask; 4492 4493 /* Commit changes */ 4494 bnx_coal_change(sc); 4495 } 4496 } 4497 4498 ifnet_deserialize_all(ifp); 4499 return error; 4500 } 4501 4502 static void 4503 bnx_coal_change(struct bnx_softc *sc) 4504 { 4505 struct ifnet *ifp = &sc->arpcom.ac_if; 4506 int i; 4507 4508 ASSERT_IFNET_SERIALIZED_ALL(ifp); 4509 4510 if (sc->bnx_coal_chg & BNX_RX_COAL_TICKS_CHG) { 4511 if (sc->bnx_rx_retcnt == 1) { 4512 CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS, 4513 sc->bnx_rx_coal_ticks); 4514 i = 0; 4515 } else { 4516 CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS, 0); 4517 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 4518 CSR_WRITE_4(sc, BGE_VEC1_RX_COAL_TICKS + 4519 (i * BGE_VEC_COALSET_SIZE), 4520 sc->bnx_rx_coal_ticks); 4521 } 4522 } 4523 for (; i < BNX_INTR_MAX - 1; ++i) { 4524 CSR_WRITE_4(sc, BGE_VEC1_RX_COAL_TICKS + 4525 (i * BGE_VEC_COALSET_SIZE), 0); 4526 } 4527 if (bootverbose) { 4528 if_printf(ifp, "rx_coal_ticks -> %u\n", 4529 sc->bnx_rx_coal_ticks); 4530 } 4531 } 4532 4533 if (sc->bnx_coal_chg & BNX_TX_COAL_TICKS_CHG) { 4534 if (sc->bnx_tx_ringcnt == 1) { 4535 CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS, 4536 sc->bnx_tx_coal_ticks); 4537 i = 0; 4538 } else { 4539 CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS, 0); 4540 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 4541 CSR_WRITE_4(sc, BGE_VEC1_TX_COAL_TICKS + 4542 (i * BGE_VEC_COALSET_SIZE), 4543 sc->bnx_tx_coal_ticks); 4544 } 4545 } 4546 for (; i < BNX_INTR_MAX - 1; ++i) { 4547 CSR_WRITE_4(sc, BGE_VEC1_TX_COAL_TICKS + 4548 (i * BGE_VEC_COALSET_SIZE), 0); 4549 } 4550 if (bootverbose) { 4551 if_printf(ifp, "tx_coal_ticks -> %u\n", 4552 sc->bnx_tx_coal_ticks); 4553 } 4554 } 4555 4556 if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_CHG) { 4557 uint32_t rx_coal_bds; 4558 4559 if (ifp->if_flags & IFF_NPOLLING) 4560 rx_coal_bds = sc->bnx_rx_coal_bds_poll; 4561 else 4562 rx_coal_bds = sc->bnx_rx_coal_bds; 4563 4564 if (sc->bnx_rx_retcnt == 1) { 4565 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, rx_coal_bds); 4566 i = 0; 4567 } else { 4568 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, 0); 4569 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 4570 CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS + 4571 (i * BGE_VEC_COALSET_SIZE), rx_coal_bds); 4572 } 4573 } 4574 for (; i < BNX_INTR_MAX - 1; ++i) { 4575 CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS + 4576 (i * BGE_VEC_COALSET_SIZE), 0); 4577 } 4578 if (bootverbose) { 4579 if_printf(ifp, "%srx_coal_bds -> %u\n", 4580 (ifp->if_flags & IFF_NPOLLING) ? "polling " : "", 4581 rx_coal_bds); 4582 } 4583 } 4584 4585 if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_CHG) { 4586 uint32_t tx_coal_bds; 4587 4588 if (ifp->if_flags & IFF_NPOLLING) 4589 tx_coal_bds = sc->bnx_tx_coal_bds_poll; 4590 else 4591 tx_coal_bds = sc->bnx_tx_coal_bds; 4592 4593 if (sc->bnx_tx_ringcnt == 1) { 4594 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, tx_coal_bds); 4595 i = 0; 4596 } else { 4597 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, 0); 4598 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 4599 CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS + 4600 (i * BGE_VEC_COALSET_SIZE), tx_coal_bds); 4601 } 4602 } 4603 for (; i < BNX_INTR_MAX - 1; ++i) { 4604 CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS + 4605 (i * BGE_VEC_COALSET_SIZE), 0); 4606 } 4607 if (bootverbose) { 4608 if_printf(ifp, "%stx_coal_bds -> %u\n", 4609 (ifp->if_flags & IFF_NPOLLING) ? "polling " : "", 4610 tx_coal_bds); 4611 } 4612 } 4613 4614 if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_INT_CHG) { 4615 if (sc->bnx_rx_retcnt == 1) { 4616 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT, 4617 sc->bnx_rx_coal_bds_int); 4618 i = 0; 4619 } else { 4620 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT, 0); 4621 for (i = 0; i < sc->bnx_rx_retcnt; ++i) { 4622 CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS_INT + 4623 (i * BGE_VEC_COALSET_SIZE), 4624 sc->bnx_rx_coal_bds_int); 4625 } 4626 } 4627 for (; i < BNX_INTR_MAX - 1; ++i) { 4628 CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS_INT + 4629 (i * BGE_VEC_COALSET_SIZE), 0); 4630 } 4631 if (bootverbose) { 4632 if_printf(ifp, "rx_coal_bds_int -> %u\n", 4633 sc->bnx_rx_coal_bds_int); 4634 } 4635 } 4636 4637 if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_INT_CHG) { 4638 if (sc->bnx_tx_ringcnt == 1) { 4639 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT, 4640 sc->bnx_tx_coal_bds_int); 4641 i = 0; 4642 } else { 4643 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT, 0); 4644 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 4645 CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS_INT + 4646 (i * BGE_VEC_COALSET_SIZE), 4647 sc->bnx_tx_coal_bds_int); 4648 } 4649 } 4650 for (; i < BNX_INTR_MAX - 1; ++i) { 4651 CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS_INT + 4652 (i * BGE_VEC_COALSET_SIZE), 0); 4653 } 4654 if (bootverbose) { 4655 if_printf(ifp, "tx_coal_bds_int -> %u\n", 4656 sc->bnx_tx_coal_bds_int); 4657 } 4658 } 4659 4660 sc->bnx_coal_chg = 0; 4661 } 4662 4663 static void 4664 bnx_check_intr_rxtx(void *xintr) 4665 { 4666 struct bnx_intr_data *intr = xintr; 4667 struct bnx_rx_ret_ring *ret; 4668 struct bnx_tx_ring *txr; 4669 struct ifnet *ifp; 4670 4671 lwkt_serialize_enter(intr->bnx_intr_serialize); 4672 4673 KKASSERT(mycpuid == intr->bnx_intr_cpuid); 4674 4675 ifp = &intr->bnx_sc->arpcom.ac_if; 4676 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) { 4677 lwkt_serialize_exit(intr->bnx_intr_serialize); 4678 return; 4679 } 4680 4681 txr = intr->bnx_txr; 4682 ret = intr->bnx_ret; 4683 4684 if (*ret->bnx_rx_considx != ret->bnx_rx_saved_considx || 4685 *txr->bnx_tx_considx != txr->bnx_tx_saved_considx) { 4686 if (intr->bnx_rx_check_considx == ret->bnx_rx_saved_considx && 4687 intr->bnx_tx_check_considx == txr->bnx_tx_saved_considx) { 4688 if (!intr->bnx_intr_maylose) { 4689 intr->bnx_intr_maylose = TRUE; 4690 goto done; 4691 } 4692 if (bootverbose) 4693 if_printf(ifp, "lost interrupt\n"); 4694 intr->bnx_intr_func(intr->bnx_intr_arg); 4695 } 4696 } 4697 intr->bnx_intr_maylose = FALSE; 4698 intr->bnx_rx_check_considx = ret->bnx_rx_saved_considx; 4699 intr->bnx_tx_check_considx = txr->bnx_tx_saved_considx; 4700 4701 done: 4702 callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL, 4703 intr->bnx_intr_check, intr); 4704 lwkt_serialize_exit(intr->bnx_intr_serialize); 4705 } 4706 4707 static void 4708 bnx_check_intr_tx(void *xintr) 4709 { 4710 struct bnx_intr_data *intr = xintr; 4711 struct bnx_tx_ring *txr; 4712 struct ifnet *ifp; 4713 4714 lwkt_serialize_enter(intr->bnx_intr_serialize); 4715 4716 KKASSERT(mycpuid == intr->bnx_intr_cpuid); 4717 4718 ifp = &intr->bnx_sc->arpcom.ac_if; 4719 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) { 4720 lwkt_serialize_exit(intr->bnx_intr_serialize); 4721 return; 4722 } 4723 4724 txr = intr->bnx_txr; 4725 4726 if (*txr->bnx_tx_considx != txr->bnx_tx_saved_considx) { 4727 if (intr->bnx_tx_check_considx == txr->bnx_tx_saved_considx) { 4728 if (!intr->bnx_intr_maylose) { 4729 intr->bnx_intr_maylose = TRUE; 4730 goto done; 4731 } 4732 if (bootverbose) 4733 if_printf(ifp, "lost interrupt\n"); 4734 intr->bnx_intr_func(intr->bnx_intr_arg); 4735 } 4736 } 4737 intr->bnx_intr_maylose = FALSE; 4738 intr->bnx_tx_check_considx = txr->bnx_tx_saved_considx; 4739 4740 done: 4741 callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL, 4742 intr->bnx_intr_check, intr); 4743 lwkt_serialize_exit(intr->bnx_intr_serialize); 4744 } 4745 4746 static void 4747 bnx_check_intr_rx(void *xintr) 4748 { 4749 struct bnx_intr_data *intr = xintr; 4750 struct bnx_rx_ret_ring *ret; 4751 struct ifnet *ifp; 4752 4753 lwkt_serialize_enter(intr->bnx_intr_serialize); 4754 4755 KKASSERT(mycpuid == intr->bnx_intr_cpuid); 4756 4757 ifp = &intr->bnx_sc->arpcom.ac_if; 4758 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) { 4759 lwkt_serialize_exit(intr->bnx_intr_serialize); 4760 return; 4761 } 4762 4763 ret = intr->bnx_ret; 4764 4765 if (*ret->bnx_rx_considx != ret->bnx_rx_saved_considx) { 4766 if (intr->bnx_rx_check_considx == ret->bnx_rx_saved_considx) { 4767 if (!intr->bnx_intr_maylose) { 4768 intr->bnx_intr_maylose = TRUE; 4769 goto done; 4770 } 4771 if (bootverbose) 4772 if_printf(ifp, "lost interrupt\n"); 4773 intr->bnx_intr_func(intr->bnx_intr_arg); 4774 } 4775 } 4776 intr->bnx_intr_maylose = FALSE; 4777 intr->bnx_rx_check_considx = ret->bnx_rx_saved_considx; 4778 4779 done: 4780 callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL, 4781 intr->bnx_intr_check, intr); 4782 lwkt_serialize_exit(intr->bnx_intr_serialize); 4783 } 4784 4785 static void 4786 bnx_enable_intr(struct bnx_softc *sc) 4787 { 4788 struct ifnet *ifp = &sc->arpcom.ac_if; 4789 int i; 4790 4791 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4792 lwkt_serialize_handler_enable( 4793 sc->bnx_intr_data[i].bnx_intr_serialize); 4794 } 4795 4796 /* 4797 * Enable interrupt. 4798 */ 4799 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4800 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 4801 4802 bnx_writembx(sc, intr->bnx_intr_mbx, 4803 (*intr->bnx_saved_status_tag) << 24); 4804 /* XXX Linux driver */ 4805 bnx_writembx(sc, intr->bnx_intr_mbx, 4806 (*intr->bnx_saved_status_tag) << 24); 4807 } 4808 4809 /* 4810 * Unmask the interrupt when we stop polling. 4811 */ 4812 PCI_CLRBIT(sc->bnx_dev, BGE_PCI_MISC_CTL, 4813 BGE_PCIMISCCTL_MASK_PCI_INTR, 4); 4814 4815 /* 4816 * Trigger another interrupt, since above writing 4817 * to interrupt mailbox0 may acknowledge pending 4818 * interrupt. 4819 */ 4820 BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET); 4821 4822 if (sc->bnx_flags & BNX_FLAG_STATUSTAG_BUG) { 4823 if (bootverbose) 4824 if_printf(ifp, "status tag bug workaround\n"); 4825 4826 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4827 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 4828 4829 if (intr->bnx_intr_check == NULL) 4830 continue; 4831 intr->bnx_intr_maylose = FALSE; 4832 intr->bnx_rx_check_considx = 0; 4833 intr->bnx_tx_check_considx = 0; 4834 callout_reset_bycpu(&intr->bnx_intr_timer, 4835 BNX_INTR_CKINTVL, intr->bnx_intr_check, intr, 4836 intr->bnx_intr_cpuid); 4837 } 4838 } 4839 } 4840 4841 static void 4842 bnx_disable_intr(struct bnx_softc *sc) 4843 { 4844 int i; 4845 4846 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4847 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 4848 4849 callout_stop(&intr->bnx_intr_timer); 4850 intr->bnx_intr_maylose = FALSE; 4851 intr->bnx_rx_check_considx = 0; 4852 intr->bnx_tx_check_considx = 0; 4853 } 4854 4855 /* 4856 * Mask the interrupt when we start polling. 4857 */ 4858 PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL, 4859 BGE_PCIMISCCTL_MASK_PCI_INTR, 4); 4860 4861 /* 4862 * Acknowledge possible asserted interrupt. 4863 */ 4864 for (i = 0; i < BNX_INTR_MAX; ++i) 4865 bnx_writembx(sc, sc->bnx_intr_data[i].bnx_intr_mbx, 1); 4866 4867 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 4868 lwkt_serialize_handler_disable( 4869 sc->bnx_intr_data[i].bnx_intr_serialize); 4870 } 4871 } 4872 4873 static int 4874 bnx_get_eaddr_mem(struct bnx_softc *sc, uint8_t ether_addr[]) 4875 { 4876 uint32_t mac_addr; 4877 int ret = 1; 4878 4879 mac_addr = bnx_readmem_ind(sc, 0x0c14); 4880 if ((mac_addr >> 16) == 0x484b) { 4881 ether_addr[0] = (uint8_t)(mac_addr >> 8); 4882 ether_addr[1] = (uint8_t)mac_addr; 4883 mac_addr = bnx_readmem_ind(sc, 0x0c18); 4884 ether_addr[2] = (uint8_t)(mac_addr >> 24); 4885 ether_addr[3] = (uint8_t)(mac_addr >> 16); 4886 ether_addr[4] = (uint8_t)(mac_addr >> 8); 4887 ether_addr[5] = (uint8_t)mac_addr; 4888 ret = 0; 4889 } 4890 return ret; 4891 } 4892 4893 static int 4894 bnx_get_eaddr_nvram(struct bnx_softc *sc, uint8_t ether_addr[]) 4895 { 4896 int mac_offset = BGE_EE_MAC_OFFSET; 4897 4898 if (BNX_IS_5717_PLUS(sc)) { 4899 int f; 4900 4901 f = pci_get_function(sc->bnx_dev); 4902 if (f & 1) 4903 mac_offset = BGE_EE_MAC_OFFSET_5717; 4904 if (f > 1) 4905 mac_offset += BGE_EE_MAC_OFFSET_5717_OFF; 4906 } 4907 4908 return bnx_read_nvram(sc, ether_addr, mac_offset + 2, ETHER_ADDR_LEN); 4909 } 4910 4911 static int 4912 bnx_get_eaddr_eeprom(struct bnx_softc *sc, uint8_t ether_addr[]) 4913 { 4914 if (sc->bnx_flags & BNX_FLAG_NO_EEPROM) 4915 return 1; 4916 4917 return bnx_read_eeprom(sc, ether_addr, BGE_EE_MAC_OFFSET + 2, 4918 ETHER_ADDR_LEN); 4919 } 4920 4921 static int 4922 bnx_get_eaddr(struct bnx_softc *sc, uint8_t eaddr[]) 4923 { 4924 static const bnx_eaddr_fcn_t bnx_eaddr_funcs[] = { 4925 /* NOTE: Order is critical */ 4926 bnx_get_eaddr_mem, 4927 bnx_get_eaddr_nvram, 4928 bnx_get_eaddr_eeprom, 4929 NULL 4930 }; 4931 const bnx_eaddr_fcn_t *func; 4932 4933 for (func = bnx_eaddr_funcs; *func != NULL; ++func) { 4934 if ((*func)(sc, eaddr) == 0) 4935 break; 4936 } 4937 return (*func == NULL ? ENXIO : 0); 4938 } 4939 4940 /* 4941 * NOTE: 'm' is not freed upon failure 4942 */ 4943 struct mbuf * 4944 bnx_defrag_shortdma(struct mbuf *m) 4945 { 4946 struct mbuf *n; 4947 int found; 4948 4949 /* 4950 * If device receive two back-to-back send BDs with less than 4951 * or equal to 8 total bytes then the device may hang. The two 4952 * back-to-back send BDs must in the same frame for this failure 4953 * to occur. Scan mbuf chains and see whether two back-to-back 4954 * send BDs are there. If this is the case, allocate new mbuf 4955 * and copy the frame to workaround the silicon bug. 4956 */ 4957 for (n = m, found = 0; n != NULL; n = n->m_next) { 4958 if (n->m_len < 8) { 4959 found++; 4960 if (found > 1) 4961 break; 4962 continue; 4963 } 4964 found = 0; 4965 } 4966 4967 if (found > 1) 4968 n = m_defrag(m, MB_DONTWAIT); 4969 else 4970 n = m; 4971 return n; 4972 } 4973 4974 static void 4975 bnx_stop_block(struct bnx_softc *sc, bus_size_t reg, uint32_t bit) 4976 { 4977 int i; 4978 4979 BNX_CLRBIT(sc, reg, bit); 4980 for (i = 0; i < BNX_TIMEOUT; i++) { 4981 if ((CSR_READ_4(sc, reg) & bit) == 0) 4982 return; 4983 DELAY(100); 4984 } 4985 } 4986 4987 static void 4988 bnx_link_poll(struct bnx_softc *sc) 4989 { 4990 uint32_t status; 4991 4992 status = CSR_READ_4(sc, BGE_MAC_STS); 4993 if ((status & sc->bnx_link_chg) || sc->bnx_link_evt) { 4994 sc->bnx_link_evt = 0; 4995 sc->bnx_link_upd(sc, status); 4996 } 4997 } 4998 4999 static void 5000 bnx_enable_msi(struct bnx_softc *sc, boolean_t is_msix) 5001 { 5002 uint32_t msi_mode; 5003 5004 msi_mode = CSR_READ_4(sc, BGE_MSI_MODE); 5005 msi_mode |= BGE_MSIMODE_ENABLE; 5006 /* 5007 * NOTE: 5008 * 5718-PG105-R says that "one shot" mode does not work 5009 * if MSI is used, however, it obviously works. 5010 */ 5011 msi_mode &= ~BGE_MSIMODE_ONESHOT_DISABLE; 5012 if (is_msix) 5013 msi_mode |= BGE_MSIMODE_MSIX_MULTIMODE; 5014 else 5015 msi_mode &= ~BGE_MSIMODE_MSIX_MULTIMODE; 5016 CSR_WRITE_4(sc, BGE_MSI_MODE, msi_mode); 5017 } 5018 5019 static uint32_t 5020 bnx_dma_swap_options(struct bnx_softc *sc) 5021 { 5022 uint32_t dma_options; 5023 5024 dma_options = BGE_MODECTL_WORDSWAP_NONFRAME | 5025 BGE_MODECTL_BYTESWAP_DATA | BGE_MODECTL_WORDSWAP_DATA; 5026 #if BYTE_ORDER == BIG_ENDIAN 5027 dma_options |= BGE_MODECTL_BYTESWAP_NONFRAME; 5028 #endif 5029 if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 || 5030 sc->bnx_asicrev == BGE_ASICREV_BCM5762) { 5031 dma_options |= BGE_MODECTL_BYTESWAP_B2HRX_DATA | 5032 BGE_MODECTL_WORDSWAP_B2HRX_DATA | BGE_MODECTL_B2HRX_ENABLE | 5033 BGE_MODECTL_HTX2B_ENABLE; 5034 } 5035 return dma_options; 5036 } 5037 5038 static int 5039 bnx_setup_tso(struct bnx_tx_ring *txr, struct mbuf **mp, 5040 uint16_t *mss0, uint16_t *flags0) 5041 { 5042 struct mbuf *m; 5043 struct ip *ip; 5044 struct tcphdr *th; 5045 int thoff, iphlen, hoff, hlen; 5046 uint16_t flags, mss; 5047 5048 m = *mp; 5049 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable")); 5050 5051 hoff = m->m_pkthdr.csum_lhlen; 5052 iphlen = m->m_pkthdr.csum_iphlen; 5053 thoff = m->m_pkthdr.csum_thlen; 5054 5055 KASSERT(hoff > 0, ("invalid ether header len")); 5056 KASSERT(iphlen > 0, ("invalid ip header len")); 5057 KASSERT(thoff > 0, ("invalid tcp header len")); 5058 5059 if (__predict_false(m->m_len < hoff + iphlen + thoff)) { 5060 m = m_pullup(m, hoff + iphlen + thoff); 5061 if (m == NULL) { 5062 *mp = NULL; 5063 return ENOBUFS; 5064 } 5065 *mp = m; 5066 } 5067 ip = mtodoff(m, struct ip *, hoff); 5068 th = mtodoff(m, struct tcphdr *, hoff + iphlen); 5069 5070 mss = m->m_pkthdr.tso_segsz; 5071 flags = BGE_TXBDFLAG_CPU_PRE_DMA | BGE_TXBDFLAG_CPU_POST_DMA; 5072 5073 ip->ip_len = htons(mss + iphlen + thoff); 5074 th->th_sum = 0; 5075 5076 hlen = (iphlen + thoff) >> 2; 5077 mss |= ((hlen & 0x3) << 14); 5078 flags |= ((hlen & 0xf8) << 7) | ((hlen & 0x4) << 2); 5079 5080 *mss0 = mss; 5081 *flags0 = flags; 5082 5083 return 0; 5084 } 5085 5086 static int 5087 bnx_create_tx_ring(struct bnx_tx_ring *txr) 5088 { 5089 bus_size_t txmaxsz, txmaxsegsz; 5090 int i, error; 5091 5092 lwkt_serialize_init(&txr->bnx_tx_serialize); 5093 5094 /* 5095 * Create DMA tag and maps for TX mbufs. 5096 */ 5097 if (txr->bnx_sc->bnx_flags & BNX_FLAG_TSO) 5098 txmaxsz = IP_MAXPACKET + sizeof(struct ether_vlan_header); 5099 else 5100 txmaxsz = BNX_JUMBO_FRAMELEN; 5101 if (txr->bnx_sc->bnx_asicrev == BGE_ASICREV_BCM57766) 5102 txmaxsegsz = MCLBYTES; 5103 else 5104 txmaxsegsz = PAGE_SIZE; 5105 error = bus_dma_tag_create(txr->bnx_sc->bnx_cdata.bnx_parent_tag, 5106 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 5107 txmaxsz, BNX_NSEG_NEW, txmaxsegsz, 5108 BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 5109 &txr->bnx_tx_mtag); 5110 if (error) { 5111 device_printf(txr->bnx_sc->bnx_dev, 5112 "could not create TX mbuf DMA tag\n"); 5113 return error; 5114 } 5115 5116 for (i = 0; i < BGE_TX_RING_CNT; i++) { 5117 error = bus_dmamap_create(txr->bnx_tx_mtag, 5118 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 5119 &txr->bnx_tx_buf[i].bnx_tx_dmamap); 5120 if (error) { 5121 int j; 5122 5123 for (j = 0; j < i; ++j) { 5124 bus_dmamap_destroy(txr->bnx_tx_mtag, 5125 txr->bnx_tx_buf[j].bnx_tx_dmamap); 5126 } 5127 bus_dma_tag_destroy(txr->bnx_tx_mtag); 5128 txr->bnx_tx_mtag = NULL; 5129 5130 device_printf(txr->bnx_sc->bnx_dev, 5131 "could not create TX mbuf DMA map\n"); 5132 return error; 5133 } 5134 } 5135 5136 /* 5137 * Create DMA stuffs for TX ring. 5138 */ 5139 error = bnx_dma_block_alloc(txr->bnx_sc, BGE_TX_RING_SZ, 5140 &txr->bnx_tx_ring_tag, 5141 &txr->bnx_tx_ring_map, 5142 (void *)&txr->bnx_tx_ring, 5143 &txr->bnx_tx_ring_paddr); 5144 if (error) { 5145 device_printf(txr->bnx_sc->bnx_dev, 5146 "could not create TX ring\n"); 5147 return error; 5148 } 5149 5150 txr->bnx_tx_flags |= BNX_TX_FLAG_SHORTDMA; 5151 txr->bnx_tx_wreg = BNX_TX_WREG_NSEGS; 5152 5153 return 0; 5154 } 5155 5156 static void 5157 bnx_destroy_tx_ring(struct bnx_tx_ring *txr) 5158 { 5159 /* Destroy TX mbuf DMA stuffs. */ 5160 if (txr->bnx_tx_mtag != NULL) { 5161 int i; 5162 5163 for (i = 0; i < BGE_TX_RING_CNT; i++) { 5164 KKASSERT(txr->bnx_tx_buf[i].bnx_tx_mbuf == NULL); 5165 bus_dmamap_destroy(txr->bnx_tx_mtag, 5166 txr->bnx_tx_buf[i].bnx_tx_dmamap); 5167 } 5168 bus_dma_tag_destroy(txr->bnx_tx_mtag); 5169 } 5170 5171 /* Destroy TX ring */ 5172 bnx_dma_block_free(txr->bnx_tx_ring_tag, 5173 txr->bnx_tx_ring_map, txr->bnx_tx_ring); 5174 } 5175 5176 static int 5177 bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS) 5178 { 5179 struct bnx_softc *sc = (void *)arg1; 5180 struct ifnet *ifp = &sc->arpcom.ac_if; 5181 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; 5182 int error, defrag, i; 5183 5184 if (txr->bnx_tx_flags & BNX_TX_FLAG_FORCE_DEFRAG) 5185 defrag = 1; 5186 else 5187 defrag = 0; 5188 5189 error = sysctl_handle_int(oidp, &defrag, 0, req); 5190 if (error || req->newptr == NULL) 5191 return error; 5192 5193 ifnet_serialize_all(ifp); 5194 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) { 5195 txr = &sc->bnx_tx_ring[i]; 5196 if (defrag) 5197 txr->bnx_tx_flags |= BNX_TX_FLAG_FORCE_DEFRAG; 5198 else 5199 txr->bnx_tx_flags &= ~BNX_TX_FLAG_FORCE_DEFRAG; 5200 } 5201 ifnet_deserialize_all(ifp); 5202 5203 return 0; 5204 } 5205 5206 static int 5207 bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS) 5208 { 5209 struct bnx_softc *sc = (void *)arg1; 5210 struct ifnet *ifp = &sc->arpcom.ac_if; 5211 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; 5212 int error, tx_wreg, i; 5213 5214 tx_wreg = txr->bnx_tx_wreg; 5215 error = sysctl_handle_int(oidp, &tx_wreg, 0, req); 5216 if (error || req->newptr == NULL) 5217 return error; 5218 5219 ifnet_serialize_all(ifp); 5220 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) 5221 sc->bnx_tx_ring[i].bnx_tx_wreg = tx_wreg; 5222 ifnet_deserialize_all(ifp); 5223 5224 return 0; 5225 } 5226 5227 static int 5228 bnx_create_rx_ret_ring(struct bnx_rx_ret_ring *ret) 5229 { 5230 int error; 5231 5232 lwkt_serialize_init(&ret->bnx_rx_ret_serialize); 5233 5234 /* 5235 * Create DMA stuffs for RX return ring. 5236 */ 5237 error = bnx_dma_block_alloc(ret->bnx_sc, 5238 BGE_RX_RTN_RING_SZ(BNX_RETURN_RING_CNT), 5239 &ret->bnx_rx_ret_ring_tag, 5240 &ret->bnx_rx_ret_ring_map, 5241 (void *)&ret->bnx_rx_ret_ring, 5242 &ret->bnx_rx_ret_ring_paddr); 5243 if (error) { 5244 device_printf(ret->bnx_sc->bnx_dev, 5245 "could not create RX ret ring\n"); 5246 return error; 5247 } 5248 5249 /* Shadow standard ring's RX mbuf DMA tag */ 5250 ret->bnx_rx_mtag = ret->bnx_std->bnx_rx_mtag; 5251 5252 /* 5253 * Create tmp DMA map for RX mbufs. 5254 */ 5255 error = bus_dmamap_create(ret->bnx_rx_mtag, BUS_DMA_WAITOK, 5256 &ret->bnx_rx_tmpmap); 5257 if (error) { 5258 device_printf(ret->bnx_sc->bnx_dev, 5259 "could not create tmp RX mbuf DMA map\n"); 5260 ret->bnx_rx_mtag = NULL; 5261 return error; 5262 } 5263 return 0; 5264 } 5265 5266 static void 5267 bnx_destroy_rx_ret_ring(struct bnx_rx_ret_ring *ret) 5268 { 5269 /* Destroy tmp RX mbuf DMA map */ 5270 if (ret->bnx_rx_mtag != NULL) 5271 bus_dmamap_destroy(ret->bnx_rx_mtag, ret->bnx_rx_tmpmap); 5272 5273 /* Destroy RX return ring */ 5274 bnx_dma_block_free(ret->bnx_rx_ret_ring_tag, 5275 ret->bnx_rx_ret_ring_map, ret->bnx_rx_ret_ring); 5276 } 5277 5278 static int 5279 bnx_alloc_intr(struct bnx_softc *sc) 5280 { 5281 struct bnx_intr_data *intr; 5282 u_int intr_flags; 5283 int error; 5284 5285 if (sc->bnx_intr_cnt > 1) { 5286 error = bnx_alloc_msix(sc); 5287 if (error) 5288 return error; 5289 KKASSERT(sc->bnx_intr_type == PCI_INTR_TYPE_MSIX); 5290 return 0; 5291 } 5292 5293 KKASSERT(sc->bnx_intr_cnt == 1); 5294 5295 intr = &sc->bnx_intr_data[0]; 5296 intr->bnx_ret = &sc->bnx_rx_ret_ring[0]; 5297 intr->bnx_txr = &sc->bnx_tx_ring[0]; 5298 intr->bnx_intr_serialize = &sc->bnx_main_serialize; 5299 intr->bnx_intr_check = bnx_check_intr_rxtx; 5300 intr->bnx_saved_status_tag = &intr->bnx_ret->bnx_saved_status_tag; 5301 5302 sc->bnx_intr_type = pci_alloc_1intr(sc->bnx_dev, bnx_msi_enable, 5303 &intr->bnx_intr_rid, &intr_flags); 5304 5305 intr->bnx_intr_res = bus_alloc_resource_any(sc->bnx_dev, SYS_RES_IRQ, 5306 &intr->bnx_intr_rid, intr_flags); 5307 if (intr->bnx_intr_res == NULL) { 5308 device_printf(sc->bnx_dev, "could not alloc interrupt\n"); 5309 return ENXIO; 5310 } 5311 5312 if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI) { 5313 bnx_enable_msi(sc, FALSE); 5314 intr->bnx_intr_func = bnx_msi; 5315 if (bootverbose) 5316 device_printf(sc->bnx_dev, "oneshot MSI\n"); 5317 } else { 5318 intr->bnx_intr_func = bnx_intr_legacy; 5319 } 5320 intr->bnx_intr_arg = sc; 5321 intr->bnx_intr_cpuid = rman_get_cpuid(intr->bnx_intr_res); 5322 5323 intr->bnx_txr->bnx_tx_cpuid = intr->bnx_intr_cpuid; 5324 5325 return 0; 5326 } 5327 5328 static int 5329 bnx_setup_intr(struct bnx_softc *sc) 5330 { 5331 int error, i; 5332 5333 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 5334 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 5335 5336 error = bus_setup_intr_descr(sc->bnx_dev, intr->bnx_intr_res, 5337 INTR_MPSAFE, intr->bnx_intr_func, intr->bnx_intr_arg, 5338 &intr->bnx_intr_hand, intr->bnx_intr_serialize, 5339 intr->bnx_intr_desc); 5340 if (error) { 5341 device_printf(sc->bnx_dev, 5342 "could not set up %dth intr\n", i); 5343 bnx_teardown_intr(sc, i); 5344 return error; 5345 } 5346 } 5347 return 0; 5348 } 5349 5350 static void 5351 bnx_teardown_intr(struct bnx_softc *sc, int cnt) 5352 { 5353 int i; 5354 5355 for (i = 0; i < cnt; ++i) { 5356 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 5357 5358 bus_teardown_intr(sc->bnx_dev, intr->bnx_intr_res, 5359 intr->bnx_intr_hand); 5360 } 5361 } 5362 5363 static void 5364 bnx_free_intr(struct bnx_softc *sc) 5365 { 5366 if (sc->bnx_intr_type != PCI_INTR_TYPE_MSIX) { 5367 struct bnx_intr_data *intr; 5368 5369 KKASSERT(sc->bnx_intr_cnt <= 1); 5370 intr = &sc->bnx_intr_data[0]; 5371 5372 if (intr->bnx_intr_res != NULL) { 5373 bus_release_resource(sc->bnx_dev, SYS_RES_IRQ, 5374 intr->bnx_intr_rid, intr->bnx_intr_res); 5375 } 5376 if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI) 5377 pci_release_msi(sc->bnx_dev); 5378 } else { 5379 bnx_free_msix(sc, TRUE); 5380 } 5381 } 5382 5383 static void 5384 bnx_setup_serialize(struct bnx_softc *sc) 5385 { 5386 int i, j; 5387 5388 /* 5389 * Allocate serializer array 5390 */ 5391 5392 /* Main + RX STD + TX + RX RET */ 5393 sc->bnx_serialize_cnt = 1 + 1 + sc->bnx_tx_ringcnt + sc->bnx_rx_retcnt; 5394 5395 sc->bnx_serialize = 5396 kmalloc(sc->bnx_serialize_cnt * sizeof(struct lwkt_serialize *), 5397 M_DEVBUF, M_WAITOK | M_ZERO); 5398 5399 /* 5400 * Setup serializers 5401 * 5402 * NOTE: Order is critical 5403 */ 5404 5405 i = 0; 5406 5407 KKASSERT(i < sc->bnx_serialize_cnt); 5408 sc->bnx_serialize[i++] = &sc->bnx_main_serialize; 5409 5410 KKASSERT(i < sc->bnx_serialize_cnt); 5411 sc->bnx_serialize[i++] = &sc->bnx_rx_std_ring.bnx_rx_std_serialize; 5412 5413 for (j = 0; j < sc->bnx_rx_retcnt; ++j) { 5414 KKASSERT(i < sc->bnx_serialize_cnt); 5415 sc->bnx_serialize[i++] = 5416 &sc->bnx_rx_ret_ring[j].bnx_rx_ret_serialize; 5417 } 5418 5419 for (j = 0; j < sc->bnx_tx_ringcnt; ++j) { 5420 KKASSERT(i < sc->bnx_serialize_cnt); 5421 sc->bnx_serialize[i++] = 5422 &sc->bnx_tx_ring[j].bnx_tx_serialize; 5423 } 5424 5425 KKASSERT(i == sc->bnx_serialize_cnt); 5426 } 5427 5428 static void 5429 bnx_serialize(struct ifnet *ifp, enum ifnet_serialize slz) 5430 { 5431 struct bnx_softc *sc = ifp->if_softc; 5432 5433 ifnet_serialize_array_enter(sc->bnx_serialize, 5434 sc->bnx_serialize_cnt, slz); 5435 } 5436 5437 static void 5438 bnx_deserialize(struct ifnet *ifp, enum ifnet_serialize slz) 5439 { 5440 struct bnx_softc *sc = ifp->if_softc; 5441 5442 ifnet_serialize_array_exit(sc->bnx_serialize, 5443 sc->bnx_serialize_cnt, slz); 5444 } 5445 5446 static int 5447 bnx_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz) 5448 { 5449 struct bnx_softc *sc = ifp->if_softc; 5450 5451 return ifnet_serialize_array_try(sc->bnx_serialize, 5452 sc->bnx_serialize_cnt, slz); 5453 } 5454 5455 #ifdef INVARIANTS 5456 5457 static void 5458 bnx_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz, 5459 boolean_t serialized) 5460 { 5461 struct bnx_softc *sc = ifp->if_softc; 5462 5463 ifnet_serialize_array_assert(sc->bnx_serialize, sc->bnx_serialize_cnt, 5464 slz, serialized); 5465 } 5466 5467 #endif /* INVARIANTS */ 5468 5469 #ifdef IFPOLL_ENABLE 5470 5471 static int 5472 bnx_sysctl_npoll_offset(SYSCTL_HANDLER_ARGS) 5473 { 5474 struct bnx_softc *sc = (void *)arg1; 5475 struct ifnet *ifp = &sc->arpcom.ac_if; 5476 int error, off; 5477 5478 off = sc->bnx_npoll_rxoff; 5479 error = sysctl_handle_int(oidp, &off, 0, req); 5480 if (error || req->newptr == NULL) 5481 return error; 5482 if (off < 0) 5483 return EINVAL; 5484 5485 ifnet_serialize_all(ifp); 5486 if (off >= ncpus2 || off % sc->bnx_rx_retcnt != 0) { 5487 error = EINVAL; 5488 } else { 5489 error = 0; 5490 sc->bnx_npoll_txoff = off; 5491 sc->bnx_npoll_rxoff = off; 5492 } 5493 ifnet_deserialize_all(ifp); 5494 5495 return error; 5496 } 5497 5498 static int 5499 bnx_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS) 5500 { 5501 struct bnx_softc *sc = (void *)arg1; 5502 struct ifnet *ifp = &sc->arpcom.ac_if; 5503 int error, off; 5504 5505 off = sc->bnx_npoll_rxoff; 5506 error = sysctl_handle_int(oidp, &off, 0, req); 5507 if (error || req->newptr == NULL) 5508 return error; 5509 if (off < 0) 5510 return EINVAL; 5511 5512 ifnet_serialize_all(ifp); 5513 if (off >= ncpus2 || off % sc->bnx_rx_retcnt != 0) { 5514 error = EINVAL; 5515 } else { 5516 error = 0; 5517 sc->bnx_npoll_rxoff = off; 5518 } 5519 ifnet_deserialize_all(ifp); 5520 5521 return error; 5522 } 5523 5524 static int 5525 bnx_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS) 5526 { 5527 struct bnx_softc *sc = (void *)arg1; 5528 struct ifnet *ifp = &sc->arpcom.ac_if; 5529 int error, off; 5530 5531 off = sc->bnx_npoll_txoff; 5532 error = sysctl_handle_int(oidp, &off, 0, req); 5533 if (error || req->newptr == NULL) 5534 return error; 5535 if (off < 0) 5536 return EINVAL; 5537 5538 ifnet_serialize_all(ifp); 5539 if (off >= ncpus2) { 5540 error = EINVAL; 5541 } else { 5542 error = 0; 5543 sc->bnx_npoll_txoff = off; 5544 } 5545 ifnet_deserialize_all(ifp); 5546 5547 return error; 5548 } 5549 5550 #endif /* IFPOLL_ENABLE */ 5551 5552 static void 5553 bnx_set_tick_cpuid(struct bnx_softc *sc, boolean_t polling) 5554 { 5555 if (polling) 5556 sc->bnx_tick_cpuid = 0; /* XXX */ 5557 else 5558 sc->bnx_tick_cpuid = sc->bnx_intr_data[0].bnx_intr_cpuid; 5559 } 5560 5561 static void 5562 bnx_rx_std_refill_ithread(void *xstd) 5563 { 5564 struct bnx_rx_std_ring *std = xstd; 5565 struct globaldata *gd = mycpu; 5566 5567 crit_enter_gd(gd); 5568 5569 while (!std->bnx_rx_std_stop) { 5570 if (std->bnx_rx_std_refill) { 5571 lwkt_serialize_handler_call( 5572 &std->bnx_rx_std_serialize, 5573 bnx_rx_std_refill, std, NULL); 5574 } 5575 5576 crit_exit_gd(gd); 5577 crit_enter_gd(gd); 5578 5579 atomic_poll_release_int(&std->bnx_rx_std_running); 5580 cpu_mfence(); 5581 5582 if (!std->bnx_rx_std_refill && !std->bnx_rx_std_stop) { 5583 lwkt_deschedule_self(gd->gd_curthread); 5584 lwkt_switch(); 5585 } 5586 } 5587 5588 crit_exit_gd(gd); 5589 5590 wakeup(std); 5591 5592 lwkt_exit(); 5593 } 5594 5595 static void 5596 bnx_rx_std_refill(void *xstd, void *frame __unused) 5597 { 5598 struct bnx_rx_std_ring *std = xstd; 5599 int cnt, refill_mask; 5600 5601 again: 5602 cnt = 0; 5603 5604 cpu_lfence(); 5605 refill_mask = std->bnx_rx_std_refill; 5606 atomic_clear_int(&std->bnx_rx_std_refill, refill_mask); 5607 5608 while (refill_mask) { 5609 uint16_t check_idx = std->bnx_rx_std; 5610 int ret_idx; 5611 5612 ret_idx = bsfl(refill_mask); 5613 for (;;) { 5614 struct bnx_rx_buf *rb; 5615 int refilled; 5616 5617 BNX_INC(check_idx, BGE_STD_RX_RING_CNT); 5618 rb = &std->bnx_rx_std_buf[check_idx]; 5619 refilled = rb->bnx_rx_refilled; 5620 cpu_lfence(); 5621 if (refilled) { 5622 bnx_setup_rxdesc_std(std, check_idx); 5623 std->bnx_rx_std = check_idx; 5624 ++cnt; 5625 if (cnt >= 8) { 5626 atomic_subtract_int( 5627 &std->bnx_rx_std_used, cnt); 5628 bnx_writembx(std->bnx_sc, 5629 BGE_MBX_RX_STD_PROD_LO, 5630 std->bnx_rx_std); 5631 cnt = 0; 5632 } 5633 } else { 5634 break; 5635 } 5636 } 5637 refill_mask &= ~(1 << ret_idx); 5638 } 5639 5640 if (cnt) { 5641 atomic_subtract_int(&std->bnx_rx_std_used, cnt); 5642 bnx_writembx(std->bnx_sc, BGE_MBX_RX_STD_PROD_LO, 5643 std->bnx_rx_std); 5644 } 5645 5646 if (std->bnx_rx_std_refill) 5647 goto again; 5648 5649 atomic_poll_release_int(&std->bnx_rx_std_running); 5650 cpu_mfence(); 5651 5652 if (std->bnx_rx_std_refill) 5653 goto again; 5654 } 5655 5656 static int 5657 bnx_sysctl_std_refill(SYSCTL_HANDLER_ARGS) 5658 { 5659 struct bnx_softc *sc = (void *)arg1; 5660 struct ifnet *ifp = &sc->arpcom.ac_if; 5661 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0]; 5662 int error, cntmax, i; 5663 5664 cntmax = ret->bnx_rx_cntmax; 5665 error = sysctl_handle_int(oidp, &cntmax, 0, req); 5666 if (error || req->newptr == NULL) 5667 return error; 5668 5669 ifnet_serialize_all(ifp); 5670 5671 if ((cntmax * sc->bnx_rx_retcnt) >= BGE_STD_RX_RING_CNT / 2) { 5672 error = EINVAL; 5673 goto back; 5674 } 5675 5676 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) 5677 sc->bnx_rx_ret_ring[i].bnx_rx_cntmax = cntmax; 5678 error = 0; 5679 5680 back: 5681 ifnet_deserialize_all(ifp); 5682 5683 return error; 5684 } 5685 5686 static void 5687 bnx_init_rss(struct bnx_softc *sc) 5688 { 5689 uint8_t key[BGE_RSS_KEYREG_CNT * BGE_RSS_KEYREG_SIZE]; 5690 int i, j, r; 5691 5692 KKASSERT(BNX_RSS_ENABLED(sc)); 5693 5694 /* 5695 * Configure RSS redirect table in following fashion: 5696 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] 5697 */ 5698 r = 0; 5699 for (j = 0; j < BGE_RSS_INDIR_TBL_CNT; ++j) { 5700 uint32_t tbl = 0; 5701 5702 for (i = 0; i < BGE_RSS_INDIR_TBLENT_CNT; ++i) { 5703 uint32_t q; 5704 5705 q = r % sc->bnx_rx_retcnt; 5706 tbl |= q << (BGE_RSS_INDIR_TBLENT_SHIFT * 5707 (BGE_RSS_INDIR_TBLENT_CNT - i - 1)); 5708 ++r; 5709 } 5710 5711 BNX_RSS_DPRINTF(sc, 1, "tbl%d %08x\n", j, tbl); 5712 CSR_WRITE_4(sc, BGE_RSS_INDIR_TBL(j), tbl); 5713 } 5714 5715 toeplitz_get_key(key, sizeof(key)); 5716 for (i = 0; i < BGE_RSS_KEYREG_CNT; ++i) { 5717 uint32_t keyreg; 5718 5719 keyreg = BGE_RSS_KEYREG_VAL(key, i); 5720 5721 BNX_RSS_DPRINTF(sc, 1, "key%d %08x\n", i, keyreg); 5722 CSR_WRITE_4(sc, BGE_RSS_KEYREG(i), keyreg); 5723 } 5724 } 5725 5726 static void 5727 bnx_setup_ring_cnt(struct bnx_softc *sc) 5728 { 5729 int msix_enable, i, msix_cnt, msix_cnt2, ring_max; 5730 5731 sc->bnx_tx_ringcnt = 1; 5732 sc->bnx_rx_retcnt = 1; 5733 sc->bnx_intr_cnt = 1; 5734 5735 msix_enable = device_getenv_int(sc->bnx_dev, "msix.enable", 5736 bnx_msix_enable); 5737 if (!msix_enable) 5738 return; 5739 5740 if (ncpus2 == 1) 5741 return; 5742 5743 msix_cnt = pci_msix_count(sc->bnx_dev); 5744 if (msix_cnt <= 1) 5745 return; 5746 5747 i = 0; 5748 while ((1 << (i + 1)) <= msix_cnt) 5749 ++i; 5750 msix_cnt2 = 1 << i; 5751 5752 /* 5753 * One MSI-X vector is dedicated to status or single TX queue, 5754 * so make sure that there are enough MSI-X vectors. 5755 */ 5756 if (msix_cnt == msix_cnt2) { 5757 /* 5758 * XXX 5759 * This probably will not happen; 57785/5718 families 5760 * come with at least 5 MSI-X vectors. 5761 */ 5762 msix_cnt2 >>= 1; 5763 if (msix_cnt2 <= 1) { 5764 device_printf(sc->bnx_dev, 5765 "MSI-X count %d could not be used\n", msix_cnt); 5766 return; 5767 } 5768 device_printf(sc->bnx_dev, "MSI-X count %d is power of 2\n", 5769 msix_cnt); 5770 } 5771 5772 /* 5773 * Setup RX ring count 5774 */ 5775 ring_max = BNX_RX_RING_MAX; 5776 if (ring_max > msix_cnt2) 5777 ring_max = msix_cnt2; 5778 sc->bnx_rx_retcnt = device_getenv_int(sc->bnx_dev, "rx_rings", 5779 bnx_rx_rings); 5780 sc->bnx_rx_retcnt = if_ring_count2(sc->bnx_rx_retcnt, ring_max); 5781 5782 if (sc->bnx_rx_retcnt == 1) 5783 return; 5784 5785 /* 5786 * We need one extra MSI-X vector for link status or 5787 * TX ring (if only one TX ring is enabled). 5788 */ 5789 sc->bnx_intr_cnt = sc->bnx_rx_retcnt + 1; 5790 5791 /* 5792 * Setup TX ring count 5793 * 5794 * Currently only BCM5719 and BCM5720 support multiple TX rings 5795 * and the TX ring count must be less than the RX ring count. 5796 */ 5797 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 || 5798 sc->bnx_asicrev == BGE_ASICREV_BCM5720) { 5799 ring_max = BNX_TX_RING_MAX; 5800 if (ring_max > msix_cnt2) 5801 ring_max = msix_cnt2; 5802 if (ring_max > sc->bnx_rx_retcnt) 5803 ring_max = sc->bnx_rx_retcnt; 5804 sc->bnx_tx_ringcnt = device_getenv_int(sc->bnx_dev, "tx_rings", 5805 bnx_tx_rings); 5806 sc->bnx_tx_ringcnt = if_ring_count2(sc->bnx_tx_ringcnt, 5807 ring_max); 5808 } 5809 } 5810 5811 static int 5812 bnx_alloc_msix(struct bnx_softc *sc) 5813 { 5814 struct bnx_intr_data *intr; 5815 boolean_t setup = FALSE; 5816 int error, i, offset, offset_def; 5817 5818 KKASSERT(sc->bnx_intr_cnt > 1); 5819 KKASSERT(sc->bnx_intr_cnt == sc->bnx_rx_retcnt + 1); 5820 5821 if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) { 5822 /* 5823 * Link status 5824 */ 5825 intr = &sc->bnx_intr_data[0]; 5826 5827 intr->bnx_intr_serialize = &sc->bnx_main_serialize; 5828 intr->bnx_saved_status_tag = &sc->bnx_saved_status_tag; 5829 5830 intr->bnx_intr_func = bnx_msix_status; 5831 intr->bnx_intr_arg = sc; 5832 intr->bnx_intr_cpuid = 0; /* XXX */ 5833 5834 ksnprintf(intr->bnx_intr_desc0, sizeof(intr->bnx_intr_desc0), 5835 "%s sts", device_get_nameunit(sc->bnx_dev)); 5836 intr->bnx_intr_desc = intr->bnx_intr_desc0; 5837 5838 /* 5839 * RX/TX rings 5840 */ 5841 if (sc->bnx_rx_retcnt == ncpus2) { 5842 offset = 0; 5843 } else { 5844 offset_def = (sc->bnx_rx_retcnt * 5845 device_get_unit(sc->bnx_dev)) % ncpus2; 5846 5847 offset = device_getenv_int(sc->bnx_dev, 5848 "msix.offset", offset_def); 5849 if (offset >= ncpus2 || 5850 offset % sc->bnx_rx_retcnt != 0) { 5851 device_printf(sc->bnx_dev, 5852 "invalid msix.offset %d, use %d\n", 5853 offset, offset_def); 5854 offset = offset_def; 5855 } 5856 } 5857 5858 for (i = 1; i < sc->bnx_intr_cnt; ++i) { 5859 int idx = i - 1; 5860 5861 intr = &sc->bnx_intr_data[i]; 5862 5863 KKASSERT(idx < sc->bnx_rx_retcnt); 5864 intr->bnx_ret = &sc->bnx_rx_ret_ring[idx]; 5865 if (idx < sc->bnx_tx_ringcnt) { 5866 intr->bnx_txr = &sc->bnx_tx_ring[idx]; 5867 intr->bnx_ret->bnx_txr = intr->bnx_txr; 5868 } 5869 5870 intr->bnx_intr_serialize = 5871 &intr->bnx_ret->bnx_rx_ret_serialize; 5872 intr->bnx_saved_status_tag = 5873 &intr->bnx_ret->bnx_saved_status_tag; 5874 5875 intr->bnx_intr_arg = intr->bnx_ret; 5876 KKASSERT(idx + offset < ncpus2); 5877 intr->bnx_intr_cpuid = idx + offset; 5878 5879 if (intr->bnx_txr == NULL) { 5880 intr->bnx_intr_check = bnx_check_intr_rx; 5881 intr->bnx_intr_func = bnx_msix_rx; 5882 ksnprintf(intr->bnx_intr_desc0, 5883 sizeof(intr->bnx_intr_desc0), "%s rx%d", 5884 device_get_nameunit(sc->bnx_dev), idx); 5885 } else { 5886 intr->bnx_intr_check = bnx_check_intr_rxtx; 5887 intr->bnx_intr_func = bnx_msix_rxtx; 5888 ksnprintf(intr->bnx_intr_desc0, 5889 sizeof(intr->bnx_intr_desc0), "%s rxtx%d", 5890 device_get_nameunit(sc->bnx_dev), idx); 5891 5892 intr->bnx_txr->bnx_tx_cpuid = 5893 intr->bnx_intr_cpuid; 5894 } 5895 intr->bnx_intr_desc = intr->bnx_intr_desc0; 5896 5897 intr->bnx_ret->bnx_msix_mbx = intr->bnx_intr_mbx; 5898 } 5899 } else { 5900 /* 5901 * TX ring and link status 5902 */ 5903 offset_def = device_get_unit(sc->bnx_dev) % ncpus2; 5904 offset = device_getenv_int(sc->bnx_dev, "msix.txoff", 5905 offset_def); 5906 if (offset >= ncpus2) { 5907 device_printf(sc->bnx_dev, 5908 "invalid msix.txoff %d, use %d\n", 5909 offset, offset_def); 5910 offset = offset_def; 5911 } 5912 5913 intr = &sc->bnx_intr_data[0]; 5914 5915 intr->bnx_txr = &sc->bnx_tx_ring[0]; 5916 intr->bnx_intr_serialize = &sc->bnx_main_serialize; 5917 intr->bnx_intr_check = bnx_check_intr_tx; 5918 intr->bnx_saved_status_tag = 5919 &intr->bnx_txr->bnx_saved_status_tag; 5920 5921 intr->bnx_intr_func = bnx_msix_tx_status; 5922 intr->bnx_intr_arg = intr->bnx_txr; 5923 intr->bnx_intr_cpuid = offset; 5924 5925 ksnprintf(intr->bnx_intr_desc0, sizeof(intr->bnx_intr_desc0), 5926 "%s ststx", device_get_nameunit(sc->bnx_dev)); 5927 intr->bnx_intr_desc = intr->bnx_intr_desc0; 5928 5929 intr->bnx_txr->bnx_tx_cpuid = intr->bnx_intr_cpuid; 5930 5931 /* 5932 * RX rings 5933 */ 5934 if (sc->bnx_rx_retcnt == ncpus2) { 5935 offset = 0; 5936 } else { 5937 offset_def = (sc->bnx_rx_retcnt * 5938 device_get_unit(sc->bnx_dev)) % ncpus2; 5939 5940 offset = device_getenv_int(sc->bnx_dev, 5941 "msix.rxoff", offset_def); 5942 if (offset >= ncpus2 || 5943 offset % sc->bnx_rx_retcnt != 0) { 5944 device_printf(sc->bnx_dev, 5945 "invalid msix.rxoff %d, use %d\n", 5946 offset, offset_def); 5947 offset = offset_def; 5948 } 5949 } 5950 5951 for (i = 1; i < sc->bnx_intr_cnt; ++i) { 5952 int idx = i - 1; 5953 5954 intr = &sc->bnx_intr_data[i]; 5955 5956 KKASSERT(idx < sc->bnx_rx_retcnt); 5957 intr->bnx_ret = &sc->bnx_rx_ret_ring[idx]; 5958 intr->bnx_intr_serialize = 5959 &intr->bnx_ret->bnx_rx_ret_serialize; 5960 intr->bnx_intr_check = bnx_check_intr_rx; 5961 intr->bnx_saved_status_tag = 5962 &intr->bnx_ret->bnx_saved_status_tag; 5963 5964 intr->bnx_intr_func = bnx_msix_rx; 5965 intr->bnx_intr_arg = intr->bnx_ret; 5966 KKASSERT(idx + offset < ncpus2); 5967 intr->bnx_intr_cpuid = idx + offset; 5968 5969 ksnprintf(intr->bnx_intr_desc0, 5970 sizeof(intr->bnx_intr_desc0), "%s rx%d", 5971 device_get_nameunit(sc->bnx_dev), idx); 5972 intr->bnx_intr_desc = intr->bnx_intr_desc0; 5973 5974 intr->bnx_ret->bnx_msix_mbx = intr->bnx_intr_mbx; 5975 } 5976 } 5977 5978 sc->bnx_msix_mem_rid = PCIR_BAR(4); 5979 sc->bnx_msix_mem_res = bus_alloc_resource_any(sc->bnx_dev, 5980 SYS_RES_MEMORY, &sc->bnx_msix_mem_rid, RF_ACTIVE); 5981 if (sc->bnx_msix_mem_res == NULL) { 5982 device_printf(sc->bnx_dev, "could not alloc MSI-X table\n"); 5983 return ENXIO; 5984 } 5985 5986 bnx_enable_msi(sc, TRUE); 5987 5988 error = pci_setup_msix(sc->bnx_dev); 5989 if (error) { 5990 device_printf(sc->bnx_dev, "could not setup MSI-X\n"); 5991 goto back; 5992 } 5993 setup = TRUE; 5994 5995 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 5996 intr = &sc->bnx_intr_data[i]; 5997 5998 error = pci_alloc_msix_vector(sc->bnx_dev, i, 5999 &intr->bnx_intr_rid, intr->bnx_intr_cpuid); 6000 if (error) { 6001 device_printf(sc->bnx_dev, 6002 "could not alloc MSI-X %d on cpu%d\n", 6003 i, intr->bnx_intr_cpuid); 6004 goto back; 6005 } 6006 6007 intr->bnx_intr_res = bus_alloc_resource_any(sc->bnx_dev, 6008 SYS_RES_IRQ, &intr->bnx_intr_rid, RF_ACTIVE); 6009 if (intr->bnx_intr_res == NULL) { 6010 device_printf(sc->bnx_dev, 6011 "could not alloc MSI-X %d resource\n", i); 6012 error = ENXIO; 6013 goto back; 6014 } 6015 } 6016 6017 pci_enable_msix(sc->bnx_dev); 6018 sc->bnx_intr_type = PCI_INTR_TYPE_MSIX; 6019 back: 6020 if (error) 6021 bnx_free_msix(sc, setup); 6022 return error; 6023 } 6024 6025 static void 6026 bnx_free_msix(struct bnx_softc *sc, boolean_t setup) 6027 { 6028 int i; 6029 6030 KKASSERT(sc->bnx_intr_cnt > 1); 6031 6032 for (i = 0; i < sc->bnx_intr_cnt; ++i) { 6033 struct bnx_intr_data *intr = &sc->bnx_intr_data[i]; 6034 6035 if (intr->bnx_intr_res != NULL) { 6036 bus_release_resource(sc->bnx_dev, SYS_RES_IRQ, 6037 intr->bnx_intr_rid, intr->bnx_intr_res); 6038 } 6039 if (intr->bnx_intr_rid >= 0) { 6040 pci_release_msix_vector(sc->bnx_dev, 6041 intr->bnx_intr_rid); 6042 } 6043 } 6044 if (setup) 6045 pci_teardown_msix(sc->bnx_dev); 6046 } 6047 6048 static void 6049 bnx_rx_std_refill_sched_ipi(void *xret) 6050 { 6051 struct bnx_rx_ret_ring *ret = xret; 6052 struct bnx_rx_std_ring *std = ret->bnx_std; 6053 struct globaldata *gd = mycpu; 6054 6055 crit_enter_gd(gd); 6056 6057 atomic_set_int(&std->bnx_rx_std_refill, ret->bnx_rx_mask); 6058 cpu_sfence(); 6059 6060 KKASSERT(std->bnx_rx_std_ithread.td_gd == gd); 6061 lwkt_schedule(&std->bnx_rx_std_ithread); 6062 6063 crit_exit_gd(gd); 6064 } 6065 6066 static void 6067 bnx_rx_std_refill_stop(void *xstd) 6068 { 6069 struct bnx_rx_std_ring *std = xstd; 6070 struct globaldata *gd = mycpu; 6071 6072 crit_enter_gd(gd); 6073 6074 std->bnx_rx_std_stop = 1; 6075 cpu_sfence(); 6076 6077 KKASSERT(std->bnx_rx_std_ithread.td_gd == gd); 6078 lwkt_schedule(&std->bnx_rx_std_ithread); 6079 6080 crit_exit_gd(gd); 6081 } 6082 6083 static void 6084 bnx_serialize_skipmain(struct bnx_softc *sc) 6085 { 6086 lwkt_serialize_array_enter(sc->bnx_serialize, 6087 sc->bnx_serialize_cnt, 1); 6088 } 6089 6090 static void 6091 bnx_deserialize_skipmain(struct bnx_softc *sc) 6092 { 6093 lwkt_serialize_array_exit(sc->bnx_serialize, 6094 sc->bnx_serialize_cnt, 1); 6095 } 6096 6097 static void 6098 bnx_rx_std_refill_sched(struct bnx_rx_ret_ring *ret, 6099 struct bnx_rx_std_ring *std) 6100 { 6101 struct globaldata *gd = mycpu; 6102 6103 ret->bnx_rx_cnt = 0; 6104 cpu_sfence(); 6105 6106 crit_enter_gd(gd); 6107 6108 atomic_set_int(&std->bnx_rx_std_refill, ret->bnx_rx_mask); 6109 cpu_sfence(); 6110 if (atomic_poll_acquire_int(&std->bnx_rx_std_running)) { 6111 if (std->bnx_rx_std_ithread.td_gd == gd) { 6112 lwkt_schedule(&std->bnx_rx_std_ithread); 6113 } else { 6114 lwkt_send_ipiq( 6115 std->bnx_rx_std_ithread.td_gd, 6116 bnx_rx_std_refill_sched_ipi, ret); 6117 } 6118 } 6119 6120 crit_exit_gd(gd); 6121 } 6122 6123 static struct pktinfo * 6124 bnx_rss_info(struct pktinfo *pi, const struct bge_rx_bd *cur_rx) 6125 { 6126 /* Don't pick up IPv6 packet */ 6127 if (cur_rx->bge_flags & BGE_RXBDFLAG_IPV6) 6128 return NULL; 6129 6130 /* Don't pick up IP packet w/o IP checksum */ 6131 if ((cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) == 0 || 6132 (cur_rx->bge_error_flag & BGE_RXERRFLAG_IP_CSUM_NOK)) 6133 return NULL; 6134 6135 /* Don't pick up IP packet w/o TCP/UDP checksum */ 6136 if ((cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) == 0) 6137 return NULL; 6138 6139 /* May be IP fragment */ 6140 if (cur_rx->bge_tcp_udp_csum != 0xffff) 6141 return NULL; 6142 6143 if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_IS_TCP) 6144 pi->pi_l3proto = IPPROTO_TCP; 6145 else 6146 pi->pi_l3proto = IPPROTO_UDP; 6147 pi->pi_netisr = NETISR_IP; 6148 pi->pi_flags = 0; 6149 6150 return pi; 6151 } 6152