1 /* 2 * Copyright (c) 2004 3 * Joerg Sonnenberger <joerg@bec.de>. All rights reserved. 4 * 5 * Copyright (c) 1997, 1998-2003 6 * Bill Paul <wpaul@windriver.com>. All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Bill Paul. 19 * 4. Neither the name of the author nor the names of any co-contributors 20 * may be used to endorse or promote products derived from this software 21 * without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 33 * THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * $FreeBSD: src/sys/dev/re/if_re.c,v 1.25 2004/06/09 14:34:01 naddy Exp $ 36 */ 37 38 /* 39 * RealTek 8169S/8110S/8168/8111/8101E PCI NIC driver 40 * 41 * Written by Bill Paul <wpaul@windriver.com> 42 * Senior Networking Software Engineer 43 * Wind River Systems 44 */ 45 46 /* 47 * This driver is designed to support RealTek's next generation of 48 * 10/100 and 10/100/1000 PCI ethernet controllers. There are currently 49 * seven devices in this family: the the RTL8169, the RTL8169S, RTL8110S, 50 * the RTL8168, the RTL8111 and the RTL8101E. 51 * 52 * The 8169 is a 64-bit 10/100/1000 gigabit ethernet MAC: 53 * 54 * o Descriptor based DMA mechanism. Each descriptor represents 55 * a single packet fragment. Data buffers may be aligned on 56 * any byte boundary. 57 * 58 * o 64-bit DMA. 59 * 60 * o TCP/IP checksum offload for both RX and TX. 61 * 62 * o High and normal priority transmit DMA rings. 63 * 64 * o VLAN tag insertion and extraction. 65 * 66 * o TCP large send (segmentation offload). 67 * 68 * o 1000Mbps mode. 69 * 70 * o Jumbo frames. 71 * 72 * o GMII and TBI ports/registers for interfacing with copper 73 * or fiber PHYs. 74 * 75 * o RX and TX DMA rings can have up to 1024 descriptors. 76 * 77 * The 8169 does not have a built-in PHY. Most reference boards use a 78 * Marvell 88E1000 'Alaska' copper gigE PHY. 8169/8110 is _no longer_ 79 * supported. 80 * 81 * The 8169S/8110S 10/100/1000 devices have built-in copper gigE PHYs 82 * (the 'S' stands for 'single-chip'). These devices have the same 83 * programming API as the older 8169, but also have some vendor-specific 84 * registers for the on-board PHY. The 8110S is a LAN-on-motherboard 85 * part designed to be pin-compatible with the RealTek 8100 10/100 chip. 86 * 87 * This driver takes advantage of the RX and TX checksum offload and 88 * VLAN tag insertion/extraction features. It also implements 89 * interrupt moderation using the timer interrupt registers, which 90 * significantly reduces interrupt load. 91 */ 92 93 #define _IP_VHL 94 95 #include "opt_ifpoll.h" 96 97 #include <sys/param.h> 98 #include <sys/bus.h> 99 #include <sys/endian.h> 100 #include <sys/kernel.h> 101 #include <sys/in_cksum.h> 102 #include <sys/interrupt.h> 103 #include <sys/malloc.h> 104 #include <sys/mbuf.h> 105 #include <sys/rman.h> 106 #include <sys/serialize.h> 107 #include <sys/socket.h> 108 #include <sys/sockio.h> 109 #include <sys/sysctl.h> 110 111 #include <net/bpf.h> 112 #include <net/ethernet.h> 113 #include <net/if.h> 114 #include <net/ifq_var.h> 115 #include <net/if_arp.h> 116 #include <net/if_dl.h> 117 #include <net/if_media.h> 118 #include <net/if_poll.h> 119 #include <net/if_types.h> 120 #include <net/vlan/if_vlan_var.h> 121 #include <net/vlan/if_vlan_ether.h> 122 123 #include <netinet/ip.h> 124 125 #include "pcidevs.h" 126 #include <bus/pci/pcireg.h> 127 #include <bus/pci/pcivar.h> 128 129 #include <dev/netif/re/if_rereg.h> 130 #include <dev/netif/re/if_revar.h> 131 #include <dev/netif/re/re.h> 132 #include <dev/netif/re/re_dragonfly.h> 133 134 /* 135 * Various supported device vendors/types and their names. 136 */ 137 static const struct re_type { 138 uint16_t re_vid; 139 uint16_t re_did; 140 const char *re_name; 141 } re_devs[] = { 142 { PCI_VENDOR_DLINK, PCI_PRODUCT_DLINK_DGE528T, 143 "D-Link DGE-528(T) Gigabit Ethernet Adapter" }, 144 145 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8101E, 146 "RealTek 810x PCIe 10/100baseTX" }, 147 148 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8168, 149 "RealTek 8111/8168 PCIe Gigabit Ethernet" }, 150 151 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8168_1, 152 "RealTek 8168 PCIe Gigabit Ethernet" }, 153 154 #ifdef notyet 155 /* 156 * This driver now only supports built-in PHYs. 157 */ 158 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8169, 159 "RealTek 8110/8169 Gigabit Ethernet" }, 160 #endif 161 162 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8169SC, 163 "RealTek 8169SC/8110SC Single-chip Gigabit Ethernet" }, 164 165 { PCI_VENDOR_COREGA, PCI_PRODUCT_COREGA_CG_LAPCIGT, 166 "Corega CG-LAPCIGT Gigabit Ethernet" }, 167 168 { PCI_VENDOR_LINKSYS, PCI_PRODUCT_LINKSYS_EG1032, 169 "Linksys EG1032 Gigabit Ethernet" }, 170 171 { PCI_VENDOR_USR2, PCI_PRODUCT_USR2_997902, 172 "US Robotics 997902 Gigabit Ethernet" }, 173 174 { PCI_VENDOR_TTTECH, PCI_PRODUCT_TTTECH_MC322, 175 "TTTech MC322 Gigabit Ethernet" }, 176 177 { 0, 0, NULL } 178 }; 179 180 static int re_probe(device_t); 181 static int re_attach(device_t); 182 static int re_detach(device_t); 183 static int re_suspend(device_t); 184 static int re_resume(device_t); 185 static void re_shutdown(device_t); 186 187 static int re_allocmem(device_t); 188 static void re_freemem(device_t); 189 static void re_freebufmem(struct re_softc *, int, int); 190 static int re_encap(struct re_softc *, struct mbuf **, int *); 191 static int re_newbuf_std(struct re_softc *, int, int); 192 #ifdef RE_JUMBO 193 static int re_newbuf_jumbo(struct re_softc *, int, int); 194 #endif 195 static void re_setup_rxdesc(struct re_softc *, int); 196 static int re_rx_list_init(struct re_softc *); 197 static int re_tx_list_init(struct re_softc *); 198 static int re_rxeof(struct re_softc *); 199 static int re_txeof(struct re_softc *); 200 static int re_tx_collect(struct re_softc *); 201 static void re_intr(void *); 202 static void re_tick(void *); 203 static void re_tick_serialized(void *); 204 static void re_disable_aspm(device_t); 205 static void re_link_up(struct re_softc *); 206 static void re_link_down(struct re_softc *); 207 208 static void re_start(struct ifnet *, struct ifaltq_subque *); 209 static int re_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 210 static void re_init(void *); 211 static void re_stop(struct re_softc *, boolean_t); 212 static void re_watchdog(struct ifnet *); 213 214 static void re_setup_hw_im(struct re_softc *); 215 static void re_setup_sim_im(struct re_softc *); 216 static void re_disable_hw_im(struct re_softc *); 217 static void re_disable_sim_im(struct re_softc *); 218 static void re_config_imtype(struct re_softc *, int); 219 static void re_setup_intr(struct re_softc *, int, int); 220 221 static int re_sysctl_hwtime(SYSCTL_HANDLER_ARGS, int *); 222 static int re_sysctl_rxtime(SYSCTL_HANDLER_ARGS); 223 static int re_sysctl_txtime(SYSCTL_HANDLER_ARGS); 224 static int re_sysctl_simtime(SYSCTL_HANDLER_ARGS); 225 static int re_sysctl_imtype(SYSCTL_HANDLER_ARGS); 226 227 static int re_jpool_alloc(struct re_softc *); 228 static void re_jpool_free(struct re_softc *); 229 #ifdef RE_JUMBO 230 static struct re_jbuf *re_jbuf_alloc(struct re_softc *); 231 static void re_jbuf_free(void *); 232 static void re_jbuf_ref(void *); 233 #endif 234 235 #ifdef IFPOLL_ENABLE 236 static void re_npoll(struct ifnet *, struct ifpoll_info *); 237 static void re_npoll_compat(struct ifnet *, void *, int); 238 #endif 239 240 static device_method_t re_methods[] = { 241 /* Device interface */ 242 DEVMETHOD(device_probe, re_probe), 243 DEVMETHOD(device_attach, re_attach), 244 DEVMETHOD(device_detach, re_detach), 245 DEVMETHOD(device_suspend, re_suspend), 246 DEVMETHOD(device_resume, re_resume), 247 DEVMETHOD(device_shutdown, re_shutdown), 248 DEVMETHOD_END 249 }; 250 251 static driver_t re_driver = { 252 "re", 253 re_methods, 254 sizeof(struct re_softc) 255 }; 256 257 static devclass_t re_devclass; 258 259 DECLARE_DUMMY_MODULE(if_re); 260 DRIVER_MODULE(if_re, pci, re_driver, re_devclass, NULL, NULL); 261 DRIVER_MODULE(if_re, cardbus, re_driver, re_devclass, NULL, NULL); 262 263 static int re_rx_desc_count = RE_RX_DESC_CNT_DEF; 264 static int re_tx_desc_count = RE_TX_DESC_CNT_DEF; 265 static int re_msi_enable = 1; 266 267 TUNABLE_INT("hw.re.rx_desc_count", &re_rx_desc_count); 268 TUNABLE_INT("hw.re.tx_desc_count", &re_tx_desc_count); 269 TUNABLE_INT("hw.re.msi.enable", &re_msi_enable); 270 271 static __inline void 272 re_free_rxchain(struct re_softc *sc) 273 { 274 if (sc->re_head != NULL) { 275 m_freem(sc->re_head); 276 sc->re_head = sc->re_tail = NULL; 277 } 278 } 279 280 static int 281 re_probe(device_t dev) 282 { 283 const struct re_type *t; 284 uint16_t vendor, product; 285 286 vendor = pci_get_vendor(dev); 287 product = pci_get_device(dev); 288 289 /* 290 * Only attach to rev.3 of the Linksys EG1032 adapter. 291 * Rev.2 is supported by sk(4). 292 */ 293 if (vendor == PCI_VENDOR_LINKSYS && 294 product == PCI_PRODUCT_LINKSYS_EG1032 && 295 pci_get_subdevice(dev) != PCI_SUBDEVICE_LINKSYS_EG1032_REV3) 296 return ENXIO; 297 298 for (t = re_devs; t->re_name != NULL; t++) { 299 if (product == t->re_did && vendor == t->re_vid) 300 break; 301 } 302 if (t->re_name == NULL) 303 return ENXIO; 304 305 device_set_desc(dev, t->re_name); 306 return 0; 307 } 308 309 static int 310 re_allocmem(device_t dev) 311 { 312 struct re_softc *sc = device_get_softc(dev); 313 bus_dmamem_t dmem; 314 int error, i; 315 316 /* 317 * Allocate list data 318 */ 319 sc->re_ldata.re_tx_mbuf = 320 kmalloc(sc->re_tx_desc_cnt * sizeof(struct mbuf *), 321 M_DEVBUF, M_ZERO | M_WAITOK); 322 323 sc->re_ldata.re_rx_mbuf = 324 kmalloc(sc->re_rx_desc_cnt * sizeof(struct mbuf *), 325 M_DEVBUF, M_ZERO | M_WAITOK); 326 327 sc->re_ldata.re_rx_paddr = 328 kmalloc(sc->re_rx_desc_cnt * sizeof(bus_addr_t), 329 M_DEVBUF, M_ZERO | M_WAITOK); 330 331 sc->re_ldata.re_tx_dmamap = 332 kmalloc(sc->re_tx_desc_cnt * sizeof(bus_dmamap_t), 333 M_DEVBUF, M_ZERO | M_WAITOK); 334 335 sc->re_ldata.re_rx_dmamap = 336 kmalloc(sc->re_rx_desc_cnt * sizeof(bus_dmamap_t), 337 M_DEVBUF, M_ZERO | M_WAITOK); 338 339 /* 340 * Allocate the parent bus DMA tag appropriate for PCI. 341 */ 342 error = bus_dma_tag_create(NULL, /* parent */ 343 1, 0, /* alignment, boundary */ 344 BUS_SPACE_MAXADDR, /* lowaddr */ 345 BUS_SPACE_MAXADDR, /* highaddr */ 346 NULL, NULL, /* filter, filterarg */ 347 BUS_SPACE_MAXSIZE_32BIT,/* maxsize */ 348 0, /* nsegments */ 349 BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */ 350 0, /* flags */ 351 &sc->re_parent_tag); 352 if (error) { 353 device_printf(dev, "could not allocate parent dma tag\n"); 354 return error; 355 } 356 357 /* Allocate TX descriptor list. */ 358 error = bus_dmamem_coherent(sc->re_parent_tag, 359 RE_RING_ALIGN, 0, 360 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 361 RE_TX_LIST_SZ(sc), BUS_DMA_WAITOK | BUS_DMA_ZERO, 362 &dmem); 363 if (error) { 364 device_printf(dev, "could not allocate TX ring\n"); 365 return error; 366 } 367 sc->re_ldata.re_tx_list_tag = dmem.dmem_tag; 368 sc->re_ldata.re_tx_list_map = dmem.dmem_map; 369 sc->re_ldata.re_tx_list = dmem.dmem_addr; 370 sc->re_ldata.re_tx_list_addr = dmem.dmem_busaddr; 371 372 /* Allocate RX descriptor list. */ 373 error = bus_dmamem_coherent(sc->re_parent_tag, 374 RE_RING_ALIGN, 0, 375 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 376 RE_RX_LIST_SZ(sc), BUS_DMA_WAITOK | BUS_DMA_ZERO, 377 &dmem); 378 if (error) { 379 device_printf(dev, "could not allocate RX ring\n"); 380 return error; 381 } 382 sc->re_ldata.re_rx_list_tag = dmem.dmem_tag; 383 sc->re_ldata.re_rx_list_map = dmem.dmem_map; 384 sc->re_ldata.re_rx_list = dmem.dmem_addr; 385 sc->re_ldata.re_rx_list_addr = dmem.dmem_busaddr; 386 387 /* Allocate maps for TX mbufs. */ 388 error = bus_dma_tag_create(sc->re_parent_tag, 389 1, 0, 390 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 391 NULL, NULL, 392 RE_FRAMELEN_MAX, RE_MAXSEGS, MCLBYTES, 393 BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 394 &sc->re_ldata.re_tx_mtag); 395 if (error) { 396 device_printf(dev, "could not allocate TX buf dma tag\n"); 397 return(error); 398 } 399 400 /* Create DMA maps for TX buffers */ 401 for (i = 0; i < sc->re_tx_desc_cnt; i++) { 402 error = bus_dmamap_create(sc->re_ldata.re_tx_mtag, 403 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 404 &sc->re_ldata.re_tx_dmamap[i]); 405 if (error) { 406 device_printf(dev, "can't create DMA map for TX buf\n"); 407 re_freebufmem(sc, i, 0); 408 return(error); 409 } 410 } 411 412 /* Allocate maps for RX mbufs. */ 413 error = bus_dma_tag_create(sc->re_parent_tag, 414 RE_RXBUF_ALIGN, 0, 415 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 416 NULL, NULL, 417 MCLBYTES, 1, MCLBYTES, 418 BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ALIGNED, 419 &sc->re_ldata.re_rx_mtag); 420 if (error) { 421 device_printf(dev, "could not allocate RX buf dma tag\n"); 422 return(error); 423 } 424 425 /* Create spare DMA map for RX */ 426 error = bus_dmamap_create(sc->re_ldata.re_rx_mtag, BUS_DMA_WAITOK, 427 &sc->re_ldata.re_rx_spare); 428 if (error) { 429 device_printf(dev, "can't create spare DMA map for RX\n"); 430 bus_dma_tag_destroy(sc->re_ldata.re_rx_mtag); 431 sc->re_ldata.re_rx_mtag = NULL; 432 return error; 433 } 434 435 /* Create DMA maps for RX buffers */ 436 for (i = 0; i < sc->re_rx_desc_cnt; i++) { 437 error = bus_dmamap_create(sc->re_ldata.re_rx_mtag, 438 BUS_DMA_WAITOK, &sc->re_ldata.re_rx_dmamap[i]); 439 if (error) { 440 device_printf(dev, "can't create DMA map for RX buf\n"); 441 re_freebufmem(sc, sc->re_tx_desc_cnt, i); 442 return(error); 443 } 444 } 445 446 /* Create jumbo buffer pool for RX if required */ 447 if (sc->re_caps & RE_C_CONTIGRX) { 448 error = re_jpool_alloc(sc); 449 if (error) { 450 re_jpool_free(sc); 451 #ifdef RE_JUMBO 452 /* Disable jumbo frame support */ 453 sc->re_maxmtu = ETHERMTU; 454 #endif 455 } 456 } 457 return(0); 458 } 459 460 static void 461 re_freebufmem(struct re_softc *sc, int tx_cnt, int rx_cnt) 462 { 463 int i; 464 465 /* Destroy all the RX and TX buffer maps */ 466 if (sc->re_ldata.re_tx_mtag) { 467 for (i = 0; i < tx_cnt; i++) { 468 bus_dmamap_destroy(sc->re_ldata.re_tx_mtag, 469 sc->re_ldata.re_tx_dmamap[i]); 470 } 471 bus_dma_tag_destroy(sc->re_ldata.re_tx_mtag); 472 sc->re_ldata.re_tx_mtag = NULL; 473 } 474 475 if (sc->re_ldata.re_rx_mtag) { 476 for (i = 0; i < rx_cnt; i++) { 477 bus_dmamap_destroy(sc->re_ldata.re_rx_mtag, 478 sc->re_ldata.re_rx_dmamap[i]); 479 } 480 bus_dmamap_destroy(sc->re_ldata.re_rx_mtag, 481 sc->re_ldata.re_rx_spare); 482 bus_dma_tag_destroy(sc->re_ldata.re_rx_mtag); 483 sc->re_ldata.re_rx_mtag = NULL; 484 } 485 } 486 487 static void 488 re_freemem(device_t dev) 489 { 490 struct re_softc *sc = device_get_softc(dev); 491 492 /* Unload and free the RX DMA ring memory and map */ 493 if (sc->re_ldata.re_rx_list_tag) { 494 bus_dmamap_unload(sc->re_ldata.re_rx_list_tag, 495 sc->re_ldata.re_rx_list_map); 496 bus_dmamem_free(sc->re_ldata.re_rx_list_tag, 497 sc->re_ldata.re_rx_list, 498 sc->re_ldata.re_rx_list_map); 499 bus_dma_tag_destroy(sc->re_ldata.re_rx_list_tag); 500 } 501 502 /* Unload and free the TX DMA ring memory and map */ 503 if (sc->re_ldata.re_tx_list_tag) { 504 bus_dmamap_unload(sc->re_ldata.re_tx_list_tag, 505 sc->re_ldata.re_tx_list_map); 506 bus_dmamem_free(sc->re_ldata.re_tx_list_tag, 507 sc->re_ldata.re_tx_list, 508 sc->re_ldata.re_tx_list_map); 509 bus_dma_tag_destroy(sc->re_ldata.re_tx_list_tag); 510 } 511 512 /* Free RX/TX buf DMA stuffs */ 513 re_freebufmem(sc, sc->re_tx_desc_cnt, sc->re_rx_desc_cnt); 514 515 /* Unload and free the stats buffer and map */ 516 if (sc->re_ldata.re_stag) { 517 bus_dmamap_unload(sc->re_ldata.re_stag, sc->re_ldata.re_smap); 518 bus_dmamem_free(sc->re_ldata.re_stag, 519 sc->re_ldata.re_stats, 520 sc->re_ldata.re_smap); 521 bus_dma_tag_destroy(sc->re_ldata.re_stag); 522 } 523 524 if (sc->re_caps & RE_C_CONTIGRX) 525 re_jpool_free(sc); 526 527 if (sc->re_parent_tag) 528 bus_dma_tag_destroy(sc->re_parent_tag); 529 530 if (sc->re_ldata.re_tx_mbuf != NULL) 531 kfree(sc->re_ldata.re_tx_mbuf, M_DEVBUF); 532 if (sc->re_ldata.re_rx_mbuf != NULL) 533 kfree(sc->re_ldata.re_rx_mbuf, M_DEVBUF); 534 if (sc->re_ldata.re_rx_paddr != NULL) 535 kfree(sc->re_ldata.re_rx_paddr, M_DEVBUF); 536 if (sc->re_ldata.re_tx_dmamap != NULL) 537 kfree(sc->re_ldata.re_tx_dmamap, M_DEVBUF); 538 if (sc->re_ldata.re_rx_dmamap != NULL) 539 kfree(sc->re_ldata.re_rx_dmamap, M_DEVBUF); 540 } 541 542 static boolean_t 543 re_is_faste(struct re_softc *sc) 544 { 545 if (pci_get_vendor(sc->dev) == PCI_VENDOR_REALTEK) { 546 switch (sc->re_device_id) { 547 case PCI_PRODUCT_REALTEK_RT8169: 548 case PCI_PRODUCT_REALTEK_RT8169SC: 549 case PCI_PRODUCT_REALTEK_RT8168: 550 case PCI_PRODUCT_REALTEK_RT8168_1: 551 return FALSE; 552 default: 553 return TRUE; 554 } 555 } else { 556 return FALSE; 557 } 558 } 559 560 /* 561 * Attach the interface. Allocate softc structures, do ifmedia 562 * setup and ethernet/BPF attach. 563 */ 564 static int 565 re_attach(device_t dev) 566 { 567 struct re_softc *sc = device_get_softc(dev); 568 struct ifnet *ifp; 569 struct sysctl_ctx_list *ctx; 570 struct sysctl_oid *tree; 571 uint8_t eaddr[ETHER_ADDR_LEN]; 572 int error = 0, qlen, msi_enable; 573 u_int irq_flags; 574 575 callout_init_mp(&sc->re_timer); 576 sc->dev = dev; 577 sc->re_device_id = pci_get_device(dev); 578 sc->re_unit = device_get_unit(dev); 579 ifmedia_init(&sc->media, IFM_IMASK, rtl_ifmedia_upd, rtl_ifmedia_sts); 580 581 sc->re_caps = RE_C_HWIM; 582 583 sc->re_rx_desc_cnt = re_rx_desc_count; 584 if (sc->re_rx_desc_cnt > RE_RX_DESC_CNT_MAX) 585 sc->re_rx_desc_cnt = RE_RX_DESC_CNT_MAX; 586 587 sc->re_tx_desc_cnt = re_tx_desc_count; 588 if (sc->re_tx_desc_cnt > RE_TX_DESC_CNT_MAX) 589 sc->re_tx_desc_cnt = RE_TX_DESC_CNT_MAX; 590 591 qlen = RE_IFQ_MAXLEN; 592 if (sc->re_tx_desc_cnt > qlen) 593 qlen = sc->re_tx_desc_cnt; 594 595 sc->re_rxbuf_size = MCLBYTES; 596 sc->re_newbuf = re_newbuf_std; 597 598 /* 599 * Hardware interrupt moderation settings. 600 * XXX does not seem correct, undocumented. 601 */ 602 sc->re_tx_time = 5; /* 125us */ 603 sc->re_rx_time = 2; /* 50us */ 604 605 /* Simulated interrupt moderation setting. */ 606 sc->re_sim_time = 150; /* 150us */ 607 608 /* Use simulated interrupt moderation by default. */ 609 sc->re_imtype = RE_IMTYPE_SIM; 610 re_config_imtype(sc, sc->re_imtype); 611 612 ctx = device_get_sysctl_ctx(dev); 613 tree = device_get_sysctl_tree(dev); 614 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, 615 "rx_desc_count", CTLFLAG_RD, &sc->re_rx_desc_cnt, 616 0, "RX desc count"); 617 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, 618 "tx_desc_count", CTLFLAG_RD, &sc->re_tx_desc_cnt, 619 0, "TX desc count"); 620 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "sim_time", 621 CTLTYPE_INT | CTLFLAG_RW, 622 sc, 0, re_sysctl_simtime, "I", 623 "Simulated interrupt moderation time (usec)."); 624 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "imtype", 625 CTLTYPE_INT | CTLFLAG_RW, 626 sc, 0, re_sysctl_imtype, "I", 627 "Interrupt moderation type -- " 628 "0:disable, 1:simulated, " 629 "2:hardware(if supported)"); 630 if (sc->re_caps & RE_C_HWIM) { 631 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 632 OID_AUTO, "hw_rxtime", 633 CTLTYPE_INT | CTLFLAG_RW, 634 sc, 0, re_sysctl_rxtime, "I", 635 "Hardware interrupt moderation time " 636 "(unit: 25usec)."); 637 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 638 OID_AUTO, "hw_txtime", 639 CTLTYPE_INT | CTLFLAG_RW, 640 sc, 0, re_sysctl_txtime, "I", 641 "Hardware interrupt moderation time " 642 "(unit: 25usec)."); 643 } 644 645 #ifndef BURN_BRIDGES 646 /* 647 * Handle power management nonsense. 648 */ 649 650 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) { 651 uint32_t membase, irq; 652 653 /* Save important PCI config data. */ 654 membase = pci_read_config(dev, RE_PCI_LOMEM, 4); 655 irq = pci_read_config(dev, PCIR_INTLINE, 4); 656 657 /* Reset the power state. */ 658 device_printf(dev, "chip is in D%d power mode " 659 "-- setting to D0\n", pci_get_powerstate(dev)); 660 661 pci_set_powerstate(dev, PCI_POWERSTATE_D0); 662 663 /* Restore PCI config data. */ 664 pci_write_config(dev, RE_PCI_LOMEM, membase, 4); 665 pci_write_config(dev, PCIR_INTLINE, irq, 4); 666 } 667 #endif 668 /* 669 * Map control/status registers. 670 */ 671 pci_enable_busmaster(dev); 672 673 if (pci_is_pcie(dev)) { 674 sc->re_res_rid = PCIR_BAR(2); 675 sc->re_res_type = SYS_RES_MEMORY; 676 } else { 677 sc->re_res_rid = PCIR_BAR(0); 678 sc->re_res_type = SYS_RES_IOPORT; 679 } 680 sc->re_res = bus_alloc_resource_any(dev, sc->re_res_type, 681 &sc->re_res_rid, RF_ACTIVE); 682 if (sc->re_res == NULL) { 683 device_printf(dev, "couldn't map IO\n"); 684 error = ENXIO; 685 goto fail; 686 } 687 688 sc->re_btag = rman_get_bustag(sc->re_res); 689 sc->re_bhandle = rman_get_bushandle(sc->re_res); 690 691 error = rtl_check_mac_version(sc); 692 if (error) { 693 device_printf(dev, "check mac version failed\n"); 694 goto fail; 695 } 696 697 rtl_init_software_variable(sc); 698 if (pci_is_pcie(dev)) 699 sc->re_if_flags |= RL_FLAG_PCIE; 700 else 701 sc->re_if_flags &= ~RL_FLAG_PCIE; 702 device_printf(dev, "MAC version 0x%08x, MACFG %u%s%s\n", 703 (CSR_READ_4(sc, RE_TXCFG) & 0xFCF00000), sc->re_type, 704 sc->re_coalesce_tx_pkt ? ", software TX defrag" : "", 705 sc->re_hw_enable_msi_msix ? ", support MSI" : ""); 706 707 /* 708 * Allocate interrupt 709 */ 710 if (pci_is_pcie(dev) && sc->re_hw_enable_msi_msix) 711 msi_enable = re_msi_enable; 712 else 713 msi_enable = 0; 714 sc->re_irq_type = pci_alloc_1intr(dev, msi_enable, 715 &sc->re_irq_rid, &irq_flags); 716 717 sc->re_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->re_irq_rid, 718 irq_flags); 719 if (sc->re_irq == NULL) { 720 device_printf(dev, "couldn't map interrupt\n"); 721 error = ENXIO; 722 goto fail; 723 } 724 725 /* Disable ASPM */ 726 re_disable_aspm(dev); 727 728 rtl_exit_oob(sc); 729 rtl_hw_init(sc); 730 731 /* Reset the adapter. */ 732 rtl_reset(sc); 733 734 rtl_get_hw_mac_address(sc, eaddr); 735 if (sc->re_type == MACFG_3) /* Change PCI Latency time*/ 736 pci_write_config(dev, PCIR_LATTIMER, 0x40, 1); 737 738 /* Allocate DMA stuffs */ 739 error = re_allocmem(dev); 740 if (error) 741 goto fail; 742 743 if (pci_is_pcie(dev)) { 744 sc->re_bus_speed = 125; 745 } else { 746 uint8_t cfg2; 747 748 cfg2 = CSR_READ_1(sc, RE_CFG2); 749 switch (cfg2 & RE_CFG2_PCICLK_MASK) { 750 case RE_CFG2_PCICLK_33MHZ: 751 sc->re_bus_speed = 33; 752 break; 753 case RE_CFG2_PCICLK_66MHZ: 754 sc->re_bus_speed = 66; 755 break; 756 default: 757 device_printf(dev, "unknown bus speed, assume 33MHz\n"); 758 sc->re_bus_speed = 33; 759 break; 760 } 761 } 762 device_printf(dev, "bus speed %dMHz\n", sc->re_bus_speed); 763 764 rtl_phy_power_up(sc); 765 rtl_hw_phy_config(sc); 766 rtl_clrwol(sc); 767 768 /* TODO: jumbo frame */ 769 CSR_WRITE_2(sc, RE_RxMaxSize, sc->re_rxbuf_size); 770 771 /* Enable hardware checksum if available. */ 772 sc->re_tx_cstag = 1; 773 sc->re_rx_cstag = 1; 774 775 ifp = &sc->arpcom.ac_if; 776 ifp->if_softc = sc; 777 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 778 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 779 ifp->if_ioctl = re_ioctl; 780 ifp->if_start = re_start; 781 #ifdef IFPOLL_ENABLE 782 ifp->if_npoll = re_npoll; 783 #endif 784 ifp->if_watchdog = re_watchdog; 785 ifp->if_init = re_init; 786 if (!re_is_faste(sc)) 787 ifp->if_baudrate = 1000000000; 788 else 789 ifp->if_baudrate = 100000000; 790 ifp->if_nmbclusters = sc->re_rx_desc_cnt; 791 ifq_set_maxlen(&ifp->if_snd, qlen); 792 ifq_set_ready(&ifp->if_snd); 793 794 ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 795 IFCAP_RXCSUM | IFCAP_TXCSUM; 796 ifp->if_capenable = ifp->if_capabilities; 797 /* NOTE: if_hwassist will be setup after the interface is up. */ 798 799 /* 800 * Call MI attach routine. 801 */ 802 ether_ifattach(ifp, eaddr, NULL); 803 804 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(sc->re_irq)); 805 806 #ifdef IFPOLL_ENABLE 807 ifpoll_compat_setup(&sc->re_npoll, ctx, (struct sysctl_oid *)tree, 808 device_get_unit(dev), ifp->if_serializer); 809 #endif 810 811 /* Hook interrupt last to avoid having to lock softc */ 812 error = bus_setup_intr(dev, sc->re_irq, INTR_MPSAFE | INTR_HIFREQ, 813 re_intr, sc, &sc->re_intrhand, ifp->if_serializer); 814 if (error) { 815 device_printf(dev, "couldn't set up irq\n"); 816 ether_ifdetach(ifp); 817 goto fail; 818 } 819 820 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 821 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 822 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 823 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 824 if (!re_is_faste(sc)) { 825 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 826 0, NULL); 827 } 828 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 829 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 830 rtl_ifmedia_upd(ifp); 831 832 fail: 833 if (error) 834 re_detach(dev); 835 836 return (error); 837 } 838 839 /* 840 * Shutdown hardware and free up resources. This can be called any 841 * time after the mutex has been initialized. It is called in both 842 * the error case in attach and the normal detach case so it needs 843 * to be careful about only freeing resources that have actually been 844 * allocated. 845 */ 846 static int 847 re_detach(device_t dev) 848 { 849 struct re_softc *sc = device_get_softc(dev); 850 struct ifnet *ifp = &sc->arpcom.ac_if; 851 852 /* These should only be active if attach succeeded */ 853 if (device_is_attached(dev)) { 854 lwkt_serialize_enter(ifp->if_serializer); 855 re_stop(sc, TRUE); 856 bus_teardown_intr(dev, sc->re_irq, sc->re_intrhand); 857 lwkt_serialize_exit(ifp->if_serializer); 858 859 ether_ifdetach(ifp); 860 } 861 ifmedia_removeall(&sc->media); 862 863 if (sc->re_irq) 864 bus_release_resource(dev, SYS_RES_IRQ, sc->re_irq_rid, 865 sc->re_irq); 866 867 if (sc->re_irq_type == PCI_INTR_TYPE_MSI) 868 pci_release_msi(dev); 869 870 if (sc->re_res) { 871 bus_release_resource(dev, sc->re_res_type, sc->re_res_rid, 872 sc->re_res); 873 } 874 rtl_cmac_unmap(sc); 875 876 /* Free DMA stuffs */ 877 re_freemem(dev); 878 879 return(0); 880 } 881 882 static void 883 re_setup_rxdesc(struct re_softc *sc, int idx) 884 { 885 bus_addr_t paddr; 886 uint32_t cmdstat; 887 struct re_desc *d; 888 889 paddr = sc->re_ldata.re_rx_paddr[idx]; 890 d = &sc->re_ldata.re_rx_list[idx]; 891 892 d->re_bufaddr_lo = htole32(RE_ADDR_LO(paddr)); 893 d->re_bufaddr_hi = htole32(RE_ADDR_HI(paddr)); 894 895 cmdstat = sc->re_rxbuf_size | RE_RDESC_CMD_OWN; 896 if (idx == (sc->re_rx_desc_cnt - 1)) 897 cmdstat |= RE_RDESC_CMD_EOR; 898 d->re_cmdstat = htole32(cmdstat); 899 } 900 901 static int 902 re_newbuf_std(struct re_softc *sc, int idx, int init) 903 { 904 bus_dma_segment_t seg; 905 bus_dmamap_t map; 906 struct mbuf *m; 907 int error, nsegs; 908 909 m = m_getcl(init ? M_WAITOK : M_NOWAIT, MT_DATA, M_PKTHDR); 910 if (m == NULL) { 911 error = ENOBUFS; 912 913 if (init) { 914 if_printf(&sc->arpcom.ac_if, "m_getcl failed\n"); 915 return error; 916 } else { 917 goto back; 918 } 919 } 920 m->m_len = m->m_pkthdr.len = MCLBYTES; 921 922 /* 923 * NOTE: 924 * re(4) chips need address of the receive buffer to be 8-byte 925 * aligned, so don't call m_adj(m, ETHER_ALIGN) here. 926 */ 927 928 error = bus_dmamap_load_mbuf_segment(sc->re_ldata.re_rx_mtag, 929 sc->re_ldata.re_rx_spare, m, 930 &seg, 1, &nsegs, BUS_DMA_NOWAIT); 931 if (error) { 932 m_freem(m); 933 if (init) { 934 if_printf(&sc->arpcom.ac_if, "can't load RX mbuf\n"); 935 return error; 936 } else { 937 goto back; 938 } 939 } 940 941 if (!init) { 942 bus_dmamap_sync(sc->re_ldata.re_rx_mtag, 943 sc->re_ldata.re_rx_dmamap[idx], 944 BUS_DMASYNC_POSTREAD); 945 bus_dmamap_unload(sc->re_ldata.re_rx_mtag, 946 sc->re_ldata.re_rx_dmamap[idx]); 947 } 948 sc->re_ldata.re_rx_mbuf[idx] = m; 949 sc->re_ldata.re_rx_paddr[idx] = seg.ds_addr; 950 951 map = sc->re_ldata.re_rx_dmamap[idx]; 952 sc->re_ldata.re_rx_dmamap[idx] = sc->re_ldata.re_rx_spare; 953 sc->re_ldata.re_rx_spare = map; 954 back: 955 re_setup_rxdesc(sc, idx); 956 return error; 957 } 958 959 #ifdef RE_JUMBO 960 static int 961 re_newbuf_jumbo(struct re_softc *sc, int idx, int init) 962 { 963 struct mbuf *m; 964 struct re_jbuf *jbuf; 965 int error = 0; 966 967 MGETHDR(m, init ? M_WAITOK : M_NOWAIT, MT_DATA); 968 if (m == NULL) { 969 error = ENOBUFS; 970 if (init) { 971 if_printf(&sc->arpcom.ac_if, "MGETHDR failed\n"); 972 return error; 973 } else { 974 goto back; 975 } 976 } 977 978 jbuf = re_jbuf_alloc(sc); 979 if (jbuf == NULL) { 980 m_freem(m); 981 982 error = ENOBUFS; 983 if (init) { 984 if_printf(&sc->arpcom.ac_if, "jpool is empty\n"); 985 return error; 986 } else { 987 goto back; 988 } 989 } 990 991 m->m_ext.ext_arg = jbuf; 992 m->m_ext.ext_buf = jbuf->re_buf; 993 m->m_ext.ext_free = re_jbuf_free; 994 m->m_ext.ext_ref = re_jbuf_ref; 995 m->m_ext.ext_size = sc->re_rxbuf_size; 996 997 m->m_data = m->m_ext.ext_buf; 998 m->m_flags |= M_EXT; 999 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 1000 1001 /* 1002 * NOTE: 1003 * Some re(4) chips(e.g. RTL8101E) need address of the receive buffer 1004 * to be 8-byte aligned, so don't call m_adj(m, ETHER_ALIGN) here. 1005 */ 1006 1007 sc->re_ldata.re_rx_mbuf[idx] = m; 1008 sc->re_ldata.re_rx_paddr[idx] = jbuf->re_paddr; 1009 back: 1010 re_setup_rxdesc(sc, idx); 1011 return error; 1012 } 1013 #endif /* RE_JUMBO */ 1014 1015 static int 1016 re_tx_list_init(struct re_softc *sc) 1017 { 1018 bzero(sc->re_ldata.re_tx_list, RE_TX_LIST_SZ(sc)); 1019 1020 sc->re_ldata.re_tx_prodidx = 0; 1021 sc->re_ldata.re_tx_considx = 0; 1022 sc->re_ldata.re_tx_free = sc->re_tx_desc_cnt; 1023 1024 return(0); 1025 } 1026 1027 static int 1028 re_rx_list_init(struct re_softc *sc) 1029 { 1030 int i, error; 1031 1032 bzero(sc->re_ldata.re_rx_list, RE_RX_LIST_SZ(sc)); 1033 1034 for (i = 0; i < sc->re_rx_desc_cnt; i++) { 1035 error = sc->re_newbuf(sc, i, 1); 1036 if (error) 1037 return(error); 1038 } 1039 1040 sc->re_ldata.re_rx_prodidx = 0; 1041 sc->re_head = sc->re_tail = NULL; 1042 1043 return(0); 1044 } 1045 1046 #define RE_IP4_PACKET 0x1 1047 #define RE_TCP_PACKET 0x2 1048 #define RE_UDP_PACKET 0x4 1049 1050 static __inline uint8_t 1051 re_packet_type(struct re_softc *sc, uint32_t rxstat, uint32_t rxctrl) 1052 { 1053 uint8_t packet_type = 0; 1054 1055 if (sc->re_if_flags & RL_FLAG_DESCV2) { 1056 if (rxctrl & RE_RDESC_CTL_PROTOIP4) 1057 packet_type |= RE_IP4_PACKET; 1058 } else { 1059 if (rxstat & RE_RDESC_STAT_PROTOID) 1060 packet_type |= RE_IP4_PACKET; 1061 } 1062 if (RE_TCPPKT(rxstat)) 1063 packet_type |= RE_TCP_PACKET; 1064 else if (RE_UDPPKT(rxstat)) 1065 packet_type |= RE_UDP_PACKET; 1066 return packet_type; 1067 } 1068 1069 /* 1070 * RX handler for C+ and 8169. For the gigE chips, we support 1071 * the reception of jumbo frames that have been fragmented 1072 * across multiple 2K mbuf cluster buffers. 1073 */ 1074 static int 1075 re_rxeof(struct re_softc *sc) 1076 { 1077 struct ifnet *ifp = &sc->arpcom.ac_if; 1078 struct mbuf *m; 1079 struct re_desc *cur_rx; 1080 uint32_t rxstat, rxctrl; 1081 int i, total_len, rx = 0; 1082 1083 for (i = sc->re_ldata.re_rx_prodidx; 1084 RE_OWN(&sc->re_ldata.re_rx_list[i]) == 0; RE_RXDESC_INC(sc, i)) { 1085 cur_rx = &sc->re_ldata.re_rx_list[i]; 1086 m = sc->re_ldata.re_rx_mbuf[i]; 1087 total_len = RE_RXBYTES(cur_rx); 1088 rxstat = le32toh(cur_rx->re_cmdstat); 1089 rxctrl = le32toh(cur_rx->re_control); 1090 1091 rx = 1; 1092 1093 #ifdef INVARIANTS 1094 if (sc->re_flags & RE_F_USE_JPOOL) 1095 KKASSERT(rxstat & RE_RDESC_STAT_EOF); 1096 #endif 1097 1098 if ((rxstat & RE_RDESC_STAT_EOF) == 0) { 1099 if (sc->re_flags & RE_F_DROP_RXFRAG) { 1100 re_setup_rxdesc(sc, i); 1101 continue; 1102 } 1103 1104 if (sc->re_newbuf(sc, i, 0)) { 1105 /* Drop upcoming fragments */ 1106 sc->re_flags |= RE_F_DROP_RXFRAG; 1107 continue; 1108 } 1109 1110 m->m_len = MCLBYTES; 1111 if (sc->re_head == NULL) { 1112 sc->re_head = sc->re_tail = m; 1113 } else { 1114 sc->re_tail->m_next = m; 1115 sc->re_tail = m; 1116 } 1117 continue; 1118 } else if (sc->re_flags & RE_F_DROP_RXFRAG) { 1119 /* 1120 * Last fragment of a multi-fragment packet. 1121 * 1122 * Since error already happened, this fragment 1123 * must be dropped as well as the fragment chain. 1124 */ 1125 re_setup_rxdesc(sc, i); 1126 re_free_rxchain(sc); 1127 sc->re_flags &= ~RE_F_DROP_RXFRAG; 1128 continue; 1129 } 1130 1131 rxstat >>= 1; 1132 if (rxstat & RE_RDESC_STAT_RXERRSUM) { 1133 IFNET_STAT_INC(ifp, ierrors, 1); 1134 /* 1135 * If this is part of a multi-fragment packet, 1136 * discard all the pieces. 1137 */ 1138 re_free_rxchain(sc); 1139 re_setup_rxdesc(sc, i); 1140 continue; 1141 } 1142 1143 /* 1144 * If allocating a replacement mbuf fails, 1145 * reload the current one. 1146 */ 1147 1148 if (sc->re_newbuf(sc, i, 0)) { 1149 IFNET_STAT_INC(ifp, ierrors, 1); 1150 continue; 1151 } 1152 1153 if (sc->re_head != NULL) { 1154 m->m_len = total_len % MCLBYTES; 1155 /* 1156 * Special case: if there's 4 bytes or less 1157 * in this buffer, the mbuf can be discarded: 1158 * the last 4 bytes is the CRC, which we don't 1159 * care about anyway. 1160 */ 1161 if (m->m_len <= ETHER_CRC_LEN) { 1162 sc->re_tail->m_len -= 1163 (ETHER_CRC_LEN - m->m_len); 1164 m_freem(m); 1165 } else { 1166 m->m_len -= ETHER_CRC_LEN; 1167 sc->re_tail->m_next = m; 1168 } 1169 m = sc->re_head; 1170 sc->re_head = sc->re_tail = NULL; 1171 m->m_pkthdr.len = total_len - ETHER_CRC_LEN; 1172 } else { 1173 m->m_pkthdr.len = m->m_len = 1174 (total_len - ETHER_CRC_LEN); 1175 } 1176 1177 IFNET_STAT_INC(ifp, ipackets, 1); 1178 m->m_pkthdr.rcvif = ifp; 1179 1180 /* Do RX checksumming if enabled */ 1181 1182 if (ifp->if_capenable & IFCAP_RXCSUM) { 1183 uint8_t packet_type; 1184 1185 packet_type = re_packet_type(sc, rxstat, rxctrl); 1186 1187 /* Check IP header checksum */ 1188 if (packet_type & RE_IP4_PACKET) { 1189 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 1190 if ((rxstat & RE_RDESC_STAT_IPSUMBAD) == 0) 1191 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 1192 } 1193 1194 /* Check TCP/UDP checksum */ 1195 if (((packet_type & RE_TCP_PACKET) && 1196 (rxstat & RE_RDESC_STAT_TCPSUMBAD) == 0) || 1197 ((packet_type & RE_UDP_PACKET) && 1198 (rxstat & RE_RDESC_STAT_UDPSUMBAD) == 0)) { 1199 m->m_pkthdr.csum_flags |= 1200 CSUM_DATA_VALID|CSUM_PSEUDO_HDR| 1201 CSUM_FRAG_NOT_CHECKED; 1202 m->m_pkthdr.csum_data = 0xffff; 1203 } 1204 } 1205 1206 if (rxctrl & RE_RDESC_CTL_HASTAG) { 1207 m->m_flags |= M_VLANTAG; 1208 m->m_pkthdr.ether_vlantag = 1209 be16toh((rxctrl & RE_RDESC_CTL_TAGDATA)); 1210 } 1211 ifp->if_input(ifp, m, NULL, -1); 1212 } 1213 1214 sc->re_ldata.re_rx_prodidx = i; 1215 1216 return rx; 1217 } 1218 1219 #undef RE_IP4_PACKET 1220 #undef RE_TCP_PACKET 1221 #undef RE_UDP_PACKET 1222 1223 static int 1224 re_tx_collect(struct re_softc *sc) 1225 { 1226 struct ifnet *ifp = &sc->arpcom.ac_if; 1227 uint32_t txstat; 1228 int idx, tx = 0; 1229 1230 for (idx = sc->re_ldata.re_tx_considx; 1231 sc->re_ldata.re_tx_free < sc->re_tx_desc_cnt; 1232 RE_TXDESC_INC(sc, idx)) { 1233 txstat = le32toh(sc->re_ldata.re_tx_list[idx].re_cmdstat); 1234 if (txstat & RE_TDESC_CMD_OWN) 1235 break; 1236 1237 tx = 1; 1238 1239 sc->re_ldata.re_tx_list[idx].re_bufaddr_lo = 0; 1240 1241 /* 1242 * We only stash mbufs in the last descriptor 1243 * in a fragment chain, which also happens to 1244 * be the only place where the TX status bits 1245 * are valid. 1246 */ 1247 if (txstat & RE_TDESC_CMD_EOF) { 1248 bus_dmamap_unload(sc->re_ldata.re_tx_mtag, 1249 sc->re_ldata.re_tx_dmamap[idx]); 1250 m_freem(sc->re_ldata.re_tx_mbuf[idx]); 1251 sc->re_ldata.re_tx_mbuf[idx] = NULL; 1252 if (txstat & (RE_TDESC_STAT_EXCESSCOL| 1253 RE_TDESC_STAT_COLCNT)) 1254 IFNET_STAT_INC(ifp, collisions, 1); 1255 if (txstat & RE_TDESC_STAT_TXERRSUM) 1256 IFNET_STAT_INC(ifp, oerrors, 1); 1257 else 1258 IFNET_STAT_INC(ifp, opackets, 1); 1259 } 1260 sc->re_ldata.re_tx_free++; 1261 } 1262 sc->re_ldata.re_tx_considx = idx; 1263 1264 return tx; 1265 } 1266 1267 static int 1268 re_txeof(struct re_softc *sc) 1269 { 1270 struct ifnet *ifp = &sc->arpcom.ac_if; 1271 int tx; 1272 1273 tx = re_tx_collect(sc); 1274 1275 /* There is enough free TX descs */ 1276 if (sc->re_ldata.re_tx_free > RE_TXDESC_SPARE) 1277 ifq_clr_oactive(&ifp->if_snd); 1278 1279 /* 1280 * Some chips will ignore a second TX request issued while an 1281 * existing transmission is in progress. If the transmitter goes 1282 * idle but there are still packets waiting to be sent, we need 1283 * to restart the channel here to flush them out. This only seems 1284 * to be required with the PCIe devices. 1285 */ 1286 if (sc->re_ldata.re_tx_free < sc->re_tx_desc_cnt) 1287 CSR_WRITE_1(sc, RE_TPPOLL, RE_NPQ); 1288 else 1289 ifp->if_timer = 0; 1290 1291 return tx; 1292 } 1293 1294 static void 1295 re_tick(void *xsc) 1296 { 1297 struct re_softc *sc = xsc; 1298 1299 lwkt_serialize_enter(sc->arpcom.ac_if.if_serializer); 1300 re_tick_serialized(xsc); 1301 lwkt_serialize_exit(sc->arpcom.ac_if.if_serializer); 1302 } 1303 1304 static void 1305 re_tick_serialized(void *xsc) 1306 { 1307 struct re_softc *sc = xsc; 1308 struct ifnet *ifp = &sc->arpcom.ac_if; 1309 1310 ASSERT_SERIALIZED(ifp->if_serializer); 1311 1312 if ((ifp->if_flags & IFF_RUNNING) == 0) 1313 return; 1314 1315 if (rtl_link_ok(sc)) { 1316 if ((sc->re_flags & RE_F_LINKED) == 0) 1317 re_link_up(sc); 1318 } else if (sc->re_flags & RE_F_LINKED) { 1319 re_link_down(sc); 1320 } 1321 callout_reset(&sc->re_timer, hz, re_tick, sc); 1322 } 1323 1324 #ifdef IFPOLL_ENABLE 1325 1326 static void 1327 re_npoll_compat(struct ifnet *ifp, void *arg __unused, int count) 1328 { 1329 struct re_softc *sc = ifp->if_softc; 1330 1331 ASSERT_SERIALIZED(ifp->if_serializer); 1332 1333 if (sc->re_npoll.ifpc_stcount-- == 0) { 1334 uint16_t status; 1335 1336 sc->re_npoll.ifpc_stcount = sc->re_npoll.ifpc_stfrac; 1337 1338 status = CSR_READ_2(sc, RE_ISR); 1339 if (status) 1340 CSR_WRITE_2(sc, RE_ISR, status); 1341 1342 /* 1343 * XXX check behaviour on receiver stalls. 1344 */ 1345 1346 if (status & RE_ISR_SYSTEM_ERR) { 1347 rtl_reset(sc); 1348 re_init(sc); 1349 /* Done! */ 1350 return; 1351 } 1352 } 1353 1354 sc->rxcycles = count; 1355 re_rxeof(sc); 1356 re_txeof(sc); 1357 1358 if (!ifq_is_empty(&ifp->if_snd)) 1359 if_devstart(ifp); 1360 } 1361 1362 static void 1363 re_npoll(struct ifnet *ifp, struct ifpoll_info *info) 1364 { 1365 struct re_softc *sc = ifp->if_softc; 1366 1367 ASSERT_SERIALIZED(ifp->if_serializer); 1368 1369 if (info != NULL) { 1370 int cpuid = sc->re_npoll.ifpc_cpuid; 1371 1372 info->ifpi_rx[cpuid].poll_func = re_npoll_compat; 1373 info->ifpi_rx[cpuid].arg = NULL; 1374 info->ifpi_rx[cpuid].serializer = ifp->if_serializer; 1375 1376 if (ifp->if_flags & IFF_RUNNING) 1377 re_setup_intr(sc, 0, RE_IMTYPE_NONE); 1378 ifq_set_cpuid(&ifp->if_snd, cpuid); 1379 } else { 1380 if (ifp->if_flags & IFF_RUNNING) 1381 re_setup_intr(sc, 1, sc->re_imtype); 1382 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(sc->re_irq)); 1383 } 1384 } 1385 #endif /* IFPOLL_ENABLE */ 1386 1387 static void 1388 re_intr(void *arg) 1389 { 1390 struct re_softc *sc = arg; 1391 struct ifnet *ifp = &sc->arpcom.ac_if; 1392 uint16_t status; 1393 int proc; 1394 1395 ASSERT_SERIALIZED(ifp->if_serializer); 1396 1397 if ((sc->re_flags & RE_F_SUSPENDED) || 1398 (ifp->if_flags & IFF_RUNNING) == 0) 1399 return; 1400 1401 /* Disable interrupts. */ 1402 CSR_WRITE_2(sc, RE_IMR, 0); 1403 1404 status = CSR_READ_2(sc, RE_ISR); 1405 again: 1406 proc = 0; 1407 if (status) 1408 CSR_WRITE_2(sc, RE_ISR, status); 1409 if (status & sc->re_intrs) { 1410 if (status & RE_ISR_SYSTEM_ERR) { 1411 rtl_reset(sc); 1412 re_init(sc); 1413 /* Done! */ 1414 return; 1415 } 1416 proc |= re_rxeof(sc); 1417 proc |= re_txeof(sc); 1418 } 1419 1420 if (sc->re_imtype == RE_IMTYPE_SIM) { 1421 if ((sc->re_flags & RE_F_TIMER_INTR)) { 1422 if (!proc) { 1423 /* 1424 * Nothing needs to be processed, fallback 1425 * to use TX/RX interrupts. 1426 * 1427 * NOTE: This will re-enable interrupts. 1428 */ 1429 re_setup_intr(sc, 1, RE_IMTYPE_NONE); 1430 1431 /* 1432 * Recollect, mainly to avoid the possible 1433 * race introduced by changing interrupt 1434 * masks. 1435 */ 1436 re_rxeof(sc); 1437 re_txeof(sc); 1438 } else { 1439 /* Re-enable interrupts. */ 1440 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 1441 CSR_WRITE_4(sc, RE_TIMERCNT, 1); /* reload */ 1442 } 1443 } else if (proc) { 1444 /* 1445 * Assume that using simulated interrupt moderation 1446 * (hardware timer based) could reduce the interript 1447 * rate. 1448 * 1449 * NOTE: This will re-enable interrupts. 1450 */ 1451 re_setup_intr(sc, 1, RE_IMTYPE_SIM); 1452 } else { 1453 /* Re-enable interrupts. */ 1454 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 1455 } 1456 } else { 1457 status = CSR_READ_2(sc, RE_ISR); 1458 if (status & sc->re_intrs) { 1459 if (!ifq_is_empty(&ifp->if_snd)) 1460 if_devstart(ifp); 1461 /* NOTE: Interrupts are still disabled. */ 1462 goto again; 1463 } 1464 /* Re-enable interrupts. */ 1465 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 1466 } 1467 1468 if (!ifq_is_empty(&ifp->if_snd)) 1469 if_devstart(ifp); 1470 } 1471 1472 static int 1473 re_encap(struct re_softc *sc, struct mbuf **m_head, int *idx0) 1474 { 1475 struct mbuf *m = *m_head; 1476 bus_dma_segment_t segs[RE_MAXSEGS]; 1477 bus_dmamap_t map; 1478 int error, maxsegs, idx, i, nsegs; 1479 struct re_desc *d, *tx_ring; 1480 uint32_t cmd_csum, ctl_csum, vlantag; 1481 1482 KASSERT(sc->re_ldata.re_tx_free > RE_TXDESC_SPARE, 1483 ("not enough free TX desc")); 1484 1485 if (sc->re_coalesce_tx_pkt && m->m_pkthdr.len != m->m_len) { 1486 struct mbuf *m_new; 1487 1488 m_new = m_defrag(m, M_NOWAIT); 1489 if (m_new == NULL) { 1490 error = ENOBUFS; 1491 goto back; 1492 } else { 1493 *m_head = m = m_new; 1494 if (m->m_pkthdr.len != m->m_len) { 1495 /* Still not configuous; give up. */ 1496 error = ENOBUFS; 1497 goto back; 1498 } 1499 } 1500 } 1501 1502 map = sc->re_ldata.re_tx_dmamap[*idx0]; 1503 1504 /* 1505 * Set up checksum offload. Note: checksum offload bits must 1506 * appear in all descriptors of a multi-descriptor transmit 1507 * attempt. (This is according to testing done with an 8169 1508 * chip. I'm not sure if this is a requirement or a bug.) 1509 */ 1510 cmd_csum = ctl_csum = 0; 1511 if (m->m_pkthdr.csum_flags & CSUM_IP) { 1512 cmd_csum |= RE_TDESC_CMD_IPCSUM; 1513 ctl_csum |= RE_TDESC_CTL_IPCSUM; 1514 } 1515 if (m->m_pkthdr.csum_flags & CSUM_TCP) { 1516 cmd_csum |= RE_TDESC_CMD_TCPCSUM; 1517 ctl_csum |= RE_TDESC_CTL_TCPCSUM; 1518 } 1519 if (m->m_pkthdr.csum_flags & CSUM_UDP) { 1520 cmd_csum |= RE_TDESC_CMD_UDPCSUM; 1521 ctl_csum |= RE_TDESC_CTL_UDPCSUM; 1522 } 1523 1524 /* For version2 descriptor, csum flags are set on re_control */ 1525 if (sc->re_if_flags & RL_FLAG_DESCV2) 1526 cmd_csum = 0; 1527 else 1528 ctl_csum = 0; 1529 1530 if (sc->re_coalesce_tx_pkt) { 1531 /* 1532 * With some of the RealTek chips, using the checksum offload 1533 * support in conjunction with the autopadding feature results 1534 * in the transmission of corrupt frames. For example, if we 1535 * need to send a really small IP fragment that's less than 60 1536 * bytes in size, and IP header checksumming is enabled, the 1537 * resulting ethernet frame that appears on the wire will 1538 * have garbled payload. To work around this, if TX checksum 1539 * offload is enabled, we always manually pad short frames out 1540 * to the minimum ethernet frame size. 1541 * 1542 * Note: this appears unnecessary for TCP, and doing it for TCP 1543 * with PCIe adapters seems to result in bad checksums. 1544 */ 1545 if ((m->m_pkthdr.csum_flags & 1546 (CSUM_DELAY_IP | CSUM_DELAY_DATA)) && 1547 (m->m_pkthdr.csum_flags & CSUM_TCP) == 0 && 1548 m->m_pkthdr.len < RE_MIN_FRAMELEN) { 1549 error = m_devpad(m, RE_MIN_FRAMELEN); 1550 if (error) 1551 goto back; 1552 } 1553 } 1554 1555 vlantag = 0; 1556 if (m->m_flags & M_VLANTAG) { 1557 vlantag = htobe16(m->m_pkthdr.ether_vlantag) | 1558 RE_TDESC_CTL_INSTAG; 1559 } 1560 1561 maxsegs = sc->re_ldata.re_tx_free; 1562 if (maxsegs > RE_MAXSEGS) 1563 maxsegs = RE_MAXSEGS; 1564 1565 error = bus_dmamap_load_mbuf_defrag(sc->re_ldata.re_tx_mtag, map, 1566 m_head, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 1567 if (error) 1568 goto back; 1569 1570 m = *m_head; 1571 bus_dmamap_sync(sc->re_ldata.re_tx_mtag, map, BUS_DMASYNC_PREWRITE); 1572 1573 /* 1574 * Map the segment array into descriptors. We also keep track 1575 * of the end of the ring and set the end-of-ring bits as needed, 1576 * and we set the ownership bits in all except the very first 1577 * descriptor, whose ownership bits will be turned on later. 1578 */ 1579 tx_ring = sc->re_ldata.re_tx_list; 1580 idx = *idx0; 1581 i = 0; 1582 for (;;) { 1583 uint32_t cmdstat; 1584 1585 d = &tx_ring[idx]; 1586 1587 cmdstat = segs[i].ds_len; 1588 d->re_bufaddr_lo = htole32(RE_ADDR_LO(segs[i].ds_addr)); 1589 d->re_bufaddr_hi = htole32(RE_ADDR_HI(segs[i].ds_addr)); 1590 if (i == 0) 1591 cmdstat |= RE_TDESC_CMD_SOF; 1592 else 1593 cmdstat |= RE_TDESC_CMD_OWN; 1594 if (idx == (sc->re_tx_desc_cnt - 1)) 1595 cmdstat |= RE_TDESC_CMD_EOR; 1596 d->re_cmdstat = htole32(cmdstat | cmd_csum); 1597 d->re_control = htole32(ctl_csum | vlantag); 1598 1599 i++; 1600 if (i == nsegs) 1601 break; 1602 RE_TXDESC_INC(sc, idx); 1603 } 1604 d->re_cmdstat |= htole32(RE_TDESC_CMD_EOF); 1605 1606 /* Transfer ownership of packet to the chip. */ 1607 d->re_cmdstat |= htole32(RE_TDESC_CMD_OWN); 1608 if (*idx0 != idx) 1609 tx_ring[*idx0].re_cmdstat |= htole32(RE_TDESC_CMD_OWN); 1610 1611 /* 1612 * Insure that the map for this transmission 1613 * is placed at the array index of the last descriptor 1614 * in this chain. 1615 */ 1616 sc->re_ldata.re_tx_dmamap[*idx0] = sc->re_ldata.re_tx_dmamap[idx]; 1617 sc->re_ldata.re_tx_dmamap[idx] = map; 1618 1619 sc->re_ldata.re_tx_mbuf[idx] = m; 1620 sc->re_ldata.re_tx_free -= nsegs; 1621 1622 RE_TXDESC_INC(sc, idx); 1623 *idx0 = idx; 1624 back: 1625 if (error) { 1626 m_freem(*m_head); 1627 *m_head = NULL; 1628 } 1629 return error; 1630 } 1631 1632 /* 1633 * Main transmit routine for C+ and gigE NICs. 1634 */ 1635 1636 static void 1637 re_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 1638 { 1639 struct re_softc *sc = ifp->if_softc; 1640 struct mbuf *m_head; 1641 int idx, need_trans, oactive, error; 1642 1643 ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); 1644 ASSERT_SERIALIZED(ifp->if_serializer); 1645 1646 if ((sc->re_flags & RE_F_LINKED) == 0) { 1647 ifq_purge(&ifp->if_snd); 1648 return; 1649 } 1650 1651 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) 1652 return; 1653 1654 idx = sc->re_ldata.re_tx_prodidx; 1655 1656 need_trans = 0; 1657 oactive = 0; 1658 while (sc->re_ldata.re_tx_mbuf[idx] == NULL) { 1659 if (sc->re_ldata.re_tx_free <= RE_TXDESC_SPARE) { 1660 if (!oactive) { 1661 if (re_tx_collect(sc)) { 1662 oactive = 1; 1663 continue; 1664 } 1665 } 1666 ifq_set_oactive(&ifp->if_snd); 1667 break; 1668 } 1669 1670 m_head = ifq_dequeue(&ifp->if_snd); 1671 if (m_head == NULL) 1672 break; 1673 1674 error = re_encap(sc, &m_head, &idx); 1675 if (error) { 1676 /* m_head is freed by re_encap(), if we reach here */ 1677 IFNET_STAT_INC(ifp, oerrors, 1); 1678 1679 if (error == EFBIG && !oactive) { 1680 if (re_tx_collect(sc)) { 1681 oactive = 1; 1682 continue; 1683 } 1684 } 1685 ifq_set_oactive(&ifp->if_snd); 1686 break; 1687 } 1688 1689 oactive = 0; 1690 need_trans = 1; 1691 1692 /* 1693 * If there's a BPF listener, bounce a copy of this frame 1694 * to him. 1695 */ 1696 ETHER_BPF_MTAP(ifp, m_head); 1697 } 1698 1699 /* 1700 * If sc->re_ldata.re_tx_mbuf[idx] is not NULL it is possible 1701 * for OACTIVE to not be properly set when we also do not 1702 * have sufficient free tx descriptors, leaving packet in 1703 * ifp->if_snd. This can cause if_start_dispatch() to loop 1704 * infinitely so make sure OACTIVE is set properly. 1705 */ 1706 if (sc->re_ldata.re_tx_free <= RE_TXDESC_SPARE) { 1707 if (!ifq_is_oactive(&ifp->if_snd)) { 1708 #if 0 1709 if_printf(ifp, "Debug: OACTIVE was not set when " 1710 "re_tx_free was below minimum!\n"); 1711 #endif 1712 ifq_set_oactive(&ifp->if_snd); 1713 } 1714 } 1715 if (!need_trans) 1716 return; 1717 1718 sc->re_ldata.re_tx_prodidx = idx; 1719 1720 /* 1721 * RealTek put the TX poll request register in a different 1722 * location on the 8169 gigE chip. I don't know why. 1723 */ 1724 CSR_WRITE_1(sc, RE_TPPOLL, RE_NPQ); 1725 1726 /* 1727 * Set a timeout in case the chip goes out to lunch. 1728 */ 1729 ifp->if_timer = 5; 1730 } 1731 1732 static void 1733 re_link_up(struct re_softc *sc) 1734 { 1735 struct ifnet *ifp = &sc->arpcom.ac_if; 1736 int error; 1737 1738 ASSERT_SERIALIZED(ifp->if_serializer); 1739 1740 rtl_link_on_patch(sc); 1741 re_stop(sc, FALSE); 1742 rtl_set_eaddr(sc); 1743 1744 error = re_rx_list_init(sc); 1745 if (error) { 1746 re_stop(sc, TRUE); 1747 return; 1748 } 1749 error = re_tx_list_init(sc); 1750 if (error) { 1751 re_stop(sc, TRUE); 1752 return; 1753 } 1754 1755 /* 1756 * Load the addresses of the RX and TX lists into the chip. 1757 */ 1758 CSR_WRITE_4(sc, RE_RXLIST_ADDR_HI, 1759 RE_ADDR_HI(sc->re_ldata.re_rx_list_addr)); 1760 CSR_WRITE_4(sc, RE_RXLIST_ADDR_LO, 1761 RE_ADDR_LO(sc->re_ldata.re_rx_list_addr)); 1762 1763 CSR_WRITE_4(sc, RE_TXLIST_ADDR_HI, 1764 RE_ADDR_HI(sc->re_ldata.re_tx_list_addr)); 1765 CSR_WRITE_4(sc, RE_TXLIST_ADDR_LO, 1766 RE_ADDR_LO(sc->re_ldata.re_tx_list_addr)); 1767 1768 rtl_hw_start(sc); 1769 1770 #ifdef IFPOLL_ENABLE 1771 /* 1772 * Disable interrupts if we are polling. 1773 */ 1774 if (ifp->if_flags & IFF_NPOLLING) 1775 re_setup_intr(sc, 0, RE_IMTYPE_NONE); 1776 else /* otherwise ... */ 1777 #endif /* IFPOLL_ENABLE */ 1778 /* 1779 * Enable interrupts. 1780 */ 1781 re_setup_intr(sc, 1, sc->re_imtype); 1782 CSR_WRITE_2(sc, RE_ISR, sc->re_intrs); 1783 1784 sc->re_flags |= RE_F_LINKED; 1785 ifp->if_link_state = LINK_STATE_UP; 1786 if_link_state_change(ifp); 1787 1788 if (bootverbose) 1789 if_printf(ifp, "link UP\n"); 1790 1791 if (!ifq_is_empty(&ifp->if_snd)) 1792 if_devstart(ifp); 1793 } 1794 1795 static void 1796 re_link_down(struct re_softc *sc) 1797 { 1798 struct ifnet *ifp = &sc->arpcom.ac_if; 1799 1800 /* NOTE: re_stop() will reset RE_F_LINKED. */ 1801 ifp->if_link_state = LINK_STATE_DOWN; 1802 if_link_state_change(ifp); 1803 1804 re_stop(sc, FALSE); 1805 rtl_ifmedia_upd(ifp); 1806 1807 if (bootverbose) 1808 if_printf(ifp, "link DOWN\n"); 1809 } 1810 1811 static void 1812 re_init(void *xsc) 1813 { 1814 struct re_softc *sc = xsc; 1815 struct ifnet *ifp = &sc->arpcom.ac_if; 1816 1817 ASSERT_SERIALIZED(ifp->if_serializer); 1818 1819 re_stop(sc, TRUE); 1820 if (rtl_link_ok(sc)) { 1821 if (bootverbose) 1822 if_printf(ifp, "link is UP in if_init\n"); 1823 re_link_up(sc); 1824 } 1825 1826 ifp->if_flags |= IFF_RUNNING; 1827 ifq_clr_oactive(&ifp->if_snd); 1828 1829 callout_reset(&sc->re_timer, hz, re_tick, sc); 1830 } 1831 1832 static int 1833 re_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) 1834 { 1835 struct re_softc *sc = ifp->if_softc; 1836 struct ifreq *ifr = (struct ifreq *)data; 1837 int error = 0, mask; 1838 1839 ASSERT_SERIALIZED(ifp->if_serializer); 1840 1841 switch(command) { 1842 case SIOCSIFMTU: 1843 #ifdef RE_JUMBO 1844 if (ifr->ifr_mtu > sc->re_maxmtu) { 1845 error = EINVAL; 1846 } else if (ifp->if_mtu != ifr->ifr_mtu) { 1847 ifp->if_mtu = ifr->ifr_mtu; 1848 if (ifp->if_flags & IFF_RUNNING) 1849 ifp->if_init(sc); 1850 } 1851 #else 1852 error = EOPNOTSUPP; 1853 #endif 1854 break; 1855 1856 case SIOCSIFFLAGS: 1857 if (ifp->if_flags & IFF_UP) { 1858 if (ifp->if_flags & IFF_RUNNING) { 1859 if ((ifp->if_flags ^ sc->re_saved_ifflags) & 1860 (IFF_PROMISC | IFF_ALLMULTI)) 1861 rtl_set_rx_packet_filter(sc); 1862 } else { 1863 re_init(sc); 1864 } 1865 } else if (ifp->if_flags & IFF_RUNNING) { 1866 re_stop(sc, TRUE); 1867 } 1868 sc->re_saved_ifflags = ifp->if_flags; 1869 break; 1870 1871 case SIOCADDMULTI: 1872 case SIOCDELMULTI: 1873 rtl_set_rx_packet_filter(sc); 1874 break; 1875 1876 case SIOCGIFMEDIA: 1877 case SIOCSIFMEDIA: 1878 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 1879 break; 1880 1881 case SIOCSIFCAP: 1882 mask = (ifr->ifr_reqcap ^ ifp->if_capenable) & 1883 ifp->if_capabilities; 1884 ifp->if_capenable ^= mask; 1885 1886 /* NOTE: re_init will setup if_hwassist. */ 1887 ifp->if_hwassist = 0; 1888 1889 /* Setup flags for the backend. */ 1890 if (ifp->if_capenable & IFCAP_RXCSUM) 1891 sc->re_rx_cstag = 1; 1892 else 1893 sc->re_rx_cstag = 0; 1894 if (ifp->if_capenable & IFCAP_TXCSUM) 1895 sc->re_tx_cstag = 1; 1896 else 1897 sc->re_tx_cstag = 0; 1898 1899 if (mask && (ifp->if_flags & IFF_RUNNING)) 1900 re_init(sc); 1901 break; 1902 1903 default: 1904 error = ether_ioctl(ifp, command, data); 1905 break; 1906 } 1907 return(error); 1908 } 1909 1910 static void 1911 re_watchdog(struct ifnet *ifp) 1912 { 1913 struct re_softc *sc = ifp->if_softc; 1914 1915 ASSERT_SERIALIZED(ifp->if_serializer); 1916 1917 IFNET_STAT_INC(ifp, oerrors, 1); 1918 1919 re_txeof(sc); 1920 re_rxeof(sc); 1921 1922 if (sc->re_ldata.re_tx_free != sc->re_tx_desc_cnt) { 1923 if_printf(ifp, "watchdog timeout, txd free %d\n", 1924 sc->re_ldata.re_tx_free); 1925 rtl_reset(sc); 1926 re_init(sc); 1927 } 1928 } 1929 1930 /* 1931 * Stop the adapter and free any mbufs allocated to the 1932 * RX and TX lists. 1933 */ 1934 static void 1935 re_stop(struct re_softc *sc, boolean_t full_stop) 1936 { 1937 struct ifnet *ifp = &sc->arpcom.ac_if; 1938 int i; 1939 1940 ASSERT_SERIALIZED(ifp->if_serializer); 1941 1942 /* Stop the adapter. */ 1943 rtl_stop(sc); 1944 1945 ifp->if_timer = 0; 1946 if (full_stop) { 1947 callout_stop(&sc->re_timer); 1948 ifp->if_flags &= ~IFF_RUNNING; 1949 } 1950 ifq_clr_oactive(&ifp->if_snd); 1951 sc->re_flags &= ~(RE_F_TIMER_INTR | RE_F_DROP_RXFRAG | RE_F_LINKED); 1952 1953 re_free_rxchain(sc); 1954 1955 /* Free the TX list buffers. */ 1956 for (i = 0; i < sc->re_tx_desc_cnt; i++) { 1957 if (sc->re_ldata.re_tx_mbuf[i] != NULL) { 1958 bus_dmamap_unload(sc->re_ldata.re_tx_mtag, 1959 sc->re_ldata.re_tx_dmamap[i]); 1960 m_freem(sc->re_ldata.re_tx_mbuf[i]); 1961 sc->re_ldata.re_tx_mbuf[i] = NULL; 1962 } 1963 } 1964 1965 /* Free the RX list buffers. */ 1966 for (i = 0; i < sc->re_rx_desc_cnt; i++) { 1967 if (sc->re_ldata.re_rx_mbuf[i] != NULL) { 1968 if ((sc->re_flags & RE_F_USE_JPOOL) == 0) { 1969 bus_dmamap_unload(sc->re_ldata.re_rx_mtag, 1970 sc->re_ldata.re_rx_dmamap[i]); 1971 } 1972 m_freem(sc->re_ldata.re_rx_mbuf[i]); 1973 sc->re_ldata.re_rx_mbuf[i] = NULL; 1974 } 1975 } 1976 } 1977 1978 /* 1979 * Device suspend routine. Stop the interface and save some PCI 1980 * settings in case the BIOS doesn't restore them properly on 1981 * resume. 1982 */ 1983 static int 1984 re_suspend(device_t dev) 1985 { 1986 #ifndef BURN_BRIDGES 1987 int i; 1988 #endif 1989 struct re_softc *sc = device_get_softc(dev); 1990 struct ifnet *ifp = &sc->arpcom.ac_if; 1991 1992 lwkt_serialize_enter(ifp->if_serializer); 1993 1994 re_stop(sc, TRUE); 1995 1996 #ifndef BURN_BRIDGES 1997 for (i = 0; i < 5; i++) 1998 sc->saved_maps[i] = pci_read_config(dev, PCIR_MAPS + i * 4, 4); 1999 sc->saved_biosaddr = pci_read_config(dev, PCIR_BIOS, 4); 2000 sc->saved_intline = pci_read_config(dev, PCIR_INTLINE, 1); 2001 sc->saved_cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1); 2002 sc->saved_lattimer = pci_read_config(dev, PCIR_LATTIMER, 1); 2003 #endif 2004 2005 sc->re_flags |= RE_F_SUSPENDED; 2006 2007 lwkt_serialize_exit(ifp->if_serializer); 2008 2009 return (0); 2010 } 2011 2012 /* 2013 * Device resume routine. Restore some PCI settings in case the BIOS 2014 * doesn't, re-enable busmastering, and restart the interface if 2015 * appropriate. 2016 */ 2017 static int 2018 re_resume(device_t dev) 2019 { 2020 struct re_softc *sc = device_get_softc(dev); 2021 struct ifnet *ifp = &sc->arpcom.ac_if; 2022 #ifndef BURN_BRIDGES 2023 int i; 2024 #endif 2025 2026 lwkt_serialize_enter(ifp->if_serializer); 2027 2028 #ifndef BURN_BRIDGES 2029 /* better way to do this? */ 2030 for (i = 0; i < 5; i++) 2031 pci_write_config(dev, PCIR_MAPS + i * 4, sc->saved_maps[i], 4); 2032 pci_write_config(dev, PCIR_BIOS, sc->saved_biosaddr, 4); 2033 pci_write_config(dev, PCIR_INTLINE, sc->saved_intline, 1); 2034 pci_write_config(dev, PCIR_CACHELNSZ, sc->saved_cachelnsz, 1); 2035 pci_write_config(dev, PCIR_LATTIMER, sc->saved_lattimer, 1); 2036 2037 /* reenable busmastering */ 2038 pci_enable_busmaster(dev); 2039 pci_enable_io(dev, SYS_RES_IOPORT); 2040 #endif 2041 2042 /* reinitialize interface if necessary */ 2043 if (ifp->if_flags & IFF_UP) 2044 re_init(sc); 2045 2046 sc->re_flags &= ~RE_F_SUSPENDED; 2047 2048 lwkt_serialize_exit(ifp->if_serializer); 2049 2050 return (0); 2051 } 2052 2053 /* 2054 * Stop all chip I/O so that the kernel's probe routines don't 2055 * get confused by errant DMAs when rebooting. 2056 */ 2057 static void 2058 re_shutdown(device_t dev) 2059 { 2060 struct re_softc *sc = device_get_softc(dev); 2061 struct ifnet *ifp = &sc->arpcom.ac_if; 2062 2063 lwkt_serialize_enter(ifp->if_serializer); 2064 re_stop(sc, TRUE); 2065 rtl_hw_d3_para(sc); 2066 rtl_phy_power_down(sc); 2067 lwkt_serialize_exit(ifp->if_serializer); 2068 } 2069 2070 static int 2071 re_sysctl_rxtime(SYSCTL_HANDLER_ARGS) 2072 { 2073 struct re_softc *sc = arg1; 2074 2075 return re_sysctl_hwtime(oidp, arg1, arg2, req, &sc->re_rx_time); 2076 } 2077 2078 static int 2079 re_sysctl_txtime(SYSCTL_HANDLER_ARGS) 2080 { 2081 struct re_softc *sc = arg1; 2082 2083 return re_sysctl_hwtime(oidp, arg1, arg2, req, &sc->re_tx_time); 2084 } 2085 2086 static int 2087 re_sysctl_hwtime(SYSCTL_HANDLER_ARGS, int *hwtime) 2088 { 2089 struct re_softc *sc = arg1; 2090 struct ifnet *ifp = &sc->arpcom.ac_if; 2091 int error, v; 2092 2093 lwkt_serialize_enter(ifp->if_serializer); 2094 2095 v = *hwtime; 2096 error = sysctl_handle_int(oidp, &v, 0, req); 2097 if (error || req->newptr == NULL) 2098 goto back; 2099 2100 if (v <= 0) { 2101 error = EINVAL; 2102 goto back; 2103 } 2104 2105 if (v != *hwtime) { 2106 *hwtime = v; 2107 2108 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) == 2109 IFF_RUNNING && sc->re_imtype == RE_IMTYPE_HW) 2110 re_setup_hw_im(sc); 2111 } 2112 back: 2113 lwkt_serialize_exit(ifp->if_serializer); 2114 return error; 2115 } 2116 2117 static int 2118 re_sysctl_simtime(SYSCTL_HANDLER_ARGS) 2119 { 2120 struct re_softc *sc = arg1; 2121 struct ifnet *ifp = &sc->arpcom.ac_if; 2122 int error, v; 2123 2124 lwkt_serialize_enter(ifp->if_serializer); 2125 2126 v = sc->re_sim_time; 2127 error = sysctl_handle_int(oidp, &v, 0, req); 2128 if (error || req->newptr == NULL) 2129 goto back; 2130 2131 if (v <= 0) { 2132 error = EINVAL; 2133 goto back; 2134 } 2135 2136 if (v != sc->re_sim_time) { 2137 sc->re_sim_time = v; 2138 2139 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) == 2140 IFF_RUNNING && sc->re_imtype == RE_IMTYPE_SIM) { 2141 #ifdef foo 2142 /* 2143 * Following code causes various strange 2144 * performance problems. Hmm ... 2145 */ 2146 CSR_WRITE_2(sc, RE_IMR, 0); 2147 CSR_WRITE_4(sc, RE_TIMERINT, 0); 2148 CSR_READ_4(sc, RE_TIMERINT); /* flush */ 2149 2150 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 2151 re_setup_sim_im(sc); 2152 #else 2153 re_setup_intr(sc, 0, RE_IMTYPE_NONE); 2154 DELAY(10); 2155 re_setup_intr(sc, 1, RE_IMTYPE_SIM); 2156 #endif 2157 } 2158 } 2159 back: 2160 lwkt_serialize_exit(ifp->if_serializer); 2161 return error; 2162 } 2163 2164 static int 2165 re_sysctl_imtype(SYSCTL_HANDLER_ARGS) 2166 { 2167 struct re_softc *sc = arg1; 2168 struct ifnet *ifp = &sc->arpcom.ac_if; 2169 int error, v; 2170 2171 lwkt_serialize_enter(ifp->if_serializer); 2172 2173 v = sc->re_imtype; 2174 error = sysctl_handle_int(oidp, &v, 0, req); 2175 if (error || req->newptr == NULL) 2176 goto back; 2177 2178 if (v != RE_IMTYPE_HW && v != RE_IMTYPE_SIM && v != RE_IMTYPE_NONE) { 2179 error = EINVAL; 2180 goto back; 2181 } 2182 if (v == RE_IMTYPE_HW && (sc->re_caps & RE_C_HWIM) == 0) { 2183 /* Can't do hardware interrupt moderation */ 2184 error = EOPNOTSUPP; 2185 goto back; 2186 } 2187 2188 if (v != sc->re_imtype) { 2189 sc->re_imtype = v; 2190 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) == 2191 IFF_RUNNING) 2192 re_setup_intr(sc, 1, sc->re_imtype); 2193 } 2194 back: 2195 lwkt_serialize_exit(ifp->if_serializer); 2196 return error; 2197 } 2198 2199 static void 2200 re_setup_hw_im(struct re_softc *sc) 2201 { 2202 KKASSERT(sc->re_caps & RE_C_HWIM); 2203 2204 /* 2205 * Interrupt moderation 2206 * 2207 * 0xABCD 2208 * A - unknown (maybe TX related) 2209 * B - TX timer (unit: 25us) 2210 * C - unknown (maybe RX related) 2211 * D - RX timer (unit: 25us) 2212 * 2213 * 2214 * re(4)'s interrupt moderation is actually controlled by 2215 * two variables, like most other NICs (bge, bce etc.) 2216 * o timer 2217 * o number of packets [P] 2218 * 2219 * The logic relationship between these two variables is 2220 * similar to other NICs too: 2221 * if (timer expire || packets > [P]) 2222 * Interrupt is delivered 2223 * 2224 * Currently we only know how to set 'timer', but not 2225 * 'number of packets', which should be ~30, as far as I 2226 * tested (sink ~900Kpps, interrupt rate is 30KHz) 2227 */ 2228 CSR_WRITE_2(sc, RE_IM, 2229 RE_IM_RXTIME(sc->re_rx_time) | 2230 RE_IM_TXTIME(sc->re_tx_time) | 2231 RE_IM_MAGIC); 2232 } 2233 2234 static void 2235 re_disable_hw_im(struct re_softc *sc) 2236 { 2237 if (sc->re_caps & RE_C_HWIM) 2238 CSR_WRITE_2(sc, RE_IM, 0); 2239 } 2240 2241 static void 2242 re_setup_sim_im(struct re_softc *sc) 2243 { 2244 uint32_t ticks; 2245 2246 if (sc->re_if_flags & RL_FLAG_PCIE) { 2247 ticks = sc->re_sim_time * sc->re_bus_speed; 2248 } else { 2249 /* 2250 * Datasheet says tick decreases at bus speed, 2251 * but it seems the clock runs a little bit 2252 * faster, so we do some compensation here. 2253 */ 2254 ticks = (sc->re_sim_time * sc->re_bus_speed * 8) / 5; 2255 } 2256 CSR_WRITE_4(sc, RE_TIMERINT, ticks); 2257 2258 CSR_WRITE_4(sc, RE_TIMERCNT, 1); /* reload */ 2259 sc->re_flags |= RE_F_TIMER_INTR; 2260 } 2261 2262 static void 2263 re_disable_sim_im(struct re_softc *sc) 2264 { 2265 CSR_WRITE_4(sc, RE_TIMERINT, 0); 2266 sc->re_flags &= ~RE_F_TIMER_INTR; 2267 } 2268 2269 static void 2270 re_config_imtype(struct re_softc *sc, int imtype) 2271 { 2272 switch (imtype) { 2273 case RE_IMTYPE_HW: 2274 KKASSERT(sc->re_caps & RE_C_HWIM); 2275 /* FALL THROUGH */ 2276 case RE_IMTYPE_NONE: 2277 sc->re_intrs = RE_INTRS; 2278 sc->re_rx_ack = RE_ISR_RX_OK | RE_ISR_FIFO_OFLOW | 2279 RE_ISR_RX_OVERRUN; 2280 sc->re_tx_ack = RE_ISR_TX_OK; 2281 break; 2282 2283 case RE_IMTYPE_SIM: 2284 sc->re_intrs = RE_INTRS_TIMER; 2285 sc->re_rx_ack = RE_ISR_PCS_TIMEOUT; 2286 sc->re_tx_ack = RE_ISR_PCS_TIMEOUT; 2287 break; 2288 2289 default: 2290 panic("%s: unknown imtype %d", 2291 sc->arpcom.ac_if.if_xname, imtype); 2292 } 2293 } 2294 2295 static void 2296 re_setup_intr(struct re_softc *sc, int enable_intrs, int imtype) 2297 { 2298 re_config_imtype(sc, imtype); 2299 2300 if (enable_intrs) 2301 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 2302 else 2303 CSR_WRITE_2(sc, RE_IMR, 0); 2304 2305 sc->re_npoll.ifpc_stcount = 0; 2306 2307 switch (imtype) { 2308 case RE_IMTYPE_NONE: 2309 re_disable_sim_im(sc); 2310 re_disable_hw_im(sc); 2311 break; 2312 2313 case RE_IMTYPE_HW: 2314 KKASSERT(sc->re_caps & RE_C_HWIM); 2315 re_disable_sim_im(sc); 2316 re_setup_hw_im(sc); 2317 break; 2318 2319 case RE_IMTYPE_SIM: 2320 re_disable_hw_im(sc); 2321 re_setup_sim_im(sc); 2322 break; 2323 2324 default: 2325 panic("%s: unknown imtype %d", 2326 sc->arpcom.ac_if.if_xname, imtype); 2327 } 2328 } 2329 2330 static int 2331 re_jpool_alloc(struct re_softc *sc) 2332 { 2333 struct re_list_data *ldata = &sc->re_ldata; 2334 struct re_jbuf *jbuf; 2335 bus_addr_t paddr; 2336 bus_size_t jpool_size; 2337 bus_dmamem_t dmem; 2338 caddr_t buf; 2339 int i, error; 2340 2341 lwkt_serialize_init(&ldata->re_jbuf_serializer); 2342 2343 ldata->re_jbuf = kmalloc(sizeof(struct re_jbuf) * RE_JBUF_COUNT(sc), 2344 M_DEVBUF, M_WAITOK | M_ZERO); 2345 2346 jpool_size = RE_JBUF_COUNT(sc) * RE_JBUF_SIZE; 2347 2348 error = bus_dmamem_coherent(sc->re_parent_tag, 2349 RE_RXBUF_ALIGN, 0, 2350 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 2351 jpool_size, BUS_DMA_WAITOK, &dmem); 2352 if (error) { 2353 device_printf(sc->dev, "could not allocate jumbo memory\n"); 2354 return error; 2355 } 2356 ldata->re_jpool_tag = dmem.dmem_tag; 2357 ldata->re_jpool_map = dmem.dmem_map; 2358 ldata->re_jpool = dmem.dmem_addr; 2359 paddr = dmem.dmem_busaddr; 2360 2361 /* ..and split it into 9KB chunks */ 2362 SLIST_INIT(&ldata->re_jbuf_free); 2363 2364 buf = ldata->re_jpool; 2365 for (i = 0; i < RE_JBUF_COUNT(sc); i++) { 2366 jbuf = &ldata->re_jbuf[i]; 2367 2368 jbuf->re_sc = sc; 2369 jbuf->re_inuse = 0; 2370 jbuf->re_slot = i; 2371 jbuf->re_buf = buf; 2372 jbuf->re_paddr = paddr; 2373 2374 SLIST_INSERT_HEAD(&ldata->re_jbuf_free, jbuf, re_link); 2375 2376 buf += RE_JBUF_SIZE; 2377 paddr += RE_JBUF_SIZE; 2378 } 2379 return 0; 2380 } 2381 2382 static void 2383 re_jpool_free(struct re_softc *sc) 2384 { 2385 struct re_list_data *ldata = &sc->re_ldata; 2386 2387 if (ldata->re_jpool_tag != NULL) { 2388 bus_dmamap_unload(ldata->re_jpool_tag, ldata->re_jpool_map); 2389 bus_dmamem_free(ldata->re_jpool_tag, ldata->re_jpool, 2390 ldata->re_jpool_map); 2391 bus_dma_tag_destroy(ldata->re_jpool_tag); 2392 ldata->re_jpool_tag = NULL; 2393 } 2394 2395 if (ldata->re_jbuf != NULL) { 2396 kfree(ldata->re_jbuf, M_DEVBUF); 2397 ldata->re_jbuf = NULL; 2398 } 2399 } 2400 2401 #ifdef RE_JUMBO 2402 static struct re_jbuf * 2403 re_jbuf_alloc(struct re_softc *sc) 2404 { 2405 struct re_list_data *ldata = &sc->re_ldata; 2406 struct re_jbuf *jbuf; 2407 2408 lwkt_serialize_enter(&ldata->re_jbuf_serializer); 2409 2410 jbuf = SLIST_FIRST(&ldata->re_jbuf_free); 2411 if (jbuf != NULL) { 2412 SLIST_REMOVE_HEAD(&ldata->re_jbuf_free, re_link); 2413 jbuf->re_inuse = 1; 2414 } 2415 2416 lwkt_serialize_exit(&ldata->re_jbuf_serializer); 2417 2418 return jbuf; 2419 } 2420 2421 static void 2422 re_jbuf_free(void *arg) 2423 { 2424 struct re_jbuf *jbuf = arg; 2425 struct re_softc *sc = jbuf->re_sc; 2426 struct re_list_data *ldata = &sc->re_ldata; 2427 2428 if (&ldata->re_jbuf[jbuf->re_slot] != jbuf) { 2429 panic("%s: free wrong jumbo buffer", 2430 sc->arpcom.ac_if.if_xname); 2431 } else if (jbuf->re_inuse == 0) { 2432 panic("%s: jumbo buffer already freed", 2433 sc->arpcom.ac_if.if_xname); 2434 } 2435 2436 lwkt_serialize_enter(&ldata->re_jbuf_serializer); 2437 atomic_subtract_int(&jbuf->re_inuse, 1); 2438 if (jbuf->re_inuse == 0) 2439 SLIST_INSERT_HEAD(&ldata->re_jbuf_free, jbuf, re_link); 2440 lwkt_serialize_exit(&ldata->re_jbuf_serializer); 2441 } 2442 2443 static void 2444 re_jbuf_ref(void *arg) 2445 { 2446 struct re_jbuf *jbuf = arg; 2447 struct re_softc *sc = jbuf->re_sc; 2448 struct re_list_data *ldata = &sc->re_ldata; 2449 2450 if (&ldata->re_jbuf[jbuf->re_slot] != jbuf) { 2451 panic("%s: ref wrong jumbo buffer", 2452 sc->arpcom.ac_if.if_xname); 2453 } else if (jbuf->re_inuse == 0) { 2454 panic("%s: jumbo buffer already freed", 2455 sc->arpcom.ac_if.if_xname); 2456 } 2457 atomic_add_int(&jbuf->re_inuse, 1); 2458 } 2459 #endif /* RE_JUMBO */ 2460 2461 static void 2462 re_disable_aspm(device_t dev) 2463 { 2464 uint16_t link_cap, link_ctrl; 2465 uint8_t pcie_ptr, reg; 2466 2467 pcie_ptr = pci_get_pciecap_ptr(dev); 2468 if (pcie_ptr == 0) 2469 return; 2470 2471 link_cap = pci_read_config(dev, pcie_ptr + PCIER_LINKCAP, 2); 2472 if ((link_cap & PCIEM_LNKCAP_ASPM_MASK) == 0) 2473 return; 2474 2475 if (bootverbose) 2476 device_printf(dev, "disable ASPM\n"); 2477 2478 reg = pcie_ptr + PCIER_LINKCTRL; 2479 link_ctrl = pci_read_config(dev, reg, 2); 2480 link_ctrl &= ~(PCIEM_LNKCTL_ASPM_L0S | PCIEM_LNKCTL_ASPM_L1); 2481 pci_write_config(dev, reg, link_ctrl, 2); 2482 } 2483