1 /* 2 * Copyright (c) 2004 3 * Joerg Sonnenberger <joerg@bec.de>. All rights reserved. 4 * 5 * Copyright (c) 1997, 1998-2003 6 * Bill Paul <wpaul@windriver.com>. All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Bill Paul. 19 * 4. Neither the name of the author nor the names of any co-contributors 20 * may be used to endorse or promote products derived from this software 21 * without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 33 * THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * $FreeBSD: src/sys/dev/re/if_re.c,v 1.25 2004/06/09 14:34:01 naddy Exp $ 36 */ 37 38 /* 39 * RealTek 8169S/8110S/8168/8111/8101E PCI NIC driver 40 * 41 * Written by Bill Paul <wpaul@windriver.com> 42 * Senior Networking Software Engineer 43 * Wind River Systems 44 */ 45 46 /* 47 * This driver is designed to support RealTek's next generation of 48 * 10/100 and 10/100/1000 PCI ethernet controllers. There are currently 49 * seven devices in this family: the the RTL8169, the RTL8169S, RTL8110S, 50 * the RTL8168, the RTL8111 and the RTL8101E. 51 * 52 * The 8169 is a 64-bit 10/100/1000 gigabit ethernet MAC: 53 * 54 * o Descriptor based DMA mechanism. Each descriptor represents 55 * a single packet fragment. Data buffers may be aligned on 56 * any byte boundary. 57 * 58 * o 64-bit DMA. 59 * 60 * o TCP/IP checksum offload for both RX and TX. 61 * 62 * o High and normal priority transmit DMA rings. 63 * 64 * o VLAN tag insertion and extraction. 65 * 66 * o TCP large send (segmentation offload). 67 * 68 * o 1000Mbps mode. 69 * 70 * o Jumbo frames. 71 * 72 * o GMII and TBI ports/registers for interfacing with copper 73 * or fiber PHYs. 74 * 75 * o RX and TX DMA rings can have up to 1024 descriptors. 76 * 77 * The 8169 does not have a built-in PHY. Most reference boards use a 78 * Marvell 88E1000 'Alaska' copper gigE PHY. 8169/8110 is _no longer_ 79 * supported. 80 * 81 * The 8169S/8110S 10/100/1000 devices have built-in copper gigE PHYs 82 * (the 'S' stands for 'single-chip'). These devices have the same 83 * programming API as the older 8169, but also have some vendor-specific 84 * registers for the on-board PHY. The 8110S is a LAN-on-motherboard 85 * part designed to be pin-compatible with the RealTek 8100 10/100 chip. 86 * 87 * This driver takes advantage of the RX and TX checksum offload and 88 * VLAN tag insertion/extraction features. It also implements 89 * interrupt moderation using the timer interrupt registers, which 90 * significantly reduces interrupt load. 91 */ 92 93 #define _IP_VHL 94 95 #include "opt_ifpoll.h" 96 97 #include <sys/param.h> 98 #include <sys/bus.h> 99 #include <sys/endian.h> 100 #include <sys/kernel.h> 101 #include <sys/in_cksum.h> 102 #include <sys/interrupt.h> 103 #include <sys/malloc.h> 104 #include <sys/mbuf.h> 105 #include <sys/rman.h> 106 #include <sys/serialize.h> 107 #include <sys/socket.h> 108 #include <sys/sockio.h> 109 #include <sys/sysctl.h> 110 111 #include <net/bpf.h> 112 #include <net/ethernet.h> 113 #include <net/if.h> 114 #include <net/ifq_var.h> 115 #include <net/if_arp.h> 116 #include <net/if_dl.h> 117 #include <net/if_media.h> 118 #include <net/if_poll.h> 119 #include <net/if_types.h> 120 #include <net/vlan/if_vlan_var.h> 121 #include <net/vlan/if_vlan_ether.h> 122 123 #include <netinet/ip.h> 124 125 #include "pcidevs.h" 126 #include <bus/pci/pcireg.h> 127 #include <bus/pci/pcivar.h> 128 129 #include <dev/netif/re/if_rereg.h> 130 #include <dev/netif/re/if_revar.h> 131 #include <dev/netif/re/re.h> 132 #include <dev/netif/re/re_dragonfly.h> 133 134 /* 135 * Various supported device vendors/types and their names. 136 */ 137 static const struct re_type { 138 uint16_t re_vid; 139 uint16_t re_did; 140 const char *re_name; 141 } re_devs[] = { 142 { PCI_VENDOR_DLINK, PCI_PRODUCT_DLINK_DGE528T, 143 "D-Link DGE-528(T) Gigabit Ethernet Adapter" }, 144 145 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8101E, 146 "RealTek 810x PCIe 10/100baseTX" }, 147 148 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8168, 149 "RealTek 8111/8168 PCIe Gigabit Ethernet" }, 150 151 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8168_1, 152 "RealTek 8168 PCIe Gigabit Ethernet" }, 153 154 #ifdef notyet 155 /* 156 * This driver now only supports built-in PHYs. 157 */ 158 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8169, 159 "RealTek 8110/8169 Gigabit Ethernet" }, 160 #endif 161 162 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8169SC, 163 "RealTek 8169SC/8110SC Single-chip Gigabit Ethernet" }, 164 165 { PCI_VENDOR_COREGA, PCI_PRODUCT_COREGA_CG_LAPCIGT, 166 "Corega CG-LAPCIGT Gigabit Ethernet" }, 167 168 { PCI_VENDOR_LINKSYS, PCI_PRODUCT_LINKSYS_EG1032, 169 "Linksys EG1032 Gigabit Ethernet" }, 170 171 { PCI_VENDOR_USR2, PCI_PRODUCT_USR2_997902, 172 "US Robotics 997902 Gigabit Ethernet" }, 173 174 { PCI_VENDOR_TTTECH, PCI_PRODUCT_TTTECH_MC322, 175 "TTTech MC322 Gigabit Ethernet" }, 176 177 { 0, 0, NULL } 178 }; 179 180 static int re_probe(device_t); 181 static int re_attach(device_t); 182 static int re_detach(device_t); 183 static int re_suspend(device_t); 184 static int re_resume(device_t); 185 static void re_shutdown(device_t); 186 187 static int re_allocmem(device_t); 188 static void re_freemem(device_t); 189 static void re_freebufmem(struct re_softc *, int, int); 190 static int re_encap(struct re_softc *, struct mbuf **, int *); 191 static int re_newbuf_std(struct re_softc *, int, int); 192 #ifdef RE_JUMBO 193 static int re_newbuf_jumbo(struct re_softc *, int, int); 194 #endif 195 static void re_setup_rxdesc(struct re_softc *, int); 196 static int re_rx_list_init(struct re_softc *); 197 static int re_tx_list_init(struct re_softc *); 198 static int re_rxeof(struct re_softc *); 199 static int re_txeof(struct re_softc *); 200 static int re_tx_collect(struct re_softc *); 201 static void re_intr(void *); 202 static void re_tick(void *); 203 static void re_tick_serialized(void *); 204 static void re_disable_aspm(device_t); 205 static void re_link_up(struct re_softc *); 206 static void re_link_down(struct re_softc *); 207 208 static void re_start(struct ifnet *, struct ifaltq_subque *); 209 static int re_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 210 static void re_init(void *); 211 static void re_stop(struct re_softc *, boolean_t); 212 static void re_watchdog(struct ifnet *); 213 214 static void re_setup_hw_im(struct re_softc *); 215 static void re_setup_sim_im(struct re_softc *); 216 static void re_disable_hw_im(struct re_softc *); 217 static void re_disable_sim_im(struct re_softc *); 218 static void re_config_imtype(struct re_softc *, int); 219 static void re_setup_intr(struct re_softc *, int, int); 220 221 static int re_sysctl_hwtime(SYSCTL_HANDLER_ARGS, int *); 222 static int re_sysctl_rxtime(SYSCTL_HANDLER_ARGS); 223 static int re_sysctl_txtime(SYSCTL_HANDLER_ARGS); 224 static int re_sysctl_simtime(SYSCTL_HANDLER_ARGS); 225 static int re_sysctl_imtype(SYSCTL_HANDLER_ARGS); 226 227 static int re_jpool_alloc(struct re_softc *); 228 static void re_jpool_free(struct re_softc *); 229 #ifdef RE_JUMBO 230 static struct re_jbuf *re_jbuf_alloc(struct re_softc *); 231 static void re_jbuf_free(void *); 232 static void re_jbuf_ref(void *); 233 #endif 234 235 #ifdef IFPOLL_ENABLE 236 static void re_npoll(struct ifnet *, struct ifpoll_info *); 237 static void re_npoll_compat(struct ifnet *, void *, int); 238 #endif 239 240 static device_method_t re_methods[] = { 241 /* Device interface */ 242 DEVMETHOD(device_probe, re_probe), 243 DEVMETHOD(device_attach, re_attach), 244 DEVMETHOD(device_detach, re_detach), 245 DEVMETHOD(device_suspend, re_suspend), 246 DEVMETHOD(device_resume, re_resume), 247 DEVMETHOD(device_shutdown, re_shutdown), 248 DEVMETHOD_END 249 }; 250 251 static driver_t re_driver = { 252 "re", 253 re_methods, 254 sizeof(struct re_softc) 255 }; 256 257 static devclass_t re_devclass; 258 259 DECLARE_DUMMY_MODULE(if_re); 260 DRIVER_MODULE(if_re, pci, re_driver, re_devclass, NULL, NULL); 261 DRIVER_MODULE(if_re, cardbus, re_driver, re_devclass, NULL, NULL); 262 263 static int re_rx_desc_count = RE_RX_DESC_CNT_DEF; 264 static int re_tx_desc_count = RE_TX_DESC_CNT_DEF; 265 static int re_msi_enable = 1; 266 267 TUNABLE_INT("hw.re.rx_desc_count", &re_rx_desc_count); 268 TUNABLE_INT("hw.re.tx_desc_count", &re_tx_desc_count); 269 TUNABLE_INT("hw.re.msi.enable", &re_msi_enable); 270 271 static __inline void 272 re_free_rxchain(struct re_softc *sc) 273 { 274 if (sc->re_head != NULL) { 275 m_freem(sc->re_head); 276 sc->re_head = sc->re_tail = NULL; 277 } 278 } 279 280 static int 281 re_probe(device_t dev) 282 { 283 const struct re_type *t; 284 uint16_t vendor, product; 285 286 vendor = pci_get_vendor(dev); 287 product = pci_get_device(dev); 288 289 /* 290 * Only attach to rev.3 of the Linksys EG1032 adapter. 291 * Rev.2 is supported by sk(4). 292 */ 293 if (vendor == PCI_VENDOR_LINKSYS && 294 product == PCI_PRODUCT_LINKSYS_EG1032 && 295 pci_get_subdevice(dev) != PCI_SUBDEVICE_LINKSYS_EG1032_REV3) 296 return ENXIO; 297 298 for (t = re_devs; t->re_name != NULL; t++) { 299 if (product == t->re_did && vendor == t->re_vid) 300 break; 301 } 302 if (t->re_name == NULL) 303 return ENXIO; 304 305 device_set_desc(dev, t->re_name); 306 return 0; 307 } 308 309 static int 310 re_allocmem(device_t dev) 311 { 312 struct re_softc *sc = device_get_softc(dev); 313 bus_dmamem_t dmem; 314 int error, i; 315 316 /* 317 * Allocate list data 318 */ 319 sc->re_ldata.re_tx_mbuf = 320 kmalloc(sc->re_tx_desc_cnt * sizeof(struct mbuf *), 321 M_DEVBUF, M_ZERO | M_WAITOK); 322 323 sc->re_ldata.re_rx_mbuf = 324 kmalloc(sc->re_rx_desc_cnt * sizeof(struct mbuf *), 325 M_DEVBUF, M_ZERO | M_WAITOK); 326 327 sc->re_ldata.re_rx_paddr = 328 kmalloc(sc->re_rx_desc_cnt * sizeof(bus_addr_t), 329 M_DEVBUF, M_ZERO | M_WAITOK); 330 331 sc->re_ldata.re_tx_dmamap = 332 kmalloc(sc->re_tx_desc_cnt * sizeof(bus_dmamap_t), 333 M_DEVBUF, M_ZERO | M_WAITOK); 334 335 sc->re_ldata.re_rx_dmamap = 336 kmalloc(sc->re_rx_desc_cnt * sizeof(bus_dmamap_t), 337 M_DEVBUF, M_ZERO | M_WAITOK); 338 339 /* 340 * Allocate the parent bus DMA tag appropriate for PCI. 341 */ 342 error = bus_dma_tag_create(NULL, /* parent */ 343 1, 0, /* alignment, boundary */ 344 BUS_SPACE_MAXADDR, /* lowaddr */ 345 BUS_SPACE_MAXADDR, /* highaddr */ 346 NULL, NULL, /* filter, filterarg */ 347 BUS_SPACE_MAXSIZE_32BIT,/* maxsize */ 348 0, /* nsegments */ 349 BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */ 350 0, /* flags */ 351 &sc->re_parent_tag); 352 if (error) { 353 device_printf(dev, "could not allocate parent dma tag\n"); 354 return error; 355 } 356 357 /* Allocate TX descriptor list. */ 358 error = bus_dmamem_coherent(sc->re_parent_tag, 359 RE_RING_ALIGN, 0, 360 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 361 RE_TX_LIST_SZ(sc), BUS_DMA_WAITOK | BUS_DMA_ZERO, 362 &dmem); 363 if (error) { 364 device_printf(dev, "could not allocate TX ring\n"); 365 return error; 366 } 367 sc->re_ldata.re_tx_list_tag = dmem.dmem_tag; 368 sc->re_ldata.re_tx_list_map = dmem.dmem_map; 369 sc->re_ldata.re_tx_list = dmem.dmem_addr; 370 sc->re_ldata.re_tx_list_addr = dmem.dmem_busaddr; 371 372 /* Allocate RX descriptor list. */ 373 error = bus_dmamem_coherent(sc->re_parent_tag, 374 RE_RING_ALIGN, 0, 375 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 376 RE_RX_LIST_SZ(sc), BUS_DMA_WAITOK | BUS_DMA_ZERO, 377 &dmem); 378 if (error) { 379 device_printf(dev, "could not allocate RX ring\n"); 380 return error; 381 } 382 sc->re_ldata.re_rx_list_tag = dmem.dmem_tag; 383 sc->re_ldata.re_rx_list_map = dmem.dmem_map; 384 sc->re_ldata.re_rx_list = dmem.dmem_addr; 385 sc->re_ldata.re_rx_list_addr = dmem.dmem_busaddr; 386 387 /* Allocate maps for TX mbufs. */ 388 error = bus_dma_tag_create(sc->re_parent_tag, 389 1, 0, 390 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 391 NULL, NULL, 392 RE_FRAMELEN_MAX, RE_MAXSEGS, MCLBYTES, 393 BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 394 &sc->re_ldata.re_tx_mtag); 395 if (error) { 396 device_printf(dev, "could not allocate TX buf dma tag\n"); 397 return(error); 398 } 399 400 /* Create DMA maps for TX buffers */ 401 for (i = 0; i < sc->re_tx_desc_cnt; i++) { 402 error = bus_dmamap_create(sc->re_ldata.re_tx_mtag, 403 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 404 &sc->re_ldata.re_tx_dmamap[i]); 405 if (error) { 406 device_printf(dev, "can't create DMA map for TX buf\n"); 407 re_freebufmem(sc, i, 0); 408 return(error); 409 } 410 } 411 412 /* Allocate maps for RX mbufs. */ 413 error = bus_dma_tag_create(sc->re_parent_tag, 414 RE_RXBUF_ALIGN, 0, 415 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 416 NULL, NULL, 417 MCLBYTES, 1, MCLBYTES, 418 BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ALIGNED, 419 &sc->re_ldata.re_rx_mtag); 420 if (error) { 421 device_printf(dev, "could not allocate RX buf dma tag\n"); 422 return(error); 423 } 424 425 /* Create spare DMA map for RX */ 426 error = bus_dmamap_create(sc->re_ldata.re_rx_mtag, BUS_DMA_WAITOK, 427 &sc->re_ldata.re_rx_spare); 428 if (error) { 429 device_printf(dev, "can't create spare DMA map for RX\n"); 430 bus_dma_tag_destroy(sc->re_ldata.re_rx_mtag); 431 sc->re_ldata.re_rx_mtag = NULL; 432 return error; 433 } 434 435 /* Create DMA maps for RX buffers */ 436 for (i = 0; i < sc->re_rx_desc_cnt; i++) { 437 error = bus_dmamap_create(sc->re_ldata.re_rx_mtag, 438 BUS_DMA_WAITOK, &sc->re_ldata.re_rx_dmamap[i]); 439 if (error) { 440 device_printf(dev, "can't create DMA map for RX buf\n"); 441 re_freebufmem(sc, sc->re_tx_desc_cnt, i); 442 return(error); 443 } 444 } 445 446 /* Create jumbo buffer pool for RX if required */ 447 if (sc->re_caps & RE_C_CONTIGRX) { 448 error = re_jpool_alloc(sc); 449 if (error) { 450 re_jpool_free(sc); 451 #ifdef RE_JUMBO 452 /* Disable jumbo frame support */ 453 sc->re_maxmtu = ETHERMTU; 454 #endif 455 } 456 } 457 return(0); 458 } 459 460 static void 461 re_freebufmem(struct re_softc *sc, int tx_cnt, int rx_cnt) 462 { 463 int i; 464 465 /* Destroy all the RX and TX buffer maps */ 466 if (sc->re_ldata.re_tx_mtag) { 467 for (i = 0; i < tx_cnt; i++) { 468 bus_dmamap_destroy(sc->re_ldata.re_tx_mtag, 469 sc->re_ldata.re_tx_dmamap[i]); 470 } 471 bus_dma_tag_destroy(sc->re_ldata.re_tx_mtag); 472 sc->re_ldata.re_tx_mtag = NULL; 473 } 474 475 if (sc->re_ldata.re_rx_mtag) { 476 for (i = 0; i < rx_cnt; i++) { 477 bus_dmamap_destroy(sc->re_ldata.re_rx_mtag, 478 sc->re_ldata.re_rx_dmamap[i]); 479 } 480 bus_dmamap_destroy(sc->re_ldata.re_rx_mtag, 481 sc->re_ldata.re_rx_spare); 482 bus_dma_tag_destroy(sc->re_ldata.re_rx_mtag); 483 sc->re_ldata.re_rx_mtag = NULL; 484 } 485 } 486 487 static void 488 re_freemem(device_t dev) 489 { 490 struct re_softc *sc = device_get_softc(dev); 491 492 /* Unload and free the RX DMA ring memory and map */ 493 if (sc->re_ldata.re_rx_list_tag) { 494 bus_dmamap_unload(sc->re_ldata.re_rx_list_tag, 495 sc->re_ldata.re_rx_list_map); 496 bus_dmamem_free(sc->re_ldata.re_rx_list_tag, 497 sc->re_ldata.re_rx_list, 498 sc->re_ldata.re_rx_list_map); 499 bus_dma_tag_destroy(sc->re_ldata.re_rx_list_tag); 500 } 501 502 /* Unload and free the TX DMA ring memory and map */ 503 if (sc->re_ldata.re_tx_list_tag) { 504 bus_dmamap_unload(sc->re_ldata.re_tx_list_tag, 505 sc->re_ldata.re_tx_list_map); 506 bus_dmamem_free(sc->re_ldata.re_tx_list_tag, 507 sc->re_ldata.re_tx_list, 508 sc->re_ldata.re_tx_list_map); 509 bus_dma_tag_destroy(sc->re_ldata.re_tx_list_tag); 510 } 511 512 /* Free RX/TX buf DMA stuffs */ 513 re_freebufmem(sc, sc->re_tx_desc_cnt, sc->re_rx_desc_cnt); 514 515 /* Unload and free the stats buffer and map */ 516 if (sc->re_ldata.re_stag) { 517 bus_dmamap_unload(sc->re_ldata.re_stag, sc->re_ldata.re_smap); 518 bus_dmamem_free(sc->re_ldata.re_stag, 519 sc->re_ldata.re_stats, 520 sc->re_ldata.re_smap); 521 bus_dma_tag_destroy(sc->re_ldata.re_stag); 522 } 523 524 if (sc->re_caps & RE_C_CONTIGRX) 525 re_jpool_free(sc); 526 527 if (sc->re_parent_tag) 528 bus_dma_tag_destroy(sc->re_parent_tag); 529 530 if (sc->re_ldata.re_tx_mbuf != NULL) 531 kfree(sc->re_ldata.re_tx_mbuf, M_DEVBUF); 532 if (sc->re_ldata.re_rx_mbuf != NULL) 533 kfree(sc->re_ldata.re_rx_mbuf, M_DEVBUF); 534 if (sc->re_ldata.re_rx_paddr != NULL) 535 kfree(sc->re_ldata.re_rx_paddr, M_DEVBUF); 536 if (sc->re_ldata.re_tx_dmamap != NULL) 537 kfree(sc->re_ldata.re_tx_dmamap, M_DEVBUF); 538 if (sc->re_ldata.re_rx_dmamap != NULL) 539 kfree(sc->re_ldata.re_rx_dmamap, M_DEVBUF); 540 } 541 542 static boolean_t 543 re_is_faste(struct re_softc *sc) 544 { 545 if (pci_get_vendor(sc->dev) == PCI_VENDOR_REALTEK) { 546 switch (sc->re_device_id) { 547 case PCI_PRODUCT_REALTEK_RT8169: 548 case PCI_PRODUCT_REALTEK_RT8169SC: 549 case PCI_PRODUCT_REALTEK_RT8168: 550 case PCI_PRODUCT_REALTEK_RT8168_1: 551 return FALSE; 552 default: 553 return TRUE; 554 } 555 } else { 556 return FALSE; 557 } 558 } 559 560 /* 561 * Attach the interface. Allocate softc structures, do ifmedia 562 * setup and ethernet/BPF attach. 563 */ 564 static int 565 re_attach(device_t dev) 566 { 567 struct re_softc *sc = device_get_softc(dev); 568 struct ifnet *ifp; 569 struct sysctl_ctx_list *ctx; 570 struct sysctl_oid *tree; 571 uint8_t eaddr[ETHER_ADDR_LEN]; 572 int error = 0, qlen, msi_enable; 573 u_int irq_flags; 574 575 callout_init_mp(&sc->re_timer); 576 sc->dev = dev; 577 sc->re_device_id = pci_get_device(dev); 578 sc->re_unit = device_get_unit(dev); 579 ifmedia_init(&sc->media, IFM_IMASK, rtl_ifmedia_upd, rtl_ifmedia_sts); 580 581 sc->re_caps = RE_C_HWIM; 582 583 sc->re_rx_desc_cnt = re_rx_desc_count; 584 if (sc->re_rx_desc_cnt > RE_RX_DESC_CNT_MAX) 585 sc->re_rx_desc_cnt = RE_RX_DESC_CNT_MAX; 586 587 sc->re_tx_desc_cnt = re_tx_desc_count; 588 if (sc->re_tx_desc_cnt > RE_TX_DESC_CNT_MAX) 589 sc->re_tx_desc_cnt = RE_TX_DESC_CNT_MAX; 590 591 qlen = RE_IFQ_MAXLEN; 592 if (sc->re_tx_desc_cnt > qlen) 593 qlen = sc->re_tx_desc_cnt; 594 595 sc->re_rxbuf_size = MCLBYTES; 596 sc->re_newbuf = re_newbuf_std; 597 598 /* 599 * Hardware interrupt moderation settings. 600 * XXX does not seem correct, undocumented. 601 */ 602 sc->re_tx_time = 5; /* 125us */ 603 sc->re_rx_time = 2; /* 50us */ 604 605 /* Simulated interrupt moderation setting. */ 606 sc->re_sim_time = 150; /* 150us */ 607 608 /* Use simulated interrupt moderation by default. */ 609 sc->re_imtype = RE_IMTYPE_SIM; 610 re_config_imtype(sc, sc->re_imtype); 611 612 ctx = device_get_sysctl_ctx(dev); 613 tree = device_get_sysctl_tree(dev); 614 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, 615 "rx_desc_count", CTLFLAG_RD, &sc->re_rx_desc_cnt, 616 0, "RX desc count"); 617 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, 618 "tx_desc_count", CTLFLAG_RD, &sc->re_tx_desc_cnt, 619 0, "TX desc count"); 620 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "sim_time", 621 CTLTYPE_INT | CTLFLAG_RW, 622 sc, 0, re_sysctl_simtime, "I", 623 "Simulated interrupt moderation time (usec)."); 624 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "imtype", 625 CTLTYPE_INT | CTLFLAG_RW, 626 sc, 0, re_sysctl_imtype, "I", 627 "Interrupt moderation type -- " 628 "0:disable, 1:simulated, " 629 "2:hardware(if supported)"); 630 if (sc->re_caps & RE_C_HWIM) { 631 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 632 OID_AUTO, "hw_rxtime", 633 CTLTYPE_INT | CTLFLAG_RW, 634 sc, 0, re_sysctl_rxtime, "I", 635 "Hardware interrupt moderation time " 636 "(unit: 25usec)."); 637 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 638 OID_AUTO, "hw_txtime", 639 CTLTYPE_INT | CTLFLAG_RW, 640 sc, 0, re_sysctl_txtime, "I", 641 "Hardware interrupt moderation time " 642 "(unit: 25usec)."); 643 } 644 645 #ifndef BURN_BRIDGES 646 /* 647 * Handle power management nonsense. 648 */ 649 650 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) { 651 uint32_t membase, irq; 652 653 /* Save important PCI config data. */ 654 membase = pci_read_config(dev, RE_PCI_LOMEM, 4); 655 irq = pci_read_config(dev, PCIR_INTLINE, 4); 656 657 /* Reset the power state. */ 658 device_printf(dev, "chip is in D%d power mode " 659 "-- setting to D0\n", pci_get_powerstate(dev)); 660 661 pci_set_powerstate(dev, PCI_POWERSTATE_D0); 662 663 /* Restore PCI config data. */ 664 pci_write_config(dev, RE_PCI_LOMEM, membase, 4); 665 pci_write_config(dev, PCIR_INTLINE, irq, 4); 666 } 667 #endif 668 /* 669 * Map control/status registers. 670 */ 671 pci_enable_busmaster(dev); 672 673 if (pci_is_pcie(dev)) { 674 sc->re_res_rid = PCIR_BAR(2); 675 sc->re_res_type = SYS_RES_MEMORY; 676 } else { 677 sc->re_res_rid = PCIR_BAR(0); 678 sc->re_res_type = SYS_RES_IOPORT; 679 } 680 sc->re_res = bus_alloc_resource_any(dev, sc->re_res_type, 681 &sc->re_res_rid, RF_ACTIVE); 682 if (sc->re_res == NULL) { 683 device_printf(dev, "couldn't map IO\n"); 684 error = ENXIO; 685 goto fail; 686 } 687 688 sc->re_btag = rman_get_bustag(sc->re_res); 689 sc->re_bhandle = rman_get_bushandle(sc->re_res); 690 691 error = rtl_check_mac_version(sc); 692 if (error) { 693 device_printf(dev, "check mac version failed\n"); 694 goto fail; 695 } 696 697 rtl_init_software_variable(sc); 698 if (pci_is_pcie(dev)) 699 sc->re_if_flags |= RL_FLAG_PCIE; 700 else 701 sc->re_if_flags &= ~RL_FLAG_PCIE; 702 device_printf(dev, "MAC version 0x%08x, MACFG %u%s%s\n", 703 (CSR_READ_4(sc, RE_TXCFG) & 0xFCF00000), sc->re_type, 704 sc->re_coalesce_tx_pkt ? ", software TX defrag" : "", 705 sc->re_hw_enable_msi_msix ? ", support MSI" : ""); 706 707 /* 708 * Allocate interrupt 709 */ 710 if (pci_is_pcie(dev) && sc->re_hw_enable_msi_msix) 711 msi_enable = re_msi_enable; 712 else 713 msi_enable = 0; 714 sc->re_irq_type = pci_alloc_1intr(dev, msi_enable, 715 &sc->re_irq_rid, &irq_flags); 716 717 sc->re_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->re_irq_rid, 718 irq_flags); 719 if (sc->re_irq == NULL) { 720 device_printf(dev, "couldn't map interrupt\n"); 721 error = ENXIO; 722 goto fail; 723 } 724 725 /* Disable ASPM */ 726 re_disable_aspm(dev); 727 728 rtl_exit_oob(sc); 729 rtl_hw_init(sc); 730 731 /* Reset the adapter. */ 732 rtl_reset(sc); 733 734 rtl_get_hw_mac_address(sc, eaddr); 735 if (sc->re_type == MACFG_3) /* Change PCI Latency time*/ 736 pci_write_config(dev, PCIR_LATTIMER, 0x40, 1); 737 738 /* Allocate DMA stuffs */ 739 error = re_allocmem(dev); 740 if (error) 741 goto fail; 742 743 if (pci_is_pcie(dev)) { 744 sc->re_bus_speed = 125; 745 } else { 746 uint8_t cfg2; 747 748 cfg2 = CSR_READ_1(sc, RE_CFG2); 749 switch (cfg2 & RE_CFG2_PCICLK_MASK) { 750 case RE_CFG2_PCICLK_33MHZ: 751 sc->re_bus_speed = 33; 752 break; 753 case RE_CFG2_PCICLK_66MHZ: 754 sc->re_bus_speed = 66; 755 break; 756 default: 757 device_printf(dev, "unknown bus speed, assume 33MHz\n"); 758 sc->re_bus_speed = 33; 759 break; 760 } 761 } 762 device_printf(dev, "bus speed %dMHz\n", sc->re_bus_speed); 763 764 rtl_phy_power_up(sc); 765 rtl_hw_phy_config(sc); 766 rtl_clrwol(sc); 767 768 /* TODO: jumbo frame */ 769 CSR_WRITE_2(sc, RE_RxMaxSize, sc->re_rxbuf_size); 770 771 /* Enable hardware checksum if available. */ 772 sc->re_tx_cstag = 1; 773 sc->re_rx_cstag = 1; 774 775 ifp = &sc->arpcom.ac_if; 776 ifp->if_softc = sc; 777 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 778 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 779 ifp->if_ioctl = re_ioctl; 780 ifp->if_start = re_start; 781 #ifdef IFPOLL_ENABLE 782 ifp->if_npoll = re_npoll; 783 #endif 784 ifp->if_watchdog = re_watchdog; 785 ifp->if_init = re_init; 786 if (!re_is_faste(sc)) 787 ifp->if_baudrate = 1000000000; 788 else 789 ifp->if_baudrate = 100000000; 790 ifp->if_nmbclusters = sc->re_rx_desc_cnt; 791 ifq_set_maxlen(&ifp->if_snd, qlen); 792 ifq_set_ready(&ifp->if_snd); 793 794 ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 795 IFCAP_RXCSUM | IFCAP_TXCSUM; 796 ifp->if_capenable = ifp->if_capabilities; 797 /* NOTE: if_hwassist will be setup after the interface is up. */ 798 799 /* 800 * Call MI attach routine. 801 */ 802 ether_ifattach(ifp, eaddr, NULL); 803 804 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(sc->re_irq)); 805 806 #ifdef IFPOLL_ENABLE 807 ifpoll_compat_setup(&sc->re_npoll, ctx, (struct sysctl_oid *)tree, 808 device_get_unit(dev), ifp->if_serializer); 809 #endif 810 811 /* Hook interrupt last to avoid having to lock softc */ 812 error = bus_setup_intr(dev, sc->re_irq, INTR_MPSAFE | INTR_HIFREQ, 813 re_intr, sc, &sc->re_intrhand, ifp->if_serializer); 814 if (error) { 815 device_printf(dev, "couldn't set up irq\n"); 816 ether_ifdetach(ifp); 817 goto fail; 818 } 819 820 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 821 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 822 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 823 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 824 if (!re_is_faste(sc)) { 825 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 826 0, NULL); 827 } 828 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 829 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 830 rtl_ifmedia_upd(ifp); 831 832 fail: 833 if (error) 834 re_detach(dev); 835 836 return (error); 837 } 838 839 /* 840 * Shutdown hardware and free up resources. This can be called any 841 * time after the mutex has been initialized. It is called in both 842 * the error case in attach and the normal detach case so it needs 843 * to be careful about only freeing resources that have actually been 844 * allocated. 845 */ 846 static int 847 re_detach(device_t dev) 848 { 849 struct re_softc *sc = device_get_softc(dev); 850 struct ifnet *ifp = &sc->arpcom.ac_if; 851 852 /* These should only be active if attach succeeded */ 853 if (device_is_attached(dev)) { 854 lwkt_serialize_enter(ifp->if_serializer); 855 re_stop(sc, TRUE); 856 bus_teardown_intr(dev, sc->re_irq, sc->re_intrhand); 857 lwkt_serialize_exit(ifp->if_serializer); 858 859 ether_ifdetach(ifp); 860 } 861 ifmedia_removeall(&sc->media); 862 863 if (sc->re_irq) 864 bus_release_resource(dev, SYS_RES_IRQ, sc->re_irq_rid, 865 sc->re_irq); 866 867 if (sc->re_irq_type == PCI_INTR_TYPE_MSI) 868 pci_release_msi(dev); 869 870 if (sc->re_res) { 871 bus_release_resource(dev, sc->re_res_type, sc->re_res_rid, 872 sc->re_res); 873 } 874 875 /* Free DMA stuffs */ 876 re_freemem(dev); 877 878 return(0); 879 } 880 881 static void 882 re_setup_rxdesc(struct re_softc *sc, int idx) 883 { 884 bus_addr_t paddr; 885 uint32_t cmdstat; 886 struct re_desc *d; 887 888 paddr = sc->re_ldata.re_rx_paddr[idx]; 889 d = &sc->re_ldata.re_rx_list[idx]; 890 891 d->re_bufaddr_lo = htole32(RE_ADDR_LO(paddr)); 892 d->re_bufaddr_hi = htole32(RE_ADDR_HI(paddr)); 893 894 cmdstat = sc->re_rxbuf_size | RE_RDESC_CMD_OWN; 895 if (idx == (sc->re_rx_desc_cnt - 1)) 896 cmdstat |= RE_RDESC_CMD_EOR; 897 d->re_cmdstat = htole32(cmdstat); 898 } 899 900 static int 901 re_newbuf_std(struct re_softc *sc, int idx, int init) 902 { 903 bus_dma_segment_t seg; 904 bus_dmamap_t map; 905 struct mbuf *m; 906 int error, nsegs; 907 908 m = m_getcl(init ? M_WAITOK : M_NOWAIT, MT_DATA, M_PKTHDR); 909 if (m == NULL) { 910 error = ENOBUFS; 911 912 if (init) { 913 if_printf(&sc->arpcom.ac_if, "m_getcl failed\n"); 914 return error; 915 } else { 916 goto back; 917 } 918 } 919 m->m_len = m->m_pkthdr.len = MCLBYTES; 920 921 /* 922 * NOTE: 923 * re(4) chips need address of the receive buffer to be 8-byte 924 * aligned, so don't call m_adj(m, ETHER_ALIGN) here. 925 */ 926 927 error = bus_dmamap_load_mbuf_segment(sc->re_ldata.re_rx_mtag, 928 sc->re_ldata.re_rx_spare, m, 929 &seg, 1, &nsegs, BUS_DMA_NOWAIT); 930 if (error) { 931 m_freem(m); 932 if (init) { 933 if_printf(&sc->arpcom.ac_if, "can't load RX mbuf\n"); 934 return error; 935 } else { 936 goto back; 937 } 938 } 939 940 if (!init) { 941 bus_dmamap_sync(sc->re_ldata.re_rx_mtag, 942 sc->re_ldata.re_rx_dmamap[idx], 943 BUS_DMASYNC_POSTREAD); 944 bus_dmamap_unload(sc->re_ldata.re_rx_mtag, 945 sc->re_ldata.re_rx_dmamap[idx]); 946 } 947 sc->re_ldata.re_rx_mbuf[idx] = m; 948 sc->re_ldata.re_rx_paddr[idx] = seg.ds_addr; 949 950 map = sc->re_ldata.re_rx_dmamap[idx]; 951 sc->re_ldata.re_rx_dmamap[idx] = sc->re_ldata.re_rx_spare; 952 sc->re_ldata.re_rx_spare = map; 953 back: 954 re_setup_rxdesc(sc, idx); 955 return error; 956 } 957 958 #ifdef RE_JUMBO 959 static int 960 re_newbuf_jumbo(struct re_softc *sc, int idx, int init) 961 { 962 struct mbuf *m; 963 struct re_jbuf *jbuf; 964 int error = 0; 965 966 MGETHDR(m, init ? M_WAITOK : M_NOWAIT, MT_DATA); 967 if (m == NULL) { 968 error = ENOBUFS; 969 if (init) { 970 if_printf(&sc->arpcom.ac_if, "MGETHDR failed\n"); 971 return error; 972 } else { 973 goto back; 974 } 975 } 976 977 jbuf = re_jbuf_alloc(sc); 978 if (jbuf == NULL) { 979 m_freem(m); 980 981 error = ENOBUFS; 982 if (init) { 983 if_printf(&sc->arpcom.ac_if, "jpool is empty\n"); 984 return error; 985 } else { 986 goto back; 987 } 988 } 989 990 m->m_ext.ext_arg = jbuf; 991 m->m_ext.ext_buf = jbuf->re_buf; 992 m->m_ext.ext_free = re_jbuf_free; 993 m->m_ext.ext_ref = re_jbuf_ref; 994 m->m_ext.ext_size = sc->re_rxbuf_size; 995 996 m->m_data = m->m_ext.ext_buf; 997 m->m_flags |= M_EXT; 998 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 999 1000 /* 1001 * NOTE: 1002 * Some re(4) chips(e.g. RTL8101E) need address of the receive buffer 1003 * to be 8-byte aligned, so don't call m_adj(m, ETHER_ALIGN) here. 1004 */ 1005 1006 sc->re_ldata.re_rx_mbuf[idx] = m; 1007 sc->re_ldata.re_rx_paddr[idx] = jbuf->re_paddr; 1008 back: 1009 re_setup_rxdesc(sc, idx); 1010 return error; 1011 } 1012 #endif /* RE_JUMBO */ 1013 1014 static int 1015 re_tx_list_init(struct re_softc *sc) 1016 { 1017 bzero(sc->re_ldata.re_tx_list, RE_TX_LIST_SZ(sc)); 1018 1019 sc->re_ldata.re_tx_prodidx = 0; 1020 sc->re_ldata.re_tx_considx = 0; 1021 sc->re_ldata.re_tx_free = sc->re_tx_desc_cnt; 1022 1023 return(0); 1024 } 1025 1026 static int 1027 re_rx_list_init(struct re_softc *sc) 1028 { 1029 int i, error; 1030 1031 bzero(sc->re_ldata.re_rx_list, RE_RX_LIST_SZ(sc)); 1032 1033 for (i = 0; i < sc->re_rx_desc_cnt; i++) { 1034 error = sc->re_newbuf(sc, i, 1); 1035 if (error) 1036 return(error); 1037 } 1038 1039 sc->re_ldata.re_rx_prodidx = 0; 1040 sc->re_head = sc->re_tail = NULL; 1041 1042 return(0); 1043 } 1044 1045 #define RE_IP4_PACKET 0x1 1046 #define RE_TCP_PACKET 0x2 1047 #define RE_UDP_PACKET 0x4 1048 1049 static __inline uint8_t 1050 re_packet_type(struct re_softc *sc, uint32_t rxstat, uint32_t rxctrl) 1051 { 1052 uint8_t packet_type = 0; 1053 1054 if (sc->re_if_flags & RL_FLAG_DESCV2) { 1055 if (rxctrl & RE_RDESC_CTL_PROTOIP4) 1056 packet_type |= RE_IP4_PACKET; 1057 } else { 1058 if (rxstat & RE_RDESC_STAT_PROTOID) 1059 packet_type |= RE_IP4_PACKET; 1060 } 1061 if (RE_TCPPKT(rxstat)) 1062 packet_type |= RE_TCP_PACKET; 1063 else if (RE_UDPPKT(rxstat)) 1064 packet_type |= RE_UDP_PACKET; 1065 return packet_type; 1066 } 1067 1068 /* 1069 * RX handler for C+ and 8169. For the gigE chips, we support 1070 * the reception of jumbo frames that have been fragmented 1071 * across multiple 2K mbuf cluster buffers. 1072 */ 1073 static int 1074 re_rxeof(struct re_softc *sc) 1075 { 1076 struct ifnet *ifp = &sc->arpcom.ac_if; 1077 struct mbuf *m; 1078 struct re_desc *cur_rx; 1079 uint32_t rxstat, rxctrl; 1080 int i, total_len, rx = 0; 1081 1082 for (i = sc->re_ldata.re_rx_prodidx; 1083 RE_OWN(&sc->re_ldata.re_rx_list[i]) == 0; RE_RXDESC_INC(sc, i)) { 1084 cur_rx = &sc->re_ldata.re_rx_list[i]; 1085 m = sc->re_ldata.re_rx_mbuf[i]; 1086 total_len = RE_RXBYTES(cur_rx); 1087 rxstat = le32toh(cur_rx->re_cmdstat); 1088 rxctrl = le32toh(cur_rx->re_control); 1089 1090 rx = 1; 1091 1092 #ifdef INVARIANTS 1093 if (sc->re_flags & RE_F_USE_JPOOL) 1094 KKASSERT(rxstat & RE_RDESC_STAT_EOF); 1095 #endif 1096 1097 if ((rxstat & RE_RDESC_STAT_EOF) == 0) { 1098 if (sc->re_flags & RE_F_DROP_RXFRAG) { 1099 re_setup_rxdesc(sc, i); 1100 continue; 1101 } 1102 1103 if (sc->re_newbuf(sc, i, 0)) { 1104 /* Drop upcoming fragments */ 1105 sc->re_flags |= RE_F_DROP_RXFRAG; 1106 continue; 1107 } 1108 1109 m->m_len = MCLBYTES; 1110 if (sc->re_head == NULL) { 1111 sc->re_head = sc->re_tail = m; 1112 } else { 1113 sc->re_tail->m_next = m; 1114 sc->re_tail = m; 1115 } 1116 continue; 1117 } else if (sc->re_flags & RE_F_DROP_RXFRAG) { 1118 /* 1119 * Last fragment of a multi-fragment packet. 1120 * 1121 * Since error already happened, this fragment 1122 * must be dropped as well as the fragment chain. 1123 */ 1124 re_setup_rxdesc(sc, i); 1125 re_free_rxchain(sc); 1126 sc->re_flags &= ~RE_F_DROP_RXFRAG; 1127 continue; 1128 } 1129 1130 rxstat >>= 1; 1131 if (rxstat & RE_RDESC_STAT_RXERRSUM) { 1132 IFNET_STAT_INC(ifp, ierrors, 1); 1133 /* 1134 * If this is part of a multi-fragment packet, 1135 * discard all the pieces. 1136 */ 1137 re_free_rxchain(sc); 1138 re_setup_rxdesc(sc, i); 1139 continue; 1140 } 1141 1142 /* 1143 * If allocating a replacement mbuf fails, 1144 * reload the current one. 1145 */ 1146 1147 if (sc->re_newbuf(sc, i, 0)) { 1148 IFNET_STAT_INC(ifp, ierrors, 1); 1149 continue; 1150 } 1151 1152 if (sc->re_head != NULL) { 1153 m->m_len = total_len % MCLBYTES; 1154 /* 1155 * Special case: if there's 4 bytes or less 1156 * in this buffer, the mbuf can be discarded: 1157 * the last 4 bytes is the CRC, which we don't 1158 * care about anyway. 1159 */ 1160 if (m->m_len <= ETHER_CRC_LEN) { 1161 sc->re_tail->m_len -= 1162 (ETHER_CRC_LEN - m->m_len); 1163 m_freem(m); 1164 } else { 1165 m->m_len -= ETHER_CRC_LEN; 1166 sc->re_tail->m_next = m; 1167 } 1168 m = sc->re_head; 1169 sc->re_head = sc->re_tail = NULL; 1170 m->m_pkthdr.len = total_len - ETHER_CRC_LEN; 1171 } else { 1172 m->m_pkthdr.len = m->m_len = 1173 (total_len - ETHER_CRC_LEN); 1174 } 1175 1176 IFNET_STAT_INC(ifp, ipackets, 1); 1177 m->m_pkthdr.rcvif = ifp; 1178 1179 /* Do RX checksumming if enabled */ 1180 1181 if (ifp->if_capenable & IFCAP_RXCSUM) { 1182 uint8_t packet_type; 1183 1184 packet_type = re_packet_type(sc, rxstat, rxctrl); 1185 1186 /* Check IP header checksum */ 1187 if (packet_type & RE_IP4_PACKET) { 1188 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 1189 if ((rxstat & RE_RDESC_STAT_IPSUMBAD) == 0) 1190 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 1191 } 1192 1193 /* Check TCP/UDP checksum */ 1194 if (((packet_type & RE_TCP_PACKET) && 1195 (rxstat & RE_RDESC_STAT_TCPSUMBAD) == 0) || 1196 ((packet_type & RE_UDP_PACKET) && 1197 (rxstat & RE_RDESC_STAT_UDPSUMBAD) == 0)) { 1198 m->m_pkthdr.csum_flags |= 1199 CSUM_DATA_VALID|CSUM_PSEUDO_HDR| 1200 CSUM_FRAG_NOT_CHECKED; 1201 m->m_pkthdr.csum_data = 0xffff; 1202 } 1203 } 1204 1205 if (rxctrl & RE_RDESC_CTL_HASTAG) { 1206 m->m_flags |= M_VLANTAG; 1207 m->m_pkthdr.ether_vlantag = 1208 be16toh((rxctrl & RE_RDESC_CTL_TAGDATA)); 1209 } 1210 ifp->if_input(ifp, m, NULL, -1); 1211 } 1212 1213 sc->re_ldata.re_rx_prodidx = i; 1214 1215 return rx; 1216 } 1217 1218 #undef RE_IP4_PACKET 1219 #undef RE_TCP_PACKET 1220 #undef RE_UDP_PACKET 1221 1222 static int 1223 re_tx_collect(struct re_softc *sc) 1224 { 1225 struct ifnet *ifp = &sc->arpcom.ac_if; 1226 uint32_t txstat; 1227 int idx, tx = 0; 1228 1229 for (idx = sc->re_ldata.re_tx_considx; 1230 sc->re_ldata.re_tx_free < sc->re_tx_desc_cnt; 1231 RE_TXDESC_INC(sc, idx)) { 1232 txstat = le32toh(sc->re_ldata.re_tx_list[idx].re_cmdstat); 1233 if (txstat & RE_TDESC_CMD_OWN) 1234 break; 1235 1236 tx = 1; 1237 1238 sc->re_ldata.re_tx_list[idx].re_bufaddr_lo = 0; 1239 1240 /* 1241 * We only stash mbufs in the last descriptor 1242 * in a fragment chain, which also happens to 1243 * be the only place where the TX status bits 1244 * are valid. 1245 */ 1246 if (txstat & RE_TDESC_CMD_EOF) { 1247 bus_dmamap_unload(sc->re_ldata.re_tx_mtag, 1248 sc->re_ldata.re_tx_dmamap[idx]); 1249 m_freem(sc->re_ldata.re_tx_mbuf[idx]); 1250 sc->re_ldata.re_tx_mbuf[idx] = NULL; 1251 if (txstat & (RE_TDESC_STAT_EXCESSCOL| 1252 RE_TDESC_STAT_COLCNT)) 1253 IFNET_STAT_INC(ifp, collisions, 1); 1254 if (txstat & RE_TDESC_STAT_TXERRSUM) 1255 IFNET_STAT_INC(ifp, oerrors, 1); 1256 else 1257 IFNET_STAT_INC(ifp, opackets, 1); 1258 } 1259 sc->re_ldata.re_tx_free++; 1260 } 1261 sc->re_ldata.re_tx_considx = idx; 1262 1263 return tx; 1264 } 1265 1266 static int 1267 re_txeof(struct re_softc *sc) 1268 { 1269 struct ifnet *ifp = &sc->arpcom.ac_if; 1270 int tx; 1271 1272 tx = re_tx_collect(sc); 1273 1274 /* There is enough free TX descs */ 1275 if (sc->re_ldata.re_tx_free > RE_TXDESC_SPARE) 1276 ifq_clr_oactive(&ifp->if_snd); 1277 1278 /* 1279 * Some chips will ignore a second TX request issued while an 1280 * existing transmission is in progress. If the transmitter goes 1281 * idle but there are still packets waiting to be sent, we need 1282 * to restart the channel here to flush them out. This only seems 1283 * to be required with the PCIe devices. 1284 */ 1285 if (sc->re_ldata.re_tx_free < sc->re_tx_desc_cnt) 1286 CSR_WRITE_1(sc, RE_TPPOLL, RE_NPQ); 1287 else 1288 ifp->if_timer = 0; 1289 1290 return tx; 1291 } 1292 1293 static void 1294 re_tick(void *xsc) 1295 { 1296 struct re_softc *sc = xsc; 1297 1298 lwkt_serialize_enter(sc->arpcom.ac_if.if_serializer); 1299 re_tick_serialized(xsc); 1300 lwkt_serialize_exit(sc->arpcom.ac_if.if_serializer); 1301 } 1302 1303 static void 1304 re_tick_serialized(void *xsc) 1305 { 1306 struct re_softc *sc = xsc; 1307 struct ifnet *ifp = &sc->arpcom.ac_if; 1308 1309 ASSERT_SERIALIZED(ifp->if_serializer); 1310 1311 if ((ifp->if_flags & IFF_RUNNING) == 0) 1312 return; 1313 1314 if (rtl_link_ok(sc)) { 1315 if ((sc->re_flags & RE_F_LINKED) == 0) 1316 re_link_up(sc); 1317 } else if (sc->re_flags & RE_F_LINKED) { 1318 re_link_down(sc); 1319 } 1320 callout_reset(&sc->re_timer, hz, re_tick, sc); 1321 } 1322 1323 #ifdef IFPOLL_ENABLE 1324 1325 static void 1326 re_npoll_compat(struct ifnet *ifp, void *arg __unused, int count) 1327 { 1328 struct re_softc *sc = ifp->if_softc; 1329 1330 ASSERT_SERIALIZED(ifp->if_serializer); 1331 1332 if (sc->re_npoll.ifpc_stcount-- == 0) { 1333 uint16_t status; 1334 1335 sc->re_npoll.ifpc_stcount = sc->re_npoll.ifpc_stfrac; 1336 1337 status = CSR_READ_2(sc, RE_ISR); 1338 if (status) 1339 CSR_WRITE_2(sc, RE_ISR, status); 1340 1341 /* 1342 * XXX check behaviour on receiver stalls. 1343 */ 1344 1345 if (status & RE_ISR_SYSTEM_ERR) { 1346 rtl_reset(sc); 1347 re_init(sc); 1348 /* Done! */ 1349 return; 1350 } 1351 } 1352 1353 sc->rxcycles = count; 1354 re_rxeof(sc); 1355 re_txeof(sc); 1356 1357 if (!ifq_is_empty(&ifp->if_snd)) 1358 if_devstart(ifp); 1359 } 1360 1361 static void 1362 re_npoll(struct ifnet *ifp, struct ifpoll_info *info) 1363 { 1364 struct re_softc *sc = ifp->if_softc; 1365 1366 ASSERT_SERIALIZED(ifp->if_serializer); 1367 1368 if (info != NULL) { 1369 int cpuid = sc->re_npoll.ifpc_cpuid; 1370 1371 info->ifpi_rx[cpuid].poll_func = re_npoll_compat; 1372 info->ifpi_rx[cpuid].arg = NULL; 1373 info->ifpi_rx[cpuid].serializer = ifp->if_serializer; 1374 1375 if (ifp->if_flags & IFF_RUNNING) 1376 re_setup_intr(sc, 0, RE_IMTYPE_NONE); 1377 ifq_set_cpuid(&ifp->if_snd, cpuid); 1378 } else { 1379 if (ifp->if_flags & IFF_RUNNING) 1380 re_setup_intr(sc, 1, sc->re_imtype); 1381 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(sc->re_irq)); 1382 } 1383 } 1384 #endif /* IFPOLL_ENABLE */ 1385 1386 static void 1387 re_intr(void *arg) 1388 { 1389 struct re_softc *sc = arg; 1390 struct ifnet *ifp = &sc->arpcom.ac_if; 1391 uint16_t status; 1392 int proc; 1393 1394 ASSERT_SERIALIZED(ifp->if_serializer); 1395 1396 if ((sc->re_flags & RE_F_SUSPENDED) || 1397 (ifp->if_flags & IFF_RUNNING) == 0) 1398 return; 1399 1400 /* Disable interrupts. */ 1401 CSR_WRITE_2(sc, RE_IMR, 0); 1402 1403 status = CSR_READ_2(sc, RE_ISR); 1404 again: 1405 proc = 0; 1406 if (status) 1407 CSR_WRITE_2(sc, RE_ISR, status); 1408 if (status & sc->re_intrs) { 1409 if (status & RE_ISR_SYSTEM_ERR) { 1410 rtl_reset(sc); 1411 re_init(sc); 1412 /* Done! */ 1413 return; 1414 } 1415 proc |= re_rxeof(sc); 1416 proc |= re_txeof(sc); 1417 } 1418 1419 if (sc->re_imtype == RE_IMTYPE_SIM) { 1420 if ((sc->re_flags & RE_F_TIMER_INTR)) { 1421 if (!proc) { 1422 /* 1423 * Nothing needs to be processed, fallback 1424 * to use TX/RX interrupts. 1425 * 1426 * NOTE: This will re-enable interrupts. 1427 */ 1428 re_setup_intr(sc, 1, RE_IMTYPE_NONE); 1429 1430 /* 1431 * Recollect, mainly to avoid the possible 1432 * race introduced by changing interrupt 1433 * masks. 1434 */ 1435 re_rxeof(sc); 1436 re_txeof(sc); 1437 } else { 1438 /* Re-enable interrupts. */ 1439 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 1440 CSR_WRITE_4(sc, RE_TIMERCNT, 1); /* reload */ 1441 } 1442 } else if (proc) { 1443 /* 1444 * Assume that using simulated interrupt moderation 1445 * (hardware timer based) could reduce the interript 1446 * rate. 1447 * 1448 * NOTE: This will re-enable interrupts. 1449 */ 1450 re_setup_intr(sc, 1, RE_IMTYPE_SIM); 1451 } else { 1452 /* Re-enable interrupts. */ 1453 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 1454 } 1455 } else { 1456 status = CSR_READ_2(sc, RE_ISR); 1457 if (status & sc->re_intrs) { 1458 if (!ifq_is_empty(&ifp->if_snd)) 1459 if_devstart(ifp); 1460 /* NOTE: Interrupts are still disabled. */ 1461 goto again; 1462 } 1463 /* Re-enable interrupts. */ 1464 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 1465 } 1466 1467 if (!ifq_is_empty(&ifp->if_snd)) 1468 if_devstart(ifp); 1469 } 1470 1471 static int 1472 re_encap(struct re_softc *sc, struct mbuf **m_head, int *idx0) 1473 { 1474 struct mbuf *m = *m_head; 1475 bus_dma_segment_t segs[RE_MAXSEGS]; 1476 bus_dmamap_t map; 1477 int error, maxsegs, idx, i, nsegs; 1478 struct re_desc *d, *tx_ring; 1479 uint32_t cmd_csum, ctl_csum, vlantag; 1480 1481 KASSERT(sc->re_ldata.re_tx_free > RE_TXDESC_SPARE, 1482 ("not enough free TX desc")); 1483 1484 if (sc->re_coalesce_tx_pkt && m->m_pkthdr.len != m->m_len) { 1485 struct mbuf *m_new; 1486 1487 m_new = m_defrag(m, M_NOWAIT); 1488 if (m_new == NULL) { 1489 error = ENOBUFS; 1490 goto back; 1491 } else { 1492 *m_head = m = m_new; 1493 if (m->m_pkthdr.len != m->m_len) { 1494 /* Still not configuous; give up. */ 1495 error = ENOBUFS; 1496 goto back; 1497 } 1498 } 1499 } 1500 1501 map = sc->re_ldata.re_tx_dmamap[*idx0]; 1502 1503 /* 1504 * Set up checksum offload. Note: checksum offload bits must 1505 * appear in all descriptors of a multi-descriptor transmit 1506 * attempt. (This is according to testing done with an 8169 1507 * chip. I'm not sure if this is a requirement or a bug.) 1508 */ 1509 cmd_csum = ctl_csum = 0; 1510 if (m->m_pkthdr.csum_flags & CSUM_IP) { 1511 cmd_csum |= RE_TDESC_CMD_IPCSUM; 1512 ctl_csum |= RE_TDESC_CTL_IPCSUM; 1513 } 1514 if (m->m_pkthdr.csum_flags & CSUM_TCP) { 1515 cmd_csum |= RE_TDESC_CMD_TCPCSUM; 1516 ctl_csum |= RE_TDESC_CTL_TCPCSUM; 1517 } 1518 if (m->m_pkthdr.csum_flags & CSUM_UDP) { 1519 cmd_csum |= RE_TDESC_CMD_UDPCSUM; 1520 ctl_csum |= RE_TDESC_CTL_UDPCSUM; 1521 } 1522 1523 /* For version2 descriptor, csum flags are set on re_control */ 1524 if (sc->re_if_flags & RL_FLAG_DESCV2) 1525 cmd_csum = 0; 1526 else 1527 ctl_csum = 0; 1528 1529 if (sc->re_coalesce_tx_pkt) { 1530 /* 1531 * With some of the RealTek chips, using the checksum offload 1532 * support in conjunction with the autopadding feature results 1533 * in the transmission of corrupt frames. For example, if we 1534 * need to send a really small IP fragment that's less than 60 1535 * bytes in size, and IP header checksumming is enabled, the 1536 * resulting ethernet frame that appears on the wire will 1537 * have garbled payload. To work around this, if TX checksum 1538 * offload is enabled, we always manually pad short frames out 1539 * to the minimum ethernet frame size. 1540 * 1541 * Note: this appears unnecessary for TCP, and doing it for TCP 1542 * with PCIe adapters seems to result in bad checksums. 1543 */ 1544 if ((m->m_pkthdr.csum_flags & 1545 (CSUM_DELAY_IP | CSUM_DELAY_DATA)) && 1546 (m->m_pkthdr.csum_flags & CSUM_TCP) == 0 && 1547 m->m_pkthdr.len < RE_MIN_FRAMELEN) { 1548 error = m_devpad(m, RE_MIN_FRAMELEN); 1549 if (error) 1550 goto back; 1551 } 1552 } 1553 1554 vlantag = 0; 1555 if (m->m_flags & M_VLANTAG) { 1556 vlantag = htobe16(m->m_pkthdr.ether_vlantag) | 1557 RE_TDESC_CTL_INSTAG; 1558 } 1559 1560 maxsegs = sc->re_ldata.re_tx_free; 1561 if (maxsegs > RE_MAXSEGS) 1562 maxsegs = RE_MAXSEGS; 1563 1564 error = bus_dmamap_load_mbuf_defrag(sc->re_ldata.re_tx_mtag, map, 1565 m_head, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 1566 if (error) 1567 goto back; 1568 1569 m = *m_head; 1570 bus_dmamap_sync(sc->re_ldata.re_tx_mtag, map, BUS_DMASYNC_PREWRITE); 1571 1572 /* 1573 * Map the segment array into descriptors. We also keep track 1574 * of the end of the ring and set the end-of-ring bits as needed, 1575 * and we set the ownership bits in all except the very first 1576 * descriptor, whose ownership bits will be turned on later. 1577 */ 1578 tx_ring = sc->re_ldata.re_tx_list; 1579 idx = *idx0; 1580 i = 0; 1581 for (;;) { 1582 uint32_t cmdstat; 1583 1584 d = &tx_ring[idx]; 1585 1586 cmdstat = segs[i].ds_len; 1587 d->re_bufaddr_lo = htole32(RE_ADDR_LO(segs[i].ds_addr)); 1588 d->re_bufaddr_hi = htole32(RE_ADDR_HI(segs[i].ds_addr)); 1589 if (i == 0) 1590 cmdstat |= RE_TDESC_CMD_SOF; 1591 else 1592 cmdstat |= RE_TDESC_CMD_OWN; 1593 if (idx == (sc->re_tx_desc_cnt - 1)) 1594 cmdstat |= RE_TDESC_CMD_EOR; 1595 d->re_cmdstat = htole32(cmdstat | cmd_csum); 1596 d->re_control = htole32(ctl_csum | vlantag); 1597 1598 i++; 1599 if (i == nsegs) 1600 break; 1601 RE_TXDESC_INC(sc, idx); 1602 } 1603 d->re_cmdstat |= htole32(RE_TDESC_CMD_EOF); 1604 1605 /* Transfer ownership of packet to the chip. */ 1606 d->re_cmdstat |= htole32(RE_TDESC_CMD_OWN); 1607 if (*idx0 != idx) 1608 tx_ring[*idx0].re_cmdstat |= htole32(RE_TDESC_CMD_OWN); 1609 1610 /* 1611 * Insure that the map for this transmission 1612 * is placed at the array index of the last descriptor 1613 * in this chain. 1614 */ 1615 sc->re_ldata.re_tx_dmamap[*idx0] = sc->re_ldata.re_tx_dmamap[idx]; 1616 sc->re_ldata.re_tx_dmamap[idx] = map; 1617 1618 sc->re_ldata.re_tx_mbuf[idx] = m; 1619 sc->re_ldata.re_tx_free -= nsegs; 1620 1621 RE_TXDESC_INC(sc, idx); 1622 *idx0 = idx; 1623 back: 1624 if (error) { 1625 m_freem(*m_head); 1626 *m_head = NULL; 1627 } 1628 return error; 1629 } 1630 1631 /* 1632 * Main transmit routine for C+ and gigE NICs. 1633 */ 1634 1635 static void 1636 re_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 1637 { 1638 struct re_softc *sc = ifp->if_softc; 1639 struct mbuf *m_head; 1640 int idx, need_trans, oactive, error; 1641 1642 ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); 1643 ASSERT_SERIALIZED(ifp->if_serializer); 1644 1645 if ((sc->re_flags & RE_F_LINKED) == 0) { 1646 ifq_purge(&ifp->if_snd); 1647 return; 1648 } 1649 1650 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) 1651 return; 1652 1653 idx = sc->re_ldata.re_tx_prodidx; 1654 1655 need_trans = 0; 1656 oactive = 0; 1657 while (sc->re_ldata.re_tx_mbuf[idx] == NULL) { 1658 if (sc->re_ldata.re_tx_free <= RE_TXDESC_SPARE) { 1659 if (!oactive) { 1660 if (re_tx_collect(sc)) { 1661 oactive = 1; 1662 continue; 1663 } 1664 } 1665 ifq_set_oactive(&ifp->if_snd); 1666 break; 1667 } 1668 1669 m_head = ifq_dequeue(&ifp->if_snd); 1670 if (m_head == NULL) 1671 break; 1672 1673 error = re_encap(sc, &m_head, &idx); 1674 if (error) { 1675 /* m_head is freed by re_encap(), if we reach here */ 1676 IFNET_STAT_INC(ifp, oerrors, 1); 1677 1678 if (error == EFBIG && !oactive) { 1679 if (re_tx_collect(sc)) { 1680 oactive = 1; 1681 continue; 1682 } 1683 } 1684 ifq_set_oactive(&ifp->if_snd); 1685 break; 1686 } 1687 1688 oactive = 0; 1689 need_trans = 1; 1690 1691 /* 1692 * If there's a BPF listener, bounce a copy of this frame 1693 * to him. 1694 */ 1695 ETHER_BPF_MTAP(ifp, m_head); 1696 } 1697 1698 /* 1699 * If sc->re_ldata.re_tx_mbuf[idx] is not NULL it is possible 1700 * for OACTIVE to not be properly set when we also do not 1701 * have sufficient free tx descriptors, leaving packet in 1702 * ifp->if_snd. This can cause if_start_dispatch() to loop 1703 * infinitely so make sure OACTIVE is set properly. 1704 */ 1705 if (sc->re_ldata.re_tx_free <= RE_TXDESC_SPARE) { 1706 if (!ifq_is_oactive(&ifp->if_snd)) { 1707 #if 0 1708 if_printf(ifp, "Debug: OACTIVE was not set when " 1709 "re_tx_free was below minimum!\n"); 1710 #endif 1711 ifq_set_oactive(&ifp->if_snd); 1712 } 1713 } 1714 if (!need_trans) 1715 return; 1716 1717 sc->re_ldata.re_tx_prodidx = idx; 1718 1719 /* 1720 * RealTek put the TX poll request register in a different 1721 * location on the 8169 gigE chip. I don't know why. 1722 */ 1723 CSR_WRITE_1(sc, RE_TPPOLL, RE_NPQ); 1724 1725 /* 1726 * Set a timeout in case the chip goes out to lunch. 1727 */ 1728 ifp->if_timer = 5; 1729 } 1730 1731 static void 1732 re_link_up(struct re_softc *sc) 1733 { 1734 struct ifnet *ifp = &sc->arpcom.ac_if; 1735 int error; 1736 1737 ASSERT_SERIALIZED(ifp->if_serializer); 1738 1739 rtl_link_on_patch(sc); 1740 re_stop(sc, FALSE); 1741 rtl_set_eaddr(sc); 1742 1743 error = re_rx_list_init(sc); 1744 if (error) { 1745 re_stop(sc, TRUE); 1746 return; 1747 } 1748 error = re_tx_list_init(sc); 1749 if (error) { 1750 re_stop(sc, TRUE); 1751 return; 1752 } 1753 1754 /* 1755 * Load the addresses of the RX and TX lists into the chip. 1756 */ 1757 CSR_WRITE_4(sc, RE_RXLIST_ADDR_HI, 1758 RE_ADDR_HI(sc->re_ldata.re_rx_list_addr)); 1759 CSR_WRITE_4(sc, RE_RXLIST_ADDR_LO, 1760 RE_ADDR_LO(sc->re_ldata.re_rx_list_addr)); 1761 1762 CSR_WRITE_4(sc, RE_TXLIST_ADDR_HI, 1763 RE_ADDR_HI(sc->re_ldata.re_tx_list_addr)); 1764 CSR_WRITE_4(sc, RE_TXLIST_ADDR_LO, 1765 RE_ADDR_LO(sc->re_ldata.re_tx_list_addr)); 1766 1767 rtl_hw_start(sc); 1768 1769 #ifdef IFPOLL_ENABLE 1770 /* 1771 * Disable interrupts if we are polling. 1772 */ 1773 if (ifp->if_flags & IFF_NPOLLING) 1774 re_setup_intr(sc, 0, RE_IMTYPE_NONE); 1775 else /* otherwise ... */ 1776 #endif /* IFPOLL_ENABLE */ 1777 /* 1778 * Enable interrupts. 1779 */ 1780 re_setup_intr(sc, 1, sc->re_imtype); 1781 CSR_WRITE_2(sc, RE_ISR, sc->re_intrs); 1782 1783 sc->re_flags |= RE_F_LINKED; 1784 ifp->if_link_state = LINK_STATE_UP; 1785 if_link_state_change(ifp); 1786 1787 if (bootverbose) 1788 if_printf(ifp, "link UP\n"); 1789 1790 if (!ifq_is_empty(&ifp->if_snd)) 1791 if_devstart(ifp); 1792 } 1793 1794 static void 1795 re_link_down(struct re_softc *sc) 1796 { 1797 struct ifnet *ifp = &sc->arpcom.ac_if; 1798 1799 /* NOTE: re_stop() will reset RE_F_LINKED. */ 1800 ifp->if_link_state = LINK_STATE_DOWN; 1801 if_link_state_change(ifp); 1802 1803 re_stop(sc, FALSE); 1804 rtl_ifmedia_upd(ifp); 1805 1806 if (bootverbose) 1807 if_printf(ifp, "link DOWN\n"); 1808 } 1809 1810 static void 1811 re_init(void *xsc) 1812 { 1813 struct re_softc *sc = xsc; 1814 struct ifnet *ifp = &sc->arpcom.ac_if; 1815 1816 ASSERT_SERIALIZED(ifp->if_serializer); 1817 1818 re_stop(sc, TRUE); 1819 if (rtl_link_ok(sc)) { 1820 if (bootverbose) 1821 if_printf(ifp, "link is UP in if_init\n"); 1822 re_link_up(sc); 1823 } 1824 1825 ifp->if_flags |= IFF_RUNNING; 1826 ifq_clr_oactive(&ifp->if_snd); 1827 1828 callout_reset(&sc->re_timer, hz, re_tick, sc); 1829 } 1830 1831 static int 1832 re_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) 1833 { 1834 struct re_softc *sc = ifp->if_softc; 1835 struct ifreq *ifr = (struct ifreq *)data; 1836 int error = 0, mask; 1837 1838 ASSERT_SERIALIZED(ifp->if_serializer); 1839 1840 switch(command) { 1841 case SIOCSIFMTU: 1842 #ifdef RE_JUMBO 1843 if (ifr->ifr_mtu > sc->re_maxmtu) { 1844 error = EINVAL; 1845 } else if (ifp->if_mtu != ifr->ifr_mtu) { 1846 ifp->if_mtu = ifr->ifr_mtu; 1847 if (ifp->if_flags & IFF_RUNNING) 1848 ifp->if_init(sc); 1849 } 1850 #else 1851 error = EOPNOTSUPP; 1852 #endif 1853 break; 1854 1855 case SIOCSIFFLAGS: 1856 if (ifp->if_flags & IFF_UP) { 1857 if (ifp->if_flags & IFF_RUNNING) { 1858 if ((ifp->if_flags ^ sc->re_saved_ifflags) & 1859 (IFF_PROMISC | IFF_ALLMULTI)) 1860 rtl_set_rx_packet_filter(sc); 1861 } else { 1862 re_init(sc); 1863 } 1864 } else if (ifp->if_flags & IFF_RUNNING) { 1865 re_stop(sc, TRUE); 1866 } 1867 sc->re_saved_ifflags = ifp->if_flags; 1868 break; 1869 1870 case SIOCADDMULTI: 1871 case SIOCDELMULTI: 1872 rtl_set_rx_packet_filter(sc); 1873 break; 1874 1875 case SIOCGIFMEDIA: 1876 case SIOCSIFMEDIA: 1877 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 1878 break; 1879 1880 case SIOCSIFCAP: 1881 mask = (ifr->ifr_reqcap ^ ifp->if_capenable) & 1882 ifp->if_capabilities; 1883 ifp->if_capenable ^= mask; 1884 1885 /* NOTE: re_init will setup if_hwassist. */ 1886 ifp->if_hwassist = 0; 1887 1888 /* Setup flags for the backend. */ 1889 if (ifp->if_capenable & IFCAP_RXCSUM) 1890 sc->re_rx_cstag = 1; 1891 else 1892 sc->re_rx_cstag = 0; 1893 if (ifp->if_capenable & IFCAP_TXCSUM) 1894 sc->re_tx_cstag = 1; 1895 else 1896 sc->re_tx_cstag = 0; 1897 1898 if (mask && (ifp->if_flags & IFF_RUNNING)) 1899 re_init(sc); 1900 break; 1901 1902 default: 1903 error = ether_ioctl(ifp, command, data); 1904 break; 1905 } 1906 return(error); 1907 } 1908 1909 static void 1910 re_watchdog(struct ifnet *ifp) 1911 { 1912 struct re_softc *sc = ifp->if_softc; 1913 1914 ASSERT_SERIALIZED(ifp->if_serializer); 1915 1916 IFNET_STAT_INC(ifp, oerrors, 1); 1917 1918 re_txeof(sc); 1919 re_rxeof(sc); 1920 1921 if (sc->re_ldata.re_tx_free != sc->re_tx_desc_cnt) { 1922 if_printf(ifp, "watchdog timeout, txd free %d\n", 1923 sc->re_ldata.re_tx_free); 1924 rtl_reset(sc); 1925 re_init(sc); 1926 } 1927 } 1928 1929 /* 1930 * Stop the adapter and free any mbufs allocated to the 1931 * RX and TX lists. 1932 */ 1933 static void 1934 re_stop(struct re_softc *sc, boolean_t full_stop) 1935 { 1936 struct ifnet *ifp = &sc->arpcom.ac_if; 1937 int i; 1938 1939 ASSERT_SERIALIZED(ifp->if_serializer); 1940 1941 /* Stop the adapter. */ 1942 rtl_stop(sc); 1943 1944 ifp->if_timer = 0; 1945 if (full_stop) { 1946 callout_stop(&sc->re_timer); 1947 ifp->if_flags &= ~IFF_RUNNING; 1948 } 1949 ifq_clr_oactive(&ifp->if_snd); 1950 sc->re_flags &= ~(RE_F_TIMER_INTR | RE_F_DROP_RXFRAG | RE_F_LINKED); 1951 1952 re_free_rxchain(sc); 1953 1954 /* Free the TX list buffers. */ 1955 for (i = 0; i < sc->re_tx_desc_cnt; i++) { 1956 if (sc->re_ldata.re_tx_mbuf[i] != NULL) { 1957 bus_dmamap_unload(sc->re_ldata.re_tx_mtag, 1958 sc->re_ldata.re_tx_dmamap[i]); 1959 m_freem(sc->re_ldata.re_tx_mbuf[i]); 1960 sc->re_ldata.re_tx_mbuf[i] = NULL; 1961 } 1962 } 1963 1964 /* Free the RX list buffers. */ 1965 for (i = 0; i < sc->re_rx_desc_cnt; i++) { 1966 if (sc->re_ldata.re_rx_mbuf[i] != NULL) { 1967 if ((sc->re_flags & RE_F_USE_JPOOL) == 0) { 1968 bus_dmamap_unload(sc->re_ldata.re_rx_mtag, 1969 sc->re_ldata.re_rx_dmamap[i]); 1970 } 1971 m_freem(sc->re_ldata.re_rx_mbuf[i]); 1972 sc->re_ldata.re_rx_mbuf[i] = NULL; 1973 } 1974 } 1975 } 1976 1977 /* 1978 * Device suspend routine. Stop the interface and save some PCI 1979 * settings in case the BIOS doesn't restore them properly on 1980 * resume. 1981 */ 1982 static int 1983 re_suspend(device_t dev) 1984 { 1985 #ifndef BURN_BRIDGES 1986 int i; 1987 #endif 1988 struct re_softc *sc = device_get_softc(dev); 1989 struct ifnet *ifp = &sc->arpcom.ac_if; 1990 1991 lwkt_serialize_enter(ifp->if_serializer); 1992 1993 re_stop(sc, TRUE); 1994 1995 #ifndef BURN_BRIDGES 1996 for (i = 0; i < 5; i++) 1997 sc->saved_maps[i] = pci_read_config(dev, PCIR_MAPS + i * 4, 4); 1998 sc->saved_biosaddr = pci_read_config(dev, PCIR_BIOS, 4); 1999 sc->saved_intline = pci_read_config(dev, PCIR_INTLINE, 1); 2000 sc->saved_cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1); 2001 sc->saved_lattimer = pci_read_config(dev, PCIR_LATTIMER, 1); 2002 #endif 2003 2004 sc->re_flags |= RE_F_SUSPENDED; 2005 2006 lwkt_serialize_exit(ifp->if_serializer); 2007 2008 return (0); 2009 } 2010 2011 /* 2012 * Device resume routine. Restore some PCI settings in case the BIOS 2013 * doesn't, re-enable busmastering, and restart the interface if 2014 * appropriate. 2015 */ 2016 static int 2017 re_resume(device_t dev) 2018 { 2019 struct re_softc *sc = device_get_softc(dev); 2020 struct ifnet *ifp = &sc->arpcom.ac_if; 2021 #ifndef BURN_BRIDGES 2022 int i; 2023 #endif 2024 2025 lwkt_serialize_enter(ifp->if_serializer); 2026 2027 #ifndef BURN_BRIDGES 2028 /* better way to do this? */ 2029 for (i = 0; i < 5; i++) 2030 pci_write_config(dev, PCIR_MAPS + i * 4, sc->saved_maps[i], 4); 2031 pci_write_config(dev, PCIR_BIOS, sc->saved_biosaddr, 4); 2032 pci_write_config(dev, PCIR_INTLINE, sc->saved_intline, 1); 2033 pci_write_config(dev, PCIR_CACHELNSZ, sc->saved_cachelnsz, 1); 2034 pci_write_config(dev, PCIR_LATTIMER, sc->saved_lattimer, 1); 2035 2036 /* reenable busmastering */ 2037 pci_enable_busmaster(dev); 2038 pci_enable_io(dev, SYS_RES_IOPORT); 2039 #endif 2040 2041 /* reinitialize interface if necessary */ 2042 if (ifp->if_flags & IFF_UP) 2043 re_init(sc); 2044 2045 sc->re_flags &= ~RE_F_SUSPENDED; 2046 2047 lwkt_serialize_exit(ifp->if_serializer); 2048 2049 return (0); 2050 } 2051 2052 /* 2053 * Stop all chip I/O so that the kernel's probe routines don't 2054 * get confused by errant DMAs when rebooting. 2055 */ 2056 static void 2057 re_shutdown(device_t dev) 2058 { 2059 struct re_softc *sc = device_get_softc(dev); 2060 struct ifnet *ifp = &sc->arpcom.ac_if; 2061 2062 lwkt_serialize_enter(ifp->if_serializer); 2063 re_stop(sc, TRUE); 2064 rtl_hw_d3_para(sc); 2065 rtl_phy_power_down(sc); 2066 lwkt_serialize_exit(ifp->if_serializer); 2067 } 2068 2069 static int 2070 re_sysctl_rxtime(SYSCTL_HANDLER_ARGS) 2071 { 2072 struct re_softc *sc = arg1; 2073 2074 return re_sysctl_hwtime(oidp, arg1, arg2, req, &sc->re_rx_time); 2075 } 2076 2077 static int 2078 re_sysctl_txtime(SYSCTL_HANDLER_ARGS) 2079 { 2080 struct re_softc *sc = arg1; 2081 2082 return re_sysctl_hwtime(oidp, arg1, arg2, req, &sc->re_tx_time); 2083 } 2084 2085 static int 2086 re_sysctl_hwtime(SYSCTL_HANDLER_ARGS, int *hwtime) 2087 { 2088 struct re_softc *sc = arg1; 2089 struct ifnet *ifp = &sc->arpcom.ac_if; 2090 int error, v; 2091 2092 lwkt_serialize_enter(ifp->if_serializer); 2093 2094 v = *hwtime; 2095 error = sysctl_handle_int(oidp, &v, 0, req); 2096 if (error || req->newptr == NULL) 2097 goto back; 2098 2099 if (v <= 0) { 2100 error = EINVAL; 2101 goto back; 2102 } 2103 2104 if (v != *hwtime) { 2105 *hwtime = v; 2106 2107 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) == 2108 IFF_RUNNING && sc->re_imtype == RE_IMTYPE_HW) 2109 re_setup_hw_im(sc); 2110 } 2111 back: 2112 lwkt_serialize_exit(ifp->if_serializer); 2113 return error; 2114 } 2115 2116 static int 2117 re_sysctl_simtime(SYSCTL_HANDLER_ARGS) 2118 { 2119 struct re_softc *sc = arg1; 2120 struct ifnet *ifp = &sc->arpcom.ac_if; 2121 int error, v; 2122 2123 lwkt_serialize_enter(ifp->if_serializer); 2124 2125 v = sc->re_sim_time; 2126 error = sysctl_handle_int(oidp, &v, 0, req); 2127 if (error || req->newptr == NULL) 2128 goto back; 2129 2130 if (v <= 0) { 2131 error = EINVAL; 2132 goto back; 2133 } 2134 2135 if (v != sc->re_sim_time) { 2136 sc->re_sim_time = v; 2137 2138 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) == 2139 IFF_RUNNING && sc->re_imtype == RE_IMTYPE_SIM) { 2140 #ifdef foo 2141 /* 2142 * Following code causes various strange 2143 * performance problems. Hmm ... 2144 */ 2145 CSR_WRITE_2(sc, RE_IMR, 0); 2146 CSR_WRITE_4(sc, RE_TIMERINT, 0); 2147 CSR_READ_4(sc, RE_TIMERINT); /* flush */ 2148 2149 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 2150 re_setup_sim_im(sc); 2151 #else 2152 re_setup_intr(sc, 0, RE_IMTYPE_NONE); 2153 DELAY(10); 2154 re_setup_intr(sc, 1, RE_IMTYPE_SIM); 2155 #endif 2156 } 2157 } 2158 back: 2159 lwkt_serialize_exit(ifp->if_serializer); 2160 return error; 2161 } 2162 2163 static int 2164 re_sysctl_imtype(SYSCTL_HANDLER_ARGS) 2165 { 2166 struct re_softc *sc = arg1; 2167 struct ifnet *ifp = &sc->arpcom.ac_if; 2168 int error, v; 2169 2170 lwkt_serialize_enter(ifp->if_serializer); 2171 2172 v = sc->re_imtype; 2173 error = sysctl_handle_int(oidp, &v, 0, req); 2174 if (error || req->newptr == NULL) 2175 goto back; 2176 2177 if (v != RE_IMTYPE_HW && v != RE_IMTYPE_SIM && v != RE_IMTYPE_NONE) { 2178 error = EINVAL; 2179 goto back; 2180 } 2181 if (v == RE_IMTYPE_HW && (sc->re_caps & RE_C_HWIM) == 0) { 2182 /* Can't do hardware interrupt moderation */ 2183 error = EOPNOTSUPP; 2184 goto back; 2185 } 2186 2187 if (v != sc->re_imtype) { 2188 sc->re_imtype = v; 2189 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) == 2190 IFF_RUNNING) 2191 re_setup_intr(sc, 1, sc->re_imtype); 2192 } 2193 back: 2194 lwkt_serialize_exit(ifp->if_serializer); 2195 return error; 2196 } 2197 2198 static void 2199 re_setup_hw_im(struct re_softc *sc) 2200 { 2201 KKASSERT(sc->re_caps & RE_C_HWIM); 2202 2203 /* 2204 * Interrupt moderation 2205 * 2206 * 0xABCD 2207 * A - unknown (maybe TX related) 2208 * B - TX timer (unit: 25us) 2209 * C - unknown (maybe RX related) 2210 * D - RX timer (unit: 25us) 2211 * 2212 * 2213 * re(4)'s interrupt moderation is actually controlled by 2214 * two variables, like most other NICs (bge, bce etc.) 2215 * o timer 2216 * o number of packets [P] 2217 * 2218 * The logic relationship between these two variables is 2219 * similar to other NICs too: 2220 * if (timer expire || packets > [P]) 2221 * Interrupt is delivered 2222 * 2223 * Currently we only know how to set 'timer', but not 2224 * 'number of packets', which should be ~30, as far as I 2225 * tested (sink ~900Kpps, interrupt rate is 30KHz) 2226 */ 2227 CSR_WRITE_2(sc, RE_IM, 2228 RE_IM_RXTIME(sc->re_rx_time) | 2229 RE_IM_TXTIME(sc->re_tx_time) | 2230 RE_IM_MAGIC); 2231 } 2232 2233 static void 2234 re_disable_hw_im(struct re_softc *sc) 2235 { 2236 if (sc->re_caps & RE_C_HWIM) 2237 CSR_WRITE_2(sc, RE_IM, 0); 2238 } 2239 2240 static void 2241 re_setup_sim_im(struct re_softc *sc) 2242 { 2243 uint32_t ticks; 2244 2245 if (sc->re_if_flags & RL_FLAG_PCIE) { 2246 ticks = sc->re_sim_time * sc->re_bus_speed; 2247 } else { 2248 /* 2249 * Datasheet says tick decreases at bus speed, 2250 * but it seems the clock runs a little bit 2251 * faster, so we do some compensation here. 2252 */ 2253 ticks = (sc->re_sim_time * sc->re_bus_speed * 8) / 5; 2254 } 2255 CSR_WRITE_4(sc, RE_TIMERINT, ticks); 2256 2257 CSR_WRITE_4(sc, RE_TIMERCNT, 1); /* reload */ 2258 sc->re_flags |= RE_F_TIMER_INTR; 2259 } 2260 2261 static void 2262 re_disable_sim_im(struct re_softc *sc) 2263 { 2264 CSR_WRITE_4(sc, RE_TIMERINT, 0); 2265 sc->re_flags &= ~RE_F_TIMER_INTR; 2266 } 2267 2268 static void 2269 re_config_imtype(struct re_softc *sc, int imtype) 2270 { 2271 switch (imtype) { 2272 case RE_IMTYPE_HW: 2273 KKASSERT(sc->re_caps & RE_C_HWIM); 2274 /* FALL THROUGH */ 2275 case RE_IMTYPE_NONE: 2276 sc->re_intrs = RE_INTRS; 2277 sc->re_rx_ack = RE_ISR_RX_OK | RE_ISR_FIFO_OFLOW | 2278 RE_ISR_RX_OVERRUN; 2279 sc->re_tx_ack = RE_ISR_TX_OK; 2280 break; 2281 2282 case RE_IMTYPE_SIM: 2283 sc->re_intrs = RE_INTRS_TIMER; 2284 sc->re_rx_ack = RE_ISR_PCS_TIMEOUT; 2285 sc->re_tx_ack = RE_ISR_PCS_TIMEOUT; 2286 break; 2287 2288 default: 2289 panic("%s: unknown imtype %d", 2290 sc->arpcom.ac_if.if_xname, imtype); 2291 } 2292 } 2293 2294 static void 2295 re_setup_intr(struct re_softc *sc, int enable_intrs, int imtype) 2296 { 2297 re_config_imtype(sc, imtype); 2298 2299 if (enable_intrs) 2300 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 2301 else 2302 CSR_WRITE_2(sc, RE_IMR, 0); 2303 2304 sc->re_npoll.ifpc_stcount = 0; 2305 2306 switch (imtype) { 2307 case RE_IMTYPE_NONE: 2308 re_disable_sim_im(sc); 2309 re_disable_hw_im(sc); 2310 break; 2311 2312 case RE_IMTYPE_HW: 2313 KKASSERT(sc->re_caps & RE_C_HWIM); 2314 re_disable_sim_im(sc); 2315 re_setup_hw_im(sc); 2316 break; 2317 2318 case RE_IMTYPE_SIM: 2319 re_disable_hw_im(sc); 2320 re_setup_sim_im(sc); 2321 break; 2322 2323 default: 2324 panic("%s: unknown imtype %d", 2325 sc->arpcom.ac_if.if_xname, imtype); 2326 } 2327 } 2328 2329 static int 2330 re_jpool_alloc(struct re_softc *sc) 2331 { 2332 struct re_list_data *ldata = &sc->re_ldata; 2333 struct re_jbuf *jbuf; 2334 bus_addr_t paddr; 2335 bus_size_t jpool_size; 2336 bus_dmamem_t dmem; 2337 caddr_t buf; 2338 int i, error; 2339 2340 lwkt_serialize_init(&ldata->re_jbuf_serializer); 2341 2342 ldata->re_jbuf = kmalloc(sizeof(struct re_jbuf) * RE_JBUF_COUNT(sc), 2343 M_DEVBUF, M_WAITOK | M_ZERO); 2344 2345 jpool_size = RE_JBUF_COUNT(sc) * RE_JBUF_SIZE; 2346 2347 error = bus_dmamem_coherent(sc->re_parent_tag, 2348 RE_RXBUF_ALIGN, 0, 2349 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 2350 jpool_size, BUS_DMA_WAITOK, &dmem); 2351 if (error) { 2352 device_printf(sc->dev, "could not allocate jumbo memory\n"); 2353 return error; 2354 } 2355 ldata->re_jpool_tag = dmem.dmem_tag; 2356 ldata->re_jpool_map = dmem.dmem_map; 2357 ldata->re_jpool = dmem.dmem_addr; 2358 paddr = dmem.dmem_busaddr; 2359 2360 /* ..and split it into 9KB chunks */ 2361 SLIST_INIT(&ldata->re_jbuf_free); 2362 2363 buf = ldata->re_jpool; 2364 for (i = 0; i < RE_JBUF_COUNT(sc); i++) { 2365 jbuf = &ldata->re_jbuf[i]; 2366 2367 jbuf->re_sc = sc; 2368 jbuf->re_inuse = 0; 2369 jbuf->re_slot = i; 2370 jbuf->re_buf = buf; 2371 jbuf->re_paddr = paddr; 2372 2373 SLIST_INSERT_HEAD(&ldata->re_jbuf_free, jbuf, re_link); 2374 2375 buf += RE_JBUF_SIZE; 2376 paddr += RE_JBUF_SIZE; 2377 } 2378 return 0; 2379 } 2380 2381 static void 2382 re_jpool_free(struct re_softc *sc) 2383 { 2384 struct re_list_data *ldata = &sc->re_ldata; 2385 2386 if (ldata->re_jpool_tag != NULL) { 2387 bus_dmamap_unload(ldata->re_jpool_tag, ldata->re_jpool_map); 2388 bus_dmamem_free(ldata->re_jpool_tag, ldata->re_jpool, 2389 ldata->re_jpool_map); 2390 bus_dma_tag_destroy(ldata->re_jpool_tag); 2391 ldata->re_jpool_tag = NULL; 2392 } 2393 2394 if (ldata->re_jbuf != NULL) { 2395 kfree(ldata->re_jbuf, M_DEVBUF); 2396 ldata->re_jbuf = NULL; 2397 } 2398 } 2399 2400 #ifdef RE_JUMBO 2401 static struct re_jbuf * 2402 re_jbuf_alloc(struct re_softc *sc) 2403 { 2404 struct re_list_data *ldata = &sc->re_ldata; 2405 struct re_jbuf *jbuf; 2406 2407 lwkt_serialize_enter(&ldata->re_jbuf_serializer); 2408 2409 jbuf = SLIST_FIRST(&ldata->re_jbuf_free); 2410 if (jbuf != NULL) { 2411 SLIST_REMOVE_HEAD(&ldata->re_jbuf_free, re_link); 2412 jbuf->re_inuse = 1; 2413 } 2414 2415 lwkt_serialize_exit(&ldata->re_jbuf_serializer); 2416 2417 return jbuf; 2418 } 2419 2420 static void 2421 re_jbuf_free(void *arg) 2422 { 2423 struct re_jbuf *jbuf = arg; 2424 struct re_softc *sc = jbuf->re_sc; 2425 struct re_list_data *ldata = &sc->re_ldata; 2426 2427 if (&ldata->re_jbuf[jbuf->re_slot] != jbuf) { 2428 panic("%s: free wrong jumbo buffer", 2429 sc->arpcom.ac_if.if_xname); 2430 } else if (jbuf->re_inuse == 0) { 2431 panic("%s: jumbo buffer already freed", 2432 sc->arpcom.ac_if.if_xname); 2433 } 2434 2435 lwkt_serialize_enter(&ldata->re_jbuf_serializer); 2436 atomic_subtract_int(&jbuf->re_inuse, 1); 2437 if (jbuf->re_inuse == 0) 2438 SLIST_INSERT_HEAD(&ldata->re_jbuf_free, jbuf, re_link); 2439 lwkt_serialize_exit(&ldata->re_jbuf_serializer); 2440 } 2441 2442 static void 2443 re_jbuf_ref(void *arg) 2444 { 2445 struct re_jbuf *jbuf = arg; 2446 struct re_softc *sc = jbuf->re_sc; 2447 struct re_list_data *ldata = &sc->re_ldata; 2448 2449 if (&ldata->re_jbuf[jbuf->re_slot] != jbuf) { 2450 panic("%s: ref wrong jumbo buffer", 2451 sc->arpcom.ac_if.if_xname); 2452 } else if (jbuf->re_inuse == 0) { 2453 panic("%s: jumbo buffer already freed", 2454 sc->arpcom.ac_if.if_xname); 2455 } 2456 atomic_add_int(&jbuf->re_inuse, 1); 2457 } 2458 #endif /* RE_JUMBO */ 2459 2460 static void 2461 re_disable_aspm(device_t dev) 2462 { 2463 uint16_t link_cap, link_ctrl; 2464 uint8_t pcie_ptr, reg; 2465 2466 pcie_ptr = pci_get_pciecap_ptr(dev); 2467 if (pcie_ptr == 0) 2468 return; 2469 2470 link_cap = pci_read_config(dev, pcie_ptr + PCIER_LINKCAP, 2); 2471 if ((link_cap & PCIEM_LNKCAP_ASPM_MASK) == 0) 2472 return; 2473 2474 if (bootverbose) 2475 device_printf(dev, "disable ASPM\n"); 2476 2477 reg = pcie_ptr + PCIER_LINKCTRL; 2478 link_ctrl = pci_read_config(dev, reg, 2); 2479 link_ctrl &= ~(PCIEM_LNKCTL_ASPM_L0S | PCIEM_LNKCTL_ASPM_L1); 2480 pci_write_config(dev, reg, link_ctrl, 2); 2481 } 2482