1 /* 2 * Copyright (c) 2004 3 * Joerg Sonnenberger <joerg@bec.de>. All rights reserved. 4 * 5 * Copyright (c) 1997, 1998-2003 6 * Bill Paul <wpaul@windriver.com>. All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Bill Paul. 19 * 4. Neither the name of the author nor the names of any co-contributors 20 * may be used to endorse or promote products derived from this software 21 * without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 33 * THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * $FreeBSD: src/sys/dev/re/if_re.c,v 1.25 2004/06/09 14:34:01 naddy Exp $ 36 */ 37 38 /* 39 * RealTek 8169S/8110S/8168/8111/8101E PCI NIC driver 40 * 41 * Written by Bill Paul <wpaul@windriver.com> 42 * Senior Networking Software Engineer 43 * Wind River Systems 44 */ 45 46 /* 47 * This driver is designed to support RealTek's next generation of 48 * 10/100 and 10/100/1000 PCI ethernet controllers. There are currently 49 * seven devices in this family: the the RTL8169, the RTL8169S, RTL8110S, 50 * the RTL8168, the RTL8111 and the RTL8101E. 51 * 52 * The 8169 is a 64-bit 10/100/1000 gigabit ethernet MAC: 53 * 54 * o Descriptor based DMA mechanism. Each descriptor represents 55 * a single packet fragment. Data buffers may be aligned on 56 * any byte boundary. 57 * 58 * o 64-bit DMA. 59 * 60 * o TCP/IP checksum offload for both RX and TX. 61 * 62 * o High and normal priority transmit DMA rings. 63 * 64 * o VLAN tag insertion and extraction. 65 * 66 * o TCP large send (segmentation offload). 67 * 68 * o 1000Mbps mode. 69 * 70 * o Jumbo frames. 71 * 72 * o GMII and TBI ports/registers for interfacing with copper 73 * or fiber PHYs. 74 * 75 * o RX and TX DMA rings can have up to 1024 descriptors. 76 * 77 * The 8169 does not have a built-in PHY. Most reference boards use a 78 * Marvell 88E1000 'Alaska' copper gigE PHY. 8169/8110 is _no longer_ 79 * supported. 80 * 81 * The 8169S/8110S 10/100/1000 devices have built-in copper gigE PHYs 82 * (the 'S' stands for 'single-chip'). These devices have the same 83 * programming API as the older 8169, but also have some vendor-specific 84 * registers for the on-board PHY. The 8110S is a LAN-on-motherboard 85 * part designed to be pin-compatible with the RealTek 8100 10/100 chip. 86 * 87 * This driver takes advantage of the RX and TX checksum offload and 88 * VLAN tag insertion/extraction features. It also implements 89 * interrupt moderation using the timer interrupt registers, which 90 * significantly reduces interrupt load. 91 */ 92 93 #define _IP_VHL 94 95 #include "opt_ifpoll.h" 96 97 #include <sys/param.h> 98 #include <sys/bus.h> 99 #include <sys/endian.h> 100 #include <sys/kernel.h> 101 #include <sys/in_cksum.h> 102 #include <sys/interrupt.h> 103 #include <sys/malloc.h> 104 #include <sys/mbuf.h> 105 #include <sys/rman.h> 106 #include <sys/serialize.h> 107 #include <sys/socket.h> 108 #include <sys/sockio.h> 109 #include <sys/sysctl.h> 110 111 #include <net/bpf.h> 112 #include <net/ethernet.h> 113 #include <net/if.h> 114 #include <net/ifq_var.h> 115 #include <net/if_arp.h> 116 #include <net/if_dl.h> 117 #include <net/if_media.h> 118 #include <net/if_poll.h> 119 #include <net/if_types.h> 120 #include <net/vlan/if_vlan_var.h> 121 #include <net/vlan/if_vlan_ether.h> 122 123 #include <netinet/ip.h> 124 125 #include "pcidevs.h" 126 #include <bus/pci/pcireg.h> 127 #include <bus/pci/pcivar.h> 128 129 #include <dev/netif/re/if_rereg.h> 130 #include <dev/netif/re/if_revar.h> 131 #include <dev/netif/re/re.h> 132 #include <dev/netif/re/re_dragonfly.h> 133 134 /* 135 * Various supported device vendors/types and their names. 136 */ 137 static const struct re_type { 138 uint16_t re_vid; 139 uint16_t re_did; 140 const char *re_name; 141 } re_devs[] = { 142 { PCI_VENDOR_DLINK, PCI_PRODUCT_DLINK_DGE528T, 143 "D-Link DGE-528(T) Gigabit Ethernet Adapter" }, 144 145 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8101E, 146 "RealTek 810x PCIe 10/100baseTX" }, 147 148 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8168, 149 "RealTek 8111/8168 PCIe Gigabit Ethernet" }, 150 151 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8168_1, 152 "RealTek 8168 PCIe Gigabit Ethernet" }, 153 154 #ifdef notyet 155 /* 156 * This driver now only supports built-in PHYs. 157 */ 158 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8169, 159 "RealTek 8110/8169 Gigabit Ethernet" }, 160 #endif 161 162 { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8169SC, 163 "RealTek 8169SC/8110SC Single-chip Gigabit Ethernet" }, 164 165 { PCI_VENDOR_COREGA, PCI_PRODUCT_COREGA_CG_LAPCIGT, 166 "Corega CG-LAPCIGT Gigabit Ethernet" }, 167 168 { PCI_VENDOR_LINKSYS, PCI_PRODUCT_LINKSYS_EG1032, 169 "Linksys EG1032 Gigabit Ethernet" }, 170 171 { PCI_VENDOR_USR2, PCI_PRODUCT_USR2_997902, 172 "US Robotics 997902 Gigabit Ethernet" }, 173 174 { PCI_VENDOR_TTTECH, PCI_PRODUCT_TTTECH_MC322, 175 "TTTech MC322 Gigabit Ethernet" }, 176 177 { 0, 0, NULL } 178 }; 179 180 static int re_probe(device_t); 181 static int re_attach(device_t); 182 static int re_detach(device_t); 183 static int re_suspend(device_t); 184 static int re_resume(device_t); 185 static void re_shutdown(device_t); 186 187 static int re_allocmem(device_t); 188 static void re_freemem(device_t); 189 static void re_freebufmem(struct re_softc *, int, int); 190 static int re_encap(struct re_softc *, struct mbuf **, int *); 191 static int re_newbuf_std(struct re_softc *, int, int); 192 #ifdef RE_JUMBO 193 static int re_newbuf_jumbo(struct re_softc *, int, int); 194 #endif 195 static void re_setup_rxdesc(struct re_softc *, int); 196 static int re_rx_list_init(struct re_softc *); 197 static int re_tx_list_init(struct re_softc *); 198 static int re_rxeof(struct re_softc *); 199 static int re_txeof(struct re_softc *); 200 static int re_tx_collect(struct re_softc *); 201 static void re_intr(void *); 202 static void re_tick(void *); 203 static void re_tick_serialized(void *); 204 static void re_disable_aspm(device_t); 205 static void re_link_up(struct re_softc *); 206 static void re_link_down(struct re_softc *); 207 208 static void re_start(struct ifnet *, struct ifaltq_subque *); 209 static int re_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 210 static void re_init(void *); 211 static void re_stop(struct re_softc *, boolean_t); 212 static void re_watchdog(struct ifnet *); 213 214 static void re_setup_hw_im(struct re_softc *); 215 static void re_setup_sim_im(struct re_softc *); 216 static void re_disable_hw_im(struct re_softc *); 217 static void re_disable_sim_im(struct re_softc *); 218 static void re_config_imtype(struct re_softc *, int); 219 static void re_setup_intr(struct re_softc *, int, int); 220 221 static int re_sysctl_hwtime(SYSCTL_HANDLER_ARGS, int *); 222 static int re_sysctl_rxtime(SYSCTL_HANDLER_ARGS); 223 static int re_sysctl_txtime(SYSCTL_HANDLER_ARGS); 224 static int re_sysctl_simtime(SYSCTL_HANDLER_ARGS); 225 static int re_sysctl_imtype(SYSCTL_HANDLER_ARGS); 226 227 static int re_jpool_alloc(struct re_softc *); 228 static void re_jpool_free(struct re_softc *); 229 #ifdef RE_JUMBO 230 static struct re_jbuf *re_jbuf_alloc(struct re_softc *); 231 static void re_jbuf_free(void *); 232 static void re_jbuf_ref(void *); 233 #endif 234 235 #ifdef IFPOLL_ENABLE 236 static void re_npoll(struct ifnet *, struct ifpoll_info *); 237 static void re_npoll_compat(struct ifnet *, void *, int); 238 #endif 239 240 static device_method_t re_methods[] = { 241 /* Device interface */ 242 DEVMETHOD(device_probe, re_probe), 243 DEVMETHOD(device_attach, re_attach), 244 DEVMETHOD(device_detach, re_detach), 245 DEVMETHOD(device_suspend, re_suspend), 246 DEVMETHOD(device_resume, re_resume), 247 DEVMETHOD(device_shutdown, re_shutdown), 248 DEVMETHOD_END 249 }; 250 251 static driver_t re_driver = { 252 "re", 253 re_methods, 254 sizeof(struct re_softc) 255 }; 256 257 static devclass_t re_devclass; 258 259 DECLARE_DUMMY_MODULE(if_re); 260 DRIVER_MODULE(if_re, pci, re_driver, re_devclass, NULL, NULL); 261 DRIVER_MODULE(if_re, cardbus, re_driver, re_devclass, NULL, NULL); 262 263 static int re_rx_desc_count = RE_RX_DESC_CNT_DEF; 264 static int re_tx_desc_count = RE_TX_DESC_CNT_DEF; 265 static int re_msi_enable = 1; 266 267 TUNABLE_INT("hw.re.rx_desc_count", &re_rx_desc_count); 268 TUNABLE_INT("hw.re.tx_desc_count", &re_tx_desc_count); 269 TUNABLE_INT("hw.re.msi.enable", &re_msi_enable); 270 271 static __inline void 272 re_free_rxchain(struct re_softc *sc) 273 { 274 if (sc->re_head != NULL) { 275 m_freem(sc->re_head); 276 sc->re_head = sc->re_tail = NULL; 277 } 278 } 279 280 static int 281 re_probe(device_t dev) 282 { 283 const struct re_type *t; 284 uint16_t vendor, product; 285 286 vendor = pci_get_vendor(dev); 287 product = pci_get_device(dev); 288 289 /* 290 * Only attach to rev.3 of the Linksys EG1032 adapter. 291 * Rev.2 is supported by sk(4). 292 */ 293 if (vendor == PCI_VENDOR_LINKSYS && 294 product == PCI_PRODUCT_LINKSYS_EG1032 && 295 pci_get_subdevice(dev) != PCI_SUBDEVICE_LINKSYS_EG1032_REV3) 296 return ENXIO; 297 298 for (t = re_devs; t->re_name != NULL; t++) { 299 if (product == t->re_did && vendor == t->re_vid) 300 break; 301 } 302 if (t->re_name == NULL) 303 return ENXIO; 304 305 device_set_desc(dev, t->re_name); 306 return 0; 307 } 308 309 static int 310 re_allocmem(device_t dev) 311 { 312 struct re_softc *sc = device_get_softc(dev); 313 bus_dmamem_t dmem; 314 int error, i; 315 316 /* 317 * Allocate list data 318 */ 319 sc->re_ldata.re_tx_mbuf = 320 kmalloc(sc->re_tx_desc_cnt * sizeof(struct mbuf *), 321 M_DEVBUF, M_ZERO | M_WAITOK); 322 323 sc->re_ldata.re_rx_mbuf = 324 kmalloc(sc->re_rx_desc_cnt * sizeof(struct mbuf *), 325 M_DEVBUF, M_ZERO | M_WAITOK); 326 327 sc->re_ldata.re_rx_paddr = 328 kmalloc(sc->re_rx_desc_cnt * sizeof(bus_addr_t), 329 M_DEVBUF, M_ZERO | M_WAITOK); 330 331 sc->re_ldata.re_tx_dmamap = 332 kmalloc(sc->re_tx_desc_cnt * sizeof(bus_dmamap_t), 333 M_DEVBUF, M_ZERO | M_WAITOK); 334 335 sc->re_ldata.re_rx_dmamap = 336 kmalloc(sc->re_rx_desc_cnt * sizeof(bus_dmamap_t), 337 M_DEVBUF, M_ZERO | M_WAITOK); 338 339 /* 340 * Allocate the parent bus DMA tag appropriate for PCI. 341 */ 342 error = bus_dma_tag_create(NULL, /* parent */ 343 1, 0, /* alignment, boundary */ 344 BUS_SPACE_MAXADDR, /* lowaddr */ 345 BUS_SPACE_MAXADDR, /* highaddr */ 346 NULL, NULL, /* filter, filterarg */ 347 BUS_SPACE_MAXSIZE_32BIT,/* maxsize */ 348 0, /* nsegments */ 349 BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */ 350 0, /* flags */ 351 &sc->re_parent_tag); 352 if (error) { 353 device_printf(dev, "could not allocate parent dma tag\n"); 354 return error; 355 } 356 357 /* Allocate TX descriptor list. */ 358 error = bus_dmamem_coherent(sc->re_parent_tag, 359 RE_RING_ALIGN, 0, 360 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 361 RE_TX_LIST_SZ(sc), BUS_DMA_WAITOK | BUS_DMA_ZERO, 362 &dmem); 363 if (error) { 364 device_printf(dev, "could not allocate TX ring\n"); 365 return error; 366 } 367 sc->re_ldata.re_tx_list_tag = dmem.dmem_tag; 368 sc->re_ldata.re_tx_list_map = dmem.dmem_map; 369 sc->re_ldata.re_tx_list = dmem.dmem_addr; 370 sc->re_ldata.re_tx_list_addr = dmem.dmem_busaddr; 371 372 /* Allocate RX descriptor list. */ 373 error = bus_dmamem_coherent(sc->re_parent_tag, 374 RE_RING_ALIGN, 0, 375 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 376 RE_RX_LIST_SZ(sc), BUS_DMA_WAITOK | BUS_DMA_ZERO, 377 &dmem); 378 if (error) { 379 device_printf(dev, "could not allocate RX ring\n"); 380 return error; 381 } 382 sc->re_ldata.re_rx_list_tag = dmem.dmem_tag; 383 sc->re_ldata.re_rx_list_map = dmem.dmem_map; 384 sc->re_ldata.re_rx_list = dmem.dmem_addr; 385 sc->re_ldata.re_rx_list_addr = dmem.dmem_busaddr; 386 387 /* Allocate maps for TX mbufs. */ 388 error = bus_dma_tag_create(sc->re_parent_tag, 389 1, 0, 390 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 391 NULL, NULL, 392 RE_FRAMELEN_MAX, RE_MAXSEGS, MCLBYTES, 393 BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 394 &sc->re_ldata.re_tx_mtag); 395 if (error) { 396 device_printf(dev, "could not allocate TX buf dma tag\n"); 397 return(error); 398 } 399 400 /* Create DMA maps for TX buffers */ 401 for (i = 0; i < sc->re_tx_desc_cnt; i++) { 402 error = bus_dmamap_create(sc->re_ldata.re_tx_mtag, 403 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 404 &sc->re_ldata.re_tx_dmamap[i]); 405 if (error) { 406 device_printf(dev, "can't create DMA map for TX buf\n"); 407 re_freebufmem(sc, i, 0); 408 return(error); 409 } 410 } 411 412 /* Allocate maps for RX mbufs. */ 413 error = bus_dma_tag_create(sc->re_parent_tag, 414 RE_RXBUF_ALIGN, 0, 415 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 416 NULL, NULL, 417 MCLBYTES, 1, MCLBYTES, 418 BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ALIGNED, 419 &sc->re_ldata.re_rx_mtag); 420 if (error) { 421 device_printf(dev, "could not allocate RX buf dma tag\n"); 422 return(error); 423 } 424 425 /* Create spare DMA map for RX */ 426 error = bus_dmamap_create(sc->re_ldata.re_rx_mtag, BUS_DMA_WAITOK, 427 &sc->re_ldata.re_rx_spare); 428 if (error) { 429 device_printf(dev, "can't create spare DMA map for RX\n"); 430 bus_dma_tag_destroy(sc->re_ldata.re_rx_mtag); 431 sc->re_ldata.re_rx_mtag = NULL; 432 return error; 433 } 434 435 /* Create DMA maps for RX buffers */ 436 for (i = 0; i < sc->re_rx_desc_cnt; i++) { 437 error = bus_dmamap_create(sc->re_ldata.re_rx_mtag, 438 BUS_DMA_WAITOK, &sc->re_ldata.re_rx_dmamap[i]); 439 if (error) { 440 device_printf(dev, "can't create DMA map for RX buf\n"); 441 re_freebufmem(sc, sc->re_tx_desc_cnt, i); 442 return(error); 443 } 444 } 445 446 /* Create jumbo buffer pool for RX if required */ 447 if (sc->re_caps & RE_C_CONTIGRX) { 448 error = re_jpool_alloc(sc); 449 if (error) { 450 re_jpool_free(sc); 451 #ifdef RE_JUMBO 452 /* Disable jumbo frame support */ 453 sc->re_maxmtu = ETHERMTU; 454 #endif 455 } 456 } 457 return(0); 458 } 459 460 static void 461 re_freebufmem(struct re_softc *sc, int tx_cnt, int rx_cnt) 462 { 463 int i; 464 465 /* Destroy all the RX and TX buffer maps */ 466 if (sc->re_ldata.re_tx_mtag) { 467 for (i = 0; i < tx_cnt; i++) { 468 bus_dmamap_destroy(sc->re_ldata.re_tx_mtag, 469 sc->re_ldata.re_tx_dmamap[i]); 470 } 471 bus_dma_tag_destroy(sc->re_ldata.re_tx_mtag); 472 sc->re_ldata.re_tx_mtag = NULL; 473 } 474 475 if (sc->re_ldata.re_rx_mtag) { 476 for (i = 0; i < rx_cnt; i++) { 477 bus_dmamap_destroy(sc->re_ldata.re_rx_mtag, 478 sc->re_ldata.re_rx_dmamap[i]); 479 } 480 bus_dmamap_destroy(sc->re_ldata.re_rx_mtag, 481 sc->re_ldata.re_rx_spare); 482 bus_dma_tag_destroy(sc->re_ldata.re_rx_mtag); 483 sc->re_ldata.re_rx_mtag = NULL; 484 } 485 } 486 487 static void 488 re_freemem(device_t dev) 489 { 490 struct re_softc *sc = device_get_softc(dev); 491 492 /* Unload and free the RX DMA ring memory and map */ 493 if (sc->re_ldata.re_rx_list_tag) { 494 bus_dmamap_unload(sc->re_ldata.re_rx_list_tag, 495 sc->re_ldata.re_rx_list_map); 496 bus_dmamem_free(sc->re_ldata.re_rx_list_tag, 497 sc->re_ldata.re_rx_list, 498 sc->re_ldata.re_rx_list_map); 499 bus_dma_tag_destroy(sc->re_ldata.re_rx_list_tag); 500 } 501 502 /* Unload and free the TX DMA ring memory and map */ 503 if (sc->re_ldata.re_tx_list_tag) { 504 bus_dmamap_unload(sc->re_ldata.re_tx_list_tag, 505 sc->re_ldata.re_tx_list_map); 506 bus_dmamem_free(sc->re_ldata.re_tx_list_tag, 507 sc->re_ldata.re_tx_list, 508 sc->re_ldata.re_tx_list_map); 509 bus_dma_tag_destroy(sc->re_ldata.re_tx_list_tag); 510 } 511 512 /* Free RX/TX buf DMA stuffs */ 513 re_freebufmem(sc, sc->re_tx_desc_cnt, sc->re_rx_desc_cnt); 514 515 /* Unload and free the stats buffer and map */ 516 if (sc->re_ldata.re_stag) { 517 bus_dmamap_unload(sc->re_ldata.re_stag, sc->re_ldata.re_smap); 518 bus_dmamem_free(sc->re_ldata.re_stag, 519 sc->re_ldata.re_stats, 520 sc->re_ldata.re_smap); 521 bus_dma_tag_destroy(sc->re_ldata.re_stag); 522 } 523 524 if (sc->re_caps & RE_C_CONTIGRX) 525 re_jpool_free(sc); 526 527 if (sc->re_parent_tag) 528 bus_dma_tag_destroy(sc->re_parent_tag); 529 530 if (sc->re_ldata.re_tx_mbuf != NULL) 531 kfree(sc->re_ldata.re_tx_mbuf, M_DEVBUF); 532 if (sc->re_ldata.re_rx_mbuf != NULL) 533 kfree(sc->re_ldata.re_rx_mbuf, M_DEVBUF); 534 if (sc->re_ldata.re_rx_paddr != NULL) 535 kfree(sc->re_ldata.re_rx_paddr, M_DEVBUF); 536 if (sc->re_ldata.re_tx_dmamap != NULL) 537 kfree(sc->re_ldata.re_tx_dmamap, M_DEVBUF); 538 if (sc->re_ldata.re_rx_dmamap != NULL) 539 kfree(sc->re_ldata.re_rx_dmamap, M_DEVBUF); 540 } 541 542 static boolean_t 543 re_is_faste(struct re_softc *sc) 544 { 545 if (pci_get_vendor(sc->dev) == PCI_VENDOR_REALTEK) { 546 switch (sc->re_device_id) { 547 case PCI_PRODUCT_REALTEK_RT8169: 548 case PCI_PRODUCT_REALTEK_RT8169SC: 549 case PCI_PRODUCT_REALTEK_RT8168: 550 case PCI_PRODUCT_REALTEK_RT8168_1: 551 return FALSE; 552 default: 553 return TRUE; 554 } 555 } else { 556 return FALSE; 557 } 558 } 559 560 /* 561 * Attach the interface. Allocate softc structures, do ifmedia 562 * setup and ethernet/BPF attach. 563 */ 564 static int 565 re_attach(device_t dev) 566 { 567 struct re_softc *sc = device_get_softc(dev); 568 struct ifnet *ifp; 569 struct sysctl_ctx_list *ctx; 570 struct sysctl_oid *tree; 571 uint8_t eaddr[ETHER_ADDR_LEN]; 572 int error = 0, qlen, msi_enable; 573 u_int irq_flags; 574 575 callout_init_mp(&sc->re_timer); 576 sc->dev = dev; 577 sc->re_device_id = pci_get_device(dev); 578 sc->re_unit = device_get_unit(dev); 579 ifmedia_init(&sc->media, IFM_IMASK, rtl_ifmedia_upd, rtl_ifmedia_sts); 580 581 sc->re_caps = RE_C_HWIM; 582 583 sc->re_rx_desc_cnt = re_rx_desc_count; 584 if (sc->re_rx_desc_cnt > RE_RX_DESC_CNT_MAX) 585 sc->re_rx_desc_cnt = RE_RX_DESC_CNT_MAX; 586 587 sc->re_tx_desc_cnt = re_tx_desc_count; 588 if (sc->re_tx_desc_cnt > RE_TX_DESC_CNT_MAX) 589 sc->re_tx_desc_cnt = RE_TX_DESC_CNT_MAX; 590 591 qlen = RE_IFQ_MAXLEN; 592 if (sc->re_tx_desc_cnt > qlen) 593 qlen = sc->re_tx_desc_cnt; 594 595 sc->re_rxbuf_size = MCLBYTES; 596 sc->re_newbuf = re_newbuf_std; 597 598 /* 599 * Hardware interrupt moderation settings. 600 * XXX does not seem correct, undocumented. 601 */ 602 sc->re_tx_time = 5; /* 125us */ 603 sc->re_rx_time = 2; /* 50us */ 604 605 /* Simulated interrupt moderation setting. */ 606 sc->re_sim_time = 150; /* 150us */ 607 608 /* Use simulated interrupt moderation by default. */ 609 sc->re_imtype = RE_IMTYPE_SIM; 610 re_config_imtype(sc, sc->re_imtype); 611 612 ctx = device_get_sysctl_ctx(dev); 613 tree = device_get_sysctl_tree(dev); 614 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, 615 "rx_desc_count", CTLFLAG_RD, &sc->re_rx_desc_cnt, 616 0, "RX desc count"); 617 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, 618 "tx_desc_count", CTLFLAG_RD, &sc->re_tx_desc_cnt, 619 0, "TX desc count"); 620 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "sim_time", 621 CTLTYPE_INT | CTLFLAG_RW, 622 sc, 0, re_sysctl_simtime, "I", 623 "Simulated interrupt moderation time (usec)."); 624 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "imtype", 625 CTLTYPE_INT | CTLFLAG_RW, 626 sc, 0, re_sysctl_imtype, "I", 627 "Interrupt moderation type -- " 628 "0:disable, 1:simulated, " 629 "2:hardware(if supported)"); 630 if (sc->re_caps & RE_C_HWIM) { 631 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 632 OID_AUTO, "hw_rxtime", 633 CTLTYPE_INT | CTLFLAG_RW, 634 sc, 0, re_sysctl_rxtime, "I", 635 "Hardware interrupt moderation time " 636 "(unit: 25usec)."); 637 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 638 OID_AUTO, "hw_txtime", 639 CTLTYPE_INT | CTLFLAG_RW, 640 sc, 0, re_sysctl_txtime, "I", 641 "Hardware interrupt moderation time " 642 "(unit: 25usec)."); 643 } 644 645 #ifndef BURN_BRIDGES 646 /* 647 * Handle power management nonsense. 648 */ 649 650 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) { 651 uint32_t membase, irq; 652 653 /* Save important PCI config data. */ 654 membase = pci_read_config(dev, RE_PCI_LOMEM, 4); 655 irq = pci_read_config(dev, PCIR_INTLINE, 4); 656 657 /* Reset the power state. */ 658 device_printf(dev, "chip is in D%d power mode " 659 "-- setting to D0\n", pci_get_powerstate(dev)); 660 661 pci_set_powerstate(dev, PCI_POWERSTATE_D0); 662 663 /* Restore PCI config data. */ 664 pci_write_config(dev, RE_PCI_LOMEM, membase, 4); 665 pci_write_config(dev, PCIR_INTLINE, irq, 4); 666 } 667 #endif 668 /* 669 * Map control/status registers. 670 */ 671 pci_enable_busmaster(dev); 672 673 if (pci_is_pcie(dev)) { 674 sc->re_res_rid = PCIR_BAR(2); 675 sc->re_res_type = SYS_RES_MEMORY; 676 } else { 677 sc->re_res_rid = PCIR_BAR(0); 678 sc->re_res_type = SYS_RES_IOPORT; 679 } 680 sc->re_res = bus_alloc_resource_any(dev, sc->re_res_type, 681 &sc->re_res_rid, RF_ACTIVE); 682 if (sc->re_res == NULL) { 683 device_printf(dev, "couldn't map IO\n"); 684 error = ENXIO; 685 goto fail; 686 } 687 688 sc->re_btag = rman_get_bustag(sc->re_res); 689 sc->re_bhandle = rman_get_bushandle(sc->re_res); 690 691 error = rtl_check_mac_version(sc); 692 if (error) { 693 device_printf(dev, "check mac version failed\n"); 694 goto fail; 695 } 696 697 rtl_init_software_variable(sc); 698 if (pci_is_pcie(dev)) 699 sc->re_if_flags |= RL_FLAG_PCIE; 700 else 701 sc->re_if_flags &= ~RL_FLAG_PCIE; 702 device_printf(dev, "MAC version 0x%08x, MACFG %u%s%s\n", 703 (CSR_READ_4(sc, RE_TXCFG) & 0xFCF00000), sc->re_type, 704 sc->re_coalesce_tx_pkt ? ", software TX defrag" : "", 705 sc->re_hw_enable_msi_msix ? ", support MSI" : ""); 706 707 /* 708 * Allocate interrupt 709 */ 710 if (pci_is_pcie(dev) && sc->re_hw_enable_msi_msix) 711 msi_enable = re_msi_enable; 712 else 713 msi_enable = 0; 714 sc->re_irq_type = pci_alloc_1intr(dev, msi_enable, 715 &sc->re_irq_rid, &irq_flags); 716 717 sc->re_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->re_irq_rid, 718 irq_flags); 719 if (sc->re_irq == NULL) { 720 device_printf(dev, "couldn't map interrupt\n"); 721 error = ENXIO; 722 goto fail; 723 } 724 725 /* Disable ASPM */ 726 re_disable_aspm(dev); 727 728 rtl_exit_oob(sc); 729 rtl_hw_init(sc); 730 731 /* Reset the adapter. */ 732 rtl_reset(sc); 733 734 rtl_get_hw_mac_address(sc, eaddr); 735 if (sc->re_type == MACFG_3) /* Change PCI Latency time*/ 736 pci_write_config(dev, PCIR_LATTIMER, 0x40, 1); 737 738 /* Allocate DMA stuffs */ 739 error = re_allocmem(dev); 740 if (error) 741 goto fail; 742 743 if (pci_is_pcie(dev)) { 744 sc->re_bus_speed = 125; 745 } else { 746 uint8_t cfg2; 747 748 cfg2 = CSR_READ_1(sc, RE_CFG2); 749 switch (cfg2 & RE_CFG2_PCICLK_MASK) { 750 case RE_CFG2_PCICLK_33MHZ: 751 sc->re_bus_speed = 33; 752 break; 753 case RE_CFG2_PCICLK_66MHZ: 754 sc->re_bus_speed = 66; 755 break; 756 default: 757 device_printf(dev, "unknown bus speed, assume 33MHz\n"); 758 sc->re_bus_speed = 33; 759 break; 760 } 761 } 762 device_printf(dev, "bus speed %dMHz\n", sc->re_bus_speed); 763 764 rtl_phy_power_up(sc); 765 rtl_hw_phy_config(sc); 766 rtl_clrwol(sc); 767 768 /* TODO: jumbo frame */ 769 CSR_WRITE_2(sc, RE_RxMaxSize, sc->re_rxbuf_size); 770 771 /* Enable hardware checksum if available. */ 772 sc->re_tx_cstag = 1; 773 sc->re_rx_cstag = 1; 774 775 ifp = &sc->arpcom.ac_if; 776 ifp->if_softc = sc; 777 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 778 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 779 ifp->if_ioctl = re_ioctl; 780 ifp->if_start = re_start; 781 #ifdef IFPOLL_ENABLE 782 ifp->if_npoll = re_npoll; 783 #endif 784 ifp->if_watchdog = re_watchdog; 785 ifp->if_init = re_init; 786 if (!re_is_faste(sc)) 787 ifp->if_baudrate = 1000000000; 788 else 789 ifp->if_baudrate = 100000000; 790 ifp->if_nmbclusters = sc->re_rx_desc_cnt; 791 ifq_set_maxlen(&ifp->if_snd, qlen); 792 ifq_set_ready(&ifp->if_snd); 793 794 ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 795 IFCAP_RXCSUM | IFCAP_TXCSUM; 796 ifp->if_capenable = ifp->if_capabilities; 797 /* NOTE: if_hwassist will be setup after the interface is up. */ 798 799 /* 800 * Call MI attach routine. 801 */ 802 ether_ifattach(ifp, eaddr, NULL); 803 804 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(sc->re_irq)); 805 806 #ifdef IFPOLL_ENABLE 807 ifpoll_compat_setup(&sc->re_npoll, ctx, (struct sysctl_oid *)tree, 808 device_get_unit(dev), ifp->if_serializer); 809 #endif 810 811 /* Hook interrupt last to avoid having to lock softc */ 812 error = bus_setup_intr(dev, sc->re_irq, INTR_MPSAFE | INTR_HIFREQ, 813 re_intr, sc, &sc->re_intrhand, ifp->if_serializer); 814 if (error) { 815 device_printf(dev, "couldn't set up irq\n"); 816 ether_ifdetach(ifp); 817 goto fail; 818 } 819 820 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 821 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 822 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 823 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 824 if (!re_is_faste(sc)) { 825 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 826 0, NULL); 827 } 828 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 829 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 830 rtl_ifmedia_upd(ifp); 831 832 fail: 833 if (error) 834 re_detach(dev); 835 836 return (error); 837 } 838 839 /* 840 * Shutdown hardware and free up resources. This can be called any 841 * time after the mutex has been initialized. It is called in both 842 * the error case in attach and the normal detach case so it needs 843 * to be careful about only freeing resources that have actually been 844 * allocated. 845 */ 846 static int 847 re_detach(device_t dev) 848 { 849 struct re_softc *sc = device_get_softc(dev); 850 struct ifnet *ifp = &sc->arpcom.ac_if; 851 852 /* These should only be active if attach succeeded */ 853 if (device_is_attached(dev)) { 854 lwkt_serialize_enter(ifp->if_serializer); 855 re_stop(sc, TRUE); 856 bus_teardown_intr(dev, sc->re_irq, sc->re_intrhand); 857 lwkt_serialize_exit(ifp->if_serializer); 858 859 ether_ifdetach(ifp); 860 } 861 ifmedia_removeall(&sc->media); 862 863 if (sc->re_irq) 864 bus_release_resource(dev, SYS_RES_IRQ, sc->re_irq_rid, 865 sc->re_irq); 866 867 if (sc->re_irq_type == PCI_INTR_TYPE_MSI) 868 pci_release_msi(dev); 869 870 if (sc->re_res) { 871 bus_release_resource(dev, sc->re_res_type, sc->re_res_rid, 872 sc->re_res); 873 } 874 875 /* Free DMA stuffs */ 876 re_freemem(dev); 877 878 return(0); 879 } 880 881 static void 882 re_setup_rxdesc(struct re_softc *sc, int idx) 883 { 884 bus_addr_t paddr; 885 uint32_t cmdstat; 886 struct re_desc *d; 887 888 paddr = sc->re_ldata.re_rx_paddr[idx]; 889 d = &sc->re_ldata.re_rx_list[idx]; 890 891 d->re_bufaddr_lo = htole32(RE_ADDR_LO(paddr)); 892 d->re_bufaddr_hi = htole32(RE_ADDR_HI(paddr)); 893 894 cmdstat = sc->re_rxbuf_size | RE_RDESC_CMD_OWN; 895 if (idx == (sc->re_rx_desc_cnt - 1)) 896 cmdstat |= RE_RDESC_CMD_EOR; 897 d->re_cmdstat = htole32(cmdstat); 898 } 899 900 static int 901 re_newbuf_std(struct re_softc *sc, int idx, int init) 902 { 903 bus_dma_segment_t seg; 904 bus_dmamap_t map; 905 struct mbuf *m; 906 int error, nsegs; 907 908 m = m_getcl(init ? M_WAITOK : M_NOWAIT, MT_DATA, M_PKTHDR); 909 if (m == NULL) { 910 error = ENOBUFS; 911 912 if (init) { 913 if_printf(&sc->arpcom.ac_if, "m_getcl failed\n"); 914 return error; 915 } else { 916 goto back; 917 } 918 } 919 m->m_len = m->m_pkthdr.len = MCLBYTES; 920 921 /* 922 * NOTE: 923 * re(4) chips need address of the receive buffer to be 8-byte 924 * aligned, so don't call m_adj(m, ETHER_ALIGN) here. 925 */ 926 927 error = bus_dmamap_load_mbuf_segment(sc->re_ldata.re_rx_mtag, 928 sc->re_ldata.re_rx_spare, m, 929 &seg, 1, &nsegs, BUS_DMA_NOWAIT); 930 if (error) { 931 m_freem(m); 932 if (init) { 933 if_printf(&sc->arpcom.ac_if, "can't load RX mbuf\n"); 934 return error; 935 } else { 936 goto back; 937 } 938 } 939 940 if (!init) { 941 bus_dmamap_sync(sc->re_ldata.re_rx_mtag, 942 sc->re_ldata.re_rx_dmamap[idx], 943 BUS_DMASYNC_POSTREAD); 944 bus_dmamap_unload(sc->re_ldata.re_rx_mtag, 945 sc->re_ldata.re_rx_dmamap[idx]); 946 } 947 sc->re_ldata.re_rx_mbuf[idx] = m; 948 sc->re_ldata.re_rx_paddr[idx] = seg.ds_addr; 949 950 map = sc->re_ldata.re_rx_dmamap[idx]; 951 sc->re_ldata.re_rx_dmamap[idx] = sc->re_ldata.re_rx_spare; 952 sc->re_ldata.re_rx_spare = map; 953 back: 954 re_setup_rxdesc(sc, idx); 955 return error; 956 } 957 958 #ifdef RE_JUMBO 959 static int 960 re_newbuf_jumbo(struct re_softc *sc, int idx, int init) 961 { 962 struct mbuf *m; 963 struct re_jbuf *jbuf; 964 int error = 0; 965 966 MGETHDR(m, init ? M_WAITOK : M_NOWAIT, MT_DATA); 967 if (m == NULL) { 968 error = ENOBUFS; 969 if (init) { 970 if_printf(&sc->arpcom.ac_if, "MGETHDR failed\n"); 971 return error; 972 } else { 973 goto back; 974 } 975 } 976 977 jbuf = re_jbuf_alloc(sc); 978 if (jbuf == NULL) { 979 m_freem(m); 980 981 error = ENOBUFS; 982 if (init) { 983 if_printf(&sc->arpcom.ac_if, "jpool is empty\n"); 984 return error; 985 } else { 986 goto back; 987 } 988 } 989 990 m->m_ext.ext_arg = jbuf; 991 m->m_ext.ext_buf = jbuf->re_buf; 992 m->m_ext.ext_free = re_jbuf_free; 993 m->m_ext.ext_ref = re_jbuf_ref; 994 m->m_ext.ext_size = sc->re_rxbuf_size; 995 996 m->m_data = m->m_ext.ext_buf; 997 m->m_flags |= M_EXT; 998 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 999 1000 /* 1001 * NOTE: 1002 * Some re(4) chips(e.g. RTL8101E) need address of the receive buffer 1003 * to be 8-byte aligned, so don't call m_adj(m, ETHER_ALIGN) here. 1004 */ 1005 1006 sc->re_ldata.re_rx_mbuf[idx] = m; 1007 sc->re_ldata.re_rx_paddr[idx] = jbuf->re_paddr; 1008 back: 1009 re_setup_rxdesc(sc, idx); 1010 return error; 1011 } 1012 #endif /* RE_JUMBO */ 1013 1014 static int 1015 re_tx_list_init(struct re_softc *sc) 1016 { 1017 bzero(sc->re_ldata.re_tx_list, RE_TX_LIST_SZ(sc)); 1018 1019 sc->re_ldata.re_tx_prodidx = 0; 1020 sc->re_ldata.re_tx_considx = 0; 1021 sc->re_ldata.re_tx_free = sc->re_tx_desc_cnt; 1022 1023 return(0); 1024 } 1025 1026 static int 1027 re_rx_list_init(struct re_softc *sc) 1028 { 1029 int i, error; 1030 1031 bzero(sc->re_ldata.re_rx_list, RE_RX_LIST_SZ(sc)); 1032 1033 for (i = 0; i < sc->re_rx_desc_cnt; i++) { 1034 error = sc->re_newbuf(sc, i, 1); 1035 if (error) 1036 return(error); 1037 } 1038 1039 sc->re_ldata.re_rx_prodidx = 0; 1040 sc->re_head = sc->re_tail = NULL; 1041 1042 return(0); 1043 } 1044 1045 #define RE_IP4_PACKET 0x1 1046 #define RE_TCP_PACKET 0x2 1047 #define RE_UDP_PACKET 0x4 1048 1049 static __inline uint8_t 1050 re_packet_type(struct re_softc *sc, uint32_t rxstat, uint32_t rxctrl) 1051 { 1052 uint8_t packet_type = 0; 1053 1054 if (sc->re_if_flags & RL_FLAG_DESCV2) { 1055 if (rxctrl & RE_RDESC_CTL_PROTOIP4) 1056 packet_type |= RE_IP4_PACKET; 1057 } else { 1058 if (rxstat & RE_RDESC_STAT_PROTOID) 1059 packet_type |= RE_IP4_PACKET; 1060 } 1061 if (RE_TCPPKT(rxstat)) 1062 packet_type |= RE_TCP_PACKET; 1063 else if (RE_UDPPKT(rxstat)) 1064 packet_type |= RE_UDP_PACKET; 1065 return packet_type; 1066 } 1067 1068 /* 1069 * RX handler for C+ and 8169. For the gigE chips, we support 1070 * the reception of jumbo frames that have been fragmented 1071 * across multiple 2K mbuf cluster buffers. 1072 */ 1073 static int 1074 re_rxeof(struct re_softc *sc) 1075 { 1076 struct ifnet *ifp = &sc->arpcom.ac_if; 1077 struct mbuf *m; 1078 struct re_desc *cur_rx; 1079 uint32_t rxstat, rxctrl; 1080 int i, total_len, rx = 0; 1081 1082 for (i = sc->re_ldata.re_rx_prodidx; 1083 RE_OWN(&sc->re_ldata.re_rx_list[i]) == 0; RE_RXDESC_INC(sc, i)) { 1084 cur_rx = &sc->re_ldata.re_rx_list[i]; 1085 m = sc->re_ldata.re_rx_mbuf[i]; 1086 total_len = RE_RXBYTES(cur_rx); 1087 rxstat = le32toh(cur_rx->re_cmdstat); 1088 rxctrl = le32toh(cur_rx->re_control); 1089 1090 rx = 1; 1091 1092 #ifdef INVARIANTS 1093 if (sc->re_flags & RE_F_USE_JPOOL) 1094 KKASSERT(rxstat & RE_RDESC_STAT_EOF); 1095 #endif 1096 1097 if ((rxstat & RE_RDESC_STAT_EOF) == 0) { 1098 if (sc->re_flags & RE_F_DROP_RXFRAG) { 1099 re_setup_rxdesc(sc, i); 1100 continue; 1101 } 1102 1103 if (sc->re_newbuf(sc, i, 0)) { 1104 /* Drop upcoming fragments */ 1105 sc->re_flags |= RE_F_DROP_RXFRAG; 1106 continue; 1107 } 1108 1109 m->m_len = MCLBYTES; 1110 if (sc->re_head == NULL) { 1111 sc->re_head = sc->re_tail = m; 1112 } else { 1113 sc->re_tail->m_next = m; 1114 sc->re_tail = m; 1115 } 1116 continue; 1117 } else if (sc->re_flags & RE_F_DROP_RXFRAG) { 1118 /* 1119 * Last fragment of a multi-fragment packet. 1120 * 1121 * Since error already happened, this fragment 1122 * must be dropped as well as the fragment chain. 1123 */ 1124 re_setup_rxdesc(sc, i); 1125 re_free_rxchain(sc); 1126 sc->re_flags &= ~RE_F_DROP_RXFRAG; 1127 continue; 1128 } 1129 1130 rxstat >>= 1; 1131 if (rxstat & RE_RDESC_STAT_RXERRSUM) { 1132 IFNET_STAT_INC(ifp, ierrors, 1); 1133 /* 1134 * If this is part of a multi-fragment packet, 1135 * discard all the pieces. 1136 */ 1137 re_free_rxchain(sc); 1138 re_setup_rxdesc(sc, i); 1139 continue; 1140 } 1141 1142 /* 1143 * If allocating a replacement mbuf fails, 1144 * reload the current one. 1145 */ 1146 1147 if (sc->re_newbuf(sc, i, 0)) { 1148 IFNET_STAT_INC(ifp, ierrors, 1); 1149 continue; 1150 } 1151 1152 if (sc->re_head != NULL) { 1153 m->m_len = total_len % MCLBYTES; 1154 /* 1155 * Special case: if there's 4 bytes or less 1156 * in this buffer, the mbuf can be discarded: 1157 * the last 4 bytes is the CRC, which we don't 1158 * care about anyway. 1159 */ 1160 if (m->m_len <= ETHER_CRC_LEN) { 1161 sc->re_tail->m_len -= 1162 (ETHER_CRC_LEN - m->m_len); 1163 m_freem(m); 1164 } else { 1165 m->m_len -= ETHER_CRC_LEN; 1166 sc->re_tail->m_next = m; 1167 } 1168 m = sc->re_head; 1169 sc->re_head = sc->re_tail = NULL; 1170 m->m_pkthdr.len = total_len - ETHER_CRC_LEN; 1171 } else { 1172 m->m_pkthdr.len = m->m_len = 1173 (total_len - ETHER_CRC_LEN); 1174 } 1175 1176 IFNET_STAT_INC(ifp, ipackets, 1); 1177 m->m_pkthdr.rcvif = ifp; 1178 1179 /* Do RX checksumming if enabled */ 1180 1181 if (ifp->if_capenable & IFCAP_RXCSUM) { 1182 uint8_t packet_type; 1183 1184 packet_type = re_packet_type(sc, rxstat, rxctrl); 1185 1186 /* Check IP header checksum */ 1187 if (packet_type & RE_IP4_PACKET) { 1188 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 1189 if ((rxstat & RE_RDESC_STAT_IPSUMBAD) == 0) 1190 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 1191 } 1192 1193 /* Check TCP/UDP checksum */ 1194 if (((packet_type & RE_TCP_PACKET) && 1195 (rxstat & RE_RDESC_STAT_TCPSUMBAD) == 0) || 1196 ((packet_type & RE_UDP_PACKET) && 1197 (rxstat & RE_RDESC_STAT_UDPSUMBAD) == 0)) { 1198 m->m_pkthdr.csum_flags |= 1199 CSUM_DATA_VALID|CSUM_PSEUDO_HDR| 1200 CSUM_FRAG_NOT_CHECKED; 1201 m->m_pkthdr.csum_data = 0xffff; 1202 } 1203 } 1204 1205 if (rxctrl & RE_RDESC_CTL_HASTAG) { 1206 m->m_flags |= M_VLANTAG; 1207 m->m_pkthdr.ether_vlantag = 1208 be16toh((rxctrl & RE_RDESC_CTL_TAGDATA)); 1209 } 1210 ifp->if_input(ifp, m, NULL, -1); 1211 } 1212 1213 sc->re_ldata.re_rx_prodidx = i; 1214 1215 return rx; 1216 } 1217 1218 #undef RE_IP4_PACKET 1219 #undef RE_TCP_PACKET 1220 #undef RE_UDP_PACKET 1221 1222 static int 1223 re_tx_collect(struct re_softc *sc) 1224 { 1225 struct ifnet *ifp = &sc->arpcom.ac_if; 1226 uint32_t txstat; 1227 int idx, tx = 0; 1228 1229 for (idx = sc->re_ldata.re_tx_considx; 1230 sc->re_ldata.re_tx_free < sc->re_tx_desc_cnt; 1231 RE_TXDESC_INC(sc, idx)) { 1232 txstat = le32toh(sc->re_ldata.re_tx_list[idx].re_cmdstat); 1233 if (txstat & RE_TDESC_CMD_OWN) 1234 break; 1235 1236 tx = 1; 1237 1238 sc->re_ldata.re_tx_list[idx].re_bufaddr_lo = 0; 1239 1240 /* 1241 * We only stash mbufs in the last descriptor 1242 * in a fragment chain, which also happens to 1243 * be the only place where the TX status bits 1244 * are valid. 1245 */ 1246 if (txstat & RE_TDESC_CMD_EOF) { 1247 bus_dmamap_unload(sc->re_ldata.re_tx_mtag, 1248 sc->re_ldata.re_tx_dmamap[idx]); 1249 m_freem(sc->re_ldata.re_tx_mbuf[idx]); 1250 sc->re_ldata.re_tx_mbuf[idx] = NULL; 1251 if (txstat & (RE_TDESC_STAT_EXCESSCOL| 1252 RE_TDESC_STAT_COLCNT)) 1253 IFNET_STAT_INC(ifp, collisions, 1); 1254 if (txstat & RE_TDESC_STAT_TXERRSUM) 1255 IFNET_STAT_INC(ifp, oerrors, 1); 1256 else 1257 IFNET_STAT_INC(ifp, opackets, 1); 1258 } 1259 sc->re_ldata.re_tx_free++; 1260 } 1261 sc->re_ldata.re_tx_considx = idx; 1262 1263 return tx; 1264 } 1265 1266 static int 1267 re_txeof(struct re_softc *sc) 1268 { 1269 struct ifnet *ifp = &sc->arpcom.ac_if; 1270 int tx; 1271 1272 tx = re_tx_collect(sc); 1273 1274 /* There is enough free TX descs */ 1275 if (sc->re_ldata.re_tx_free > RE_TXDESC_SPARE) 1276 ifq_clr_oactive(&ifp->if_snd); 1277 1278 /* 1279 * Some chips will ignore a second TX request issued while an 1280 * existing transmission is in progress. If the transmitter goes 1281 * idle but there are still packets waiting to be sent, we need 1282 * to restart the channel here to flush them out. This only seems 1283 * to be required with the PCIe devices. 1284 */ 1285 if (sc->re_ldata.re_tx_free < sc->re_tx_desc_cnt) 1286 CSR_WRITE_1(sc, RE_TPPOLL, RE_NPQ); 1287 else 1288 ifp->if_timer = 0; 1289 1290 return tx; 1291 } 1292 1293 static void 1294 re_tick(void *xsc) 1295 { 1296 struct re_softc *sc = xsc; 1297 1298 lwkt_serialize_enter(sc->arpcom.ac_if.if_serializer); 1299 re_tick_serialized(xsc); 1300 lwkt_serialize_exit(sc->arpcom.ac_if.if_serializer); 1301 } 1302 1303 static void 1304 re_tick_serialized(void *xsc) 1305 { 1306 struct re_softc *sc = xsc; 1307 struct ifnet *ifp = &sc->arpcom.ac_if; 1308 1309 ASSERT_SERIALIZED(ifp->if_serializer); 1310 1311 if ((ifp->if_flags & IFF_RUNNING) == 0) 1312 return; 1313 1314 if (rtl_link_ok(sc)) { 1315 if ((sc->re_flags & RE_F_LINKED) == 0) 1316 re_link_up(sc); 1317 } else if (sc->re_flags & RE_F_LINKED) { 1318 re_link_down(sc); 1319 } 1320 callout_reset(&sc->re_timer, hz, re_tick, sc); 1321 } 1322 1323 #ifdef IFPOLL_ENABLE 1324 1325 static void 1326 re_npoll_compat(struct ifnet *ifp, void *arg __unused, int count) 1327 { 1328 struct re_softc *sc = ifp->if_softc; 1329 1330 ASSERT_SERIALIZED(ifp->if_serializer); 1331 1332 if (sc->re_npoll.ifpc_stcount-- == 0) { 1333 uint16_t status; 1334 1335 sc->re_npoll.ifpc_stcount = sc->re_npoll.ifpc_stfrac; 1336 1337 status = CSR_READ_2(sc, RE_ISR); 1338 if (status) 1339 CSR_WRITE_2(sc, RE_ISR, status); 1340 1341 /* 1342 * XXX check behaviour on receiver stalls. 1343 */ 1344 1345 if (status & RE_ISR_SYSTEM_ERR) { 1346 rtl_reset(sc); 1347 re_init(sc); 1348 /* Done! */ 1349 return; 1350 } 1351 } 1352 1353 sc->rxcycles = count; 1354 re_rxeof(sc); 1355 re_txeof(sc); 1356 1357 if (!ifq_is_empty(&ifp->if_snd)) 1358 if_devstart(ifp); 1359 } 1360 1361 static void 1362 re_npoll(struct ifnet *ifp, struct ifpoll_info *info) 1363 { 1364 struct re_softc *sc = ifp->if_softc; 1365 1366 ASSERT_SERIALIZED(ifp->if_serializer); 1367 1368 if (info != NULL) { 1369 int cpuid = sc->re_npoll.ifpc_cpuid; 1370 1371 info->ifpi_rx[cpuid].poll_func = re_npoll_compat; 1372 info->ifpi_rx[cpuid].arg = NULL; 1373 info->ifpi_rx[cpuid].serializer = ifp->if_serializer; 1374 1375 if (ifp->if_flags & IFF_RUNNING) 1376 re_setup_intr(sc, 0, RE_IMTYPE_NONE); 1377 ifq_set_cpuid(&ifp->if_snd, cpuid); 1378 } else { 1379 if (ifp->if_flags & IFF_RUNNING) 1380 re_setup_intr(sc, 1, sc->re_imtype); 1381 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(sc->re_irq)); 1382 } 1383 } 1384 #endif /* IFPOLL_ENABLE */ 1385 1386 static void 1387 re_intr(void *arg) 1388 { 1389 struct re_softc *sc = arg; 1390 struct ifnet *ifp = &sc->arpcom.ac_if; 1391 uint16_t status; 1392 int proc; 1393 1394 ASSERT_SERIALIZED(ifp->if_serializer); 1395 1396 if ((sc->re_flags & RE_F_SUSPENDED) || 1397 (ifp->if_flags & IFF_RUNNING) == 0) 1398 return; 1399 1400 /* Disable interrupts. */ 1401 CSR_WRITE_2(sc, RE_IMR, 0); 1402 1403 status = CSR_READ_2(sc, RE_ISR); 1404 again: 1405 proc = 0; 1406 if (status) 1407 CSR_WRITE_2(sc, RE_ISR, status); 1408 if (status & sc->re_intrs) { 1409 if (status & RE_ISR_SYSTEM_ERR) { 1410 rtl_reset(sc); 1411 re_init(sc); 1412 /* Done! */ 1413 return; 1414 } 1415 proc |= re_rxeof(sc); 1416 proc |= re_txeof(sc); 1417 } 1418 1419 if (sc->re_imtype == RE_IMTYPE_SIM) { 1420 if ((sc->re_flags & RE_F_TIMER_INTR)) { 1421 if (!proc) { 1422 /* 1423 * Nothing needs to be processed, fallback 1424 * to use TX/RX interrupts. 1425 * 1426 * NOTE: This will re-enable interrupts. 1427 */ 1428 re_setup_intr(sc, 1, RE_IMTYPE_NONE); 1429 1430 /* 1431 * Recollect, mainly to avoid the possible 1432 * race introduced by changing interrupt 1433 * masks. 1434 */ 1435 re_rxeof(sc); 1436 re_txeof(sc); 1437 } else { 1438 /* Re-enable interrupts. */ 1439 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 1440 CSR_WRITE_4(sc, RE_TIMERCNT, 1); /* reload */ 1441 } 1442 } else if (proc) { 1443 /* 1444 * Assume that using simulated interrupt moderation 1445 * (hardware timer based) could reduce the interript 1446 * rate. 1447 * 1448 * NOTE: This will re-enable interrupts. 1449 */ 1450 re_setup_intr(sc, 1, RE_IMTYPE_SIM); 1451 } else { 1452 /* Re-enable interrupts. */ 1453 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 1454 } 1455 } else { 1456 status = CSR_READ_2(sc, RE_ISR); 1457 if (status & sc->re_intrs) { 1458 if (!ifq_is_empty(&ifp->if_snd)) 1459 if_devstart(ifp); 1460 /* NOTE: Interrupts are still disabled. */ 1461 goto again; 1462 } 1463 /* Re-enable interrupts. */ 1464 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 1465 } 1466 1467 if (!ifq_is_empty(&ifp->if_snd)) 1468 if_devstart(ifp); 1469 } 1470 1471 static int 1472 re_encap(struct re_softc *sc, struct mbuf **m_head, int *idx0) 1473 { 1474 struct mbuf *m = *m_head; 1475 bus_dma_segment_t segs[RE_MAXSEGS]; 1476 bus_dmamap_t map; 1477 int error, maxsegs, idx, i, nsegs; 1478 struct re_desc *d, *tx_ring; 1479 uint32_t cmd_csum, ctl_csum, vlantag; 1480 1481 KASSERT(sc->re_ldata.re_tx_free > RE_TXDESC_SPARE, 1482 ("not enough free TX desc")); 1483 1484 if (sc->re_coalesce_tx_pkt && m->m_pkthdr.len != m->m_len) { 1485 struct mbuf *m_new; 1486 1487 m_new = m_defrag(m, M_NOWAIT); 1488 if (m_new == NULL) { 1489 error = ENOBUFS; 1490 goto back; 1491 } else { 1492 *m_head = m = m_new; 1493 if (m->m_pkthdr.len != m->m_len) { 1494 /* Still not configuous; give up. */ 1495 error = ENOBUFS; 1496 goto back; 1497 } 1498 } 1499 } 1500 1501 map = sc->re_ldata.re_tx_dmamap[*idx0]; 1502 1503 /* 1504 * Set up checksum offload. Note: checksum offload bits must 1505 * appear in all descriptors of a multi-descriptor transmit 1506 * attempt. (This is according to testing done with an 8169 1507 * chip. I'm not sure if this is a requirement or a bug.) 1508 */ 1509 cmd_csum = ctl_csum = 0; 1510 if (m->m_pkthdr.csum_flags & CSUM_IP) { 1511 cmd_csum |= RE_TDESC_CMD_IPCSUM; 1512 ctl_csum |= RE_TDESC_CTL_IPCSUM; 1513 } 1514 if (m->m_pkthdr.csum_flags & CSUM_TCP) { 1515 cmd_csum |= RE_TDESC_CMD_TCPCSUM; 1516 ctl_csum |= RE_TDESC_CTL_TCPCSUM; 1517 } 1518 if (m->m_pkthdr.csum_flags & CSUM_UDP) { 1519 cmd_csum |= RE_TDESC_CMD_UDPCSUM; 1520 ctl_csum |= RE_TDESC_CTL_UDPCSUM; 1521 } 1522 1523 /* For version2 descriptor, csum flags are set on re_control */ 1524 if (sc->re_if_flags & RL_FLAG_DESCV2) 1525 cmd_csum = 0; 1526 else 1527 ctl_csum = 0; 1528 1529 /* 1530 * With some of the RealTek chips, using the checksum offload 1531 * support in conjunction with the autopadding feature results 1532 * in the transmission of corrupt frames. For example, if we 1533 * need to send a really small IP fragment that's less than 60 1534 * bytes in size, and IP header checksumming is enabled, the 1535 * resulting ethernet frame that appears on the wire will 1536 * have garbled payload. To work around this, if TX checksum 1537 * offload is enabled, we always manually pad short frames out 1538 * to the minimum ethernet frame size. 1539 * 1540 * Note: this appears unnecessary for TCP, and doing it for TCP 1541 * with PCIe adapters seems to result in bad checksums. 1542 */ 1543 if ((m->m_pkthdr.csum_flags & (CSUM_DELAY_IP | CSUM_DELAY_DATA)) && 1544 (m->m_pkthdr.csum_flags & CSUM_TCP) == 0 && 1545 m->m_pkthdr.len < RE_MIN_FRAMELEN) { 1546 error = m_devpad(m, RE_MIN_FRAMELEN); 1547 if (error) 1548 goto back; 1549 } 1550 1551 vlantag = 0; 1552 if (m->m_flags & M_VLANTAG) { 1553 vlantag = htobe16(m->m_pkthdr.ether_vlantag) | 1554 RE_TDESC_CTL_INSTAG; 1555 } 1556 1557 maxsegs = sc->re_ldata.re_tx_free; 1558 if (maxsegs > RE_MAXSEGS) 1559 maxsegs = RE_MAXSEGS; 1560 1561 error = bus_dmamap_load_mbuf_defrag(sc->re_ldata.re_tx_mtag, map, 1562 m_head, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 1563 if (error) 1564 goto back; 1565 1566 m = *m_head; 1567 bus_dmamap_sync(sc->re_ldata.re_tx_mtag, map, BUS_DMASYNC_PREWRITE); 1568 1569 /* 1570 * Map the segment array into descriptors. We also keep track 1571 * of the end of the ring and set the end-of-ring bits as needed, 1572 * and we set the ownership bits in all except the very first 1573 * descriptor, whose ownership bits will be turned on later. 1574 */ 1575 tx_ring = sc->re_ldata.re_tx_list; 1576 idx = *idx0; 1577 i = 0; 1578 for (;;) { 1579 uint32_t cmdstat; 1580 1581 d = &tx_ring[idx]; 1582 1583 cmdstat = segs[i].ds_len; 1584 d->re_bufaddr_lo = htole32(RE_ADDR_LO(segs[i].ds_addr)); 1585 d->re_bufaddr_hi = htole32(RE_ADDR_HI(segs[i].ds_addr)); 1586 if (i == 0) 1587 cmdstat |= RE_TDESC_CMD_SOF; 1588 else 1589 cmdstat |= RE_TDESC_CMD_OWN; 1590 if (idx == (sc->re_tx_desc_cnt - 1)) 1591 cmdstat |= RE_TDESC_CMD_EOR; 1592 d->re_cmdstat = htole32(cmdstat | cmd_csum); 1593 d->re_control = htole32(ctl_csum | vlantag); 1594 1595 i++; 1596 if (i == nsegs) 1597 break; 1598 RE_TXDESC_INC(sc, idx); 1599 } 1600 d->re_cmdstat |= htole32(RE_TDESC_CMD_EOF); 1601 1602 /* Transfer ownership of packet to the chip. */ 1603 d->re_cmdstat |= htole32(RE_TDESC_CMD_OWN); 1604 if (*idx0 != idx) 1605 tx_ring[*idx0].re_cmdstat |= htole32(RE_TDESC_CMD_OWN); 1606 1607 /* 1608 * Insure that the map for this transmission 1609 * is placed at the array index of the last descriptor 1610 * in this chain. 1611 */ 1612 sc->re_ldata.re_tx_dmamap[*idx0] = sc->re_ldata.re_tx_dmamap[idx]; 1613 sc->re_ldata.re_tx_dmamap[idx] = map; 1614 1615 sc->re_ldata.re_tx_mbuf[idx] = m; 1616 sc->re_ldata.re_tx_free -= nsegs; 1617 1618 RE_TXDESC_INC(sc, idx); 1619 *idx0 = idx; 1620 back: 1621 if (error) { 1622 m_freem(*m_head); 1623 *m_head = NULL; 1624 } 1625 return error; 1626 } 1627 1628 /* 1629 * Main transmit routine for C+ and gigE NICs. 1630 */ 1631 1632 static void 1633 re_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 1634 { 1635 struct re_softc *sc = ifp->if_softc; 1636 struct mbuf *m_head; 1637 int idx, need_trans, oactive, error; 1638 1639 ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); 1640 ASSERT_SERIALIZED(ifp->if_serializer); 1641 1642 if ((sc->re_flags & RE_F_LINKED) == 0) { 1643 ifq_purge(&ifp->if_snd); 1644 return; 1645 } 1646 1647 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) 1648 return; 1649 1650 idx = sc->re_ldata.re_tx_prodidx; 1651 1652 need_trans = 0; 1653 oactive = 0; 1654 while (sc->re_ldata.re_tx_mbuf[idx] == NULL) { 1655 if (sc->re_ldata.re_tx_free <= RE_TXDESC_SPARE) { 1656 if (!oactive) { 1657 if (re_tx_collect(sc)) { 1658 oactive = 1; 1659 continue; 1660 } 1661 } 1662 ifq_set_oactive(&ifp->if_snd); 1663 break; 1664 } 1665 1666 m_head = ifq_dequeue(&ifp->if_snd); 1667 if (m_head == NULL) 1668 break; 1669 1670 error = re_encap(sc, &m_head, &idx); 1671 if (error) { 1672 /* m_head is freed by re_encap(), if we reach here */ 1673 IFNET_STAT_INC(ifp, oerrors, 1); 1674 1675 if (error == EFBIG && !oactive) { 1676 if (re_tx_collect(sc)) { 1677 oactive = 1; 1678 continue; 1679 } 1680 } 1681 ifq_set_oactive(&ifp->if_snd); 1682 break; 1683 } 1684 1685 oactive = 0; 1686 need_trans = 1; 1687 1688 /* 1689 * If there's a BPF listener, bounce a copy of this frame 1690 * to him. 1691 */ 1692 ETHER_BPF_MTAP(ifp, m_head); 1693 } 1694 1695 /* 1696 * If sc->re_ldata.re_tx_mbuf[idx] is not NULL it is possible 1697 * for OACTIVE to not be properly set when we also do not 1698 * have sufficient free tx descriptors, leaving packet in 1699 * ifp->if_snd. This can cause if_start_dispatch() to loop 1700 * infinitely so make sure OACTIVE is set properly. 1701 */ 1702 if (sc->re_ldata.re_tx_free <= RE_TXDESC_SPARE) { 1703 if (!ifq_is_oactive(&ifp->if_snd)) { 1704 #if 0 1705 if_printf(ifp, "Debug: OACTIVE was not set when " 1706 "re_tx_free was below minimum!\n"); 1707 #endif 1708 ifq_set_oactive(&ifp->if_snd); 1709 } 1710 } 1711 if (!need_trans) 1712 return; 1713 1714 sc->re_ldata.re_tx_prodidx = idx; 1715 1716 /* 1717 * RealTek put the TX poll request register in a different 1718 * location on the 8169 gigE chip. I don't know why. 1719 */ 1720 CSR_WRITE_1(sc, RE_TPPOLL, RE_NPQ); 1721 1722 /* 1723 * Set a timeout in case the chip goes out to lunch. 1724 */ 1725 ifp->if_timer = 5; 1726 } 1727 1728 static void 1729 re_link_up(struct re_softc *sc) 1730 { 1731 struct ifnet *ifp = &sc->arpcom.ac_if; 1732 int error; 1733 1734 ASSERT_SERIALIZED(ifp->if_serializer); 1735 1736 rtl_link_on_patch(sc); 1737 re_stop(sc, FALSE); 1738 rtl_set_eaddr(sc); 1739 1740 error = re_rx_list_init(sc); 1741 if (error) { 1742 re_stop(sc, TRUE); 1743 return; 1744 } 1745 error = re_tx_list_init(sc); 1746 if (error) { 1747 re_stop(sc, TRUE); 1748 return; 1749 } 1750 1751 /* 1752 * Load the addresses of the RX and TX lists into the chip. 1753 */ 1754 CSR_WRITE_4(sc, RE_RXLIST_ADDR_HI, 1755 RE_ADDR_HI(sc->re_ldata.re_rx_list_addr)); 1756 CSR_WRITE_4(sc, RE_RXLIST_ADDR_LO, 1757 RE_ADDR_LO(sc->re_ldata.re_rx_list_addr)); 1758 1759 CSR_WRITE_4(sc, RE_TXLIST_ADDR_HI, 1760 RE_ADDR_HI(sc->re_ldata.re_tx_list_addr)); 1761 CSR_WRITE_4(sc, RE_TXLIST_ADDR_LO, 1762 RE_ADDR_LO(sc->re_ldata.re_tx_list_addr)); 1763 1764 rtl_hw_start(sc); 1765 1766 #ifdef IFPOLL_ENABLE 1767 /* 1768 * Disable interrupts if we are polling. 1769 */ 1770 if (ifp->if_flags & IFF_NPOLLING) 1771 re_setup_intr(sc, 0, RE_IMTYPE_NONE); 1772 else /* otherwise ... */ 1773 #endif /* IFPOLL_ENABLE */ 1774 /* 1775 * Enable interrupts. 1776 */ 1777 re_setup_intr(sc, 1, sc->re_imtype); 1778 CSR_WRITE_2(sc, RE_ISR, sc->re_intrs); 1779 1780 sc->re_flags |= RE_F_LINKED; 1781 ifp->if_link_state = LINK_STATE_UP; 1782 if_link_state_change(ifp); 1783 1784 if (bootverbose) 1785 if_printf(ifp, "link UP\n"); 1786 1787 if (!ifq_is_empty(&ifp->if_snd)) 1788 if_devstart(ifp); 1789 } 1790 1791 static void 1792 re_link_down(struct re_softc *sc) 1793 { 1794 struct ifnet *ifp = &sc->arpcom.ac_if; 1795 1796 /* NOTE: re_stop() will reset RE_F_LINKED. */ 1797 ifp->if_link_state = LINK_STATE_DOWN; 1798 if_link_state_change(ifp); 1799 1800 re_stop(sc, FALSE); 1801 rtl_ifmedia_upd(ifp); 1802 1803 if (bootverbose) 1804 if_printf(ifp, "link DOWN\n"); 1805 } 1806 1807 static void 1808 re_init(void *xsc) 1809 { 1810 struct re_softc *sc = xsc; 1811 struct ifnet *ifp = &sc->arpcom.ac_if; 1812 1813 ASSERT_SERIALIZED(ifp->if_serializer); 1814 1815 re_stop(sc, TRUE); 1816 if (rtl_link_ok(sc)) { 1817 if (bootverbose) 1818 if_printf(ifp, "link is UP in if_init\n"); 1819 re_link_up(sc); 1820 } 1821 1822 ifp->if_flags |= IFF_RUNNING; 1823 ifq_clr_oactive(&ifp->if_snd); 1824 1825 callout_reset(&sc->re_timer, hz, re_tick, sc); 1826 } 1827 1828 static int 1829 re_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) 1830 { 1831 struct re_softc *sc = ifp->if_softc; 1832 struct ifreq *ifr = (struct ifreq *)data; 1833 int error = 0, mask; 1834 1835 ASSERT_SERIALIZED(ifp->if_serializer); 1836 1837 switch(command) { 1838 case SIOCSIFMTU: 1839 #ifdef RE_JUMBO 1840 if (ifr->ifr_mtu > sc->re_maxmtu) { 1841 error = EINVAL; 1842 } else if (ifp->if_mtu != ifr->ifr_mtu) { 1843 ifp->if_mtu = ifr->ifr_mtu; 1844 if (ifp->if_flags & IFF_RUNNING) 1845 ifp->if_init(sc); 1846 } 1847 #else 1848 error = EOPNOTSUPP; 1849 #endif 1850 break; 1851 1852 case SIOCSIFFLAGS: 1853 if (ifp->if_flags & IFF_UP) { 1854 if (ifp->if_flags & IFF_RUNNING) { 1855 if ((ifp->if_flags ^ sc->re_saved_ifflags) & 1856 (IFF_PROMISC | IFF_ALLMULTI)) 1857 rtl_set_rx_packet_filter(sc); 1858 } else { 1859 re_init(sc); 1860 } 1861 } else if (ifp->if_flags & IFF_RUNNING) { 1862 re_stop(sc, TRUE); 1863 } 1864 sc->re_saved_ifflags = ifp->if_flags; 1865 break; 1866 1867 case SIOCADDMULTI: 1868 case SIOCDELMULTI: 1869 rtl_set_rx_packet_filter(sc); 1870 break; 1871 1872 case SIOCGIFMEDIA: 1873 case SIOCSIFMEDIA: 1874 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 1875 break; 1876 1877 case SIOCSIFCAP: 1878 mask = (ifr->ifr_reqcap ^ ifp->if_capenable) & 1879 ifp->if_capabilities; 1880 ifp->if_capenable ^= mask; 1881 1882 /* NOTE: re_init will setup if_hwassist. */ 1883 ifp->if_hwassist = 0; 1884 1885 /* Setup flags for the backend. */ 1886 if (ifp->if_capenable & IFCAP_RXCSUM) 1887 sc->re_rx_cstag = 1; 1888 else 1889 sc->re_rx_cstag = 0; 1890 if (ifp->if_capenable & IFCAP_TXCSUM) 1891 sc->re_tx_cstag = 1; 1892 else 1893 sc->re_rx_cstag = 0; 1894 1895 if (mask && (ifp->if_flags & IFF_RUNNING)) 1896 re_init(sc); 1897 break; 1898 1899 default: 1900 error = ether_ioctl(ifp, command, data); 1901 break; 1902 } 1903 return(error); 1904 } 1905 1906 static void 1907 re_watchdog(struct ifnet *ifp) 1908 { 1909 struct re_softc *sc = ifp->if_softc; 1910 1911 ASSERT_SERIALIZED(ifp->if_serializer); 1912 1913 IFNET_STAT_INC(ifp, oerrors, 1); 1914 1915 re_txeof(sc); 1916 re_rxeof(sc); 1917 1918 if (sc->re_ldata.re_tx_free != sc->re_tx_desc_cnt) { 1919 if_printf(ifp, "watchdog timeout, txd free %d\n", 1920 sc->re_ldata.re_tx_free); 1921 rtl_reset(sc); 1922 re_init(sc); 1923 } 1924 } 1925 1926 /* 1927 * Stop the adapter and free any mbufs allocated to the 1928 * RX and TX lists. 1929 */ 1930 static void 1931 re_stop(struct re_softc *sc, boolean_t full_stop) 1932 { 1933 struct ifnet *ifp = &sc->arpcom.ac_if; 1934 int i; 1935 1936 ASSERT_SERIALIZED(ifp->if_serializer); 1937 1938 /* Stop the adapter. */ 1939 rtl_stop(sc); 1940 1941 ifp->if_timer = 0; 1942 if (full_stop) { 1943 callout_stop(&sc->re_timer); 1944 ifp->if_flags &= ~IFF_RUNNING; 1945 } 1946 ifq_clr_oactive(&ifp->if_snd); 1947 sc->re_flags &= ~(RE_F_TIMER_INTR | RE_F_DROP_RXFRAG | RE_F_LINKED); 1948 1949 re_free_rxchain(sc); 1950 1951 /* Free the TX list buffers. */ 1952 for (i = 0; i < sc->re_tx_desc_cnt; i++) { 1953 if (sc->re_ldata.re_tx_mbuf[i] != NULL) { 1954 bus_dmamap_unload(sc->re_ldata.re_tx_mtag, 1955 sc->re_ldata.re_tx_dmamap[i]); 1956 m_freem(sc->re_ldata.re_tx_mbuf[i]); 1957 sc->re_ldata.re_tx_mbuf[i] = NULL; 1958 } 1959 } 1960 1961 /* Free the RX list buffers. */ 1962 for (i = 0; i < sc->re_rx_desc_cnt; i++) { 1963 if (sc->re_ldata.re_rx_mbuf[i] != NULL) { 1964 if ((sc->re_flags & RE_F_USE_JPOOL) == 0) { 1965 bus_dmamap_unload(sc->re_ldata.re_rx_mtag, 1966 sc->re_ldata.re_rx_dmamap[i]); 1967 } 1968 m_freem(sc->re_ldata.re_rx_mbuf[i]); 1969 sc->re_ldata.re_rx_mbuf[i] = NULL; 1970 } 1971 } 1972 } 1973 1974 /* 1975 * Device suspend routine. Stop the interface and save some PCI 1976 * settings in case the BIOS doesn't restore them properly on 1977 * resume. 1978 */ 1979 static int 1980 re_suspend(device_t dev) 1981 { 1982 #ifndef BURN_BRIDGES 1983 int i; 1984 #endif 1985 struct re_softc *sc = device_get_softc(dev); 1986 struct ifnet *ifp = &sc->arpcom.ac_if; 1987 1988 lwkt_serialize_enter(ifp->if_serializer); 1989 1990 re_stop(sc, TRUE); 1991 1992 #ifndef BURN_BRIDGES 1993 for (i = 0; i < 5; i++) 1994 sc->saved_maps[i] = pci_read_config(dev, PCIR_MAPS + i * 4, 4); 1995 sc->saved_biosaddr = pci_read_config(dev, PCIR_BIOS, 4); 1996 sc->saved_intline = pci_read_config(dev, PCIR_INTLINE, 1); 1997 sc->saved_cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1); 1998 sc->saved_lattimer = pci_read_config(dev, PCIR_LATTIMER, 1); 1999 #endif 2000 2001 sc->re_flags |= RE_F_SUSPENDED; 2002 2003 lwkt_serialize_exit(ifp->if_serializer); 2004 2005 return (0); 2006 } 2007 2008 /* 2009 * Device resume routine. Restore some PCI settings in case the BIOS 2010 * doesn't, re-enable busmastering, and restart the interface if 2011 * appropriate. 2012 */ 2013 static int 2014 re_resume(device_t dev) 2015 { 2016 struct re_softc *sc = device_get_softc(dev); 2017 struct ifnet *ifp = &sc->arpcom.ac_if; 2018 #ifndef BURN_BRIDGES 2019 int i; 2020 #endif 2021 2022 lwkt_serialize_enter(ifp->if_serializer); 2023 2024 #ifndef BURN_BRIDGES 2025 /* better way to do this? */ 2026 for (i = 0; i < 5; i++) 2027 pci_write_config(dev, PCIR_MAPS + i * 4, sc->saved_maps[i], 4); 2028 pci_write_config(dev, PCIR_BIOS, sc->saved_biosaddr, 4); 2029 pci_write_config(dev, PCIR_INTLINE, sc->saved_intline, 1); 2030 pci_write_config(dev, PCIR_CACHELNSZ, sc->saved_cachelnsz, 1); 2031 pci_write_config(dev, PCIR_LATTIMER, sc->saved_lattimer, 1); 2032 2033 /* reenable busmastering */ 2034 pci_enable_busmaster(dev); 2035 pci_enable_io(dev, SYS_RES_IOPORT); 2036 #endif 2037 2038 /* reinitialize interface if necessary */ 2039 if (ifp->if_flags & IFF_UP) 2040 re_init(sc); 2041 2042 sc->re_flags &= ~RE_F_SUSPENDED; 2043 2044 lwkt_serialize_exit(ifp->if_serializer); 2045 2046 return (0); 2047 } 2048 2049 /* 2050 * Stop all chip I/O so that the kernel's probe routines don't 2051 * get confused by errant DMAs when rebooting. 2052 */ 2053 static void 2054 re_shutdown(device_t dev) 2055 { 2056 struct re_softc *sc = device_get_softc(dev); 2057 struct ifnet *ifp = &sc->arpcom.ac_if; 2058 2059 lwkt_serialize_enter(ifp->if_serializer); 2060 re_stop(sc, TRUE); 2061 rtl_hw_d3_para(sc); 2062 rtl_phy_power_down(sc); 2063 lwkt_serialize_exit(ifp->if_serializer); 2064 } 2065 2066 static int 2067 re_sysctl_rxtime(SYSCTL_HANDLER_ARGS) 2068 { 2069 struct re_softc *sc = arg1; 2070 2071 return re_sysctl_hwtime(oidp, arg1, arg2, req, &sc->re_rx_time); 2072 } 2073 2074 static int 2075 re_sysctl_txtime(SYSCTL_HANDLER_ARGS) 2076 { 2077 struct re_softc *sc = arg1; 2078 2079 return re_sysctl_hwtime(oidp, arg1, arg2, req, &sc->re_tx_time); 2080 } 2081 2082 static int 2083 re_sysctl_hwtime(SYSCTL_HANDLER_ARGS, int *hwtime) 2084 { 2085 struct re_softc *sc = arg1; 2086 struct ifnet *ifp = &sc->arpcom.ac_if; 2087 int error, v; 2088 2089 lwkt_serialize_enter(ifp->if_serializer); 2090 2091 v = *hwtime; 2092 error = sysctl_handle_int(oidp, &v, 0, req); 2093 if (error || req->newptr == NULL) 2094 goto back; 2095 2096 if (v <= 0) { 2097 error = EINVAL; 2098 goto back; 2099 } 2100 2101 if (v != *hwtime) { 2102 *hwtime = v; 2103 2104 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) == 2105 IFF_RUNNING && sc->re_imtype == RE_IMTYPE_HW) 2106 re_setup_hw_im(sc); 2107 } 2108 back: 2109 lwkt_serialize_exit(ifp->if_serializer); 2110 return error; 2111 } 2112 2113 static int 2114 re_sysctl_simtime(SYSCTL_HANDLER_ARGS) 2115 { 2116 struct re_softc *sc = arg1; 2117 struct ifnet *ifp = &sc->arpcom.ac_if; 2118 int error, v; 2119 2120 lwkt_serialize_enter(ifp->if_serializer); 2121 2122 v = sc->re_sim_time; 2123 error = sysctl_handle_int(oidp, &v, 0, req); 2124 if (error || req->newptr == NULL) 2125 goto back; 2126 2127 if (v <= 0) { 2128 error = EINVAL; 2129 goto back; 2130 } 2131 2132 if (v != sc->re_sim_time) { 2133 sc->re_sim_time = v; 2134 2135 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) == 2136 IFF_RUNNING && sc->re_imtype == RE_IMTYPE_SIM) { 2137 #ifdef foo 2138 /* 2139 * Following code causes various strange 2140 * performance problems. Hmm ... 2141 */ 2142 CSR_WRITE_2(sc, RE_IMR, 0); 2143 CSR_WRITE_4(sc, RE_TIMERINT, 0); 2144 CSR_READ_4(sc, RE_TIMERINT); /* flush */ 2145 2146 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 2147 re_setup_sim_im(sc); 2148 #else 2149 re_setup_intr(sc, 0, RE_IMTYPE_NONE); 2150 DELAY(10); 2151 re_setup_intr(sc, 1, RE_IMTYPE_SIM); 2152 #endif 2153 } 2154 } 2155 back: 2156 lwkt_serialize_exit(ifp->if_serializer); 2157 return error; 2158 } 2159 2160 static int 2161 re_sysctl_imtype(SYSCTL_HANDLER_ARGS) 2162 { 2163 struct re_softc *sc = arg1; 2164 struct ifnet *ifp = &sc->arpcom.ac_if; 2165 int error, v; 2166 2167 lwkt_serialize_enter(ifp->if_serializer); 2168 2169 v = sc->re_imtype; 2170 error = sysctl_handle_int(oidp, &v, 0, req); 2171 if (error || req->newptr == NULL) 2172 goto back; 2173 2174 if (v != RE_IMTYPE_HW && v != RE_IMTYPE_SIM && v != RE_IMTYPE_NONE) { 2175 error = EINVAL; 2176 goto back; 2177 } 2178 if (v == RE_IMTYPE_HW && (sc->re_caps & RE_C_HWIM) == 0) { 2179 /* Can't do hardware interrupt moderation */ 2180 error = EOPNOTSUPP; 2181 goto back; 2182 } 2183 2184 if (v != sc->re_imtype) { 2185 sc->re_imtype = v; 2186 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) == 2187 IFF_RUNNING) 2188 re_setup_intr(sc, 1, sc->re_imtype); 2189 } 2190 back: 2191 lwkt_serialize_exit(ifp->if_serializer); 2192 return error; 2193 } 2194 2195 static void 2196 re_setup_hw_im(struct re_softc *sc) 2197 { 2198 KKASSERT(sc->re_caps & RE_C_HWIM); 2199 2200 /* 2201 * Interrupt moderation 2202 * 2203 * 0xABCD 2204 * A - unknown (maybe TX related) 2205 * B - TX timer (unit: 25us) 2206 * C - unknown (maybe RX related) 2207 * D - RX timer (unit: 25us) 2208 * 2209 * 2210 * re(4)'s interrupt moderation is actually controlled by 2211 * two variables, like most other NICs (bge, bce etc.) 2212 * o timer 2213 * o number of packets [P] 2214 * 2215 * The logic relationship between these two variables is 2216 * similar to other NICs too: 2217 * if (timer expire || packets > [P]) 2218 * Interrupt is delivered 2219 * 2220 * Currently we only know how to set 'timer', but not 2221 * 'number of packets', which should be ~30, as far as I 2222 * tested (sink ~900Kpps, interrupt rate is 30KHz) 2223 */ 2224 CSR_WRITE_2(sc, RE_IM, 2225 RE_IM_RXTIME(sc->re_rx_time) | 2226 RE_IM_TXTIME(sc->re_tx_time) | 2227 RE_IM_MAGIC); 2228 } 2229 2230 static void 2231 re_disable_hw_im(struct re_softc *sc) 2232 { 2233 if (sc->re_caps & RE_C_HWIM) 2234 CSR_WRITE_2(sc, RE_IM, 0); 2235 } 2236 2237 static void 2238 re_setup_sim_im(struct re_softc *sc) 2239 { 2240 uint32_t ticks; 2241 2242 if (sc->re_if_flags & RL_FLAG_PCIE) { 2243 ticks = sc->re_sim_time * sc->re_bus_speed; 2244 } else { 2245 /* 2246 * Datasheet says tick decreases at bus speed, 2247 * but it seems the clock runs a little bit 2248 * faster, so we do some compensation here. 2249 */ 2250 ticks = (sc->re_sim_time * sc->re_bus_speed * 8) / 5; 2251 } 2252 CSR_WRITE_4(sc, RE_TIMERINT, ticks); 2253 2254 CSR_WRITE_4(sc, RE_TIMERCNT, 1); /* reload */ 2255 sc->re_flags |= RE_F_TIMER_INTR; 2256 } 2257 2258 static void 2259 re_disable_sim_im(struct re_softc *sc) 2260 { 2261 CSR_WRITE_4(sc, RE_TIMERINT, 0); 2262 sc->re_flags &= ~RE_F_TIMER_INTR; 2263 } 2264 2265 static void 2266 re_config_imtype(struct re_softc *sc, int imtype) 2267 { 2268 switch (imtype) { 2269 case RE_IMTYPE_HW: 2270 KKASSERT(sc->re_caps & RE_C_HWIM); 2271 /* FALL THROUGH */ 2272 case RE_IMTYPE_NONE: 2273 sc->re_intrs = RE_INTRS; 2274 sc->re_rx_ack = RE_ISR_RX_OK | RE_ISR_FIFO_OFLOW | 2275 RE_ISR_RX_OVERRUN; 2276 sc->re_tx_ack = RE_ISR_TX_OK; 2277 break; 2278 2279 case RE_IMTYPE_SIM: 2280 sc->re_intrs = RE_INTRS_TIMER; 2281 sc->re_rx_ack = RE_ISR_PCS_TIMEOUT; 2282 sc->re_tx_ack = RE_ISR_PCS_TIMEOUT; 2283 break; 2284 2285 default: 2286 panic("%s: unknown imtype %d", 2287 sc->arpcom.ac_if.if_xname, imtype); 2288 } 2289 } 2290 2291 static void 2292 re_setup_intr(struct re_softc *sc, int enable_intrs, int imtype) 2293 { 2294 re_config_imtype(sc, imtype); 2295 2296 if (enable_intrs) 2297 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs); 2298 else 2299 CSR_WRITE_2(sc, RE_IMR, 0); 2300 2301 sc->re_npoll.ifpc_stcount = 0; 2302 2303 switch (imtype) { 2304 case RE_IMTYPE_NONE: 2305 re_disable_sim_im(sc); 2306 re_disable_hw_im(sc); 2307 break; 2308 2309 case RE_IMTYPE_HW: 2310 KKASSERT(sc->re_caps & RE_C_HWIM); 2311 re_disable_sim_im(sc); 2312 re_setup_hw_im(sc); 2313 break; 2314 2315 case RE_IMTYPE_SIM: 2316 re_disable_hw_im(sc); 2317 re_setup_sim_im(sc); 2318 break; 2319 2320 default: 2321 panic("%s: unknown imtype %d", 2322 sc->arpcom.ac_if.if_xname, imtype); 2323 } 2324 } 2325 2326 static int 2327 re_jpool_alloc(struct re_softc *sc) 2328 { 2329 struct re_list_data *ldata = &sc->re_ldata; 2330 struct re_jbuf *jbuf; 2331 bus_addr_t paddr; 2332 bus_size_t jpool_size; 2333 bus_dmamem_t dmem; 2334 caddr_t buf; 2335 int i, error; 2336 2337 lwkt_serialize_init(&ldata->re_jbuf_serializer); 2338 2339 ldata->re_jbuf = kmalloc(sizeof(struct re_jbuf) * RE_JBUF_COUNT(sc), 2340 M_DEVBUF, M_WAITOK | M_ZERO); 2341 2342 jpool_size = RE_JBUF_COUNT(sc) * RE_JBUF_SIZE; 2343 2344 error = bus_dmamem_coherent(sc->re_parent_tag, 2345 RE_RXBUF_ALIGN, 0, 2346 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 2347 jpool_size, BUS_DMA_WAITOK, &dmem); 2348 if (error) { 2349 device_printf(sc->dev, "could not allocate jumbo memory\n"); 2350 return error; 2351 } 2352 ldata->re_jpool_tag = dmem.dmem_tag; 2353 ldata->re_jpool_map = dmem.dmem_map; 2354 ldata->re_jpool = dmem.dmem_addr; 2355 paddr = dmem.dmem_busaddr; 2356 2357 /* ..and split it into 9KB chunks */ 2358 SLIST_INIT(&ldata->re_jbuf_free); 2359 2360 buf = ldata->re_jpool; 2361 for (i = 0; i < RE_JBUF_COUNT(sc); i++) { 2362 jbuf = &ldata->re_jbuf[i]; 2363 2364 jbuf->re_sc = sc; 2365 jbuf->re_inuse = 0; 2366 jbuf->re_slot = i; 2367 jbuf->re_buf = buf; 2368 jbuf->re_paddr = paddr; 2369 2370 SLIST_INSERT_HEAD(&ldata->re_jbuf_free, jbuf, re_link); 2371 2372 buf += RE_JBUF_SIZE; 2373 paddr += RE_JBUF_SIZE; 2374 } 2375 return 0; 2376 } 2377 2378 static void 2379 re_jpool_free(struct re_softc *sc) 2380 { 2381 struct re_list_data *ldata = &sc->re_ldata; 2382 2383 if (ldata->re_jpool_tag != NULL) { 2384 bus_dmamap_unload(ldata->re_jpool_tag, ldata->re_jpool_map); 2385 bus_dmamem_free(ldata->re_jpool_tag, ldata->re_jpool, 2386 ldata->re_jpool_map); 2387 bus_dma_tag_destroy(ldata->re_jpool_tag); 2388 ldata->re_jpool_tag = NULL; 2389 } 2390 2391 if (ldata->re_jbuf != NULL) { 2392 kfree(ldata->re_jbuf, M_DEVBUF); 2393 ldata->re_jbuf = NULL; 2394 } 2395 } 2396 2397 #ifdef RE_JUMBO 2398 static struct re_jbuf * 2399 re_jbuf_alloc(struct re_softc *sc) 2400 { 2401 struct re_list_data *ldata = &sc->re_ldata; 2402 struct re_jbuf *jbuf; 2403 2404 lwkt_serialize_enter(&ldata->re_jbuf_serializer); 2405 2406 jbuf = SLIST_FIRST(&ldata->re_jbuf_free); 2407 if (jbuf != NULL) { 2408 SLIST_REMOVE_HEAD(&ldata->re_jbuf_free, re_link); 2409 jbuf->re_inuse = 1; 2410 } 2411 2412 lwkt_serialize_exit(&ldata->re_jbuf_serializer); 2413 2414 return jbuf; 2415 } 2416 2417 static void 2418 re_jbuf_free(void *arg) 2419 { 2420 struct re_jbuf *jbuf = arg; 2421 struct re_softc *sc = jbuf->re_sc; 2422 struct re_list_data *ldata = &sc->re_ldata; 2423 2424 if (&ldata->re_jbuf[jbuf->re_slot] != jbuf) { 2425 panic("%s: free wrong jumbo buffer", 2426 sc->arpcom.ac_if.if_xname); 2427 } else if (jbuf->re_inuse == 0) { 2428 panic("%s: jumbo buffer already freed", 2429 sc->arpcom.ac_if.if_xname); 2430 } 2431 2432 lwkt_serialize_enter(&ldata->re_jbuf_serializer); 2433 atomic_subtract_int(&jbuf->re_inuse, 1); 2434 if (jbuf->re_inuse == 0) 2435 SLIST_INSERT_HEAD(&ldata->re_jbuf_free, jbuf, re_link); 2436 lwkt_serialize_exit(&ldata->re_jbuf_serializer); 2437 } 2438 2439 static void 2440 re_jbuf_ref(void *arg) 2441 { 2442 struct re_jbuf *jbuf = arg; 2443 struct re_softc *sc = jbuf->re_sc; 2444 struct re_list_data *ldata = &sc->re_ldata; 2445 2446 if (&ldata->re_jbuf[jbuf->re_slot] != jbuf) { 2447 panic("%s: ref wrong jumbo buffer", 2448 sc->arpcom.ac_if.if_xname); 2449 } else if (jbuf->re_inuse == 0) { 2450 panic("%s: jumbo buffer already freed", 2451 sc->arpcom.ac_if.if_xname); 2452 } 2453 atomic_add_int(&jbuf->re_inuse, 1); 2454 } 2455 #endif /* RE_JUMBO */ 2456 2457 static void 2458 re_disable_aspm(device_t dev) 2459 { 2460 uint16_t link_cap, link_ctrl; 2461 uint8_t pcie_ptr, reg; 2462 2463 pcie_ptr = pci_get_pciecap_ptr(dev); 2464 if (pcie_ptr == 0) 2465 return; 2466 2467 link_cap = pci_read_config(dev, pcie_ptr + PCIER_LINKCAP, 2); 2468 if ((link_cap & PCIEM_LNKCAP_ASPM_MASK) == 0) 2469 return; 2470 2471 if (bootverbose) 2472 device_printf(dev, "disable ASPM\n"); 2473 2474 reg = pcie_ptr + PCIER_LINKCTRL; 2475 link_ctrl = pci_read_config(dev, reg, 2); 2476 link_ctrl &= ~(PCIEM_LNKCTL_ASPM_L0S | PCIEM_LNKCTL_ASPM_L1); 2477 pci_write_config(dev, reg, link_ctrl, 2); 2478 } 2479