1 /*- 2 * Copyright (c) 2013 Tsubai Masanari 3 * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org> 4 * 5 * Permission to use, copy, modify, and distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 * 17 * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $ 18 */ 19 20 /* Driver for VMware vmxnet3 virtual ethernet devices. */ 21 22 #include <sys/cdefs.h> 23 __FBSDID("$FreeBSD$"); 24 25 #include <sys/param.h> 26 #include <sys/systm.h> 27 #include <sys/eventhandler.h> 28 #include <sys/kernel.h> 29 #include <sys/endian.h> 30 #include <sys/sockio.h> 31 #include <sys/mbuf.h> 32 #include <sys/malloc.h> 33 #include <sys/module.h> 34 #include <sys/socket.h> 35 #include <sys/sysctl.h> 36 #include <sys/smp.h> 37 #include <sys/taskqueue.h> 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 41 #include <net/ethernet.h> 42 #include <net/if.h> 43 #include <net/if_var.h> 44 #include <net/if_arp.h> 45 #include <net/if_dl.h> 46 #include <net/if_types.h> 47 #include <net/if_media.h> 48 #include <net/if_vlan_var.h> 49 50 #include <net/bpf.h> 51 52 #include <netinet/in_systm.h> 53 #include <netinet/in.h> 54 #include <netinet/ip.h> 55 #include <netinet/ip6.h> 56 #include <netinet6/ip6_var.h> 57 #include <netinet/udp.h> 58 #include <netinet/tcp.h> 59 60 #include <machine/in_cksum.h> 61 62 #include <machine/bus.h> 63 #include <machine/resource.h> 64 #include <sys/bus.h> 65 #include <sys/rman.h> 66 67 #include <dev/pci/pcireg.h> 68 #include <dev/pci/pcivar.h> 69 70 #include "if_vmxreg.h" 71 #include "if_vmxvar.h" 72 73 #include "opt_inet.h" 74 #include "opt_inet6.h" 75 76 #ifdef VMXNET3_FAILPOINTS 77 #include <sys/fail.h> 78 static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0, 79 "vmxnet3 fail points"); 80 #define VMXNET3_FP _debug_fail_point_vmxnet3 81 #endif 82 83 static int vmxnet3_probe(device_t); 84 static int vmxnet3_attach(device_t); 85 static int vmxnet3_detach(device_t); 86 static int vmxnet3_shutdown(device_t); 87 88 static int vmxnet3_alloc_resources(struct vmxnet3_softc *); 89 static void vmxnet3_free_resources(struct vmxnet3_softc *); 90 static int vmxnet3_check_version(struct vmxnet3_softc *); 91 static void vmxnet3_initial_config(struct vmxnet3_softc *); 92 static void vmxnet3_check_multiqueue(struct vmxnet3_softc *); 93 94 static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 95 static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 96 static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 97 static int vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int, 98 struct vmxnet3_interrupt *); 99 static int vmxnet3_alloc_intr_resources(struct vmxnet3_softc *); 100 static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 101 static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 102 static int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 103 static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 104 105 static void vmxnet3_free_interrupt(struct vmxnet3_softc *, 106 struct vmxnet3_interrupt *); 107 static void vmxnet3_free_interrupts(struct vmxnet3_softc *); 108 109 #ifndef VMXNET3_LEGACY_TX 110 static int vmxnet3_alloc_taskqueue(struct vmxnet3_softc *); 111 static void vmxnet3_start_taskqueue(struct vmxnet3_softc *); 112 static void vmxnet3_drain_taskqueue(struct vmxnet3_softc *); 113 static void vmxnet3_free_taskqueue(struct vmxnet3_softc *); 114 #endif 115 116 static int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 117 static int vmxnet3_init_txq(struct vmxnet3_softc *, int); 118 static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 119 static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 120 static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 121 static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 122 123 static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 124 static void vmxnet3_free_shared_data(struct vmxnet3_softc *); 125 static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 126 static void vmxnet3_free_txq_data(struct vmxnet3_softc *); 127 static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 128 static void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 129 static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 130 static void vmxnet3_free_queue_data(struct vmxnet3_softc *); 131 static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 132 static void vmxnet3_init_shared_data(struct vmxnet3_softc *); 133 static void vmxnet3_reinit_interface(struct vmxnet3_softc *); 134 static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); 135 static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 136 static int vmxnet3_alloc_data(struct vmxnet3_softc *); 137 static void vmxnet3_free_data(struct vmxnet3_softc *); 138 static int vmxnet3_setup_interface(struct vmxnet3_softc *); 139 140 static void vmxnet3_evintr(struct vmxnet3_softc *); 141 static void vmxnet3_txq_eof(struct vmxnet3_txqueue *); 142 static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 143 static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); 144 static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 145 struct vmxnet3_rxring *, int); 146 static void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *); 147 static void vmxnet3_legacy_intr(void *); 148 static void vmxnet3_txq_intr(void *); 149 static void vmxnet3_rxq_intr(void *); 150 static void vmxnet3_event_intr(void *); 151 152 static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 153 static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 154 static void vmxnet3_stop(struct vmxnet3_softc *); 155 156 static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 157 static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 158 static int vmxnet3_reinit_queues(struct vmxnet3_softc *); 159 static int vmxnet3_enable_device(struct vmxnet3_softc *); 160 static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 161 static int vmxnet3_reinit(struct vmxnet3_softc *); 162 static void vmxnet3_init_locked(struct vmxnet3_softc *); 163 static void vmxnet3_init(void *); 164 165 static int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *, 166 int *, int *, int *); 167 static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, 168 bus_dmamap_t, bus_dma_segment_t [], int *); 169 static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 170 static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 171 #ifdef VMXNET3_LEGACY_TX 172 static void vmxnet3_start_locked(struct ifnet *); 173 static void vmxnet3_start(struct ifnet *); 174 #else 175 static int vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *, 176 struct mbuf *); 177 static int vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *); 178 static void vmxnet3_txq_tq_deferred(void *, int); 179 #endif 180 static void vmxnet3_txq_start(struct vmxnet3_txqueue *); 181 static void vmxnet3_tx_start_all(struct vmxnet3_softc *); 182 183 static void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int, 184 uint16_t); 185 static void vmxnet3_register_vlan(void *, struct ifnet *, uint16_t); 186 static void vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t); 187 static void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 188 static int vmxnet3_change_mtu(struct vmxnet3_softc *, int); 189 static int vmxnet3_ioctl(struct ifnet *, u_long, caddr_t); 190 static uint64_t vmxnet3_get_counter(struct ifnet *, ift_counter); 191 192 #ifndef VMXNET3_LEGACY_TX 193 static void vmxnet3_qflush(struct ifnet *); 194 #endif 195 196 static int vmxnet3_watchdog(struct vmxnet3_txqueue *); 197 static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); 198 static void vmxnet3_tick(void *); 199 static void vmxnet3_link_status(struct vmxnet3_softc *); 200 static void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); 201 static int vmxnet3_media_change(struct ifnet *); 202 static void vmxnet3_set_lladdr(struct vmxnet3_softc *); 203 static void vmxnet3_get_lladdr(struct vmxnet3_softc *); 204 205 static void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *, 206 struct sysctl_ctx_list *, struct sysctl_oid_list *); 207 static void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *, 208 struct sysctl_ctx_list *, struct sysctl_oid_list *); 209 static void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *, 210 struct sysctl_ctx_list *, struct sysctl_oid_list *); 211 static void vmxnet3_setup_sysctl(struct vmxnet3_softc *); 212 213 static void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t, 214 uint32_t); 215 static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t); 216 static void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t, 217 uint32_t); 218 static void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t); 219 static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t); 220 221 static void vmxnet3_enable_intr(struct vmxnet3_softc *, int); 222 static void vmxnet3_disable_intr(struct vmxnet3_softc *, int); 223 static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 224 static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 225 226 static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, 227 bus_size_t, struct vmxnet3_dma_alloc *); 228 static void vmxnet3_dma_free(struct vmxnet3_softc *, 229 struct vmxnet3_dma_alloc *); 230 static int vmxnet3_tunable_int(struct vmxnet3_softc *, 231 const char *, int); 232 233 typedef enum { 234 VMXNET3_BARRIER_RD, 235 VMXNET3_BARRIER_WR, 236 VMXNET3_BARRIER_RDWR, 237 } vmxnet3_barrier_t; 238 239 static void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t); 240 241 /* Tunables. */ 242 static int vmxnet3_mq_disable = 0; 243 TUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable); 244 static int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES; 245 TUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue); 246 static int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES; 247 TUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue); 248 static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC; 249 TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc); 250 static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC; 251 TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc); 252 253 static device_method_t vmxnet3_methods[] = { 254 /* Device interface. */ 255 DEVMETHOD(device_probe, vmxnet3_probe), 256 DEVMETHOD(device_attach, vmxnet3_attach), 257 DEVMETHOD(device_detach, vmxnet3_detach), 258 DEVMETHOD(device_shutdown, vmxnet3_shutdown), 259 260 DEVMETHOD_END 261 }; 262 263 static driver_t vmxnet3_driver = { 264 "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc) 265 }; 266 267 static devclass_t vmxnet3_devclass; 268 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0); 269 270 MODULE_DEPEND(vmx, pci, 1, 1, 1); 271 MODULE_DEPEND(vmx, ether, 1, 1, 1); 272 273 #define VMXNET3_VMWARE_VENDOR_ID 0x15AD 274 #define VMXNET3_VMWARE_DEVICE_ID 0x07B0 275 276 static int 277 vmxnet3_probe(device_t dev) 278 { 279 280 if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID && 281 pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) { 282 device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter"); 283 return (BUS_PROBE_DEFAULT); 284 } 285 286 return (ENXIO); 287 } 288 289 static int 290 vmxnet3_attach(device_t dev) 291 { 292 struct vmxnet3_softc *sc; 293 int error; 294 295 sc = device_get_softc(dev); 296 sc->vmx_dev = dev; 297 298 pci_enable_busmaster(dev); 299 300 VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev)); 301 callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0); 302 303 vmxnet3_initial_config(sc); 304 305 error = vmxnet3_alloc_resources(sc); 306 if (error) 307 goto fail; 308 309 error = vmxnet3_check_version(sc); 310 if (error) 311 goto fail; 312 313 error = vmxnet3_alloc_rxtx_queues(sc); 314 if (error) 315 goto fail; 316 317 #ifndef VMXNET3_LEGACY_TX 318 error = vmxnet3_alloc_taskqueue(sc); 319 if (error) 320 goto fail; 321 #endif 322 323 error = vmxnet3_alloc_interrupts(sc); 324 if (error) 325 goto fail; 326 327 vmxnet3_check_multiqueue(sc); 328 329 error = vmxnet3_alloc_data(sc); 330 if (error) 331 goto fail; 332 333 error = vmxnet3_setup_interface(sc); 334 if (error) 335 goto fail; 336 337 error = vmxnet3_setup_interrupts(sc); 338 if (error) { 339 ether_ifdetach(sc->vmx_ifp); 340 device_printf(dev, "could not set up interrupt\n"); 341 goto fail; 342 } 343 344 vmxnet3_setup_sysctl(sc); 345 #ifndef VMXNET3_LEGACY_TX 346 vmxnet3_start_taskqueue(sc); 347 #endif 348 349 fail: 350 if (error) 351 vmxnet3_detach(dev); 352 353 return (error); 354 } 355 356 static int 357 vmxnet3_detach(device_t dev) 358 { 359 struct vmxnet3_softc *sc; 360 struct ifnet *ifp; 361 362 sc = device_get_softc(dev); 363 ifp = sc->vmx_ifp; 364 365 if (device_is_attached(dev)) { 366 VMXNET3_CORE_LOCK(sc); 367 vmxnet3_stop(sc); 368 VMXNET3_CORE_UNLOCK(sc); 369 370 callout_drain(&sc->vmx_tick); 371 #ifndef VMXNET3_LEGACY_TX 372 vmxnet3_drain_taskqueue(sc); 373 #endif 374 375 ether_ifdetach(ifp); 376 } 377 378 if (sc->vmx_vlan_attach != NULL) { 379 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach); 380 sc->vmx_vlan_attach = NULL; 381 } 382 if (sc->vmx_vlan_detach != NULL) { 383 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach); 384 sc->vmx_vlan_detach = NULL; 385 } 386 387 #ifndef VMXNET3_LEGACY_TX 388 vmxnet3_free_taskqueue(sc); 389 #endif 390 vmxnet3_free_interrupts(sc); 391 392 if (ifp != NULL) { 393 if_free(ifp); 394 sc->vmx_ifp = NULL; 395 } 396 397 ifmedia_removeall(&sc->vmx_media); 398 399 vmxnet3_free_data(sc); 400 vmxnet3_free_resources(sc); 401 vmxnet3_free_rxtx_queues(sc); 402 403 VMXNET3_CORE_LOCK_DESTROY(sc); 404 405 return (0); 406 } 407 408 static int 409 vmxnet3_shutdown(device_t dev) 410 { 411 412 return (0); 413 } 414 415 static int 416 vmxnet3_alloc_resources(struct vmxnet3_softc *sc) 417 { 418 device_t dev; 419 int rid; 420 421 dev = sc->vmx_dev; 422 423 rid = PCIR_BAR(0); 424 sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 425 RF_ACTIVE); 426 if (sc->vmx_res0 == NULL) { 427 device_printf(dev, 428 "could not map BAR0 memory\n"); 429 return (ENXIO); 430 } 431 432 sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0); 433 sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0); 434 435 rid = PCIR_BAR(1); 436 sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 437 RF_ACTIVE); 438 if (sc->vmx_res1 == NULL) { 439 device_printf(dev, 440 "could not map BAR1 memory\n"); 441 return (ENXIO); 442 } 443 444 sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1); 445 sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1); 446 447 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 448 rid = PCIR_BAR(2); 449 sc->vmx_msix_res = bus_alloc_resource_any(dev, 450 SYS_RES_MEMORY, &rid, RF_ACTIVE); 451 } 452 453 if (sc->vmx_msix_res == NULL) 454 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 455 456 return (0); 457 } 458 459 static void 460 vmxnet3_free_resources(struct vmxnet3_softc *sc) 461 { 462 device_t dev; 463 int rid; 464 465 dev = sc->vmx_dev; 466 467 if (sc->vmx_res0 != NULL) { 468 rid = PCIR_BAR(0); 469 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0); 470 sc->vmx_res0 = NULL; 471 } 472 473 if (sc->vmx_res1 != NULL) { 474 rid = PCIR_BAR(1); 475 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1); 476 sc->vmx_res1 = NULL; 477 } 478 479 if (sc->vmx_msix_res != NULL) { 480 rid = PCIR_BAR(2); 481 bus_release_resource(dev, SYS_RES_MEMORY, rid, 482 sc->vmx_msix_res); 483 sc->vmx_msix_res = NULL; 484 } 485 } 486 487 static int 488 vmxnet3_check_version(struct vmxnet3_softc *sc) 489 { 490 device_t dev; 491 uint32_t version; 492 493 dev = sc->vmx_dev; 494 495 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 496 if ((version & 0x01) == 0) { 497 device_printf(dev, "unsupported hardware version %#x\n", 498 version); 499 return (ENOTSUP); 500 } 501 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 502 503 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 504 if ((version & 0x01) == 0) { 505 device_printf(dev, "unsupported UPT version %#x\n", version); 506 return (ENOTSUP); 507 } 508 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 509 510 return (0); 511 } 512 513 static void 514 vmxnet3_initial_config(struct vmxnet3_softc *sc) 515 { 516 int nqueue, ndesc; 517 518 nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue); 519 if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1) 520 nqueue = VMXNET3_DEF_TX_QUEUES; 521 if (nqueue > mp_ncpus) 522 nqueue = mp_ncpus; 523 sc->vmx_max_ntxqueues = nqueue; 524 525 nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue); 526 if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1) 527 nqueue = VMXNET3_DEF_RX_QUEUES; 528 if (nqueue > mp_ncpus) 529 nqueue = mp_ncpus; 530 sc->vmx_max_nrxqueues = nqueue; 531 532 if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) { 533 sc->vmx_max_nrxqueues = 1; 534 sc->vmx_max_ntxqueues = 1; 535 } 536 537 ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc); 538 if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC) 539 ndesc = VMXNET3_DEF_TX_NDESC; 540 if (ndesc & VMXNET3_MASK_TX_NDESC) 541 ndesc &= ~VMXNET3_MASK_TX_NDESC; 542 sc->vmx_ntxdescs = ndesc; 543 544 ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc); 545 if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC) 546 ndesc = VMXNET3_DEF_RX_NDESC; 547 if (ndesc & VMXNET3_MASK_RX_NDESC) 548 ndesc &= ~VMXNET3_MASK_RX_NDESC; 549 sc->vmx_nrxdescs = ndesc; 550 sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 551 } 552 553 static void 554 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc) 555 { 556 557 if (sc->vmx_intr_type != VMXNET3_IT_MSIX) 558 goto out; 559 560 /* BMV: Just use the maximum configured for now. */ 561 sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; 562 sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; 563 564 if (sc->vmx_nrxqueues > 1) 565 sc->vmx_flags |= VMXNET3_FLAG_RSS; 566 567 return; 568 569 out: 570 sc->vmx_ntxqueues = 1; 571 sc->vmx_nrxqueues = 1; 572 } 573 574 static int 575 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 576 { 577 device_t dev; 578 int nmsix, cnt, required; 579 580 dev = sc->vmx_dev; 581 582 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 583 return (1); 584 585 /* Allocate an additional vector for the events interrupt. */ 586 required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1; 587 588 nmsix = pci_msix_count(dev); 589 if (nmsix < required) 590 return (1); 591 592 cnt = required; 593 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 594 sc->vmx_nintrs = required; 595 return (0); 596 } else 597 pci_release_msi(dev); 598 599 /* BMV TODO Fallback to sharing MSIX vectors if possible. */ 600 601 return (1); 602 } 603 604 static int 605 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 606 { 607 device_t dev; 608 int nmsi, cnt, required; 609 610 dev = sc->vmx_dev; 611 required = 1; 612 613 nmsi = pci_msi_count(dev); 614 if (nmsi < required) 615 return (1); 616 617 cnt = required; 618 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) { 619 sc->vmx_nintrs = 1; 620 return (0); 621 } else 622 pci_release_msi(dev); 623 624 return (1); 625 } 626 627 static int 628 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 629 { 630 631 sc->vmx_nintrs = 1; 632 return (0); 633 } 634 635 static int 636 vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags, 637 struct vmxnet3_interrupt *intr) 638 { 639 struct resource *irq; 640 641 irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags); 642 if (irq == NULL) 643 return (ENXIO); 644 645 intr->vmxi_irq = irq; 646 intr->vmxi_rid = rid; 647 648 return (0); 649 } 650 651 static int 652 vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc) 653 { 654 int i, rid, flags, error; 655 656 rid = 0; 657 flags = RF_ACTIVE; 658 659 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) 660 flags |= RF_SHAREABLE; 661 else 662 rid = 1; 663 664 for (i = 0; i < sc->vmx_nintrs; i++, rid++) { 665 error = vmxnet3_alloc_interrupt(sc, rid, flags, 666 &sc->vmx_intrs[i]); 667 if (error) 668 return (error); 669 } 670 671 return (0); 672 } 673 674 static int 675 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 676 { 677 device_t dev; 678 struct vmxnet3_txqueue *txq; 679 struct vmxnet3_rxqueue *rxq; 680 struct vmxnet3_interrupt *intr; 681 enum intr_type type; 682 int i, error; 683 684 dev = sc->vmx_dev; 685 intr = &sc->vmx_intrs[0]; 686 type = INTR_TYPE_NET | INTR_MPSAFE; 687 688 for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) { 689 txq = &sc->vmx_txq[i]; 690 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 691 vmxnet3_txq_intr, txq, &intr->vmxi_handler); 692 if (error) 693 return (error); 694 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 695 "tq%d", i); 696 txq->vxtxq_intr_idx = intr->vmxi_rid - 1; 697 } 698 699 for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) { 700 rxq = &sc->vmx_rxq[i]; 701 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 702 vmxnet3_rxq_intr, rxq, &intr->vmxi_handler); 703 if (error) 704 return (error); 705 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 706 "rq%d", i); 707 rxq->vxrxq_intr_idx = intr->vmxi_rid - 1; 708 } 709 710 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 711 vmxnet3_event_intr, sc, &intr->vmxi_handler); 712 if (error) 713 return (error); 714 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, "event"); 715 sc->vmx_event_intr_idx = intr->vmxi_rid - 1; 716 717 return (0); 718 } 719 720 static int 721 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 722 { 723 struct vmxnet3_interrupt *intr; 724 int i, error; 725 726 intr = &sc->vmx_intrs[0]; 727 error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq, 728 INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc, 729 &intr->vmxi_handler); 730 731 for (i = 0; i < sc->vmx_ntxqueues; i++) 732 sc->vmx_txq[i].vxtxq_intr_idx = 0; 733 for (i = 0; i < sc->vmx_nrxqueues; i++) 734 sc->vmx_rxq[i].vxrxq_intr_idx = 0; 735 sc->vmx_event_intr_idx = 0; 736 737 return (error); 738 } 739 740 static void 741 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 742 { 743 struct vmxnet3_txqueue *txq; 744 struct vmxnet3_txq_shared *txs; 745 struct vmxnet3_rxqueue *rxq; 746 struct vmxnet3_rxq_shared *rxs; 747 int i; 748 749 sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 750 751 for (i = 0; i < sc->vmx_ntxqueues; i++) { 752 txq = &sc->vmx_txq[i]; 753 txs = txq->vxtxq_ts; 754 txs->intr_idx = txq->vxtxq_intr_idx; 755 } 756 757 for (i = 0; i < sc->vmx_nrxqueues; i++) { 758 rxq = &sc->vmx_rxq[i]; 759 rxs = rxq->vxrxq_rs; 760 rxs->intr_idx = rxq->vxrxq_intr_idx; 761 } 762 } 763 764 static int 765 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 766 { 767 int error; 768 769 error = vmxnet3_alloc_intr_resources(sc); 770 if (error) 771 return (error); 772 773 switch (sc->vmx_intr_type) { 774 case VMXNET3_IT_MSIX: 775 error = vmxnet3_setup_msix_interrupts(sc); 776 break; 777 case VMXNET3_IT_MSI: 778 case VMXNET3_IT_LEGACY: 779 error = vmxnet3_setup_legacy_interrupt(sc); 780 break; 781 default: 782 panic("%s: invalid interrupt type %d", __func__, 783 sc->vmx_intr_type); 784 } 785 786 if (error == 0) 787 vmxnet3_set_interrupt_idx(sc); 788 789 return (error); 790 } 791 792 static int 793 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 794 { 795 device_t dev; 796 uint32_t config; 797 int error; 798 799 dev = sc->vmx_dev; 800 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 801 802 sc->vmx_intr_type = config & 0x03; 803 sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 804 805 switch (sc->vmx_intr_type) { 806 case VMXNET3_IT_AUTO: 807 sc->vmx_intr_type = VMXNET3_IT_MSIX; 808 /* FALLTHROUGH */ 809 case VMXNET3_IT_MSIX: 810 error = vmxnet3_alloc_msix_interrupts(sc); 811 if (error == 0) 812 break; 813 sc->vmx_intr_type = VMXNET3_IT_MSI; 814 /* FALLTHROUGH */ 815 case VMXNET3_IT_MSI: 816 error = vmxnet3_alloc_msi_interrupts(sc); 817 if (error == 0) 818 break; 819 sc->vmx_intr_type = VMXNET3_IT_LEGACY; 820 /* FALLTHROUGH */ 821 case VMXNET3_IT_LEGACY: 822 error = vmxnet3_alloc_legacy_interrupts(sc); 823 if (error == 0) 824 break; 825 /* FALLTHROUGH */ 826 default: 827 sc->vmx_intr_type = -1; 828 device_printf(dev, "cannot allocate any interrupt resources\n"); 829 return (ENXIO); 830 } 831 832 return (error); 833 } 834 835 static void 836 vmxnet3_free_interrupt(struct vmxnet3_softc *sc, 837 struct vmxnet3_interrupt *intr) 838 { 839 device_t dev; 840 841 dev = sc->vmx_dev; 842 843 if (intr->vmxi_handler != NULL) { 844 bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler); 845 intr->vmxi_handler = NULL; 846 } 847 848 if (intr->vmxi_irq != NULL) { 849 bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid, 850 intr->vmxi_irq); 851 intr->vmxi_irq = NULL; 852 intr->vmxi_rid = -1; 853 } 854 } 855 856 static void 857 vmxnet3_free_interrupts(struct vmxnet3_softc *sc) 858 { 859 int i; 860 861 for (i = 0; i < sc->vmx_nintrs; i++) 862 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); 863 864 if (sc->vmx_intr_type == VMXNET3_IT_MSI || 865 sc->vmx_intr_type == VMXNET3_IT_MSIX) 866 pci_release_msi(sc->vmx_dev); 867 } 868 869 #ifndef VMXNET3_LEGACY_TX 870 static int 871 vmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc) 872 { 873 device_t dev; 874 875 dev = sc->vmx_dev; 876 877 sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT, 878 taskqueue_thread_enqueue, &sc->vmx_tq); 879 if (sc->vmx_tq == NULL) 880 return (ENOMEM); 881 882 return (0); 883 } 884 885 static void 886 vmxnet3_start_taskqueue(struct vmxnet3_softc *sc) 887 { 888 device_t dev; 889 int nthreads, error; 890 891 dev = sc->vmx_dev; 892 893 /* 894 * The taskqueue is typically not frequently used, so a dedicated 895 * thread for each queue is unnecessary. 896 */ 897 nthreads = MAX(1, sc->vmx_ntxqueues / 2); 898 899 /* 900 * Most drivers just ignore the return value - it only fails 901 * with ENOMEM so an error is not likely. It is hard for us 902 * to recover from an error here. 903 */ 904 error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET, 905 "%s taskq", device_get_nameunit(dev)); 906 if (error) 907 device_printf(dev, "failed to start taskqueue: %d", error); 908 } 909 910 static void 911 vmxnet3_drain_taskqueue(struct vmxnet3_softc *sc) 912 { 913 struct vmxnet3_txqueue *txq; 914 int i; 915 916 if (sc->vmx_tq != NULL) { 917 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 918 txq = &sc->vmx_txq[i]; 919 taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask); 920 } 921 } 922 } 923 924 static void 925 vmxnet3_free_taskqueue(struct vmxnet3_softc *sc) 926 { 927 if (sc->vmx_tq != NULL) { 928 taskqueue_free(sc->vmx_tq); 929 sc->vmx_tq = NULL; 930 } 931 } 932 #endif 933 934 static int 935 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 936 { 937 struct vmxnet3_rxqueue *rxq; 938 struct vmxnet3_rxring *rxr; 939 int i; 940 941 rxq = &sc->vmx_rxq[q]; 942 943 snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 944 device_get_nameunit(sc->vmx_dev), q); 945 mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF); 946 947 rxq->vxrxq_sc = sc; 948 rxq->vxrxq_id = q; 949 950 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 951 rxr = &rxq->vxrxq_cmd_ring[i]; 952 rxr->vxrxr_rid = i; 953 rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 954 rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc * 955 sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 956 if (rxr->vxrxr_rxbuf == NULL) 957 return (ENOMEM); 958 959 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 960 } 961 962 return (0); 963 } 964 965 static int 966 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 967 { 968 struct vmxnet3_txqueue *txq; 969 struct vmxnet3_txring *txr; 970 971 txq = &sc->vmx_txq[q]; 972 txr = &txq->vxtxq_cmd_ring; 973 974 snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 975 device_get_nameunit(sc->vmx_dev), q); 976 mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF); 977 978 txq->vxtxq_sc = sc; 979 txq->vxtxq_id = q; 980 981 txr->vxtxr_ndesc = sc->vmx_ntxdescs; 982 txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc * 983 sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 984 if (txr->vxtxr_txbuf == NULL) 985 return (ENOMEM); 986 987 txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 988 989 #ifndef VMXNET3_LEGACY_TX 990 TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq); 991 992 txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF, 993 M_NOWAIT, &txq->vxtxq_mtx); 994 if (txq->vxtxq_br == NULL) 995 return (ENOMEM); 996 #endif 997 998 return (0); 999 } 1000 1001 static int 1002 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 1003 { 1004 int i, error; 1005 1006 /* 1007 * Only attempt to create multiple queues if MSIX is available. MSIX is 1008 * disabled by default because its apparently broken for devices passed 1009 * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable 1010 * must be set to zero for MSIX. This check prevents us from allocating 1011 * queue structures that we will not use. 1012 */ 1013 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { 1014 sc->vmx_max_nrxqueues = 1; 1015 sc->vmx_max_ntxqueues = 1; 1016 } 1017 1018 sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) * 1019 sc->vmx_max_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1020 sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) * 1021 sc->vmx_max_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1022 if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL) 1023 return (ENOMEM); 1024 1025 for (i = 0; i < sc->vmx_max_nrxqueues; i++) { 1026 error = vmxnet3_init_rxq(sc, i); 1027 if (error) 1028 return (error); 1029 } 1030 1031 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1032 error = vmxnet3_init_txq(sc, i); 1033 if (error) 1034 return (error); 1035 } 1036 1037 return (0); 1038 } 1039 1040 static void 1041 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 1042 { 1043 struct vmxnet3_rxring *rxr; 1044 int i; 1045 1046 rxq->vxrxq_sc = NULL; 1047 rxq->vxrxq_id = -1; 1048 1049 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1050 rxr = &rxq->vxrxq_cmd_ring[i]; 1051 1052 if (rxr->vxrxr_rxbuf != NULL) { 1053 free(rxr->vxrxr_rxbuf, M_DEVBUF); 1054 rxr->vxrxr_rxbuf = NULL; 1055 } 1056 } 1057 1058 if (mtx_initialized(&rxq->vxrxq_mtx) != 0) 1059 mtx_destroy(&rxq->vxrxq_mtx); 1060 } 1061 1062 static void 1063 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 1064 { 1065 struct vmxnet3_txring *txr; 1066 1067 txr = &txq->vxtxq_cmd_ring; 1068 1069 txq->vxtxq_sc = NULL; 1070 txq->vxtxq_id = -1; 1071 1072 #ifndef VMXNET3_LEGACY_TX 1073 if (txq->vxtxq_br != NULL) { 1074 buf_ring_free(txq->vxtxq_br, M_DEVBUF); 1075 txq->vxtxq_br = NULL; 1076 } 1077 #endif 1078 1079 if (txr->vxtxr_txbuf != NULL) { 1080 free(txr->vxtxr_txbuf, M_DEVBUF); 1081 txr->vxtxr_txbuf = NULL; 1082 } 1083 1084 if (mtx_initialized(&txq->vxtxq_mtx) != 0) 1085 mtx_destroy(&txq->vxtxq_mtx); 1086 } 1087 1088 static void 1089 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 1090 { 1091 int i; 1092 1093 if (sc->vmx_rxq != NULL) { 1094 for (i = 0; i < sc->vmx_max_nrxqueues; i++) 1095 vmxnet3_destroy_rxq(&sc->vmx_rxq[i]); 1096 free(sc->vmx_rxq, M_DEVBUF); 1097 sc->vmx_rxq = NULL; 1098 } 1099 1100 if (sc->vmx_txq != NULL) { 1101 for (i = 0; i < sc->vmx_max_ntxqueues; i++) 1102 vmxnet3_destroy_txq(&sc->vmx_txq[i]); 1103 free(sc->vmx_txq, M_DEVBUF); 1104 sc->vmx_txq = NULL; 1105 } 1106 } 1107 1108 static int 1109 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 1110 { 1111 device_t dev; 1112 uint8_t *kva; 1113 size_t size; 1114 int i, error; 1115 1116 dev = sc->vmx_dev; 1117 1118 size = sizeof(struct vmxnet3_driver_shared); 1119 error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 1120 if (error) { 1121 device_printf(dev, "cannot alloc shared memory\n"); 1122 return (error); 1123 } 1124 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 1125 1126 size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 1127 sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 1128 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 1129 if (error) { 1130 device_printf(dev, "cannot alloc queue shared memory\n"); 1131 return (error); 1132 } 1133 sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 1134 kva = sc->vmx_qs; 1135 1136 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1137 sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; 1138 kva += sizeof(struct vmxnet3_txq_shared); 1139 } 1140 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1141 sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; 1142 kva += sizeof(struct vmxnet3_rxq_shared); 1143 } 1144 1145 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1146 size = sizeof(struct vmxnet3_rss_shared); 1147 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); 1148 if (error) { 1149 device_printf(dev, "cannot alloc rss shared memory\n"); 1150 return (error); 1151 } 1152 sc->vmx_rss = 1153 (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; 1154 } 1155 1156 return (0); 1157 } 1158 1159 static void 1160 vmxnet3_free_shared_data(struct vmxnet3_softc *sc) 1161 { 1162 1163 if (sc->vmx_rss != NULL) { 1164 vmxnet3_dma_free(sc, &sc->vmx_rss_dma); 1165 sc->vmx_rss = NULL; 1166 } 1167 1168 if (sc->vmx_qs != NULL) { 1169 vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 1170 sc->vmx_qs = NULL; 1171 } 1172 1173 if (sc->vmx_ds != NULL) { 1174 vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 1175 sc->vmx_ds = NULL; 1176 } 1177 } 1178 1179 static int 1180 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 1181 { 1182 device_t dev; 1183 struct vmxnet3_txqueue *txq; 1184 struct vmxnet3_txring *txr; 1185 struct vmxnet3_comp_ring *txc; 1186 size_t descsz, compsz; 1187 int i, q, error; 1188 1189 dev = sc->vmx_dev; 1190 1191 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1192 txq = &sc->vmx_txq[q]; 1193 txr = &txq->vxtxq_cmd_ring; 1194 txc = &txq->vxtxq_comp_ring; 1195 1196 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 1197 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 1198 1199 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1200 1, 0, /* alignment, boundary */ 1201 BUS_SPACE_MAXADDR, /* lowaddr */ 1202 BUS_SPACE_MAXADDR, /* highaddr */ 1203 NULL, NULL, /* filter, filterarg */ 1204 VMXNET3_TX_MAXSIZE, /* maxsize */ 1205 VMXNET3_TX_MAXSEGS, /* nsegments */ 1206 VMXNET3_TX_MAXSEGSIZE, /* maxsegsize */ 1207 0, /* flags */ 1208 NULL, NULL, /* lockfunc, lockarg */ 1209 &txr->vxtxr_txtag); 1210 if (error) { 1211 device_printf(dev, 1212 "unable to create Tx buffer tag for queue %d\n", q); 1213 return (error); 1214 } 1215 1216 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 1217 if (error) { 1218 device_printf(dev, "cannot alloc Tx descriptors for " 1219 "queue %d error %d\n", q, error); 1220 return (error); 1221 } 1222 txr->vxtxr_txd = 1223 (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1224 1225 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1226 if (error) { 1227 device_printf(dev, "cannot alloc Tx comp descriptors " 1228 "for queue %d error %d\n", q, error); 1229 return (error); 1230 } 1231 txc->vxcr_u.txcd = 1232 (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1233 1234 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1235 error = bus_dmamap_create(txr->vxtxr_txtag, 0, 1236 &txr->vxtxr_txbuf[i].vtxb_dmamap); 1237 if (error) { 1238 device_printf(dev, "unable to create Tx buf " 1239 "dmamap for queue %d idx %d\n", q, i); 1240 return (error); 1241 } 1242 } 1243 } 1244 1245 return (0); 1246 } 1247 1248 static void 1249 vmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1250 { 1251 device_t dev; 1252 struct vmxnet3_txqueue *txq; 1253 struct vmxnet3_txring *txr; 1254 struct vmxnet3_comp_ring *txc; 1255 struct vmxnet3_txbuf *txb; 1256 int i, q; 1257 1258 dev = sc->vmx_dev; 1259 1260 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1261 txq = &sc->vmx_txq[q]; 1262 txr = &txq->vxtxq_cmd_ring; 1263 txc = &txq->vxtxq_comp_ring; 1264 1265 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1266 txb = &txr->vxtxr_txbuf[i]; 1267 if (txb->vtxb_dmamap != NULL) { 1268 bus_dmamap_destroy(txr->vxtxr_txtag, 1269 txb->vtxb_dmamap); 1270 txb->vtxb_dmamap = NULL; 1271 } 1272 } 1273 1274 if (txc->vxcr_u.txcd != NULL) { 1275 vmxnet3_dma_free(sc, &txc->vxcr_dma); 1276 txc->vxcr_u.txcd = NULL; 1277 } 1278 1279 if (txr->vxtxr_txd != NULL) { 1280 vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1281 txr->vxtxr_txd = NULL; 1282 } 1283 1284 if (txr->vxtxr_txtag != NULL) { 1285 bus_dma_tag_destroy(txr->vxtxr_txtag); 1286 txr->vxtxr_txtag = NULL; 1287 } 1288 } 1289 } 1290 1291 static int 1292 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1293 { 1294 device_t dev; 1295 struct vmxnet3_rxqueue *rxq; 1296 struct vmxnet3_rxring *rxr; 1297 struct vmxnet3_comp_ring *rxc; 1298 int descsz, compsz; 1299 int i, j, q, error; 1300 1301 dev = sc->vmx_dev; 1302 1303 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1304 rxq = &sc->vmx_rxq[q]; 1305 rxc = &rxq->vxrxq_comp_ring; 1306 compsz = 0; 1307 1308 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1309 rxr = &rxq->vxrxq_cmd_ring[i]; 1310 1311 descsz = rxr->vxrxr_ndesc * 1312 sizeof(struct vmxnet3_rxdesc); 1313 compsz += rxr->vxrxr_ndesc * 1314 sizeof(struct vmxnet3_rxcompdesc); 1315 1316 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1317 1, 0, /* alignment, boundary */ 1318 BUS_SPACE_MAXADDR, /* lowaddr */ 1319 BUS_SPACE_MAXADDR, /* highaddr */ 1320 NULL, NULL, /* filter, filterarg */ 1321 MJUMPAGESIZE, /* maxsize */ 1322 1, /* nsegments */ 1323 MJUMPAGESIZE, /* maxsegsize */ 1324 0, /* flags */ 1325 NULL, NULL, /* lockfunc, lockarg */ 1326 &rxr->vxrxr_rxtag); 1327 if (error) { 1328 device_printf(dev, 1329 "unable to create Rx buffer tag for " 1330 "queue %d\n", q); 1331 return (error); 1332 } 1333 1334 error = vmxnet3_dma_malloc(sc, descsz, 512, 1335 &rxr->vxrxr_dma); 1336 if (error) { 1337 device_printf(dev, "cannot allocate Rx " 1338 "descriptors for queue %d/%d error %d\n", 1339 i, q, error); 1340 return (error); 1341 } 1342 rxr->vxrxr_rxd = 1343 (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1344 } 1345 1346 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1347 if (error) { 1348 device_printf(dev, "cannot alloc Rx comp descriptors " 1349 "for queue %d error %d\n", q, error); 1350 return (error); 1351 } 1352 rxc->vxcr_u.rxcd = 1353 (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1354 1355 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1356 rxr = &rxq->vxrxq_cmd_ring[i]; 1357 1358 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1359 &rxr->vxrxr_spare_dmap); 1360 if (error) { 1361 device_printf(dev, "unable to create spare " 1362 "dmamap for queue %d/%d error %d\n", 1363 q, i, error); 1364 return (error); 1365 } 1366 1367 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1368 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1369 &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1370 if (error) { 1371 device_printf(dev, "unable to create " 1372 "dmamap for queue %d/%d slot %d " 1373 "error %d\n", 1374 q, i, j, error); 1375 return (error); 1376 } 1377 } 1378 } 1379 } 1380 1381 return (0); 1382 } 1383 1384 static void 1385 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1386 { 1387 device_t dev; 1388 struct vmxnet3_rxqueue *rxq; 1389 struct vmxnet3_rxring *rxr; 1390 struct vmxnet3_comp_ring *rxc; 1391 struct vmxnet3_rxbuf *rxb; 1392 int i, j, q; 1393 1394 dev = sc->vmx_dev; 1395 1396 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1397 rxq = &sc->vmx_rxq[q]; 1398 rxc = &rxq->vxrxq_comp_ring; 1399 1400 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1401 rxr = &rxq->vxrxq_cmd_ring[i]; 1402 1403 if (rxr->vxrxr_spare_dmap != NULL) { 1404 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1405 rxr->vxrxr_spare_dmap); 1406 rxr->vxrxr_spare_dmap = NULL; 1407 } 1408 1409 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1410 rxb = &rxr->vxrxr_rxbuf[j]; 1411 if (rxb->vrxb_dmamap != NULL) { 1412 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1413 rxb->vrxb_dmamap); 1414 rxb->vrxb_dmamap = NULL; 1415 } 1416 } 1417 } 1418 1419 if (rxc->vxcr_u.rxcd != NULL) { 1420 vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1421 rxc->vxcr_u.rxcd = NULL; 1422 } 1423 1424 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1425 rxr = &rxq->vxrxq_cmd_ring[i]; 1426 1427 if (rxr->vxrxr_rxd != NULL) { 1428 vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1429 rxr->vxrxr_rxd = NULL; 1430 } 1431 1432 if (rxr->vxrxr_rxtag != NULL) { 1433 bus_dma_tag_destroy(rxr->vxrxr_rxtag); 1434 rxr->vxrxr_rxtag = NULL; 1435 } 1436 } 1437 } 1438 } 1439 1440 static int 1441 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1442 { 1443 int error; 1444 1445 error = vmxnet3_alloc_txq_data(sc); 1446 if (error) 1447 return (error); 1448 1449 error = vmxnet3_alloc_rxq_data(sc); 1450 if (error) 1451 return (error); 1452 1453 return (0); 1454 } 1455 1456 static void 1457 vmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1458 { 1459 1460 if (sc->vmx_rxq != NULL) 1461 vmxnet3_free_rxq_data(sc); 1462 1463 if (sc->vmx_txq != NULL) 1464 vmxnet3_free_txq_data(sc); 1465 } 1466 1467 static int 1468 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1469 { 1470 int error; 1471 1472 error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1473 32, &sc->vmx_mcast_dma); 1474 if (error) 1475 device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1476 else 1477 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1478 1479 return (error); 1480 } 1481 1482 static void 1483 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1484 { 1485 1486 if (sc->vmx_mcast != NULL) { 1487 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1488 sc->vmx_mcast = NULL; 1489 } 1490 } 1491 1492 static void 1493 vmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1494 { 1495 struct vmxnet3_driver_shared *ds; 1496 struct vmxnet3_txqueue *txq; 1497 struct vmxnet3_txq_shared *txs; 1498 struct vmxnet3_rxqueue *rxq; 1499 struct vmxnet3_rxq_shared *rxs; 1500 int i; 1501 1502 ds = sc->vmx_ds; 1503 1504 /* 1505 * Initialize fields of the shared data that remains the same across 1506 * reinits. Note the shared data is zero'd when allocated. 1507 */ 1508 1509 ds->magic = VMXNET3_REV1_MAGIC; 1510 1511 /* DriverInfo */ 1512 ds->version = VMXNET3_DRIVER_VERSION; 1513 ds->guest = VMXNET3_GOS_FREEBSD | 1514 #ifdef __LP64__ 1515 VMXNET3_GOS_64BIT; 1516 #else 1517 VMXNET3_GOS_32BIT; 1518 #endif 1519 ds->vmxnet3_revision = 1; 1520 ds->upt_version = 1; 1521 1522 /* Misc. conf */ 1523 ds->driver_data = vtophys(sc); 1524 ds->driver_data_len = sizeof(struct vmxnet3_softc); 1525 ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1526 ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1527 ds->nrxsg_max = sc->vmx_max_rxsegs; 1528 1529 /* RSS conf */ 1530 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1531 ds->rss.version = 1; 1532 ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; 1533 ds->rss.len = sc->vmx_rss_dma.dma_size; 1534 } 1535 1536 /* Interrupt control. */ 1537 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1538 ds->nintr = sc->vmx_nintrs; 1539 ds->evintr = sc->vmx_event_intr_idx; 1540 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1541 1542 for (i = 0; i < sc->vmx_nintrs; i++) 1543 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1544 1545 /* Receive filter. */ 1546 ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1547 ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1548 1549 /* Tx queues */ 1550 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1551 txq = &sc->vmx_txq[i]; 1552 txs = txq->vxtxq_ts; 1553 1554 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1555 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1556 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1557 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1558 txs->driver_data = vtophys(txq); 1559 txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1560 } 1561 1562 /* Rx queues */ 1563 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1564 rxq = &sc->vmx_rxq[i]; 1565 rxs = rxq->vxrxq_rs; 1566 1567 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1568 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1569 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1570 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1571 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1572 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1573 rxs->driver_data = vtophys(rxq); 1574 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1575 } 1576 } 1577 1578 static void 1579 vmxnet3_reinit_interface(struct vmxnet3_softc *sc) 1580 { 1581 struct ifnet *ifp; 1582 1583 ifp = sc->vmx_ifp; 1584 1585 /* Use the current MAC address. */ 1586 bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN); 1587 vmxnet3_set_lladdr(sc); 1588 1589 ifp->if_hwassist = 0; 1590 if (ifp->if_capenable & IFCAP_TXCSUM) 1591 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD; 1592 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 1593 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6; 1594 if (ifp->if_capenable & IFCAP_TSO4) 1595 ifp->if_hwassist |= CSUM_IP_TSO; 1596 if (ifp->if_capenable & IFCAP_TSO6) 1597 ifp->if_hwassist |= CSUM_IP6_TSO; 1598 } 1599 1600 static void 1601 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) 1602 { 1603 /* 1604 * Use the same key as the Linux driver until FreeBSD can do 1605 * RSS (presumably Toeplitz) in software. 1606 */ 1607 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 1608 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 1609 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 1610 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 1611 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 1612 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, 1613 }; 1614 1615 struct vmxnet3_driver_shared *ds; 1616 struct vmxnet3_rss_shared *rss; 1617 int i; 1618 1619 ds = sc->vmx_ds; 1620 rss = sc->vmx_rss; 1621 1622 rss->hash_type = 1623 UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | 1624 UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; 1625 rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 1626 rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; 1627 rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 1628 memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); 1629 1630 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) 1631 rss->ind_table[i] = i % sc->vmx_nrxqueues; 1632 } 1633 1634 static void 1635 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1636 { 1637 struct ifnet *ifp; 1638 struct vmxnet3_driver_shared *ds; 1639 1640 ifp = sc->vmx_ifp; 1641 ds = sc->vmx_ds; 1642 1643 ds->mtu = ifp->if_mtu; 1644 ds->ntxqueue = sc->vmx_ntxqueues; 1645 ds->nrxqueue = sc->vmx_nrxqueues; 1646 1647 ds->upt_features = 0; 1648 if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 1649 ds->upt_features |= UPT1_F_CSUM; 1650 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1651 ds->upt_features |= UPT1_F_VLAN; 1652 if (ifp->if_capenable & IFCAP_LRO) 1653 ds->upt_features |= UPT1_F_LRO; 1654 1655 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1656 ds->upt_features |= UPT1_F_RSS; 1657 vmxnet3_reinit_rss_shared_data(sc); 1658 } 1659 1660 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1661 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1662 (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1663 } 1664 1665 static int 1666 vmxnet3_alloc_data(struct vmxnet3_softc *sc) 1667 { 1668 int error; 1669 1670 error = vmxnet3_alloc_shared_data(sc); 1671 if (error) 1672 return (error); 1673 1674 error = vmxnet3_alloc_queue_data(sc); 1675 if (error) 1676 return (error); 1677 1678 error = vmxnet3_alloc_mcast_table(sc); 1679 if (error) 1680 return (error); 1681 1682 vmxnet3_init_shared_data(sc); 1683 1684 return (0); 1685 } 1686 1687 static void 1688 vmxnet3_free_data(struct vmxnet3_softc *sc) 1689 { 1690 1691 vmxnet3_free_mcast_table(sc); 1692 vmxnet3_free_queue_data(sc); 1693 vmxnet3_free_shared_data(sc); 1694 } 1695 1696 static int 1697 vmxnet3_setup_interface(struct vmxnet3_softc *sc) 1698 { 1699 device_t dev; 1700 struct ifnet *ifp; 1701 1702 dev = sc->vmx_dev; 1703 1704 ifp = sc->vmx_ifp = if_alloc(IFT_ETHER); 1705 if (ifp == NULL) { 1706 device_printf(dev, "cannot allocate ifnet structure\n"); 1707 return (ENOSPC); 1708 } 1709 1710 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1711 #if __FreeBSD_version < 1000025 1712 ifp->if_baudrate = 1000000000; 1713 #elif __FreeBSD_version < 1100011 1714 if_initbaudrate(ifp, IF_Gbps(10)); 1715 #else 1716 ifp->if_baudrate = IF_Gbps(10); 1717 #endif 1718 ifp->if_softc = sc; 1719 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1720 ifp->if_init = vmxnet3_init; 1721 ifp->if_ioctl = vmxnet3_ioctl; 1722 ifp->if_get_counter = vmxnet3_get_counter; 1723 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1724 ifp->if_hw_tsomaxsegcount = VMXNET3_TX_MAXSEGS; 1725 ifp->if_hw_tsomaxsegsize = VMXNET3_TX_MAXSEGSIZE; 1726 1727 #ifdef VMXNET3_LEGACY_TX 1728 ifp->if_start = vmxnet3_start; 1729 ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1; 1730 IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1); 1731 IFQ_SET_READY(&ifp->if_snd); 1732 #else 1733 ifp->if_transmit = vmxnet3_txq_mq_start; 1734 ifp->if_qflush = vmxnet3_qflush; 1735 #endif 1736 1737 vmxnet3_get_lladdr(sc); 1738 ether_ifattach(ifp, sc->vmx_lladdr); 1739 1740 ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; 1741 ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6; 1742 ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 1743 ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 1744 IFCAP_VLAN_HWCSUM; 1745 ifp->if_capenable = ifp->if_capabilities; 1746 1747 /* These capabilities are not enabled by default. */ 1748 ifp->if_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER; 1749 1750 sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 1751 vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 1752 sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config, 1753 vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1754 1755 ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change, 1756 vmxnet3_media_status); 1757 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1758 ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1759 1760 return (0); 1761 } 1762 1763 static void 1764 vmxnet3_evintr(struct vmxnet3_softc *sc) 1765 { 1766 device_t dev; 1767 struct ifnet *ifp; 1768 struct vmxnet3_txq_shared *ts; 1769 struct vmxnet3_rxq_shared *rs; 1770 uint32_t event; 1771 int reset; 1772 1773 dev = sc->vmx_dev; 1774 ifp = sc->vmx_ifp; 1775 reset = 0; 1776 1777 VMXNET3_CORE_LOCK(sc); 1778 1779 /* Clear events. */ 1780 event = sc->vmx_ds->event; 1781 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 1782 1783 if (event & VMXNET3_EVENT_LINK) { 1784 vmxnet3_link_status(sc); 1785 if (sc->vmx_link_active != 0) 1786 vmxnet3_tx_start_all(sc); 1787 } 1788 1789 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 1790 reset = 1; 1791 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 1792 ts = sc->vmx_txq[0].vxtxq_ts; 1793 if (ts->stopped != 0) 1794 device_printf(dev, "Tx queue error %#x\n", ts->error); 1795 rs = sc->vmx_rxq[0].vxrxq_rs; 1796 if (rs->stopped != 0) 1797 device_printf(dev, "Rx queue error %#x\n", rs->error); 1798 device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 1799 } 1800 1801 if (event & VMXNET3_EVENT_DIC) 1802 device_printf(dev, "device implementation change event\n"); 1803 if (event & VMXNET3_EVENT_DEBUG) 1804 device_printf(dev, "debug event\n"); 1805 1806 if (reset != 0) { 1807 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1808 vmxnet3_init_locked(sc); 1809 } 1810 1811 VMXNET3_CORE_UNLOCK(sc); 1812 } 1813 1814 static void 1815 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq) 1816 { 1817 struct vmxnet3_softc *sc; 1818 struct ifnet *ifp; 1819 struct vmxnet3_txring *txr; 1820 struct vmxnet3_comp_ring *txc; 1821 struct vmxnet3_txcompdesc *txcd; 1822 struct vmxnet3_txbuf *txb; 1823 struct mbuf *m; 1824 u_int sop; 1825 1826 sc = txq->vxtxq_sc; 1827 ifp = sc->vmx_ifp; 1828 txr = &txq->vxtxq_cmd_ring; 1829 txc = &txq->vxtxq_comp_ring; 1830 1831 VMXNET3_TXQ_LOCK_ASSERT(txq); 1832 1833 for (;;) { 1834 txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 1835 if (txcd->gen != txc->vxcr_gen) 1836 break; 1837 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1838 1839 if (++txc->vxcr_next == txc->vxcr_ndesc) { 1840 txc->vxcr_next = 0; 1841 txc->vxcr_gen ^= 1; 1842 } 1843 1844 sop = txr->vxtxr_next; 1845 txb = &txr->vxtxr_txbuf[sop]; 1846 1847 if ((m = txb->vtxb_m) != NULL) { 1848 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 1849 BUS_DMASYNC_POSTWRITE); 1850 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 1851 1852 txq->vxtxq_stats.vmtxs_opackets++; 1853 txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len; 1854 if (m->m_flags & M_MCAST) 1855 txq->vxtxq_stats.vmtxs_omcasts++; 1856 1857 m_freem(m); 1858 txb->vtxb_m = NULL; 1859 } 1860 1861 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 1862 } 1863 1864 if (txr->vxtxr_head == txr->vxtxr_next) 1865 txq->vxtxq_watchdog = 0; 1866 } 1867 1868 static int 1869 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr) 1870 { 1871 struct ifnet *ifp; 1872 struct mbuf *m; 1873 struct vmxnet3_rxdesc *rxd; 1874 struct vmxnet3_rxbuf *rxb; 1875 bus_dma_tag_t tag; 1876 bus_dmamap_t dmap; 1877 bus_dma_segment_t segs[1]; 1878 int idx, clsize, btype, flags, nsegs, error; 1879 1880 ifp = sc->vmx_ifp; 1881 tag = rxr->vxrxr_rxtag; 1882 dmap = rxr->vxrxr_spare_dmap; 1883 idx = rxr->vxrxr_fill; 1884 rxd = &rxr->vxrxr_rxd[idx]; 1885 rxb = &rxr->vxrxr_rxbuf[idx]; 1886 1887 #ifdef VMXNET3_FAILPOINTS 1888 KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS); 1889 if (rxr->vxrxr_rid != 0) 1890 KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS); 1891 #endif 1892 1893 if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) { 1894 flags = M_PKTHDR; 1895 clsize = MCLBYTES; 1896 btype = VMXNET3_BTYPE_HEAD; 1897 } else { 1898 #if __FreeBSD_version < 902001 1899 /* 1900 * These mbufs will never be used for the start of a frame. 1901 * Roughly prior to branching releng/9.2, the load_mbuf_sg() 1902 * required the mbuf to always be a packet header. Avoid 1903 * unnecessary mbuf initialization in newer versions where 1904 * that is not the case. 1905 */ 1906 flags = M_PKTHDR; 1907 #else 1908 flags = 0; 1909 #endif 1910 clsize = MJUMPAGESIZE; 1911 btype = VMXNET3_BTYPE_BODY; 1912 } 1913 1914 m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize); 1915 if (m == NULL) { 1916 sc->vmx_stats.vmst_mgetcl_failed++; 1917 return (ENOBUFS); 1918 } 1919 1920 if (btype == VMXNET3_BTYPE_HEAD) { 1921 m->m_len = m->m_pkthdr.len = clsize; 1922 m_adj(m, ETHER_ALIGN); 1923 } else 1924 m->m_len = clsize; 1925 1926 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs, 1927 BUS_DMA_NOWAIT); 1928 if (error) { 1929 m_freem(m); 1930 sc->vmx_stats.vmst_mbuf_load_failed++; 1931 return (error); 1932 } 1933 KASSERT(nsegs == 1, 1934 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 1935 #if __FreeBSD_version < 902001 1936 if (btype == VMXNET3_BTYPE_BODY) 1937 m->m_flags &= ~M_PKTHDR; 1938 #endif 1939 1940 if (rxb->vrxb_m != NULL) { 1941 bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD); 1942 bus_dmamap_unload(tag, rxb->vrxb_dmamap); 1943 } 1944 1945 rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 1946 rxb->vrxb_dmamap = dmap; 1947 rxb->vrxb_m = m; 1948 1949 rxd->addr = segs[0].ds_addr; 1950 rxd->len = segs[0].ds_len; 1951 rxd->btype = btype; 1952 rxd->gen = rxr->vxrxr_gen; 1953 1954 vmxnet3_rxr_increment_fill(rxr); 1955 return (0); 1956 } 1957 1958 static void 1959 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 1960 struct vmxnet3_rxring *rxr, int idx) 1961 { 1962 struct vmxnet3_rxdesc *rxd; 1963 1964 rxd = &rxr->vxrxr_rxd[idx]; 1965 rxd->gen = rxr->vxrxr_gen; 1966 vmxnet3_rxr_increment_fill(rxr); 1967 } 1968 1969 static void 1970 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 1971 { 1972 struct vmxnet3_softc *sc; 1973 struct vmxnet3_rxring *rxr; 1974 struct vmxnet3_comp_ring *rxc; 1975 struct vmxnet3_rxcompdesc *rxcd; 1976 int idx, eof; 1977 1978 sc = rxq->vxrxq_sc; 1979 rxc = &rxq->vxrxq_comp_ring; 1980 1981 do { 1982 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 1983 if (rxcd->gen != rxc->vxcr_gen) 1984 break; /* Not expected. */ 1985 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1986 1987 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 1988 rxc->vxcr_next = 0; 1989 rxc->vxcr_gen ^= 1; 1990 } 1991 1992 idx = rxcd->rxd_idx; 1993 eof = rxcd->eop; 1994 if (rxcd->qid < sc->vmx_nrxqueues) 1995 rxr = &rxq->vxrxq_cmd_ring[0]; 1996 else 1997 rxr = &rxq->vxrxq_cmd_ring[1]; 1998 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 1999 } while (!eof); 2000 } 2001 2002 static void 2003 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2004 { 2005 2006 if (rxcd->ipv4) { 2007 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 2008 if (rxcd->ipcsum_ok) 2009 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2010 } 2011 2012 if (!rxcd->fragment) { 2013 if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) { 2014 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 2015 CSUM_PSEUDO_HDR; 2016 m->m_pkthdr.csum_data = 0xFFFF; 2017 } 2018 } 2019 } 2020 2021 static void 2022 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 2023 struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2024 { 2025 struct vmxnet3_softc *sc; 2026 struct ifnet *ifp; 2027 2028 sc = rxq->vxrxq_sc; 2029 ifp = sc->vmx_ifp; 2030 2031 if (rxcd->error) { 2032 rxq->vxrxq_stats.vmrxs_ierrors++; 2033 m_freem(m); 2034 return; 2035 } 2036 2037 #ifdef notyet 2038 switch (rxcd->rss_type) { 2039 case VMXNET3_RCD_RSS_TYPE_IPV4: 2040 m->m_pkthdr.flowid = rxcd->rss_hash; 2041 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4); 2042 break; 2043 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 2044 m->m_pkthdr.flowid = rxcd->rss_hash; 2045 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4); 2046 break; 2047 case VMXNET3_RCD_RSS_TYPE_IPV6: 2048 m->m_pkthdr.flowid = rxcd->rss_hash; 2049 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6); 2050 break; 2051 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 2052 m->m_pkthdr.flowid = rxcd->rss_hash; 2053 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6); 2054 break; 2055 default: /* VMXNET3_RCD_RSS_TYPE_NONE */ 2056 m->m_pkthdr.flowid = rxq->vxrxq_id; 2057 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2058 break; 2059 } 2060 #else 2061 m->m_pkthdr.flowid = rxq->vxrxq_id; 2062 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2063 #endif 2064 2065 if (!rxcd->no_csum) 2066 vmxnet3_rx_csum(rxcd, m); 2067 if (rxcd->vlan) { 2068 m->m_flags |= M_VLANTAG; 2069 m->m_pkthdr.ether_vtag = rxcd->vtag; 2070 } 2071 2072 rxq->vxrxq_stats.vmrxs_ipackets++; 2073 rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len; 2074 2075 VMXNET3_RXQ_UNLOCK(rxq); 2076 (*ifp->if_input)(ifp, m); 2077 VMXNET3_RXQ_LOCK(rxq); 2078 } 2079 2080 static void 2081 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq) 2082 { 2083 struct vmxnet3_softc *sc; 2084 struct ifnet *ifp; 2085 struct vmxnet3_rxring *rxr; 2086 struct vmxnet3_comp_ring *rxc; 2087 struct vmxnet3_rxdesc *rxd; 2088 struct vmxnet3_rxcompdesc *rxcd; 2089 struct mbuf *m, *m_head, *m_tail; 2090 int idx, length; 2091 2092 sc = rxq->vxrxq_sc; 2093 ifp = sc->vmx_ifp; 2094 rxc = &rxq->vxrxq_comp_ring; 2095 2096 VMXNET3_RXQ_LOCK_ASSERT(rxq); 2097 2098 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2099 return; 2100 2101 m_head = rxq->vxrxq_mhead; 2102 rxq->vxrxq_mhead = NULL; 2103 m_tail = rxq->vxrxq_mtail; 2104 rxq->vxrxq_mtail = NULL; 2105 MPASS(m_head == NULL || m_tail != NULL); 2106 2107 for (;;) { 2108 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2109 if (rxcd->gen != rxc->vxcr_gen) { 2110 rxq->vxrxq_mhead = m_head; 2111 rxq->vxrxq_mtail = m_tail; 2112 break; 2113 } 2114 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2115 2116 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2117 rxc->vxcr_next = 0; 2118 rxc->vxcr_gen ^= 1; 2119 } 2120 2121 idx = rxcd->rxd_idx; 2122 length = rxcd->len; 2123 if (rxcd->qid < sc->vmx_nrxqueues) 2124 rxr = &rxq->vxrxq_cmd_ring[0]; 2125 else 2126 rxr = &rxq->vxrxq_cmd_ring[1]; 2127 rxd = &rxr->vxrxr_rxd[idx]; 2128 2129 m = rxr->vxrxr_rxbuf[idx].vrxb_m; 2130 KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf", 2131 __func__, rxcd->qid, idx)); 2132 2133 /* 2134 * The host may skip descriptors. We detect this when this 2135 * descriptor does not match the previous fill index. Catch 2136 * up with the host now. 2137 */ 2138 if (__predict_false(rxr->vxrxr_fill != idx)) { 2139 while (rxr->vxrxr_fill != idx) { 2140 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 2141 rxr->vxrxr_gen; 2142 vmxnet3_rxr_increment_fill(rxr); 2143 } 2144 } 2145 2146 if (rxcd->sop) { 2147 KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD, 2148 ("%s: start of frame w/o head buffer", __func__)); 2149 KASSERT(rxr == &rxq->vxrxq_cmd_ring[0], 2150 ("%s: start of frame not in ring 0", __func__)); 2151 KASSERT((idx % sc->vmx_rx_max_chain) == 0, 2152 ("%s: start of frame at unexcepted index %d (%d)", 2153 __func__, idx, sc->vmx_rx_max_chain)); 2154 KASSERT(m_head == NULL, 2155 ("%s: duplicate start of frame?", __func__)); 2156 2157 if (length == 0) { 2158 /* Just ignore this descriptor. */ 2159 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2160 goto nextp; 2161 } 2162 2163 if (vmxnet3_newbuf(sc, rxr) != 0) { 2164 rxq->vxrxq_stats.vmrxs_iqdrops++; 2165 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2166 if (!rxcd->eop) 2167 vmxnet3_rxq_discard_chain(rxq); 2168 goto nextp; 2169 } 2170 2171 m->m_pkthdr.rcvif = ifp; 2172 m->m_pkthdr.len = m->m_len = length; 2173 m->m_pkthdr.csum_flags = 0; 2174 m_head = m_tail = m; 2175 2176 } else { 2177 KASSERT(rxd->btype == VMXNET3_BTYPE_BODY, 2178 ("%s: non start of frame w/o body buffer", __func__)); 2179 KASSERT(m_head != NULL, 2180 ("%s: frame not started?", __func__)); 2181 2182 if (vmxnet3_newbuf(sc, rxr) != 0) { 2183 rxq->vxrxq_stats.vmrxs_iqdrops++; 2184 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2185 if (!rxcd->eop) 2186 vmxnet3_rxq_discard_chain(rxq); 2187 m_freem(m_head); 2188 m_head = m_tail = NULL; 2189 goto nextp; 2190 } 2191 2192 m->m_len = length; 2193 m_head->m_pkthdr.len += length; 2194 m_tail->m_next = m; 2195 m_tail = m; 2196 } 2197 2198 if (rxcd->eop) { 2199 vmxnet3_rxq_input(rxq, rxcd, m_head); 2200 m_head = m_tail = NULL; 2201 2202 /* Must recheck after dropping the Rx lock. */ 2203 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2204 break; 2205 } 2206 2207 nextp: 2208 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 2209 int qid = rxcd->qid; 2210 bus_size_t r; 2211 2212 idx = (idx + 1) % rxr->vxrxr_ndesc; 2213 if (qid >= sc->vmx_nrxqueues) { 2214 qid -= sc->vmx_nrxqueues; 2215 r = VMXNET3_BAR0_RXH2(qid); 2216 } else 2217 r = VMXNET3_BAR0_RXH1(qid); 2218 vmxnet3_write_bar0(sc, r, idx); 2219 } 2220 } 2221 } 2222 2223 static void 2224 vmxnet3_legacy_intr(void *xsc) 2225 { 2226 struct vmxnet3_softc *sc; 2227 struct vmxnet3_rxqueue *rxq; 2228 struct vmxnet3_txqueue *txq; 2229 2230 sc = xsc; 2231 rxq = &sc->vmx_rxq[0]; 2232 txq = &sc->vmx_txq[0]; 2233 2234 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 2235 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 2236 return; 2237 } 2238 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2239 vmxnet3_disable_all_intrs(sc); 2240 2241 if (sc->vmx_ds->event != 0) 2242 vmxnet3_evintr(sc); 2243 2244 VMXNET3_RXQ_LOCK(rxq); 2245 vmxnet3_rxq_eof(rxq); 2246 VMXNET3_RXQ_UNLOCK(rxq); 2247 2248 VMXNET3_TXQ_LOCK(txq); 2249 vmxnet3_txq_eof(txq); 2250 vmxnet3_txq_start(txq); 2251 VMXNET3_TXQ_UNLOCK(txq); 2252 2253 vmxnet3_enable_all_intrs(sc); 2254 } 2255 2256 static void 2257 vmxnet3_txq_intr(void *xtxq) 2258 { 2259 struct vmxnet3_softc *sc; 2260 struct vmxnet3_txqueue *txq; 2261 2262 txq = xtxq; 2263 sc = txq->vxtxq_sc; 2264 2265 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2266 vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx); 2267 2268 VMXNET3_TXQ_LOCK(txq); 2269 vmxnet3_txq_eof(txq); 2270 vmxnet3_txq_start(txq); 2271 VMXNET3_TXQ_UNLOCK(txq); 2272 2273 vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx); 2274 } 2275 2276 static void 2277 vmxnet3_rxq_intr(void *xrxq) 2278 { 2279 struct vmxnet3_softc *sc; 2280 struct vmxnet3_rxqueue *rxq; 2281 2282 rxq = xrxq; 2283 sc = rxq->vxrxq_sc; 2284 2285 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2286 vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx); 2287 2288 VMXNET3_RXQ_LOCK(rxq); 2289 vmxnet3_rxq_eof(rxq); 2290 VMXNET3_RXQ_UNLOCK(rxq); 2291 2292 vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx); 2293 } 2294 2295 static void 2296 vmxnet3_event_intr(void *xsc) 2297 { 2298 struct vmxnet3_softc *sc; 2299 2300 sc = xsc; 2301 2302 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2303 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 2304 2305 if (sc->vmx_ds->event != 0) 2306 vmxnet3_evintr(sc); 2307 2308 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2309 } 2310 2311 static void 2312 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2313 { 2314 struct vmxnet3_txring *txr; 2315 struct vmxnet3_txbuf *txb; 2316 int i; 2317 2318 txr = &txq->vxtxq_cmd_ring; 2319 2320 for (i = 0; i < txr->vxtxr_ndesc; i++) { 2321 txb = &txr->vxtxr_txbuf[i]; 2322 2323 if (txb->vtxb_m == NULL) 2324 continue; 2325 2326 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 2327 BUS_DMASYNC_POSTWRITE); 2328 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 2329 m_freem(txb->vtxb_m); 2330 txb->vtxb_m = NULL; 2331 } 2332 } 2333 2334 static void 2335 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2336 { 2337 struct vmxnet3_rxring *rxr; 2338 struct vmxnet3_rxbuf *rxb; 2339 int i, j; 2340 2341 if (rxq->vxrxq_mhead != NULL) { 2342 m_freem(rxq->vxrxq_mhead); 2343 rxq->vxrxq_mhead = NULL; 2344 rxq->vxrxq_mtail = NULL; 2345 } 2346 2347 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2348 rxr = &rxq->vxrxq_cmd_ring[i]; 2349 2350 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2351 rxb = &rxr->vxrxr_rxbuf[j]; 2352 2353 if (rxb->vrxb_m == NULL) 2354 continue; 2355 2356 bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap, 2357 BUS_DMASYNC_POSTREAD); 2358 bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap); 2359 m_freem(rxb->vrxb_m); 2360 rxb->vrxb_m = NULL; 2361 } 2362 } 2363 } 2364 2365 static void 2366 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2367 { 2368 struct vmxnet3_rxqueue *rxq; 2369 struct vmxnet3_txqueue *txq; 2370 int i; 2371 2372 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2373 rxq = &sc->vmx_rxq[i]; 2374 VMXNET3_RXQ_LOCK(rxq); 2375 VMXNET3_RXQ_UNLOCK(rxq); 2376 } 2377 2378 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2379 txq = &sc->vmx_txq[i]; 2380 VMXNET3_TXQ_LOCK(txq); 2381 VMXNET3_TXQ_UNLOCK(txq); 2382 } 2383 } 2384 2385 static void 2386 vmxnet3_stop(struct vmxnet3_softc *sc) 2387 { 2388 struct ifnet *ifp; 2389 int q; 2390 2391 ifp = sc->vmx_ifp; 2392 VMXNET3_CORE_LOCK_ASSERT(sc); 2393 2394 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2395 sc->vmx_link_active = 0; 2396 callout_stop(&sc->vmx_tick); 2397 2398 /* Disable interrupts. */ 2399 vmxnet3_disable_all_intrs(sc); 2400 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2401 2402 vmxnet3_stop_rendezvous(sc); 2403 2404 for (q = 0; q < sc->vmx_ntxqueues; q++) 2405 vmxnet3_txstop(sc, &sc->vmx_txq[q]); 2406 for (q = 0; q < sc->vmx_nrxqueues; q++) 2407 vmxnet3_rxstop(sc, &sc->vmx_rxq[q]); 2408 2409 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2410 } 2411 2412 static void 2413 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2414 { 2415 struct vmxnet3_txring *txr; 2416 struct vmxnet3_comp_ring *txc; 2417 2418 txr = &txq->vxtxq_cmd_ring; 2419 txr->vxtxr_head = 0; 2420 txr->vxtxr_next = 0; 2421 txr->vxtxr_gen = VMXNET3_INIT_GEN; 2422 bzero(txr->vxtxr_txd, 2423 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2424 2425 txc = &txq->vxtxq_comp_ring; 2426 txc->vxcr_next = 0; 2427 txc->vxcr_gen = VMXNET3_INIT_GEN; 2428 bzero(txc->vxcr_u.txcd, 2429 txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2430 } 2431 2432 static int 2433 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2434 { 2435 struct ifnet *ifp; 2436 struct vmxnet3_rxring *rxr; 2437 struct vmxnet3_comp_ring *rxc; 2438 int i, populate, idx, frame_size, error; 2439 2440 ifp = sc->vmx_ifp; 2441 frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) + 2442 ifp->if_mtu; 2443 2444 /* 2445 * If the MTU causes us to exceed what a regular sized cluster can 2446 * handle, we allocate a second MJUMPAGESIZE cluster after it in 2447 * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters. 2448 * 2449 * Keep rx_max_chain a divisor of the maximum Rx ring size to make 2450 * our life easier. We do not support changing the ring size after 2451 * the attach. 2452 */ 2453 if (frame_size <= MCLBYTES) 2454 sc->vmx_rx_max_chain = 1; 2455 else 2456 sc->vmx_rx_max_chain = 2; 2457 2458 /* 2459 * Only populate ring 1 if the configuration will take advantage 2460 * of it. That is either when LRO is enabled or the frame size 2461 * exceeds what ring 0 can contain. 2462 */ 2463 if ((ifp->if_capenable & IFCAP_LRO) == 0 && 2464 frame_size <= MCLBYTES + MJUMPAGESIZE) 2465 populate = 1; 2466 else 2467 populate = VMXNET3_RXRINGS_PERQ; 2468 2469 for (i = 0; i < populate; i++) { 2470 rxr = &rxq->vxrxq_cmd_ring[i]; 2471 rxr->vxrxr_fill = 0; 2472 rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2473 bzero(rxr->vxrxr_rxd, 2474 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2475 2476 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2477 error = vmxnet3_newbuf(sc, rxr); 2478 if (error) 2479 return (error); 2480 } 2481 } 2482 2483 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2484 rxr = &rxq->vxrxq_cmd_ring[i]; 2485 rxr->vxrxr_fill = 0; 2486 rxr->vxrxr_gen = 0; 2487 bzero(rxr->vxrxr_rxd, 2488 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2489 } 2490 2491 rxc = &rxq->vxrxq_comp_ring; 2492 rxc->vxcr_next = 0; 2493 rxc->vxcr_gen = VMXNET3_INIT_GEN; 2494 bzero(rxc->vxcr_u.rxcd, 2495 rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2496 2497 return (0); 2498 } 2499 2500 static int 2501 vmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2502 { 2503 device_t dev; 2504 int q, error; 2505 2506 dev = sc->vmx_dev; 2507 2508 for (q = 0; q < sc->vmx_ntxqueues; q++) 2509 vmxnet3_txinit(sc, &sc->vmx_txq[q]); 2510 2511 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2512 error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); 2513 if (error) { 2514 device_printf(dev, "cannot populate Rx queue %d\n", q); 2515 return (error); 2516 } 2517 } 2518 2519 return (0); 2520 } 2521 2522 static int 2523 vmxnet3_enable_device(struct vmxnet3_softc *sc) 2524 { 2525 int q; 2526 2527 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2528 device_printf(sc->vmx_dev, "device enable command failed!\n"); 2529 return (1); 2530 } 2531 2532 /* Reset the Rx queue heads. */ 2533 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2534 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2535 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2536 } 2537 2538 return (0); 2539 } 2540 2541 static void 2542 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2543 { 2544 struct ifnet *ifp; 2545 2546 ifp = sc->vmx_ifp; 2547 2548 vmxnet3_set_rxfilter(sc); 2549 2550 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2551 bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter, 2552 sizeof(sc->vmx_ds->vlan_filter)); 2553 else 2554 bzero(sc->vmx_ds->vlan_filter, 2555 sizeof(sc->vmx_ds->vlan_filter)); 2556 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2557 } 2558 2559 static int 2560 vmxnet3_reinit(struct vmxnet3_softc *sc) 2561 { 2562 2563 vmxnet3_reinit_interface(sc); 2564 vmxnet3_reinit_shared_data(sc); 2565 2566 if (vmxnet3_reinit_queues(sc) != 0) 2567 return (ENXIO); 2568 2569 if (vmxnet3_enable_device(sc) != 0) 2570 return (ENXIO); 2571 2572 vmxnet3_reinit_rxfilters(sc); 2573 2574 return (0); 2575 } 2576 2577 static void 2578 vmxnet3_init_locked(struct vmxnet3_softc *sc) 2579 { 2580 struct ifnet *ifp; 2581 2582 ifp = sc->vmx_ifp; 2583 2584 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2585 return; 2586 2587 vmxnet3_stop(sc); 2588 2589 if (vmxnet3_reinit(sc) != 0) { 2590 vmxnet3_stop(sc); 2591 return; 2592 } 2593 2594 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2595 vmxnet3_link_status(sc); 2596 2597 vmxnet3_enable_all_intrs(sc); 2598 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2599 } 2600 2601 static void 2602 vmxnet3_init(void *xsc) 2603 { 2604 struct vmxnet3_softc *sc; 2605 2606 sc = xsc; 2607 2608 VMXNET3_CORE_LOCK(sc); 2609 vmxnet3_init_locked(sc); 2610 VMXNET3_CORE_UNLOCK(sc); 2611 } 2612 2613 /* 2614 * BMV: Much of this can go away once we finally have offsets in 2615 * the mbuf packet header. Bug andre@. 2616 */ 2617 static int 2618 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, 2619 int *etype, int *proto, int *start) 2620 { 2621 struct ether_vlan_header *evh; 2622 int offset; 2623 #if defined(INET) 2624 struct ip *ip = NULL; 2625 struct ip iphdr; 2626 #endif 2627 #if defined(INET6) 2628 struct ip6_hdr *ip6 = NULL; 2629 struct ip6_hdr ip6hdr; 2630 #endif 2631 2632 evh = mtod(m, struct ether_vlan_header *); 2633 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2634 /* BMV: We should handle nested VLAN tags too. */ 2635 *etype = ntohs(evh->evl_proto); 2636 offset = sizeof(struct ether_vlan_header); 2637 } else { 2638 *etype = ntohs(evh->evl_encap_proto); 2639 offset = sizeof(struct ether_header); 2640 } 2641 2642 switch (*etype) { 2643 #if defined(INET) 2644 case ETHERTYPE_IP: 2645 if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 2646 m_copydata(m, offset, sizeof(struct ip), 2647 (caddr_t) &iphdr); 2648 ip = &iphdr; 2649 } else 2650 ip = mtodo(m, offset); 2651 *proto = ip->ip_p; 2652 *start = offset + (ip->ip_hl << 2); 2653 break; 2654 #endif 2655 #if defined(INET6) 2656 case ETHERTYPE_IPV6: 2657 if (__predict_false(m->m_len < 2658 offset + sizeof(struct ip6_hdr))) { 2659 m_copydata(m, offset, sizeof(struct ip6_hdr), 2660 (caddr_t) &ip6hdr); 2661 ip6 = &ip6hdr; 2662 } else 2663 ip6 = mtodo(m, offset); 2664 *proto = -1; 2665 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 2666 /* Assert the network stack sent us a valid packet. */ 2667 KASSERT(*start > offset, 2668 ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 2669 *start, offset, *proto)); 2670 break; 2671 #endif 2672 default: 2673 return (EINVAL); 2674 } 2675 2676 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2677 struct tcphdr *tcp, tcphdr; 2678 uint16_t sum; 2679 2680 if (__predict_false(*proto != IPPROTO_TCP)) { 2681 /* Likely failed to correctly parse the mbuf. */ 2682 return (EINVAL); 2683 } 2684 2685 txq->vxtxq_stats.vmtxs_tso++; 2686 2687 switch (*etype) { 2688 #if defined(INET) 2689 case ETHERTYPE_IP: 2690 sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, 2691 htons(IPPROTO_TCP)); 2692 break; 2693 #endif 2694 #if defined(INET6) 2695 case ETHERTYPE_IPV6: 2696 sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 2697 break; 2698 #endif 2699 default: 2700 sum = 0; 2701 break; 2702 } 2703 2704 if (m->m_len < *start + sizeof(struct tcphdr)) { 2705 m_copyback(m, *start + offsetof(struct tcphdr, th_sum), 2706 sizeof(uint16_t), (caddr_t) &sum); 2707 m_copydata(m, *start, sizeof(struct tcphdr), 2708 (caddr_t) &tcphdr); 2709 tcp = &tcphdr; 2710 } else { 2711 tcp = mtodo(m, *start); 2712 tcp->th_sum = sum; 2713 } 2714 2715 /* 2716 * For TSO, the size of the protocol header is also 2717 * included in the descriptor header size. 2718 */ 2719 *start += (tcp->th_off << 2); 2720 } else 2721 txq->vxtxq_stats.vmtxs_csum++; 2722 2723 return (0); 2724 } 2725 2726 static int 2727 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 2728 bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs) 2729 { 2730 struct vmxnet3_txring *txr; 2731 struct mbuf *m; 2732 bus_dma_tag_t tag; 2733 int error; 2734 2735 txr = &txq->vxtxq_cmd_ring; 2736 m = *m0; 2737 tag = txr->vxtxr_txtag; 2738 2739 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2740 if (error == 0 || error != EFBIG) 2741 return (error); 2742 2743 m = m_defrag(m, M_NOWAIT); 2744 if (m != NULL) { 2745 *m0 = m; 2746 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2747 } else 2748 error = ENOBUFS; 2749 2750 if (error) { 2751 m_freem(*m0); 2752 *m0 = NULL; 2753 txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++; 2754 } else 2755 txq->vxtxq_sc->vmx_stats.vmst_defragged++; 2756 2757 return (error); 2758 } 2759 2760 static void 2761 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 2762 { 2763 struct vmxnet3_txring *txr; 2764 2765 txr = &txq->vxtxq_cmd_ring; 2766 bus_dmamap_unload(txr->vxtxr_txtag, dmap); 2767 } 2768 2769 static int 2770 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 2771 { 2772 struct vmxnet3_softc *sc; 2773 struct vmxnet3_txring *txr; 2774 struct vmxnet3_txdesc *txd, *sop; 2775 struct mbuf *m; 2776 bus_dmamap_t dmap; 2777 bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS]; 2778 int i, gen, nsegs, etype, proto, start, error; 2779 2780 sc = txq->vxtxq_sc; 2781 start = 0; 2782 txd = NULL; 2783 txr = &txq->vxtxq_cmd_ring; 2784 dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 2785 2786 error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs); 2787 if (error) 2788 return (error); 2789 2790 m = *m0; 2791 M_ASSERTPKTHDR(m); 2792 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS, 2793 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 2794 2795 if (VMXNET3_TXRING_AVAIL(txr) < nsegs) { 2796 txq->vxtxq_stats.vmtxs_full++; 2797 vmxnet3_txq_unload_mbuf(txq, dmap); 2798 return (ENOSPC); 2799 } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 2800 error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start); 2801 if (error) { 2802 txq->vxtxq_stats.vmtxs_offload_failed++; 2803 vmxnet3_txq_unload_mbuf(txq, dmap); 2804 m_freem(m); 2805 *m0 = NULL; 2806 return (error); 2807 } 2808 } 2809 2810 txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m; 2811 sop = &txr->vxtxr_txd[txr->vxtxr_head]; 2812 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 2813 2814 for (i = 0; i < nsegs; i++) { 2815 txd = &txr->vxtxr_txd[txr->vxtxr_head]; 2816 2817 txd->addr = segs[i].ds_addr; 2818 txd->len = segs[i].ds_len; 2819 txd->gen = gen; 2820 txd->dtype = 0; 2821 txd->offload_mode = VMXNET3_OM_NONE; 2822 txd->offload_pos = 0; 2823 txd->hlen = 0; 2824 txd->eop = 0; 2825 txd->compreq = 0; 2826 txd->vtag_mode = 0; 2827 txd->vtag = 0; 2828 2829 if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 2830 txr->vxtxr_head = 0; 2831 txr->vxtxr_gen ^= 1; 2832 } 2833 gen = txr->vxtxr_gen; 2834 } 2835 txd->eop = 1; 2836 txd->compreq = 1; 2837 2838 if (m->m_flags & M_VLANTAG) { 2839 sop->vtag_mode = 1; 2840 sop->vtag = m->m_pkthdr.ether_vtag; 2841 } 2842 2843 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2844 sop->offload_mode = VMXNET3_OM_TSO; 2845 sop->hlen = start; 2846 sop->offload_pos = m->m_pkthdr.tso_segsz; 2847 } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 2848 VMXNET3_CSUM_OFFLOAD_IPV6)) { 2849 sop->offload_mode = VMXNET3_OM_CSUM; 2850 sop->hlen = start; 2851 sop->offload_pos = start + m->m_pkthdr.csum_data; 2852 } 2853 2854 /* Finally, change the ownership. */ 2855 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 2856 sop->gen ^= 1; 2857 2858 txq->vxtxq_ts->npending += nsegs; 2859 if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 2860 txq->vxtxq_ts->npending = 0; 2861 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), 2862 txr->vxtxr_head); 2863 } 2864 2865 return (0); 2866 } 2867 2868 #ifdef VMXNET3_LEGACY_TX 2869 2870 static void 2871 vmxnet3_start_locked(struct ifnet *ifp) 2872 { 2873 struct vmxnet3_softc *sc; 2874 struct vmxnet3_txqueue *txq; 2875 struct vmxnet3_txring *txr; 2876 struct mbuf *m_head; 2877 int tx, avail; 2878 2879 sc = ifp->if_softc; 2880 txq = &sc->vmx_txq[0]; 2881 txr = &txq->vxtxq_cmd_ring; 2882 tx = 0; 2883 2884 VMXNET3_TXQ_LOCK_ASSERT(txq); 2885 2886 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2887 sc->vmx_link_active == 0) 2888 return; 2889 2890 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 2891 if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2) 2892 break; 2893 2894 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 2895 if (m_head == NULL) 2896 break; 2897 2898 /* Assume worse case if this mbuf is the head of a chain. */ 2899 if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2900 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2901 break; 2902 } 2903 2904 if (vmxnet3_txq_encap(txq, &m_head) != 0) { 2905 if (m_head != NULL) 2906 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2907 break; 2908 } 2909 2910 tx++; 2911 ETHER_BPF_MTAP(ifp, m_head); 2912 } 2913 2914 if (tx > 0) 2915 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 2916 } 2917 2918 static void 2919 vmxnet3_start(struct ifnet *ifp) 2920 { 2921 struct vmxnet3_softc *sc; 2922 struct vmxnet3_txqueue *txq; 2923 2924 sc = ifp->if_softc; 2925 txq = &sc->vmx_txq[0]; 2926 2927 VMXNET3_TXQ_LOCK(txq); 2928 vmxnet3_start_locked(ifp); 2929 VMXNET3_TXQ_UNLOCK(txq); 2930 } 2931 2932 #else /* !VMXNET3_LEGACY_TX */ 2933 2934 static int 2935 vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m) 2936 { 2937 struct vmxnet3_softc *sc; 2938 struct vmxnet3_txring *txr; 2939 struct buf_ring *br; 2940 struct ifnet *ifp; 2941 int tx, avail, error; 2942 2943 sc = txq->vxtxq_sc; 2944 br = txq->vxtxq_br; 2945 ifp = sc->vmx_ifp; 2946 txr = &txq->vxtxq_cmd_ring; 2947 tx = 0; 2948 error = 0; 2949 2950 VMXNET3_TXQ_LOCK_ASSERT(txq); 2951 2952 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2953 sc->vmx_link_active == 0) { 2954 if (m != NULL) 2955 error = drbr_enqueue(ifp, br, m); 2956 return (error); 2957 } 2958 2959 if (m != NULL) { 2960 error = drbr_enqueue(ifp, br, m); 2961 if (error) 2962 return (error); 2963 } 2964 2965 while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) { 2966 m = drbr_peek(ifp, br); 2967 if (m == NULL) 2968 break; 2969 2970 /* Assume worse case if this mbuf is the head of a chain. */ 2971 if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2972 drbr_putback(ifp, br, m); 2973 break; 2974 } 2975 2976 if (vmxnet3_txq_encap(txq, &m) != 0) { 2977 if (m != NULL) 2978 drbr_putback(ifp, br, m); 2979 else 2980 drbr_advance(ifp, br); 2981 break; 2982 } 2983 drbr_advance(ifp, br); 2984 2985 tx++; 2986 ETHER_BPF_MTAP(ifp, m); 2987 } 2988 2989 if (tx > 0) 2990 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 2991 2992 return (0); 2993 } 2994 2995 static int 2996 vmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m) 2997 { 2998 struct vmxnet3_softc *sc; 2999 struct vmxnet3_txqueue *txq; 3000 int i, ntxq, error; 3001 3002 sc = ifp->if_softc; 3003 ntxq = sc->vmx_ntxqueues; 3004 3005 /* check if flowid is set */ 3006 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 3007 i = m->m_pkthdr.flowid % ntxq; 3008 else 3009 i = curcpu % ntxq; 3010 3011 txq = &sc->vmx_txq[i]; 3012 3013 if (VMXNET3_TXQ_TRYLOCK(txq) != 0) { 3014 error = vmxnet3_txq_mq_start_locked(txq, m); 3015 VMXNET3_TXQ_UNLOCK(txq); 3016 } else { 3017 error = drbr_enqueue(ifp, txq->vxtxq_br, m); 3018 taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask); 3019 } 3020 3021 return (error); 3022 } 3023 3024 static void 3025 vmxnet3_txq_tq_deferred(void *xtxq, int pending) 3026 { 3027 struct vmxnet3_softc *sc; 3028 struct vmxnet3_txqueue *txq; 3029 3030 txq = xtxq; 3031 sc = txq->vxtxq_sc; 3032 3033 VMXNET3_TXQ_LOCK(txq); 3034 if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br)) 3035 vmxnet3_txq_mq_start_locked(txq, NULL); 3036 VMXNET3_TXQ_UNLOCK(txq); 3037 } 3038 3039 #endif /* VMXNET3_LEGACY_TX */ 3040 3041 static void 3042 vmxnet3_txq_start(struct vmxnet3_txqueue *txq) 3043 { 3044 struct vmxnet3_softc *sc; 3045 struct ifnet *ifp; 3046 3047 sc = txq->vxtxq_sc; 3048 ifp = sc->vmx_ifp; 3049 3050 #ifdef VMXNET3_LEGACY_TX 3051 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 3052 vmxnet3_start_locked(ifp); 3053 #else 3054 if (!drbr_empty(ifp, txq->vxtxq_br)) 3055 vmxnet3_txq_mq_start_locked(txq, NULL); 3056 #endif 3057 } 3058 3059 static void 3060 vmxnet3_tx_start_all(struct vmxnet3_softc *sc) 3061 { 3062 struct vmxnet3_txqueue *txq; 3063 int i; 3064 3065 VMXNET3_CORE_LOCK_ASSERT(sc); 3066 3067 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3068 txq = &sc->vmx_txq[i]; 3069 3070 VMXNET3_TXQ_LOCK(txq); 3071 vmxnet3_txq_start(txq); 3072 VMXNET3_TXQ_UNLOCK(txq); 3073 } 3074 } 3075 3076 static void 3077 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag) 3078 { 3079 struct ifnet *ifp; 3080 int idx, bit; 3081 3082 ifp = sc->vmx_ifp; 3083 idx = (tag >> 5) & 0x7F; 3084 bit = tag & 0x1F; 3085 3086 if (tag == 0 || tag > 4095) 3087 return; 3088 3089 VMXNET3_CORE_LOCK(sc); 3090 3091 /* Update our private VLAN bitvector. */ 3092 if (add) 3093 sc->vmx_vlan_filter[idx] |= (1 << bit); 3094 else 3095 sc->vmx_vlan_filter[idx] &= ~(1 << bit); 3096 3097 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3098 if (add) 3099 sc->vmx_ds->vlan_filter[idx] |= (1 << bit); 3100 else 3101 sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit); 3102 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 3103 } 3104 3105 VMXNET3_CORE_UNLOCK(sc); 3106 } 3107 3108 static void 3109 vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3110 { 3111 3112 if (ifp->if_softc == arg) 3113 vmxnet3_update_vlan_filter(arg, 1, tag); 3114 } 3115 3116 static void 3117 vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3118 { 3119 3120 if (ifp->if_softc == arg) 3121 vmxnet3_update_vlan_filter(arg, 0, tag); 3122 } 3123 3124 static void 3125 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 3126 { 3127 struct ifnet *ifp; 3128 struct vmxnet3_driver_shared *ds; 3129 struct ifmultiaddr *ifma; 3130 u_int mode; 3131 3132 ifp = sc->vmx_ifp; 3133 ds = sc->vmx_ds; 3134 3135 mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST; 3136 if (ifp->if_flags & IFF_PROMISC) 3137 mode |= VMXNET3_RXMODE_PROMISC; 3138 if (ifp->if_flags & IFF_ALLMULTI) 3139 mode |= VMXNET3_RXMODE_ALLMULTI; 3140 else { 3141 int cnt = 0, overflow = 0; 3142 3143 if_maddr_rlock(ifp); 3144 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3145 if (ifma->ifma_addr->sa_family != AF_LINK) 3146 continue; 3147 else if (cnt == VMXNET3_MULTICAST_MAX) { 3148 overflow = 1; 3149 break; 3150 } 3151 3152 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 3153 &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN); 3154 cnt++; 3155 } 3156 if_maddr_runlock(ifp); 3157 3158 if (overflow != 0) { 3159 cnt = 0; 3160 mode |= VMXNET3_RXMODE_ALLMULTI; 3161 } else if (cnt > 0) 3162 mode |= VMXNET3_RXMODE_MCAST; 3163 ds->mcast_tablelen = cnt * ETHER_ADDR_LEN; 3164 } 3165 3166 ds->rxmode = mode; 3167 3168 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 3169 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 3170 } 3171 3172 static int 3173 vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) 3174 { 3175 struct ifnet *ifp; 3176 3177 ifp = sc->vmx_ifp; 3178 3179 if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) 3180 return (EINVAL); 3181 3182 ifp->if_mtu = mtu; 3183 3184 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 3185 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3186 vmxnet3_init_locked(sc); 3187 } 3188 3189 return (0); 3190 } 3191 3192 static int 3193 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 3194 { 3195 struct vmxnet3_softc *sc; 3196 struct ifreq *ifr; 3197 int reinit, mask, error; 3198 3199 sc = ifp->if_softc; 3200 ifr = (struct ifreq *) data; 3201 error = 0; 3202 3203 switch (cmd) { 3204 case SIOCSIFMTU: 3205 if (ifp->if_mtu != ifr->ifr_mtu) { 3206 VMXNET3_CORE_LOCK(sc); 3207 error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); 3208 VMXNET3_CORE_UNLOCK(sc); 3209 } 3210 break; 3211 3212 case SIOCSIFFLAGS: 3213 VMXNET3_CORE_LOCK(sc); 3214 if (ifp->if_flags & IFF_UP) { 3215 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3216 if ((ifp->if_flags ^ sc->vmx_if_flags) & 3217 (IFF_PROMISC | IFF_ALLMULTI)) { 3218 vmxnet3_set_rxfilter(sc); 3219 } 3220 } else 3221 vmxnet3_init_locked(sc); 3222 } else { 3223 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3224 vmxnet3_stop(sc); 3225 } 3226 sc->vmx_if_flags = ifp->if_flags; 3227 VMXNET3_CORE_UNLOCK(sc); 3228 break; 3229 3230 case SIOCADDMULTI: 3231 case SIOCDELMULTI: 3232 VMXNET3_CORE_LOCK(sc); 3233 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3234 vmxnet3_set_rxfilter(sc); 3235 VMXNET3_CORE_UNLOCK(sc); 3236 break; 3237 3238 case SIOCSIFMEDIA: 3239 case SIOCGIFMEDIA: 3240 error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd); 3241 break; 3242 3243 case SIOCSIFCAP: 3244 VMXNET3_CORE_LOCK(sc); 3245 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 3246 3247 if (mask & IFCAP_TXCSUM) 3248 ifp->if_capenable ^= IFCAP_TXCSUM; 3249 if (mask & IFCAP_TXCSUM_IPV6) 3250 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 3251 if (mask & IFCAP_TSO4) 3252 ifp->if_capenable ^= IFCAP_TSO4; 3253 if (mask & IFCAP_TSO6) 3254 ifp->if_capenable ^= IFCAP_TSO6; 3255 3256 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO | 3257 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER)) { 3258 /* Changing these features requires us to reinit. */ 3259 reinit = 1; 3260 3261 if (mask & IFCAP_RXCSUM) 3262 ifp->if_capenable ^= IFCAP_RXCSUM; 3263 if (mask & IFCAP_RXCSUM_IPV6) 3264 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 3265 if (mask & IFCAP_LRO) 3266 ifp->if_capenable ^= IFCAP_LRO; 3267 if (mask & IFCAP_VLAN_HWTAGGING) 3268 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 3269 if (mask & IFCAP_VLAN_HWFILTER) 3270 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 3271 } else 3272 reinit = 0; 3273 3274 if (mask & IFCAP_VLAN_HWTSO) 3275 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 3276 3277 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3278 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3279 vmxnet3_init_locked(sc); 3280 } 3281 3282 VMXNET3_CORE_UNLOCK(sc); 3283 VLAN_CAPABILITIES(ifp); 3284 break; 3285 3286 default: 3287 error = ether_ioctl(ifp, cmd, data); 3288 break; 3289 } 3290 3291 VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc); 3292 3293 return (error); 3294 } 3295 3296 #ifndef VMXNET3_LEGACY_TX 3297 static void 3298 vmxnet3_qflush(struct ifnet *ifp) 3299 { 3300 struct vmxnet3_softc *sc; 3301 struct vmxnet3_txqueue *txq; 3302 struct mbuf *m; 3303 int i; 3304 3305 sc = ifp->if_softc; 3306 3307 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3308 txq = &sc->vmx_txq[i]; 3309 3310 VMXNET3_TXQ_LOCK(txq); 3311 while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL) 3312 m_freem(m); 3313 VMXNET3_TXQ_UNLOCK(txq); 3314 } 3315 3316 if_qflush(ifp); 3317 } 3318 #endif 3319 3320 static int 3321 vmxnet3_watchdog(struct vmxnet3_txqueue *txq) 3322 { 3323 struct vmxnet3_softc *sc; 3324 3325 sc = txq->vxtxq_sc; 3326 3327 VMXNET3_TXQ_LOCK(txq); 3328 if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 3329 VMXNET3_TXQ_UNLOCK(txq); 3330 return (0); 3331 } 3332 VMXNET3_TXQ_UNLOCK(txq); 3333 3334 if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n", 3335 txq->vxtxq_id); 3336 return (1); 3337 } 3338 3339 static void 3340 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) 3341 { 3342 3343 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 3344 } 3345 3346 static uint64_t 3347 vmxnet3_get_counter(struct ifnet *ifp, ift_counter cnt) 3348 { 3349 struct vmxnet3_softc *sc; 3350 uint64_t rv; 3351 3352 sc = if_getsoftc(ifp); 3353 rv = 0; 3354 3355 /* 3356 * With the exception of if_ierrors, these ifnet statistics are 3357 * only updated in the driver, so just set them to our accumulated 3358 * values. if_ierrors is updated in ether_input() for malformed 3359 * frames that we should have already discarded. 3360 */ 3361 switch (cnt) { 3362 case IFCOUNTER_IPACKETS: 3363 for (int i = 0; i < sc->vmx_nrxqueues; i++) 3364 rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_ipackets; 3365 return (rv); 3366 case IFCOUNTER_IQDROPS: 3367 for (int i = 0; i < sc->vmx_nrxqueues; i++) 3368 rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_iqdrops; 3369 return (rv); 3370 case IFCOUNTER_IERRORS: 3371 for (int i = 0; i < sc->vmx_nrxqueues; i++) 3372 rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_ierrors; 3373 return (rv); 3374 case IFCOUNTER_OPACKETS: 3375 for (int i = 0; i < sc->vmx_ntxqueues; i++) 3376 rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_opackets; 3377 return (rv); 3378 #ifndef VMXNET3_LEGACY_TX 3379 case IFCOUNTER_OBYTES: 3380 for (int i = 0; i < sc->vmx_ntxqueues; i++) 3381 rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_obytes; 3382 return (rv); 3383 case IFCOUNTER_OMCASTS: 3384 for (int i = 0; i < sc->vmx_ntxqueues; i++) 3385 rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_omcasts; 3386 return (rv); 3387 #endif 3388 default: 3389 return (if_get_counter_default(ifp, cnt)); 3390 } 3391 } 3392 3393 static void 3394 vmxnet3_tick(void *xsc) 3395 { 3396 struct vmxnet3_softc *sc; 3397 struct ifnet *ifp; 3398 int i, timedout; 3399 3400 sc = xsc; 3401 ifp = sc->vmx_ifp; 3402 timedout = 0; 3403 3404 VMXNET3_CORE_LOCK_ASSERT(sc); 3405 3406 vmxnet3_refresh_host_stats(sc); 3407 3408 for (i = 0; i < sc->vmx_ntxqueues; i++) 3409 timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]); 3410 3411 if (timedout != 0) { 3412 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3413 vmxnet3_init_locked(sc); 3414 } else 3415 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 3416 } 3417 3418 static int 3419 vmxnet3_link_is_up(struct vmxnet3_softc *sc) 3420 { 3421 uint32_t status; 3422 3423 /* Also update the link speed while here. */ 3424 status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 3425 sc->vmx_link_speed = status >> 16; 3426 return !!(status & 0x1); 3427 } 3428 3429 static void 3430 vmxnet3_link_status(struct vmxnet3_softc *sc) 3431 { 3432 struct ifnet *ifp; 3433 int link; 3434 3435 ifp = sc->vmx_ifp; 3436 link = vmxnet3_link_is_up(sc); 3437 3438 if (link != 0 && sc->vmx_link_active == 0) { 3439 sc->vmx_link_active = 1; 3440 if_link_state_change(ifp, LINK_STATE_UP); 3441 } else if (link == 0 && sc->vmx_link_active != 0) { 3442 sc->vmx_link_active = 0; 3443 if_link_state_change(ifp, LINK_STATE_DOWN); 3444 } 3445 } 3446 3447 static void 3448 vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 3449 { 3450 struct vmxnet3_softc *sc; 3451 3452 sc = ifp->if_softc; 3453 3454 ifmr->ifm_active = IFM_ETHER | IFM_AUTO; 3455 ifmr->ifm_status = IFM_AVALID; 3456 3457 VMXNET3_CORE_LOCK(sc); 3458 if (vmxnet3_link_is_up(sc) != 0) 3459 ifmr->ifm_status |= IFM_ACTIVE; 3460 else 3461 ifmr->ifm_status |= IFM_NONE; 3462 VMXNET3_CORE_UNLOCK(sc); 3463 } 3464 3465 static int 3466 vmxnet3_media_change(struct ifnet *ifp) 3467 { 3468 3469 /* Ignore. */ 3470 return (0); 3471 } 3472 3473 static void 3474 vmxnet3_set_lladdr(struct vmxnet3_softc *sc) 3475 { 3476 uint32_t ml, mh; 3477 3478 ml = sc->vmx_lladdr[0]; 3479 ml |= sc->vmx_lladdr[1] << 8; 3480 ml |= sc->vmx_lladdr[2] << 16; 3481 ml |= sc->vmx_lladdr[3] << 24; 3482 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 3483 3484 mh = sc->vmx_lladdr[4]; 3485 mh |= sc->vmx_lladdr[5] << 8; 3486 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 3487 } 3488 3489 static void 3490 vmxnet3_get_lladdr(struct vmxnet3_softc *sc) 3491 { 3492 uint32_t ml, mh; 3493 3494 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 3495 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 3496 3497 sc->vmx_lladdr[0] = ml; 3498 sc->vmx_lladdr[1] = ml >> 8; 3499 sc->vmx_lladdr[2] = ml >> 16; 3500 sc->vmx_lladdr[3] = ml >> 24; 3501 sc->vmx_lladdr[4] = mh; 3502 sc->vmx_lladdr[5] = mh >> 8; 3503 } 3504 3505 static void 3506 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq, 3507 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3508 { 3509 struct sysctl_oid *node, *txsnode; 3510 struct sysctl_oid_list *list, *txslist; 3511 struct vmxnet3_txq_stats *stats; 3512 struct UPT1_TxStats *txstats; 3513 char namebuf[16]; 3514 3515 stats = &txq->vxtxq_stats; 3516 txstats = &txq->vxtxq_ts->stats; 3517 3518 snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id); 3519 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3520 NULL, "Transmit Queue"); 3521 txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node); 3522 3523 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, 3524 &stats->vmtxs_opackets, "Transmit packets"); 3525 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, 3526 &stats->vmtxs_obytes, "Transmit bytes"); 3527 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, 3528 &stats->vmtxs_omcasts, "Transmit multicasts"); 3529 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3530 &stats->vmtxs_csum, "Transmit checksum offloaded"); 3531 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, 3532 &stats->vmtxs_tso, "Transmit TCP segmentation offloaded"); 3533 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD, 3534 &stats->vmtxs_full, "Transmit ring full"); 3535 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD, 3536 &stats->vmtxs_offload_failed, "Transmit checksum offload failed"); 3537 3538 /* 3539 * Add statistics reported by the host. These are updated once 3540 * per second. 3541 */ 3542 txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3543 NULL, "Host Statistics"); 3544 txslist = SYSCTL_CHILDREN(txsnode); 3545 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD, 3546 &txstats->TSO_packets, "TSO packets"); 3547 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD, 3548 &txstats->TSO_bytes, "TSO bytes"); 3549 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3550 &txstats->ucast_packets, "Unicast packets"); 3551 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3552 &txstats->ucast_bytes, "Unicast bytes"); 3553 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3554 &txstats->mcast_packets, "Multicast packets"); 3555 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3556 &txstats->mcast_bytes, "Multicast bytes"); 3557 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD, 3558 &txstats->error, "Errors"); 3559 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD, 3560 &txstats->discard, "Discards"); 3561 } 3562 3563 static void 3564 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq, 3565 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3566 { 3567 struct sysctl_oid *node, *rxsnode; 3568 struct sysctl_oid_list *list, *rxslist; 3569 struct vmxnet3_rxq_stats *stats; 3570 struct UPT1_RxStats *rxstats; 3571 char namebuf[16]; 3572 3573 stats = &rxq->vxrxq_stats; 3574 rxstats = &rxq->vxrxq_rs->stats; 3575 3576 snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id); 3577 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3578 NULL, "Receive Queue"); 3579 rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node); 3580 3581 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, 3582 &stats->vmrxs_ipackets, "Receive packets"); 3583 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, 3584 &stats->vmrxs_ibytes, "Receive bytes"); 3585 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, 3586 &stats->vmrxs_iqdrops, "Receive drops"); 3587 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, 3588 &stats->vmrxs_ierrors, "Receive errors"); 3589 3590 /* 3591 * Add statistics reported by the host. These are updated once 3592 * per second. 3593 */ 3594 rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3595 NULL, "Host Statistics"); 3596 rxslist = SYSCTL_CHILDREN(rxsnode); 3597 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD, 3598 &rxstats->LRO_packets, "LRO packets"); 3599 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD, 3600 &rxstats->LRO_bytes, "LRO bytes"); 3601 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3602 &rxstats->ucast_packets, "Unicast packets"); 3603 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3604 &rxstats->ucast_bytes, "Unicast bytes"); 3605 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3606 &rxstats->mcast_packets, "Multicast packets"); 3607 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3608 &rxstats->mcast_bytes, "Multicast bytes"); 3609 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD, 3610 &rxstats->bcast_packets, "Broadcast packets"); 3611 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD, 3612 &rxstats->bcast_bytes, "Broadcast bytes"); 3613 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD, 3614 &rxstats->nobuffer, "No buffer"); 3615 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD, 3616 &rxstats->error, "Errors"); 3617 } 3618 3619 static void 3620 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc, 3621 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3622 { 3623 struct sysctl_oid *node; 3624 struct sysctl_oid_list *list; 3625 int i; 3626 3627 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3628 struct vmxnet3_txqueue *txq = &sc->vmx_txq[i]; 3629 3630 node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO, 3631 "debug", CTLFLAG_RD, NULL, ""); 3632 list = SYSCTL_CHILDREN(node); 3633 3634 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD, 3635 &txq->vxtxq_cmd_ring.vxtxr_head, 0, ""); 3636 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD, 3637 &txq->vxtxq_cmd_ring.vxtxr_next, 0, ""); 3638 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD, 3639 &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, ""); 3640 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD, 3641 &txq->vxtxq_cmd_ring.vxtxr_gen, 0, ""); 3642 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3643 &txq->vxtxq_comp_ring.vxcr_next, 0, ""); 3644 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3645 &txq->vxtxq_comp_ring.vxcr_ndesc, 0,""); 3646 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3647 &txq->vxtxq_comp_ring.vxcr_gen, 0, ""); 3648 } 3649 3650 for (i = 0; i < sc->vmx_nrxqueues; i++) { 3651 struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i]; 3652 3653 node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO, 3654 "debug", CTLFLAG_RD, NULL, ""); 3655 list = SYSCTL_CHILDREN(node); 3656 3657 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD, 3658 &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, ""); 3659 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD, 3660 &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, ""); 3661 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD, 3662 &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, ""); 3663 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD, 3664 &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, ""); 3665 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD, 3666 &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, ""); 3667 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD, 3668 &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, ""); 3669 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3670 &rxq->vxrxq_comp_ring.vxcr_next, 0, ""); 3671 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3672 &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,""); 3673 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3674 &rxq->vxrxq_comp_ring.vxcr_gen, 0, ""); 3675 } 3676 } 3677 3678 static void 3679 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc, 3680 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3681 { 3682 int i; 3683 3684 for (i = 0; i < sc->vmx_ntxqueues; i++) 3685 vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child); 3686 for (i = 0; i < sc->vmx_nrxqueues; i++) 3687 vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child); 3688 3689 vmxnet3_setup_debug_sysctl(sc, ctx, child); 3690 } 3691 3692 static void 3693 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 3694 { 3695 device_t dev; 3696 struct vmxnet3_statistics *stats; 3697 struct sysctl_ctx_list *ctx; 3698 struct sysctl_oid *tree; 3699 struct sysctl_oid_list *child; 3700 3701 dev = sc->vmx_dev; 3702 ctx = device_get_sysctl_ctx(dev); 3703 tree = device_get_sysctl_tree(dev); 3704 child = SYSCTL_CHILDREN(tree); 3705 3706 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD, 3707 &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues"); 3708 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD, 3709 &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues"); 3710 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD, 3711 &sc->vmx_ntxqueues, 0, "Number of Tx queues"); 3712 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD, 3713 &sc->vmx_nrxqueues, 0, "Number of Rx queues"); 3714 3715 stats = &sc->vmx_stats; 3716 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD, 3717 &stats->vmst_defragged, 0, "Tx mbuf chains defragged"); 3718 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD, 3719 &stats->vmst_defrag_failed, 0, 3720 "Tx mbuf dropped because defrag failed"); 3721 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD, 3722 &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed"); 3723 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD, 3724 &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed"); 3725 3726 vmxnet3_setup_queue_sysctl(sc, ctx, child); 3727 } 3728 3729 static void 3730 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3731 { 3732 3733 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 3734 } 3735 3736 static uint32_t 3737 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 3738 { 3739 3740 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 3741 } 3742 3743 static void 3744 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3745 { 3746 3747 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 3748 } 3749 3750 static void 3751 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3752 { 3753 3754 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 3755 } 3756 3757 static uint32_t 3758 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3759 { 3760 3761 vmxnet3_write_cmd(sc, cmd); 3762 bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0, 3763 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 3764 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 3765 } 3766 3767 static void 3768 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 3769 { 3770 3771 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 3772 } 3773 3774 static void 3775 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 3776 { 3777 3778 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 3779 } 3780 3781 static void 3782 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3783 { 3784 int i; 3785 3786 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3787 for (i = 0; i < sc->vmx_nintrs; i++) 3788 vmxnet3_enable_intr(sc, i); 3789 } 3790 3791 static void 3792 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3793 { 3794 int i; 3795 3796 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3797 for (i = 0; i < sc->vmx_nintrs; i++) 3798 vmxnet3_disable_intr(sc, i); 3799 } 3800 3801 static void 3802 vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3803 { 3804 bus_addr_t *baddr = arg; 3805 3806 if (error == 0) 3807 *baddr = segs->ds_addr; 3808 } 3809 3810 static int 3811 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3812 struct vmxnet3_dma_alloc *dma) 3813 { 3814 device_t dev; 3815 int error; 3816 3817 dev = sc->vmx_dev; 3818 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3819 3820 error = bus_dma_tag_create(bus_get_dma_tag(dev), 3821 align, 0, /* alignment, bounds */ 3822 BUS_SPACE_MAXADDR, /* lowaddr */ 3823 BUS_SPACE_MAXADDR, /* highaddr */ 3824 NULL, NULL, /* filter, filterarg */ 3825 size, /* maxsize */ 3826 1, /* nsegments */ 3827 size, /* maxsegsize */ 3828 BUS_DMA_ALLOCNOW, /* flags */ 3829 NULL, /* lockfunc */ 3830 NULL, /* lockfuncarg */ 3831 &dma->dma_tag); 3832 if (error) { 3833 device_printf(dev, "bus_dma_tag_create failed: %d\n", error); 3834 goto fail; 3835 } 3836 3837 error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 3838 BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); 3839 if (error) { 3840 device_printf(dev, "bus_dmamem_alloc failed: %d\n", error); 3841 goto fail; 3842 } 3843 3844 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3845 size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); 3846 if (error) { 3847 device_printf(dev, "bus_dmamap_load failed: %d\n", error); 3848 goto fail; 3849 } 3850 3851 dma->dma_size = size; 3852 3853 fail: 3854 if (error) 3855 vmxnet3_dma_free(sc, dma); 3856 3857 return (error); 3858 } 3859 3860 static void 3861 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 3862 { 3863 3864 if (dma->dma_tag != NULL) { 3865 if (dma->dma_paddr != 0) { 3866 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3867 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3868 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3869 } 3870 3871 if (dma->dma_vaddr != NULL) { 3872 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, 3873 dma->dma_map); 3874 } 3875 3876 bus_dma_tag_destroy(dma->dma_tag); 3877 } 3878 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3879 } 3880 3881 static int 3882 vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def) 3883 { 3884 char path[64]; 3885 3886 snprintf(path, sizeof(path), 3887 "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob); 3888 TUNABLE_INT_FETCH(path, &def); 3889 3890 return (def); 3891 } 3892 3893 /* 3894 * Since this is a purely paravirtualized device, we do not have 3895 * to worry about DMA coherency. But at times, we must make sure 3896 * both the compiler and CPU do not reorder memory operations. 3897 */ 3898 static inline void 3899 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 3900 { 3901 3902 switch (type) { 3903 case VMXNET3_BARRIER_RD: 3904 rmb(); 3905 break; 3906 case VMXNET3_BARRIER_WR: 3907 wmb(); 3908 break; 3909 case VMXNET3_BARRIER_RDWR: 3910 mb(); 3911 break; 3912 default: 3913 panic("%s: bad barrier type %d", __func__, type); 3914 } 3915 } 3916