1 /*- 2 * Copyright (c) 2013 Tsubai Masanari 3 * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org> 4 * 5 * Permission to use, copy, modify, and distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 * 17 * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $ 18 * $FreeBSD: head/sys/dev/vmware/vmxnet3/if_vmx.c 318867 2017-05-25 10:49:56Z avg $ 19 */ 20 21 /* Driver for VMware vmxnet3 virtual ethernet devices. */ 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/device.h> 26 #include <sys/eventhandler.h> 27 #include <sys/kernel.h> 28 #include <sys/endian.h> 29 #include <sys/sockio.h> 30 #include <sys/mbuf.h> 31 #include <sys/malloc.h> 32 #include <sys/module.h> 33 #include <sys/socket.h> 34 #include <sys/sysctl.h> 35 #include <sys/taskqueue.h> 36 #include <vm/vm.h> 37 #include <vm/pmap.h> 38 39 #include <net/ethernet.h> 40 #include <net/if.h> 41 #include <net/if_var.h> 42 #include <net/ifq_var.h> 43 #include <net/if_arp.h> 44 #include <net/if_dl.h> 45 #include <net/if_types.h> 46 #include <net/if_media.h> 47 #include <net/vlan/if_vlan_ether.h> 48 #include <net/vlan/if_vlan_var.h> 49 50 #include <net/bpf.h> 51 52 #include <netinet/in_systm.h> 53 #include <netinet/in.h> 54 #include <netinet/ip.h> 55 #include <netinet/ip6.h> 56 #include <netinet6/ip6_var.h> 57 #include <netinet/udp.h> 58 #include <netinet/tcp.h> 59 60 #include <sys/in_cksum.h> 61 62 #include <sys/bus.h> 63 #include <sys/rman.h> 64 65 #include <bus/pci/pcireg.h> 66 #include <bus/pci/pcivar.h> 67 68 #define VMXNET3_LEGACY_TX 1 /* XXX we need this at the moment */ 69 #include "if_vmxreg.h" 70 #include "if_vmxvar.h" 71 72 #include "opt_inet.h" 73 #include "opt_inet6.h" 74 75 #ifdef VMXNET3_FAILPOINTS 76 #include <sys/fail.h> 77 static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0, 78 "vmxnet3 fail points"); 79 #define VMXNET3_FP _debug_fail_point_vmxnet3 80 #endif 81 82 static int vmxnet3_probe(device_t); 83 static int vmxnet3_attach(device_t); 84 static int vmxnet3_detach(device_t); 85 static int vmxnet3_shutdown(device_t); 86 87 static int vmxnet3_alloc_resources(struct vmxnet3_softc *); 88 static void vmxnet3_free_resources(struct vmxnet3_softc *); 89 static int vmxnet3_check_version(struct vmxnet3_softc *); 90 static void vmxnet3_initial_config(struct vmxnet3_softc *); 91 static void vmxnet3_check_multiqueue(struct vmxnet3_softc *); 92 93 #ifdef __FreeBSD__ 94 static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 95 static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 96 #else 97 static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 98 #endif 99 static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 100 static int vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int, 101 struct vmxnet3_interrupt *); 102 static int vmxnet3_alloc_intr_resources(struct vmxnet3_softc *); 103 #ifdef __FreeBSD__ 104 static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 105 #endif 106 static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 107 static int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 108 static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 109 110 static void vmxnet3_free_interrupt(struct vmxnet3_softc *, 111 struct vmxnet3_interrupt *); 112 static void vmxnet3_free_interrupts(struct vmxnet3_softc *); 113 114 #ifndef VMXNET3_LEGACY_TX 115 static int vmxnet3_alloc_taskqueue(struct vmxnet3_softc *); 116 static void vmxnet3_start_taskqueue(struct vmxnet3_softc *); 117 static void vmxnet3_drain_taskqueue(struct vmxnet3_softc *); 118 static void vmxnet3_free_taskqueue(struct vmxnet3_softc *); 119 #endif 120 121 static int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 122 static int vmxnet3_init_txq(struct vmxnet3_softc *, int); 123 static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 124 static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 125 static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 126 static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 127 128 static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 129 static void vmxnet3_free_shared_data(struct vmxnet3_softc *); 130 static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 131 static void vmxnet3_free_txq_data(struct vmxnet3_softc *); 132 static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 133 static void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 134 static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 135 static void vmxnet3_free_queue_data(struct vmxnet3_softc *); 136 static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 137 static void vmxnet3_init_shared_data(struct vmxnet3_softc *); 138 static void vmxnet3_init_hwassist(struct vmxnet3_softc *); 139 static void vmxnet3_reinit_interface(struct vmxnet3_softc *); 140 static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); 141 static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 142 static int vmxnet3_alloc_data(struct vmxnet3_softc *); 143 static void vmxnet3_free_data(struct vmxnet3_softc *); 144 static int vmxnet3_setup_interface(struct vmxnet3_softc *); 145 146 static void vmxnet3_evintr(struct vmxnet3_softc *); 147 static void vmxnet3_txq_eof(struct vmxnet3_txqueue *); 148 static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 149 static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); 150 static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 151 struct vmxnet3_rxring *, int); 152 static void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *); 153 static void vmxnet3_legacy_intr(void *); 154 #ifdef __FreeBSD__ 155 static void vmxnet3_txq_intr(void *); 156 static void vmxnet3_rxq_intr(void *); 157 static void vmxnet3_event_intr(void *); 158 #endif 159 160 static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 161 static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 162 static void vmxnet3_stop(struct vmxnet3_softc *); 163 164 static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 165 static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 166 static int vmxnet3_reinit_queues(struct vmxnet3_softc *); 167 static int vmxnet3_enable_device(struct vmxnet3_softc *); 168 static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 169 static int vmxnet3_reinit(struct vmxnet3_softc *); 170 static void vmxnet3_init_locked(struct vmxnet3_softc *); 171 static void vmxnet3_init(void *); 172 173 static int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *, 174 int *, int *, int *); 175 static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, 176 bus_dmamap_t, bus_dma_segment_t [], int *); 177 static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 178 static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 179 #ifdef VMXNET3_LEGACY_TX 180 static void vmxnet3_start_locked(struct ifnet *); 181 static void vmxnet3_start(struct ifnet *, struct ifaltq_subque *); 182 #else 183 static int vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *, 184 struct mbuf *); 185 static int vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *); 186 static void vmxnet3_txq_tq_deferred(void *, int); 187 #endif 188 static void vmxnet3_txq_start(struct vmxnet3_txqueue *); 189 static void vmxnet3_tx_start_all(struct vmxnet3_softc *); 190 191 static void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int, 192 uint16_t); 193 static void vmxnet3_register_vlan(void *, struct ifnet *, uint16_t); 194 static void vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t); 195 static void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 196 static int vmxnet3_change_mtu(struct vmxnet3_softc *, int); 197 static int vmxnet3_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 198 199 #ifndef VMXNET3_LEGACY_TX 200 static void vmxnet3_qflush(struct ifnet *); 201 #endif 202 203 static int vmxnet3_watchdog(struct vmxnet3_txqueue *); 204 static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); 205 static void vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *, 206 struct vmxnet3_txq_stats *); 207 static void vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *, 208 struct vmxnet3_rxq_stats *); 209 static void vmxnet3_tick(void *); 210 static void vmxnet3_link_status(struct vmxnet3_softc *); 211 static void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); 212 static int vmxnet3_media_change(struct ifnet *); 213 static void vmxnet3_set_lladdr(struct vmxnet3_softc *); 214 static void vmxnet3_get_lladdr(struct vmxnet3_softc *); 215 216 static void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *, 217 struct sysctl_ctx_list *, struct sysctl_oid_list *); 218 static void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *, 219 struct sysctl_ctx_list *, struct sysctl_oid_list *); 220 static void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *, 221 struct sysctl_ctx_list *, struct sysctl_oid_list *); 222 static void vmxnet3_setup_sysctl(struct vmxnet3_softc *); 223 224 static void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t, 225 uint32_t); 226 static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t); 227 static void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t, 228 uint32_t); 229 static void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t); 230 static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t); 231 232 static void vmxnet3_enable_intr(struct vmxnet3_softc *, int); 233 static void vmxnet3_disable_intr(struct vmxnet3_softc *, int); 234 static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 235 static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 236 237 static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, 238 bus_size_t, struct vmxnet3_dma_alloc *); 239 static void vmxnet3_dma_free(struct vmxnet3_softc *, 240 struct vmxnet3_dma_alloc *); 241 static int vmxnet3_tunable_int(struct vmxnet3_softc *, 242 const char *, int); 243 244 typedef enum { 245 VMXNET3_BARRIER_RD, 246 VMXNET3_BARRIER_WR, 247 VMXNET3_BARRIER_RDWR, 248 } vmxnet3_barrier_t; 249 250 static void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t); 251 252 /* Tunables. */ 253 static int vmxnet3_mq_disable = 0; 254 TUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable); 255 static int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES; 256 TUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue); 257 static int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES; 258 TUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue); 259 static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC; 260 TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc); 261 static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC; 262 TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc); 263 264 static device_method_t vmxnet3_methods[] = { 265 /* Device interface. */ 266 DEVMETHOD(device_probe, vmxnet3_probe), 267 DEVMETHOD(device_attach, vmxnet3_attach), 268 DEVMETHOD(device_detach, vmxnet3_detach), 269 DEVMETHOD(device_shutdown, vmxnet3_shutdown), 270 271 DEVMETHOD_END 272 }; 273 274 static driver_t vmxnet3_driver = { 275 "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc) 276 }; 277 278 static devclass_t vmxnet3_devclass; 279 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0); 280 281 MODULE_DEPEND(vmx, pci, 1, 1, 1); 282 MODULE_DEPEND(vmx, ether, 1, 1, 1); 283 284 #define VMXNET3_VMWARE_VENDOR_ID 0x15AD 285 #define VMXNET3_VMWARE_DEVICE_ID 0x07B0 286 287 static int 288 vmxnet3_probe(device_t dev) 289 { 290 291 if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID && 292 pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) { 293 device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter"); 294 return (BUS_PROBE_DEFAULT); 295 } 296 297 return (ENXIO); 298 } 299 300 static int 301 vmxnet3_attach(device_t dev) 302 { 303 struct vmxnet3_softc *sc; 304 int error; 305 306 sc = device_get_softc(dev); 307 sc->vmx_dev = dev; 308 309 pci_enable_busmaster(dev); 310 311 VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev)); 312 callout_init_lk(&sc->vmx_tick, &sc->vmx_lock); 313 314 vmxnet3_initial_config(sc); 315 316 error = vmxnet3_alloc_resources(sc); 317 if (error) 318 goto fail; 319 320 error = vmxnet3_check_version(sc); 321 if (error) 322 goto fail; 323 324 error = vmxnet3_alloc_rxtx_queues(sc); 325 if (error) 326 goto fail; 327 328 #ifndef VMXNET3_LEGACY_TX 329 error = vmxnet3_alloc_taskqueue(sc); 330 if (error) 331 goto fail; 332 #endif 333 334 error = vmxnet3_alloc_interrupts(sc); 335 if (error) 336 goto fail; 337 338 vmxnet3_check_multiqueue(sc); 339 340 error = vmxnet3_alloc_data(sc); 341 if (error) 342 goto fail; 343 344 error = vmxnet3_setup_interface(sc); 345 if (error) 346 goto fail; 347 348 error = vmxnet3_setup_interrupts(sc); 349 if (error) { 350 ether_ifdetach(sc->vmx_ifp); 351 device_printf(dev, "could not set up interrupt\n"); 352 goto fail; 353 } 354 355 vmxnet3_setup_sysctl(sc); 356 #ifndef VMXNET3_LEGACY_TX 357 vmxnet3_start_taskqueue(sc); 358 #endif 359 360 fail: 361 if (error) 362 vmxnet3_detach(dev); 363 364 return (error); 365 } 366 367 static int 368 vmxnet3_detach(device_t dev) 369 { 370 struct vmxnet3_softc *sc; 371 struct ifnet *ifp; 372 373 sc = device_get_softc(dev); 374 ifp = sc->vmx_ifp; 375 376 if (device_is_attached(dev)) { 377 VMXNET3_CORE_LOCK(sc); 378 vmxnet3_stop(sc); 379 VMXNET3_CORE_UNLOCK(sc); 380 381 callout_stop_sync(&sc->vmx_tick); 382 #ifndef VMXNET3_LEGACY_TX 383 vmxnet3_drain_taskqueue(sc); 384 #endif 385 386 ether_ifdetach(ifp); 387 } 388 389 if (sc->vmx_vlan_attach != NULL) { 390 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach); 391 sc->vmx_vlan_attach = NULL; 392 } 393 if (sc->vmx_vlan_detach != NULL) { 394 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach); 395 sc->vmx_vlan_detach = NULL; 396 } 397 398 #ifndef VMXNET3_LEGACY_TX 399 vmxnet3_free_taskqueue(sc); 400 #endif 401 vmxnet3_free_interrupts(sc); 402 403 if (ifp != NULL) { 404 if_free(ifp); 405 sc->vmx_ifp = NULL; 406 } 407 408 ifmedia_removeall(&sc->vmx_media); 409 410 vmxnet3_free_data(sc); 411 vmxnet3_free_resources(sc); 412 vmxnet3_free_rxtx_queues(sc); 413 414 VMXNET3_CORE_LOCK_DESTROY(sc); 415 416 return (0); 417 } 418 419 static int 420 vmxnet3_shutdown(device_t dev) 421 { 422 423 return (0); 424 } 425 426 static int 427 vmxnet3_alloc_resources(struct vmxnet3_softc *sc) 428 { 429 device_t dev; 430 int rid; 431 432 dev = sc->vmx_dev; 433 434 rid = PCIR_BAR(0); 435 sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 436 RF_ACTIVE); 437 if (sc->vmx_res0 == NULL) { 438 device_printf(dev, 439 "could not map BAR0 memory\n"); 440 return (ENXIO); 441 } 442 443 sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0); 444 sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0); 445 446 rid = PCIR_BAR(1); 447 sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 448 RF_ACTIVE); 449 if (sc->vmx_res1 == NULL) { 450 device_printf(dev, 451 "could not map BAR1 memory\n"); 452 return (ENXIO); 453 } 454 455 sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1); 456 sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1); 457 458 if (pci_find_extcap(dev, PCIY_MSIX, NULL) == 0) { 459 rid = PCIR_BAR(2); 460 sc->vmx_msix_res = bus_alloc_resource_any(dev, 461 SYS_RES_MEMORY, &rid, RF_ACTIVE); 462 } 463 464 if (sc->vmx_msix_res == NULL) 465 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 466 467 return (0); 468 } 469 470 static void 471 vmxnet3_free_resources(struct vmxnet3_softc *sc) 472 { 473 device_t dev; 474 int rid; 475 476 dev = sc->vmx_dev; 477 478 if (sc->vmx_res0 != NULL) { 479 rid = PCIR_BAR(0); 480 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0); 481 sc->vmx_res0 = NULL; 482 } 483 484 if (sc->vmx_res1 != NULL) { 485 rid = PCIR_BAR(1); 486 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1); 487 sc->vmx_res1 = NULL; 488 } 489 490 if (sc->vmx_msix_res != NULL) { 491 rid = PCIR_BAR(2); 492 bus_release_resource(dev, SYS_RES_MEMORY, rid, 493 sc->vmx_msix_res); 494 sc->vmx_msix_res = NULL; 495 } 496 } 497 498 static int 499 vmxnet3_check_version(struct vmxnet3_softc *sc) 500 { 501 device_t dev; 502 uint32_t version; 503 504 dev = sc->vmx_dev; 505 506 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 507 if ((version & 0x01) == 0) { 508 device_printf(dev, "unsupported hardware version %#x\n", 509 version); 510 return (ENOTSUP); 511 } 512 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 513 514 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 515 if ((version & 0x01) == 0) { 516 device_printf(dev, "unsupported UPT version %#x\n", version); 517 return (ENOTSUP); 518 } 519 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 520 521 return (0); 522 } 523 524 static int 525 trunc_powerof2(int val) 526 { 527 528 return (1U << (fls(val) - 1)); 529 } 530 531 static void 532 vmxnet3_initial_config(struct vmxnet3_softc *sc) 533 { 534 int nqueue, ndesc; 535 536 nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue); 537 if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1) 538 nqueue = VMXNET3_DEF_TX_QUEUES; 539 if (nqueue > ncpus) 540 nqueue = ncpus; 541 sc->vmx_max_ntxqueues = trunc_powerof2(nqueue); 542 543 nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue); 544 if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1) 545 nqueue = VMXNET3_DEF_RX_QUEUES; 546 if (nqueue > ncpus) 547 nqueue = ncpus; 548 sc->vmx_max_nrxqueues = trunc_powerof2(nqueue); 549 550 if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) { 551 sc->vmx_max_nrxqueues = 1; 552 sc->vmx_max_ntxqueues = 1; 553 } 554 555 ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc); 556 if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC) 557 ndesc = VMXNET3_DEF_TX_NDESC; 558 if (ndesc & VMXNET3_MASK_TX_NDESC) 559 ndesc &= ~VMXNET3_MASK_TX_NDESC; 560 sc->vmx_ntxdescs = ndesc; 561 562 ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc); 563 if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC) 564 ndesc = VMXNET3_DEF_RX_NDESC; 565 if (ndesc & VMXNET3_MASK_RX_NDESC) 566 ndesc &= ~VMXNET3_MASK_RX_NDESC; 567 sc->vmx_nrxdescs = ndesc; 568 sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 569 } 570 571 static void 572 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc) 573 { 574 575 if (sc->vmx_intr_type != VMXNET3_IT_MSIX) 576 goto out; 577 578 /* BMV: Just use the maximum configured for now. */ 579 sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; 580 sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; 581 582 if (sc->vmx_nrxqueues > 1) 583 sc->vmx_flags |= VMXNET3_FLAG_RSS; 584 585 return; 586 587 out: 588 sc->vmx_ntxqueues = 1; 589 sc->vmx_nrxqueues = 1; 590 } 591 592 #ifdef __FreeBSD__ 593 static int 594 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 595 { 596 device_t dev; 597 int nmsix, cnt, required; 598 599 dev = sc->vmx_dev; 600 601 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 602 return (1); 603 604 /* Allocate an additional vector for the events interrupt. */ 605 required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1; 606 607 nmsix = pci_msix_count(dev); 608 if (nmsix < required) 609 return (1); 610 611 cnt = required; 612 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 613 sc->vmx_nintrs = required; 614 return (0); 615 } else 616 pci_release_msi(dev); 617 618 /* BMV TODO Fallback to sharing MSIX vectors if possible. */ 619 620 return (1); 621 } 622 623 static int 624 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 625 { 626 device_t dev; 627 int nmsi, cnt, required; 628 629 dev = sc->vmx_dev; 630 required = 1; 631 632 nmsi = pci_msi_count(dev); 633 if (nmsi < required) 634 return (1); 635 636 cnt = required; 637 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) { 638 sc->vmx_nintrs = 1; 639 return (0); 640 } else 641 pci_release_msi(dev); 642 643 return (1); 644 } 645 #else 646 static int 647 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 648 { 649 int irq_flags, rid; 650 int enable = 1; 651 652 sc->vmx_irq_type = pci_alloc_1intr(sc->vmx_dev, enable, &rid, 653 &irq_flags); 654 sc->vmx_irq_flags = irq_flags; 655 sc->vmx_nintrs = 1; 656 return (0); 657 } 658 #endif 659 660 static int 661 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 662 { 663 664 sc->vmx_nintrs = 1; 665 return (0); 666 } 667 668 static int 669 vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags, 670 struct vmxnet3_interrupt *intr) 671 { 672 struct resource *irq; 673 674 irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, 675 sc->vmx_irq_flags); 676 if (irq == NULL) 677 return (ENXIO); 678 679 intr->vmxi_irq = irq; 680 intr->vmxi_rid = rid; 681 682 return (0); 683 } 684 685 static int 686 vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc) 687 { 688 int i, rid, flags, error; 689 690 rid = 0; 691 flags = RF_ACTIVE; 692 693 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) 694 flags |= RF_SHAREABLE; 695 else 696 rid = 1; 697 698 for (i = 0; i < sc->vmx_nintrs; i++, rid++) { 699 error = vmxnet3_alloc_interrupt(sc, rid, flags, 700 &sc->vmx_intrs[i]); 701 if (error) 702 return (error); 703 } 704 705 return (0); 706 } 707 708 #ifdef __FreeBSD__ 709 static int 710 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 711 { 712 device_t dev; 713 struct vmxnet3_txqueue *txq; 714 struct vmxnet3_rxqueue *rxq; 715 struct vmxnet3_interrupt *intr; 716 int i, error; 717 718 dev = sc->vmx_dev; 719 intr = &sc->vmx_intrs[0]; 720 721 for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) { 722 txq = &sc->vmx_txq[i]; 723 error = bus_setup_intr(dev, intr->vmxi_irq, INTR_MPSAFE, 724 vmxnet3_txq_intr, txq, &intr->vmxi_handler, NULL); 725 if (error) 726 return (error); 727 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 728 "tq%d", i); 729 txq->vxtxq_intr_idx = intr->vmxi_rid - 1; 730 } 731 732 for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) { 733 rxq = &sc->vmx_rxq[i]; 734 error = bus_setup_intr(dev, intr->vmxi_irq, INTR_MPSAFE, 735 vmxnet3_rxq_intr, rxq, &intr->vmxi_handler, NULL); 736 if (error) 737 return (error); 738 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 739 "rq%d", i); 740 rxq->vxrxq_intr_idx = intr->vmxi_rid - 1; 741 } 742 743 error = bus_setup_intr(dev, intr->vmxi_irq, INTR_MPSAFE, 744 vmxnet3_event_intr, sc, &intr->vmxi_handler, NULL); 745 if (error) 746 return (error); 747 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, "event"); 748 sc->vmx_event_intr_idx = intr->vmxi_rid - 1; 749 750 return (0); 751 } 752 #endif 753 754 static int 755 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 756 { 757 struct vmxnet3_interrupt *intr; 758 int i, error; 759 760 intr = &sc->vmx_intrs[0]; 761 error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq, 762 INTR_MPSAFE, vmxnet3_legacy_intr, sc, 763 &intr->vmxi_handler, NULL); 764 765 for (i = 0; i < sc->vmx_ntxqueues; i++) 766 sc->vmx_txq[i].vxtxq_intr_idx = 0; 767 for (i = 0; i < sc->vmx_nrxqueues; i++) 768 sc->vmx_rxq[i].vxrxq_intr_idx = 0; 769 sc->vmx_event_intr_idx = 0; 770 771 return (error); 772 } 773 774 static void 775 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 776 { 777 struct vmxnet3_txqueue *txq; 778 struct vmxnet3_txq_shared *txs; 779 struct vmxnet3_rxqueue *rxq; 780 struct vmxnet3_rxq_shared *rxs; 781 int i; 782 783 sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 784 785 for (i = 0; i < sc->vmx_ntxqueues; i++) { 786 txq = &sc->vmx_txq[i]; 787 txs = txq->vxtxq_ts; 788 txs->intr_idx = txq->vxtxq_intr_idx; 789 } 790 791 for (i = 0; i < sc->vmx_nrxqueues; i++) { 792 rxq = &sc->vmx_rxq[i]; 793 rxs = rxq->vxrxq_rs; 794 rxs->intr_idx = rxq->vxrxq_intr_idx; 795 } 796 } 797 798 static int 799 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 800 { 801 int error; 802 803 error = vmxnet3_alloc_intr_resources(sc); 804 if (error) 805 return (error); 806 807 switch (sc->vmx_intr_type) { 808 case VMXNET3_IT_MSIX: 809 #ifdef __FreeBSD__ 810 error = vmxnet3_setup_msix_interrupts(sc); 811 #else 812 device_printf(sc->vmx_dev, "VMXNET3_IT_MSIX unsupported\n"); 813 error = ENXIO; 814 #endif 815 break; 816 case VMXNET3_IT_MSI: 817 case VMXNET3_IT_LEGACY: 818 error = vmxnet3_setup_legacy_interrupt(sc); 819 break; 820 default: 821 panic("%s: invalid interrupt type %d", __func__, 822 sc->vmx_intr_type); 823 } 824 825 if (error == 0) 826 vmxnet3_set_interrupt_idx(sc); 827 828 return (error); 829 } 830 831 #ifdef __FreeBSD__ 832 static int 833 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 834 { 835 device_t dev; 836 uint32_t config; 837 int error; 838 839 dev = sc->vmx_dev; 840 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 841 842 sc->vmx_intr_type = config & 0x03; 843 sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 844 845 switch (sc->vmx_intr_type) { 846 case VMXNET3_IT_AUTO: 847 sc->vmx_intr_type = VMXNET3_IT_MSIX; 848 /* FALLTHROUGH */ 849 case VMXNET3_IT_MSIX: 850 error = vmxnet3_alloc_msix_interrupts(sc); 851 if (error == 0) 852 break; 853 sc->vmx_intr_type = VMXNET3_IT_MSI; 854 /* FALLTHROUGH */ 855 case VMXNET3_IT_MSI: 856 error = vmxnet3_alloc_msi_interrupts(sc); 857 if (error == 0) 858 break; 859 sc->vmx_intr_type = VMXNET3_IT_LEGACY; 860 /* FALLTHROUGH */ 861 case VMXNET3_IT_LEGACY: 862 error = vmxnet3_alloc_legacy_interrupts(sc); 863 if (error == 0) 864 break; 865 /* FALLTHROUGH */ 866 default: 867 sc->vmx_intr_type = -1; 868 device_printf(dev, "cannot allocate any interrupt resources\n"); 869 return (ENXIO); 870 } 871 872 return (error); 873 } 874 #else 875 static int 876 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 877 { 878 device_t dev; 879 uint32_t config; 880 int error; 881 882 dev = sc->vmx_dev; 883 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 884 885 sc->vmx_intr_type = config & 0x03; 886 sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 887 888 switch (sc->vmx_intr_type) { 889 case VMXNET3_IT_AUTO: 890 sc->vmx_intr_type = VMXNET3_IT_MSI; 891 /* FALLTHROUGH */ 892 case VMXNET3_IT_MSI: 893 error = vmxnet3_alloc_msi_interrupts(sc); 894 if (error == 0) 895 break; 896 sc->vmx_intr_type = VMXNET3_IT_LEGACY; 897 case VMXNET3_IT_LEGACY: 898 error = vmxnet3_alloc_legacy_interrupts(sc); 899 if (error == 0) 900 break; 901 /* FALLTHROUGH */ 902 case VMXNET3_IT_MSIX: 903 /* FALLTHROUGH */ 904 default: 905 sc->vmx_intr_type = -1; 906 device_printf(dev, "cannot allocate any interrupt resources\n"); 907 return (ENXIO); 908 } 909 910 return (error); 911 } 912 #endif 913 914 static void 915 vmxnet3_free_interrupt(struct vmxnet3_softc *sc, 916 struct vmxnet3_interrupt *intr) 917 { 918 device_t dev; 919 920 dev = sc->vmx_dev; 921 922 if (intr->vmxi_handler != NULL) { 923 bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler); 924 intr->vmxi_handler = NULL; 925 } 926 927 if (intr->vmxi_irq != NULL) { 928 bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid, 929 intr->vmxi_irq); 930 intr->vmxi_irq = NULL; 931 intr->vmxi_rid = -1; 932 } 933 } 934 935 #ifdef __FreeBSD__ 936 static void 937 vmxnet3_free_interrupts(struct vmxnet3_softc *sc) 938 { 939 int i; 940 941 for (i = 0; i < sc->vmx_nintrs; i++) 942 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); 943 944 if (sc->vmx_intr_type == VMXNET3_IT_MSI || 945 sc->vmx_intr_type == VMXNET3_IT_MSIX) 946 pci_release_msi(sc->vmx_dev); 947 } 948 #else 949 static void 950 vmxnet3_free_interrupts(struct vmxnet3_softc *sc) 951 { 952 int i; 953 954 for (i = 0; i < sc->vmx_nintrs; i++) 955 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); 956 957 if (sc->vmx_irq_type == PCI_INTR_TYPE_MSI) 958 pci_release_msi(sc->vmx_dev); 959 } 960 #endif 961 962 #ifndef VMXNET3_LEGACY_TX 963 static int 964 vmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc) 965 { 966 device_t dev; 967 968 dev = sc->vmx_dev; 969 970 sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT, 971 taskqueue_thread_enqueue, &sc->vmx_tq); 972 if (sc->vmx_tq == NULL) 973 return (ENOMEM); 974 975 return (0); 976 } 977 978 static void 979 vmxnet3_start_taskqueue(struct vmxnet3_softc *sc) 980 { 981 device_t dev; 982 int nthreads, error; 983 984 dev = sc->vmx_dev; 985 986 /* 987 * The taskqueue is typically not frequently used, so a dedicated 988 * thread for each queue is unnecessary. 989 */ 990 nthreads = MAX(1, sc->vmx_ntxqueues / 2); 991 992 /* 993 * Most drivers just ignore the return value - it only fails 994 * with ENOMEM so an error is not likely. It is hard for us 995 * to recover from an error here. 996 */ 997 error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET, 998 "%s taskq", device_get_nameunit(dev)); 999 if (error) 1000 device_printf(dev, "failed to start taskqueue: %d", error); 1001 } 1002 1003 static void 1004 vmxnet3_drain_taskqueue(struct vmxnet3_softc *sc) 1005 { 1006 struct vmxnet3_txqueue *txq; 1007 int i; 1008 1009 if (sc->vmx_tq != NULL) { 1010 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1011 txq = &sc->vmx_txq[i]; 1012 taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask); 1013 } 1014 } 1015 } 1016 1017 static void 1018 vmxnet3_free_taskqueue(struct vmxnet3_softc *sc) 1019 { 1020 if (sc->vmx_tq != NULL) { 1021 taskqueue_free(sc->vmx_tq); 1022 sc->vmx_tq = NULL; 1023 } 1024 } 1025 #endif 1026 1027 static int 1028 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 1029 { 1030 struct vmxnet3_rxqueue *rxq; 1031 struct vmxnet3_rxring *rxr; 1032 int i; 1033 1034 rxq = &sc->vmx_rxq[q]; 1035 1036 ksnprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 1037 device_get_nameunit(sc->vmx_dev), q); 1038 lockinit(&rxq->vxrxq_lock, rxq->vxrxq_name, 0, 0); 1039 1040 rxq->vxrxq_sc = sc; 1041 rxq->vxrxq_id = q; 1042 1043 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1044 rxr = &rxq->vxrxq_cmd_ring[i]; 1045 rxr->vxrxr_rid = i; 1046 rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 1047 rxr->vxrxr_rxbuf = kmalloc(rxr->vxrxr_ndesc * 1048 sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_INTWAIT | M_ZERO); 1049 if (rxr->vxrxr_rxbuf == NULL) 1050 return (ENOMEM); 1051 1052 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 1053 } 1054 1055 return (0); 1056 } 1057 1058 static int 1059 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 1060 { 1061 struct vmxnet3_txqueue *txq; 1062 struct vmxnet3_txring *txr; 1063 1064 txq = &sc->vmx_txq[q]; 1065 txr = &txq->vxtxq_cmd_ring; 1066 1067 ksnprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 1068 device_get_nameunit(sc->vmx_dev), q); 1069 lockinit(&txq->vxtxq_lock, txq->vxtxq_name, 0, 0); 1070 1071 txq->vxtxq_sc = sc; 1072 txq->vxtxq_id = q; 1073 1074 txr->vxtxr_ndesc = sc->vmx_ntxdescs; 1075 txr->vxtxr_txbuf = kmalloc(txr->vxtxr_ndesc * 1076 sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_INTWAIT | M_ZERO); 1077 if (txr->vxtxr_txbuf == NULL) 1078 return (ENOMEM); 1079 1080 txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 1081 1082 #ifndef VMXNET3_LEGACY_TX 1083 TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq); 1084 1085 txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF, 1086 M_NOWAIT, &txq->vxtxq_lock); 1087 if (txq->vxtxq_br == NULL) 1088 return (ENOMEM); 1089 #endif 1090 1091 return (0); 1092 } 1093 1094 static int 1095 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 1096 { 1097 int i, error; 1098 1099 /* 1100 * Only attempt to create multiple queues if MSIX is available. MSIX is 1101 * disabled by default because its apparently broken for devices passed 1102 * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable 1103 * must be set to zero for MSIX. This check prevents us from allocating 1104 * queue structures that we will not use. 1105 */ 1106 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { 1107 sc->vmx_max_nrxqueues = 1; 1108 sc->vmx_max_ntxqueues = 1; 1109 } 1110 1111 sc->vmx_rxq = kmalloc(sizeof(struct vmxnet3_rxqueue) * 1112 sc->vmx_max_nrxqueues, M_DEVBUF, M_INTWAIT | M_ZERO); 1113 sc->vmx_txq = kmalloc(sizeof(struct vmxnet3_txqueue) * 1114 sc->vmx_max_ntxqueues, M_DEVBUF, M_INTWAIT | M_ZERO); 1115 if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL) 1116 return (ENOMEM); 1117 1118 for (i = 0; i < sc->vmx_max_nrxqueues; i++) { 1119 error = vmxnet3_init_rxq(sc, i); 1120 if (error) 1121 return (error); 1122 } 1123 1124 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1125 error = vmxnet3_init_txq(sc, i); 1126 if (error) 1127 return (error); 1128 } 1129 1130 return (0); 1131 } 1132 1133 static void 1134 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 1135 { 1136 struct vmxnet3_rxring *rxr; 1137 int i; 1138 1139 rxq->vxrxq_sc = NULL; 1140 rxq->vxrxq_id = -1; 1141 1142 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1143 rxr = &rxq->vxrxq_cmd_ring[i]; 1144 1145 if (rxr->vxrxr_rxbuf != NULL) { 1146 kfree(rxr->vxrxr_rxbuf, M_DEVBUF); 1147 rxr->vxrxr_rxbuf = NULL; 1148 } 1149 } 1150 1151 #if 0 /* XXX */ 1152 if (mtx_initialized(&rxq->vxrxq_lock) != 0) 1153 #endif 1154 lockuninit(&rxq->vxrxq_lock); 1155 } 1156 1157 static void 1158 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 1159 { 1160 struct vmxnet3_txring *txr; 1161 1162 txr = &txq->vxtxq_cmd_ring; 1163 1164 txq->vxtxq_sc = NULL; 1165 txq->vxtxq_id = -1; 1166 1167 #ifndef VMXNET3_LEGACY_TX 1168 if (txq->vxtxq_br != NULL) { 1169 buf_ring_free(txq->vxtxq_br, M_DEVBUF); 1170 txq->vxtxq_br = NULL; 1171 } 1172 #endif 1173 1174 if (txr->vxtxr_txbuf != NULL) { 1175 kfree(txr->vxtxr_txbuf, M_DEVBUF); 1176 txr->vxtxr_txbuf = NULL; 1177 } 1178 1179 #if 0 /* XXX */ 1180 if (mtx_initialized(&txq->vxtxq_lock) != 0) 1181 #endif 1182 lockuninit(&txq->vxtxq_lock); 1183 } 1184 1185 static void 1186 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 1187 { 1188 int i; 1189 1190 if (sc->vmx_rxq != NULL) { 1191 for (i = 0; i < sc->vmx_max_nrxqueues; i++) 1192 vmxnet3_destroy_rxq(&sc->vmx_rxq[i]); 1193 kfree(sc->vmx_rxq, M_DEVBUF); 1194 sc->vmx_rxq = NULL; 1195 } 1196 1197 if (sc->vmx_txq != NULL) { 1198 for (i = 0; i < sc->vmx_max_ntxqueues; i++) 1199 vmxnet3_destroy_txq(&sc->vmx_txq[i]); 1200 kfree(sc->vmx_txq, M_DEVBUF); 1201 sc->vmx_txq = NULL; 1202 } 1203 } 1204 1205 static int 1206 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 1207 { 1208 device_t dev; 1209 uint8_t *kva; 1210 size_t size; 1211 int i, error; 1212 1213 dev = sc->vmx_dev; 1214 1215 size = sizeof(struct vmxnet3_driver_shared); 1216 error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 1217 if (error) { 1218 device_printf(dev, "cannot alloc shared memory\n"); 1219 return (error); 1220 } 1221 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 1222 1223 size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 1224 sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 1225 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 1226 if (error) { 1227 device_printf(dev, "cannot alloc queue shared memory\n"); 1228 return (error); 1229 } 1230 sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 1231 kva = sc->vmx_qs; 1232 1233 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1234 sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; 1235 kva += sizeof(struct vmxnet3_txq_shared); 1236 } 1237 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1238 sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; 1239 kva += sizeof(struct vmxnet3_rxq_shared); 1240 } 1241 1242 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1243 size = sizeof(struct vmxnet3_rss_shared); 1244 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); 1245 if (error) { 1246 device_printf(dev, "cannot alloc rss shared memory\n"); 1247 return (error); 1248 } 1249 sc->vmx_rss = 1250 (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; 1251 } 1252 1253 return (0); 1254 } 1255 1256 static void 1257 vmxnet3_free_shared_data(struct vmxnet3_softc *sc) 1258 { 1259 1260 if (sc->vmx_rss != NULL) { 1261 vmxnet3_dma_free(sc, &sc->vmx_rss_dma); 1262 sc->vmx_rss = NULL; 1263 } 1264 1265 if (sc->vmx_qs != NULL) { 1266 vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 1267 sc->vmx_qs = NULL; 1268 } 1269 1270 if (sc->vmx_ds != NULL) { 1271 vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 1272 sc->vmx_ds = NULL; 1273 } 1274 } 1275 1276 static int 1277 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 1278 { 1279 device_t dev; 1280 struct vmxnet3_txqueue *txq; 1281 struct vmxnet3_txring *txr; 1282 struct vmxnet3_comp_ring *txc; 1283 size_t descsz, compsz; 1284 int i, q, error; 1285 1286 dev = sc->vmx_dev; 1287 1288 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1289 txq = &sc->vmx_txq[q]; 1290 txr = &txq->vxtxq_cmd_ring; 1291 txc = &txq->vxtxq_comp_ring; 1292 1293 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 1294 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 1295 1296 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1297 1, 0, /* alignment, boundary */ 1298 BUS_SPACE_MAXADDR, /* lowaddr */ 1299 BUS_SPACE_MAXADDR, /* highaddr */ 1300 NULL, NULL, /* filter, filterarg */ 1301 VMXNET3_TX_MAXSIZE, /* maxsize */ 1302 VMXNET3_TX_MAXSEGS, /* nsegments */ 1303 VMXNET3_TX_MAXSEGSIZE, /* maxsegsize */ 1304 0, /* flags */ 1305 &txr->vxtxr_txtag); 1306 if (error) { 1307 device_printf(dev, 1308 "unable to create Tx buffer tag for queue %d\n", q); 1309 return (error); 1310 } 1311 1312 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 1313 if (error) { 1314 device_printf(dev, "cannot alloc Tx descriptors for " 1315 "queue %d error %d\n", q, error); 1316 return (error); 1317 } 1318 txr->vxtxr_txd = 1319 (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1320 1321 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1322 if (error) { 1323 device_printf(dev, "cannot alloc Tx comp descriptors " 1324 "for queue %d error %d\n", q, error); 1325 return (error); 1326 } 1327 txc->vxcr_u.txcd = 1328 (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1329 1330 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1331 error = bus_dmamap_create(txr->vxtxr_txtag, 0, 1332 &txr->vxtxr_txbuf[i].vtxb_dmamap); 1333 if (error) { 1334 device_printf(dev, "unable to create Tx buf " 1335 "dmamap for queue %d idx %d\n", q, i); 1336 return (error); 1337 } 1338 } 1339 } 1340 1341 return (0); 1342 } 1343 1344 static void 1345 vmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1346 { 1347 device_t dev; 1348 struct vmxnet3_txqueue *txq; 1349 struct vmxnet3_txring *txr; 1350 struct vmxnet3_comp_ring *txc; 1351 struct vmxnet3_txbuf *txb; 1352 int i, q; 1353 1354 dev = sc->vmx_dev; 1355 1356 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1357 txq = &sc->vmx_txq[q]; 1358 txr = &txq->vxtxq_cmd_ring; 1359 txc = &txq->vxtxq_comp_ring; 1360 1361 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1362 txb = &txr->vxtxr_txbuf[i]; 1363 if (txb->vtxb_dmamap != NULL) { 1364 bus_dmamap_destroy(txr->vxtxr_txtag, 1365 txb->vtxb_dmamap); 1366 txb->vtxb_dmamap = NULL; 1367 } 1368 } 1369 1370 if (txc->vxcr_u.txcd != NULL) { 1371 vmxnet3_dma_free(sc, &txc->vxcr_dma); 1372 txc->vxcr_u.txcd = NULL; 1373 } 1374 1375 if (txr->vxtxr_txd != NULL) { 1376 vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1377 txr->vxtxr_txd = NULL; 1378 } 1379 1380 if (txr->vxtxr_txtag != NULL) { 1381 bus_dma_tag_destroy(txr->vxtxr_txtag); 1382 txr->vxtxr_txtag = NULL; 1383 } 1384 } 1385 } 1386 1387 static int 1388 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1389 { 1390 device_t dev; 1391 struct vmxnet3_rxqueue *rxq; 1392 struct vmxnet3_rxring *rxr; 1393 struct vmxnet3_comp_ring *rxc; 1394 int descsz, compsz; 1395 int i, j, q, error; 1396 1397 dev = sc->vmx_dev; 1398 1399 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1400 rxq = &sc->vmx_rxq[q]; 1401 rxc = &rxq->vxrxq_comp_ring; 1402 compsz = 0; 1403 1404 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1405 rxr = &rxq->vxrxq_cmd_ring[i]; 1406 1407 descsz = rxr->vxrxr_ndesc * 1408 sizeof(struct vmxnet3_rxdesc); 1409 compsz += rxr->vxrxr_ndesc * 1410 sizeof(struct vmxnet3_rxcompdesc); 1411 1412 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1413 1, 0, /* alignment, boundary */ 1414 BUS_SPACE_MAXADDR, /* lowaddr */ 1415 BUS_SPACE_MAXADDR, /* highaddr */ 1416 NULL, NULL, /* filter, filterarg */ 1417 MJUMPAGESIZE, /* maxsize */ 1418 1, /* nsegments */ 1419 MJUMPAGESIZE, /* maxsegsize */ 1420 0, /* flags */ 1421 &rxr->vxrxr_rxtag); 1422 if (error) { 1423 device_printf(dev, 1424 "unable to create Rx buffer tag for " 1425 "queue %d\n", q); 1426 return (error); 1427 } 1428 1429 error = vmxnet3_dma_malloc(sc, descsz, 512, 1430 &rxr->vxrxr_dma); 1431 if (error) { 1432 device_printf(dev, "cannot allocate Rx " 1433 "descriptors for queue %d/%d error %d\n", 1434 i, q, error); 1435 return (error); 1436 } 1437 rxr->vxrxr_rxd = 1438 (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1439 } 1440 1441 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1442 if (error) { 1443 device_printf(dev, "cannot alloc Rx comp descriptors " 1444 "for queue %d error %d\n", q, error); 1445 return (error); 1446 } 1447 rxc->vxcr_u.rxcd = 1448 (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1449 1450 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1451 rxr = &rxq->vxrxq_cmd_ring[i]; 1452 1453 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1454 &rxr->vxrxr_spare_dmap); 1455 if (error) { 1456 device_printf(dev, "unable to create spare " 1457 "dmamap for queue %d/%d error %d\n", 1458 q, i, error); 1459 return (error); 1460 } 1461 1462 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1463 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1464 &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1465 if (error) { 1466 device_printf(dev, "unable to create " 1467 "dmamap for queue %d/%d slot %d " 1468 "error %d\n", 1469 q, i, j, error); 1470 return (error); 1471 } 1472 } 1473 } 1474 } 1475 1476 return (0); 1477 } 1478 1479 static void 1480 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1481 { 1482 device_t dev; 1483 struct vmxnet3_rxqueue *rxq; 1484 struct vmxnet3_rxring *rxr; 1485 struct vmxnet3_comp_ring *rxc; 1486 struct vmxnet3_rxbuf *rxb; 1487 int i, j, q; 1488 1489 dev = sc->vmx_dev; 1490 1491 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1492 rxq = &sc->vmx_rxq[q]; 1493 rxc = &rxq->vxrxq_comp_ring; 1494 1495 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1496 rxr = &rxq->vxrxq_cmd_ring[i]; 1497 1498 if (rxr->vxrxr_spare_dmap != NULL) { 1499 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1500 rxr->vxrxr_spare_dmap); 1501 rxr->vxrxr_spare_dmap = NULL; 1502 } 1503 1504 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1505 rxb = &rxr->vxrxr_rxbuf[j]; 1506 if (rxb->vrxb_dmamap != NULL) { 1507 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1508 rxb->vrxb_dmamap); 1509 rxb->vrxb_dmamap = NULL; 1510 } 1511 } 1512 } 1513 1514 if (rxc->vxcr_u.rxcd != NULL) { 1515 vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1516 rxc->vxcr_u.rxcd = NULL; 1517 } 1518 1519 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1520 rxr = &rxq->vxrxq_cmd_ring[i]; 1521 1522 if (rxr->vxrxr_rxd != NULL) { 1523 vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1524 rxr->vxrxr_rxd = NULL; 1525 } 1526 1527 if (rxr->vxrxr_rxtag != NULL) { 1528 bus_dma_tag_destroy(rxr->vxrxr_rxtag); 1529 rxr->vxrxr_rxtag = NULL; 1530 } 1531 } 1532 } 1533 } 1534 1535 static int 1536 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1537 { 1538 int error; 1539 1540 error = vmxnet3_alloc_txq_data(sc); 1541 if (error) 1542 return (error); 1543 1544 error = vmxnet3_alloc_rxq_data(sc); 1545 if (error) 1546 return (error); 1547 1548 return (0); 1549 } 1550 1551 static void 1552 vmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1553 { 1554 1555 if (sc->vmx_rxq != NULL) 1556 vmxnet3_free_rxq_data(sc); 1557 1558 if (sc->vmx_txq != NULL) 1559 vmxnet3_free_txq_data(sc); 1560 } 1561 1562 static int 1563 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1564 { 1565 int error; 1566 1567 error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1568 32, &sc->vmx_mcast_dma); 1569 if (error) 1570 device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1571 else 1572 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1573 1574 return (error); 1575 } 1576 1577 static void 1578 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1579 { 1580 1581 if (sc->vmx_mcast != NULL) { 1582 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1583 sc->vmx_mcast = NULL; 1584 } 1585 } 1586 1587 static void 1588 vmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1589 { 1590 struct vmxnet3_driver_shared *ds; 1591 struct vmxnet3_txqueue *txq; 1592 struct vmxnet3_txq_shared *txs; 1593 struct vmxnet3_rxqueue *rxq; 1594 struct vmxnet3_rxq_shared *rxs; 1595 int i; 1596 1597 ds = sc->vmx_ds; 1598 1599 /* 1600 * Initialize fields of the shared data that remains the same across 1601 * reinits. Note the shared data is zero'd when allocated. 1602 */ 1603 1604 ds->magic = VMXNET3_REV1_MAGIC; 1605 1606 /* DriverInfo */ 1607 ds->version = VMXNET3_DRIVER_VERSION; 1608 ds->guest = VMXNET3_GOS_FREEBSD | 1609 #ifdef __LP64__ 1610 VMXNET3_GOS_64BIT; 1611 #else 1612 VMXNET3_GOS_32BIT; 1613 #endif 1614 ds->vmxnet3_revision = 1; 1615 ds->upt_version = 1; 1616 1617 /* Misc. conf */ 1618 ds->driver_data = vtophys(sc); 1619 ds->driver_data_len = sizeof(struct vmxnet3_softc); 1620 ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1621 ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1622 ds->nrxsg_max = sc->vmx_max_rxsegs; 1623 1624 /* RSS conf */ 1625 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1626 ds->rss.version = 1; 1627 ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; 1628 ds->rss.len = sc->vmx_rss_dma.dma_size; 1629 } 1630 1631 /* Interrupt control. */ 1632 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1633 ds->nintr = sc->vmx_nintrs; 1634 ds->evintr = sc->vmx_event_intr_idx; 1635 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1636 1637 for (i = 0; i < sc->vmx_nintrs; i++) 1638 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1639 1640 /* Receive filter. */ 1641 ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1642 ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1643 1644 /* Tx queues */ 1645 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1646 txq = &sc->vmx_txq[i]; 1647 txs = txq->vxtxq_ts; 1648 1649 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1650 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1651 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1652 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1653 txs->driver_data = vtophys(txq); 1654 txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1655 } 1656 1657 /* Rx queues */ 1658 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1659 rxq = &sc->vmx_rxq[i]; 1660 rxs = rxq->vxrxq_rs; 1661 1662 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1663 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1664 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1665 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1666 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1667 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1668 rxs->driver_data = vtophys(rxq); 1669 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1670 } 1671 } 1672 1673 static void 1674 vmxnet3_init_hwassist(struct vmxnet3_softc *sc) 1675 { 1676 struct ifnet *ifp = sc->vmx_ifp; 1677 uint64_t hwassist; 1678 1679 hwassist = 0; 1680 if (ifp->if_capenable & IFCAP_TXCSUM) 1681 hwassist |= VMXNET3_CSUM_OFFLOAD; 1682 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 1683 hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6; 1684 #if 0 /* XXX TSO */ 1685 if (ifp->if_capenable & IFCAP_TSO4) 1686 hwassist |= CSUM_IP_TSO; 1687 if (ifp->if_capenable & IFCAP_TSO6) 1688 hwassist |= CSUM_IP6_TSO; 1689 #endif 1690 ifp->if_hwassist = hwassist; 1691 } 1692 1693 static void 1694 vmxnet3_reinit_interface(struct vmxnet3_softc *sc) 1695 { 1696 struct ifnet *ifp; 1697 1698 ifp = sc->vmx_ifp; 1699 1700 /* Use the current MAC address. */ 1701 bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN); 1702 vmxnet3_set_lladdr(sc); 1703 1704 vmxnet3_init_hwassist(sc); 1705 } 1706 1707 static void 1708 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) 1709 { 1710 /* 1711 * Use the same key as the Linux driver until FreeBSD can do 1712 * RSS (presumably Toeplitz) in software. 1713 */ 1714 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 1715 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 1716 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 1717 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 1718 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 1719 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, 1720 }; 1721 1722 struct vmxnet3_driver_shared *ds; 1723 struct vmxnet3_rss_shared *rss; 1724 int i; 1725 1726 ds = sc->vmx_ds; 1727 rss = sc->vmx_rss; 1728 1729 rss->hash_type = 1730 UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | 1731 UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; 1732 rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 1733 rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; 1734 rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 1735 memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); 1736 1737 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) 1738 rss->ind_table[i] = i % sc->vmx_nrxqueues; 1739 } 1740 1741 static void 1742 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1743 { 1744 struct ifnet *ifp; 1745 struct vmxnet3_driver_shared *ds; 1746 1747 ifp = sc->vmx_ifp; 1748 ds = sc->vmx_ds; 1749 1750 ds->mtu = ifp->if_mtu; 1751 ds->ntxqueue = sc->vmx_ntxqueues; 1752 ds->nrxqueue = sc->vmx_nrxqueues; 1753 1754 ds->upt_features = 0; 1755 if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 1756 ds->upt_features |= UPT1_F_CSUM; 1757 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1758 ds->upt_features |= UPT1_F_VLAN; 1759 #if 0 /* XXX LRO */ 1760 if (ifp->if_capenable & IFCAP_LRO) 1761 ds->upt_features |= UPT1_F_LRO; 1762 #endif 1763 1764 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1765 ds->upt_features |= UPT1_F_RSS; 1766 vmxnet3_reinit_rss_shared_data(sc); 1767 } 1768 1769 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1770 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1771 (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1772 } 1773 1774 static int 1775 vmxnet3_alloc_data(struct vmxnet3_softc *sc) 1776 { 1777 int error; 1778 1779 error = vmxnet3_alloc_shared_data(sc); 1780 if (error) 1781 return (error); 1782 1783 error = vmxnet3_alloc_queue_data(sc); 1784 if (error) 1785 return (error); 1786 1787 error = vmxnet3_alloc_mcast_table(sc); 1788 if (error) 1789 return (error); 1790 1791 vmxnet3_init_shared_data(sc); 1792 1793 return (0); 1794 } 1795 1796 static void 1797 vmxnet3_free_data(struct vmxnet3_softc *sc) 1798 { 1799 1800 vmxnet3_free_mcast_table(sc); 1801 vmxnet3_free_queue_data(sc); 1802 vmxnet3_free_shared_data(sc); 1803 } 1804 1805 static int 1806 vmxnet3_setup_interface(struct vmxnet3_softc *sc) 1807 { 1808 device_t dev; 1809 struct ifnet *ifp; 1810 1811 dev = sc->vmx_dev; 1812 1813 ifp = sc->vmx_ifp = if_alloc(IFT_ETHER); 1814 if (ifp == NULL) { 1815 device_printf(dev, "cannot allocate ifnet structure\n"); 1816 return (ENOSPC); 1817 } 1818 1819 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1820 ifp->if_baudrate = IF_Gbps(10ULL); 1821 ifp->if_softc = sc; 1822 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1823 ifp->if_init = vmxnet3_init; 1824 ifp->if_ioctl = vmxnet3_ioctl; 1825 #if 0 /* XXX TSO */ 1826 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1827 ifp->if_hw_tsomaxsegcount = VMXNET3_TX_MAXSEGS; 1828 ifp->if_hw_tsomaxsegsize = VMXNET3_TX_MAXSEGSIZE; 1829 #endif 1830 1831 #ifdef VMXNET3_LEGACY_TX 1832 ifp->if_start = vmxnet3_start; 1833 ifq_set_maxlen(&ifp->if_snd, sc->vmx_ntxdescs - 1); 1834 ifq_set_ready(&ifp->if_snd); 1835 #else 1836 ifp->if_transmit = vmxnet3_txq_mq_start; 1837 ifp->if_qflush = vmxnet3_qflush; 1838 #endif 1839 1840 vmxnet3_get_lladdr(sc); 1841 ether_ifattach(ifp, sc->vmx_lladdr, NULL); 1842 1843 ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; 1844 ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6; 1845 #if 0 /* XXX TSO */ 1846 ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 1847 #endif 1848 ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 1849 IFCAP_VLAN_HWCSUM; 1850 ifp->if_capenable = ifp->if_capabilities; 1851 1852 #if 0 /* XXX LRO / VLAN_HWFILTER */ 1853 /* These capabilities are not enabled by default. */ 1854 ifp->if_capabilities |= /* IFCAP_LRO | */ IFCAP_VLAN_HWFILTER; 1855 #endif 1856 1857 sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 1858 vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 1859 sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config, 1860 vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1861 1862 ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change, 1863 vmxnet3_media_status); 1864 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1865 ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1866 1867 return (0); 1868 } 1869 1870 static void 1871 vmxnet3_evintr(struct vmxnet3_softc *sc) 1872 { 1873 device_t dev; 1874 struct ifnet *ifp; 1875 struct vmxnet3_txq_shared *ts; 1876 struct vmxnet3_rxq_shared *rs; 1877 uint32_t event; 1878 int reset; 1879 1880 dev = sc->vmx_dev; 1881 ifp = sc->vmx_ifp; 1882 reset = 0; 1883 1884 VMXNET3_CORE_LOCK(sc); 1885 1886 /* Clear events. */ 1887 event = sc->vmx_ds->event; 1888 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 1889 1890 if (event & VMXNET3_EVENT_LINK) { 1891 vmxnet3_link_status(sc); 1892 if (sc->vmx_link_active != 0) 1893 vmxnet3_tx_start_all(sc); 1894 } 1895 1896 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 1897 reset = 1; 1898 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 1899 ts = sc->vmx_txq[0].vxtxq_ts; 1900 if (ts->stopped != 0) 1901 device_printf(dev, "Tx queue error %#x\n", ts->error); 1902 rs = sc->vmx_rxq[0].vxrxq_rs; 1903 if (rs->stopped != 0) 1904 device_printf(dev, "Rx queue error %#x\n", rs->error); 1905 device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 1906 } 1907 1908 if (event & VMXNET3_EVENT_DIC) 1909 device_printf(dev, "device implementation change event\n"); 1910 if (event & VMXNET3_EVENT_DEBUG) 1911 device_printf(dev, "debug event\n"); 1912 1913 if (reset != 0) { 1914 ifp->if_flags &= ~IFF_RUNNING; 1915 vmxnet3_init_locked(sc); 1916 } 1917 1918 VMXNET3_CORE_UNLOCK(sc); 1919 } 1920 1921 static void 1922 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq) 1923 { 1924 struct vmxnet3_softc *sc; 1925 struct ifnet *ifp; 1926 struct vmxnet3_txring *txr; 1927 struct vmxnet3_comp_ring *txc; 1928 struct vmxnet3_txcompdesc *txcd; 1929 struct vmxnet3_txbuf *txb; 1930 struct mbuf *m; 1931 u_int sop; 1932 1933 sc = txq->vxtxq_sc; 1934 ifp = sc->vmx_ifp; 1935 txr = &txq->vxtxq_cmd_ring; 1936 txc = &txq->vxtxq_comp_ring; 1937 1938 VMXNET3_TXQ_LOCK_ASSERT(txq); 1939 1940 for (;;) { 1941 txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 1942 if (txcd->gen != txc->vxcr_gen) 1943 break; 1944 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1945 1946 if (++txc->vxcr_next == txc->vxcr_ndesc) { 1947 txc->vxcr_next = 0; 1948 txc->vxcr_gen ^= 1; 1949 } 1950 1951 sop = txr->vxtxr_next; 1952 txb = &txr->vxtxr_txbuf[sop]; 1953 1954 if ((m = txb->vtxb_m) != NULL) { 1955 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 1956 BUS_DMASYNC_POSTWRITE); 1957 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 1958 1959 txq->vxtxq_stats.vmtxs_opackets++; 1960 txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len; 1961 if (m->m_flags & M_MCAST) 1962 txq->vxtxq_stats.vmtxs_omcasts++; 1963 1964 m_freem(m); 1965 txb->vtxb_m = NULL; 1966 } 1967 1968 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 1969 } 1970 1971 if (txr->vxtxr_head == txr->vxtxr_next) 1972 txq->vxtxq_watchdog = 0; 1973 } 1974 1975 static int 1976 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr) 1977 { 1978 struct ifnet *ifp; 1979 struct mbuf *m; 1980 struct vmxnet3_rxdesc *rxd; 1981 struct vmxnet3_rxbuf *rxb; 1982 bus_dma_tag_t tag; 1983 bus_dmamap_t dmap; 1984 bus_dma_segment_t segs[1]; 1985 int idx, clsize, btype, flags, nsegs, error; 1986 1987 ifp = sc->vmx_ifp; 1988 tag = rxr->vxrxr_rxtag; 1989 dmap = rxr->vxrxr_spare_dmap; 1990 idx = rxr->vxrxr_fill; 1991 rxd = &rxr->vxrxr_rxd[idx]; 1992 rxb = &rxr->vxrxr_rxbuf[idx]; 1993 1994 #ifdef VMXNET3_FAILPOINTS 1995 KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS); 1996 if (rxr->vxrxr_rid != 0) 1997 KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS); 1998 #endif 1999 2000 if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) { 2001 flags = M_PKTHDR; 2002 clsize = MCLBYTES; 2003 btype = VMXNET3_BTYPE_HEAD; 2004 } else { 2005 #if __FreeBSD_version < 902001 2006 /* 2007 * These mbufs will never be used for the start of a frame. 2008 * Roughly prior to branching releng/9.2, the load_mbuf_sg() 2009 * required the mbuf to always be a packet header. Avoid 2010 * unnecessary mbuf initialization in newer versions where 2011 * that is not the case. 2012 */ 2013 flags = M_PKTHDR; 2014 #else 2015 flags = 0; 2016 #endif 2017 clsize = MJUMPAGESIZE; 2018 btype = VMXNET3_BTYPE_BODY; 2019 } 2020 2021 m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize); 2022 if (m == NULL) { 2023 sc->vmx_stats.vmst_mgetcl_failed++; 2024 return (ENOBUFS); 2025 } 2026 2027 if (btype == VMXNET3_BTYPE_HEAD) { 2028 m->m_len = m->m_pkthdr.len = clsize; 2029 m_adj(m, ETHER_ALIGN); 2030 } else 2031 m->m_len = clsize; 2032 2033 error = bus_dmamap_load_mbuf_segment(tag, dmap, m, &segs[0], 1, &nsegs, 2034 BUS_DMA_NOWAIT); 2035 if (error) { 2036 m_freem(m); 2037 sc->vmx_stats.vmst_mbuf_load_failed++; 2038 return (error); 2039 } 2040 KASSERT(nsegs == 1, 2041 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 2042 #if __FreeBSD_version < 902001 2043 if (btype == VMXNET3_BTYPE_BODY) 2044 m->m_flags &= ~M_PKTHDR; 2045 #endif 2046 2047 if (rxb->vrxb_m != NULL) { 2048 bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD); 2049 bus_dmamap_unload(tag, rxb->vrxb_dmamap); 2050 } 2051 2052 rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 2053 rxb->vrxb_dmamap = dmap; 2054 rxb->vrxb_m = m; 2055 2056 rxd->addr = segs[0].ds_addr; 2057 rxd->len = segs[0].ds_len; 2058 rxd->btype = btype; 2059 rxd->gen = rxr->vxrxr_gen; 2060 2061 vmxnet3_rxr_increment_fill(rxr); 2062 return (0); 2063 } 2064 2065 static void 2066 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 2067 struct vmxnet3_rxring *rxr, int idx) 2068 { 2069 struct vmxnet3_rxdesc *rxd; 2070 2071 rxd = &rxr->vxrxr_rxd[idx]; 2072 rxd->gen = rxr->vxrxr_gen; 2073 vmxnet3_rxr_increment_fill(rxr); 2074 } 2075 2076 static void 2077 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 2078 { 2079 struct vmxnet3_softc *sc; 2080 struct vmxnet3_rxring *rxr; 2081 struct vmxnet3_comp_ring *rxc; 2082 struct vmxnet3_rxcompdesc *rxcd; 2083 int idx, eof; 2084 2085 sc = rxq->vxrxq_sc; 2086 rxc = &rxq->vxrxq_comp_ring; 2087 2088 do { 2089 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2090 if (rxcd->gen != rxc->vxcr_gen) 2091 break; /* Not expected. */ 2092 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2093 2094 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2095 rxc->vxcr_next = 0; 2096 rxc->vxcr_gen ^= 1; 2097 } 2098 2099 idx = rxcd->rxd_idx; 2100 eof = rxcd->eop; 2101 if (rxcd->qid < sc->vmx_nrxqueues) 2102 rxr = &rxq->vxrxq_cmd_ring[0]; 2103 else 2104 rxr = &rxq->vxrxq_cmd_ring[1]; 2105 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2106 } while (!eof); 2107 } 2108 2109 static void 2110 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2111 { 2112 2113 if (rxcd->ipv4) { 2114 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 2115 if (rxcd->ipcsum_ok) 2116 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2117 } 2118 2119 if (!rxcd->fragment) { 2120 if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) { 2121 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 2122 CSUM_PSEUDO_HDR; 2123 m->m_pkthdr.csum_data = 0xFFFF; 2124 } 2125 } 2126 } 2127 2128 static void 2129 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 2130 struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2131 { 2132 struct vmxnet3_softc *sc; 2133 struct ifnet *ifp; 2134 2135 sc = rxq->vxrxq_sc; 2136 ifp = sc->vmx_ifp; 2137 2138 if (rxcd->error) { 2139 rxq->vxrxq_stats.vmrxs_ierrors++; 2140 m_freem(m); 2141 return; 2142 } 2143 2144 #if 0 2145 #ifdef notyet 2146 switch (rxcd->rss_type) { 2147 case VMXNET3_RCD_RSS_TYPE_IPV4: 2148 m->m_pkthdr.flowid = rxcd->rss_hash; 2149 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4); 2150 break; 2151 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 2152 m->m_pkthdr.flowid = rxcd->rss_hash; 2153 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4); 2154 break; 2155 case VMXNET3_RCD_RSS_TYPE_IPV6: 2156 m->m_pkthdr.flowid = rxcd->rss_hash; 2157 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6); 2158 break; 2159 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 2160 m->m_pkthdr.flowid = rxcd->rss_hash; 2161 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6); 2162 break; 2163 default: /* VMXNET3_RCD_RSS_TYPE_NONE */ 2164 m->m_pkthdr.flowid = rxq->vxrxq_id; 2165 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2166 break; 2167 } 2168 #else 2169 m->m_pkthdr.flowid = rxq->vxrxq_id; 2170 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2171 #endif 2172 #endif 2173 2174 if (!rxcd->no_csum) 2175 vmxnet3_rx_csum(rxcd, m); 2176 if (rxcd->vlan) { 2177 m->m_flags |= M_VLANTAG; 2178 m->m_pkthdr.ether_vlantag = rxcd->vtag; 2179 } 2180 2181 rxq->vxrxq_stats.vmrxs_ipackets++; 2182 rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len; 2183 2184 VMXNET3_RXQ_UNLOCK(rxq); 2185 (*ifp->if_input)(ifp, m, NULL, -1); 2186 VMXNET3_RXQ_LOCK(rxq); 2187 } 2188 2189 static void 2190 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq) 2191 { 2192 struct vmxnet3_softc *sc; 2193 struct ifnet *ifp; 2194 struct vmxnet3_rxring *rxr; 2195 struct vmxnet3_comp_ring *rxc; 2196 struct vmxnet3_rxdesc *rxd; 2197 struct vmxnet3_rxcompdesc *rxcd; 2198 struct mbuf *m, *m_head, *m_tail; 2199 int idx, length; 2200 2201 sc = rxq->vxrxq_sc; 2202 ifp = sc->vmx_ifp; 2203 rxc = &rxq->vxrxq_comp_ring; 2204 2205 VMXNET3_RXQ_LOCK_ASSERT(rxq); 2206 2207 if ((ifp->if_flags & IFF_RUNNING) == 0) 2208 return; 2209 2210 m_head = rxq->vxrxq_mhead; 2211 rxq->vxrxq_mhead = NULL; 2212 m_tail = rxq->vxrxq_mtail; 2213 rxq->vxrxq_mtail = NULL; 2214 KKASSERT(m_head == NULL || m_tail != NULL); 2215 2216 for (;;) { 2217 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2218 if (rxcd->gen != rxc->vxcr_gen) { 2219 rxq->vxrxq_mhead = m_head; 2220 rxq->vxrxq_mtail = m_tail; 2221 break; 2222 } 2223 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2224 2225 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2226 rxc->vxcr_next = 0; 2227 rxc->vxcr_gen ^= 1; 2228 } 2229 2230 idx = rxcd->rxd_idx; 2231 length = rxcd->len; 2232 if (rxcd->qid < sc->vmx_nrxqueues) 2233 rxr = &rxq->vxrxq_cmd_ring[0]; 2234 else 2235 rxr = &rxq->vxrxq_cmd_ring[1]; 2236 rxd = &rxr->vxrxr_rxd[idx]; 2237 2238 m = rxr->vxrxr_rxbuf[idx].vrxb_m; 2239 KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf", 2240 __func__, rxcd->qid, idx)); 2241 2242 /* 2243 * The host may skip descriptors. We detect this when this 2244 * descriptor does not match the previous fill index. Catch 2245 * up with the host now. 2246 */ 2247 if (__predict_false(rxr->vxrxr_fill != idx)) { 2248 while (rxr->vxrxr_fill != idx) { 2249 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 2250 rxr->vxrxr_gen; 2251 vmxnet3_rxr_increment_fill(rxr); 2252 } 2253 } 2254 2255 if (rxcd->sop) { 2256 KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD, 2257 ("%s: start of frame w/o head buffer", __func__)); 2258 KASSERT(rxr == &rxq->vxrxq_cmd_ring[0], 2259 ("%s: start of frame not in ring 0", __func__)); 2260 KASSERT((idx % sc->vmx_rx_max_chain) == 0, 2261 ("%s: start of frame at unexcepted index %d (%d)", 2262 __func__, idx, sc->vmx_rx_max_chain)); 2263 KASSERT(m_head == NULL, 2264 ("%s: duplicate start of frame?", __func__)); 2265 2266 if (length == 0) { 2267 /* Just ignore this descriptor. */ 2268 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2269 goto nextp; 2270 } 2271 2272 if (vmxnet3_newbuf(sc, rxr) != 0) { 2273 rxq->vxrxq_stats.vmrxs_iqdrops++; 2274 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2275 if (!rxcd->eop) 2276 vmxnet3_rxq_discard_chain(rxq); 2277 goto nextp; 2278 } 2279 2280 m->m_pkthdr.rcvif = ifp; 2281 m->m_pkthdr.len = m->m_len = length; 2282 m->m_pkthdr.csum_flags = 0; 2283 m_head = m_tail = m; 2284 2285 } else { 2286 KASSERT(rxd->btype == VMXNET3_BTYPE_BODY, 2287 ("%s: non start of frame w/o body buffer", __func__)); 2288 2289 if (m_head == NULL && m_tail == NULL) { 2290 /* 2291 * This is a continuation of a packet that we 2292 * started to drop, but could not drop entirely 2293 * because this segment was still owned by the 2294 * host. So, drop the remainder now. 2295 */ 2296 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2297 if (!rxcd->eop) 2298 vmxnet3_rxq_discard_chain(rxq); 2299 goto nextp; 2300 } 2301 2302 KASSERT(m_head != NULL, 2303 ("%s: frame not started?", __func__)); 2304 2305 if (vmxnet3_newbuf(sc, rxr) != 0) { 2306 rxq->vxrxq_stats.vmrxs_iqdrops++; 2307 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2308 if (!rxcd->eop) 2309 vmxnet3_rxq_discard_chain(rxq); 2310 m_freem(m_head); 2311 m_head = m_tail = NULL; 2312 goto nextp; 2313 } 2314 2315 m->m_len = length; 2316 m_head->m_pkthdr.len += length; 2317 m_tail->m_next = m; 2318 m_tail = m; 2319 } 2320 2321 if (rxcd->eop) { 2322 vmxnet3_rxq_input(rxq, rxcd, m_head); 2323 m_head = m_tail = NULL; 2324 2325 /* Must recheck after dropping the Rx lock. */ 2326 if ((ifp->if_flags & IFF_RUNNING) == 0) 2327 break; 2328 } 2329 2330 nextp: 2331 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 2332 int qid = rxcd->qid; 2333 bus_size_t r; 2334 2335 idx = (idx + 1) % rxr->vxrxr_ndesc; 2336 if (qid >= sc->vmx_nrxqueues) { 2337 qid -= sc->vmx_nrxqueues; 2338 r = VMXNET3_BAR0_RXH2(qid); 2339 } else 2340 r = VMXNET3_BAR0_RXH1(qid); 2341 vmxnet3_write_bar0(sc, r, idx); 2342 } 2343 } 2344 } 2345 2346 static void 2347 vmxnet3_legacy_intr(void *xsc) 2348 { 2349 struct vmxnet3_softc *sc; 2350 struct vmxnet3_rxqueue *rxq; 2351 struct vmxnet3_txqueue *txq; 2352 2353 sc = xsc; 2354 rxq = &sc->vmx_rxq[0]; 2355 txq = &sc->vmx_txq[0]; 2356 2357 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 2358 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 2359 return; 2360 } 2361 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2362 vmxnet3_disable_all_intrs(sc); 2363 2364 if (sc->vmx_ds->event != 0) 2365 vmxnet3_evintr(sc); 2366 2367 VMXNET3_RXQ_LOCK(rxq); 2368 vmxnet3_rxq_eof(rxq); 2369 VMXNET3_RXQ_UNLOCK(rxq); 2370 2371 VMXNET3_TXQ_LOCK(txq); 2372 vmxnet3_txq_eof(txq); 2373 vmxnet3_txq_start(txq); 2374 VMXNET3_TXQ_UNLOCK(txq); 2375 2376 vmxnet3_enable_all_intrs(sc); 2377 } 2378 2379 #ifdef __FreeBSD__ 2380 static void 2381 vmxnet3_txq_intr(void *xtxq) 2382 { 2383 struct vmxnet3_softc *sc; 2384 struct vmxnet3_txqueue *txq; 2385 2386 txq = xtxq; 2387 sc = txq->vxtxq_sc; 2388 2389 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2390 vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx); 2391 2392 VMXNET3_TXQ_LOCK(txq); 2393 vmxnet3_txq_eof(txq); 2394 vmxnet3_txq_start(txq); 2395 VMXNET3_TXQ_UNLOCK(txq); 2396 2397 vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx); 2398 } 2399 2400 static void 2401 vmxnet3_rxq_intr(void *xrxq) 2402 { 2403 struct vmxnet3_softc *sc; 2404 struct vmxnet3_rxqueue *rxq; 2405 2406 rxq = xrxq; 2407 sc = rxq->vxrxq_sc; 2408 2409 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2410 vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx); 2411 2412 VMXNET3_RXQ_LOCK(rxq); 2413 vmxnet3_rxq_eof(rxq); 2414 VMXNET3_RXQ_UNLOCK(rxq); 2415 2416 vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx); 2417 } 2418 2419 static void 2420 vmxnet3_event_intr(void *xsc) 2421 { 2422 struct vmxnet3_softc *sc; 2423 2424 sc = xsc; 2425 2426 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2427 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 2428 2429 if (sc->vmx_ds->event != 0) 2430 vmxnet3_evintr(sc); 2431 2432 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2433 } 2434 #endif 2435 2436 static void 2437 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2438 { 2439 struct vmxnet3_txring *txr; 2440 struct vmxnet3_txbuf *txb; 2441 int i; 2442 2443 txr = &txq->vxtxq_cmd_ring; 2444 2445 for (i = 0; i < txr->vxtxr_ndesc; i++) { 2446 txb = &txr->vxtxr_txbuf[i]; 2447 2448 if (txb->vtxb_m == NULL) 2449 continue; 2450 2451 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 2452 BUS_DMASYNC_POSTWRITE); 2453 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 2454 m_freem(txb->vtxb_m); 2455 txb->vtxb_m = NULL; 2456 } 2457 } 2458 2459 static void 2460 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2461 { 2462 struct vmxnet3_rxring *rxr; 2463 struct vmxnet3_rxbuf *rxb; 2464 int i, j; 2465 2466 if (rxq->vxrxq_mhead != NULL) { 2467 m_freem(rxq->vxrxq_mhead); 2468 rxq->vxrxq_mhead = NULL; 2469 rxq->vxrxq_mtail = NULL; 2470 } 2471 2472 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2473 rxr = &rxq->vxrxq_cmd_ring[i]; 2474 2475 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2476 rxb = &rxr->vxrxr_rxbuf[j]; 2477 2478 if (rxb->vrxb_m == NULL) 2479 continue; 2480 2481 bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap, 2482 BUS_DMASYNC_POSTREAD); 2483 bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap); 2484 m_freem(rxb->vrxb_m); 2485 rxb->vrxb_m = NULL; 2486 } 2487 } 2488 } 2489 2490 static void 2491 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2492 { 2493 struct vmxnet3_rxqueue *rxq; 2494 struct vmxnet3_txqueue *txq; 2495 int i; 2496 2497 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2498 rxq = &sc->vmx_rxq[i]; 2499 VMXNET3_RXQ_LOCK(rxq); 2500 VMXNET3_RXQ_UNLOCK(rxq); 2501 } 2502 2503 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2504 txq = &sc->vmx_txq[i]; 2505 VMXNET3_TXQ_LOCK(txq); 2506 VMXNET3_TXQ_UNLOCK(txq); 2507 } 2508 } 2509 2510 static void 2511 vmxnet3_stop(struct vmxnet3_softc *sc) 2512 { 2513 struct ifnet *ifp; 2514 int q; 2515 2516 ifp = sc->vmx_ifp; 2517 VMXNET3_CORE_LOCK_ASSERT(sc); 2518 2519 ifp->if_flags &= ~IFF_RUNNING; 2520 sc->vmx_link_active = 0; 2521 callout_stop(&sc->vmx_tick); 2522 2523 /* Disable interrupts. */ 2524 vmxnet3_disable_all_intrs(sc); 2525 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2526 2527 vmxnet3_stop_rendezvous(sc); 2528 2529 for (q = 0; q < sc->vmx_ntxqueues; q++) 2530 vmxnet3_txstop(sc, &sc->vmx_txq[q]); 2531 for (q = 0; q < sc->vmx_nrxqueues; q++) 2532 vmxnet3_rxstop(sc, &sc->vmx_rxq[q]); 2533 2534 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2535 } 2536 2537 static void 2538 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2539 { 2540 struct vmxnet3_txring *txr; 2541 struct vmxnet3_comp_ring *txc; 2542 2543 txr = &txq->vxtxq_cmd_ring; 2544 txr->vxtxr_head = 0; 2545 txr->vxtxr_next = 0; 2546 txr->vxtxr_gen = VMXNET3_INIT_GEN; 2547 bzero(txr->vxtxr_txd, 2548 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2549 2550 txc = &txq->vxtxq_comp_ring; 2551 txc->vxcr_next = 0; 2552 txc->vxcr_gen = VMXNET3_INIT_GEN; 2553 bzero(txc->vxcr_u.txcd, 2554 txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2555 } 2556 2557 static int 2558 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2559 { 2560 struct ifnet *ifp; 2561 struct vmxnet3_rxring *rxr; 2562 struct vmxnet3_comp_ring *rxc; 2563 int i, populate, idx, frame_size, error; 2564 2565 ifp = sc->vmx_ifp; 2566 frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) + 2567 ifp->if_mtu; 2568 2569 /* 2570 * If the MTU causes us to exceed what a regular sized cluster can 2571 * handle, we allocate a second MJUMPAGESIZE cluster after it in 2572 * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters. 2573 * 2574 * Keep rx_max_chain a divisor of the maximum Rx ring size to make 2575 * our life easier. We do not support changing the ring size after 2576 * the attach. 2577 */ 2578 if (frame_size <= MCLBYTES) 2579 sc->vmx_rx_max_chain = 1; 2580 else 2581 sc->vmx_rx_max_chain = 2; 2582 2583 /* 2584 * Only populate ring 1 if the configuration will take advantage 2585 * of it. That is either when LRO is enabled or the frame size 2586 * exceeds what ring 0 can contain. 2587 */ 2588 #if 0 /* XXX LRO */ 2589 if ((ifp->if_capenable & IFCAP_LRO) == 0 && 2590 #else 2591 if ( 2592 #endif 2593 frame_size <= MCLBYTES + MJUMPAGESIZE) 2594 populate = 1; 2595 else 2596 populate = VMXNET3_RXRINGS_PERQ; 2597 2598 for (i = 0; i < populate; i++) { 2599 rxr = &rxq->vxrxq_cmd_ring[i]; 2600 rxr->vxrxr_fill = 0; 2601 rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2602 bzero(rxr->vxrxr_rxd, 2603 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2604 2605 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2606 error = vmxnet3_newbuf(sc, rxr); 2607 if (error) 2608 return (error); 2609 } 2610 } 2611 2612 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2613 rxr = &rxq->vxrxq_cmd_ring[i]; 2614 rxr->vxrxr_fill = 0; 2615 rxr->vxrxr_gen = 0; 2616 bzero(rxr->vxrxr_rxd, 2617 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2618 } 2619 2620 rxc = &rxq->vxrxq_comp_ring; 2621 rxc->vxcr_next = 0; 2622 rxc->vxcr_gen = VMXNET3_INIT_GEN; 2623 bzero(rxc->vxcr_u.rxcd, 2624 rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2625 2626 return (0); 2627 } 2628 2629 static int 2630 vmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2631 { 2632 device_t dev; 2633 int q, error; 2634 2635 dev = sc->vmx_dev; 2636 2637 for (q = 0; q < sc->vmx_ntxqueues; q++) 2638 vmxnet3_txinit(sc, &sc->vmx_txq[q]); 2639 2640 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2641 error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); 2642 if (error) { 2643 device_printf(dev, "cannot populate Rx queue %d\n", q); 2644 return (error); 2645 } 2646 } 2647 2648 return (0); 2649 } 2650 2651 static int 2652 vmxnet3_enable_device(struct vmxnet3_softc *sc) 2653 { 2654 int q; 2655 2656 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2657 device_printf(sc->vmx_dev, "device enable command failed!\n"); 2658 return (1); 2659 } 2660 2661 /* Reset the Rx queue heads. */ 2662 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2663 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2664 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2665 } 2666 2667 return (0); 2668 } 2669 2670 static void 2671 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2672 { 2673 struct ifnet *ifp; 2674 2675 ifp = sc->vmx_ifp; 2676 2677 vmxnet3_set_rxfilter(sc); 2678 2679 #if 0 /* VLAN_HWFILTER */ 2680 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2681 bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter, 2682 sizeof(sc->vmx_ds->vlan_filter)); 2683 else 2684 #endif 2685 bzero(sc->vmx_ds->vlan_filter, 2686 sizeof(sc->vmx_ds->vlan_filter)); 2687 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2688 } 2689 2690 static int 2691 vmxnet3_reinit(struct vmxnet3_softc *sc) 2692 { 2693 2694 vmxnet3_reinit_interface(sc); 2695 vmxnet3_reinit_shared_data(sc); 2696 2697 if (vmxnet3_reinit_queues(sc) != 0) 2698 return (ENXIO); 2699 2700 if (vmxnet3_enable_device(sc) != 0) 2701 return (ENXIO); 2702 2703 vmxnet3_reinit_rxfilters(sc); 2704 2705 return (0); 2706 } 2707 2708 static void 2709 vmxnet3_init_locked(struct vmxnet3_softc *sc) 2710 { 2711 struct ifnet *ifp; 2712 2713 ifp = sc->vmx_ifp; 2714 2715 if (ifp->if_flags & IFF_RUNNING) 2716 return; 2717 2718 vmxnet3_stop(sc); 2719 2720 if (vmxnet3_reinit(sc) != 0) { 2721 vmxnet3_stop(sc); 2722 return; 2723 } 2724 2725 ifp->if_flags |= IFF_RUNNING; 2726 vmxnet3_link_status(sc); 2727 2728 vmxnet3_enable_all_intrs(sc); 2729 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2730 } 2731 2732 static void 2733 vmxnet3_init(void *xsc) 2734 { 2735 struct vmxnet3_softc *sc; 2736 2737 sc = xsc; 2738 2739 VMXNET3_CORE_LOCK(sc); 2740 vmxnet3_init_locked(sc); 2741 VMXNET3_CORE_UNLOCK(sc); 2742 } 2743 2744 /* 2745 * BMV: Much of this can go away once we finally have offsets in 2746 * the mbuf packet header. Bug andre@. 2747 */ 2748 static int 2749 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, 2750 int *etype, int *proto, int *start) 2751 { 2752 struct ether_vlan_header *evh; 2753 int offset; 2754 #if defined(INET) 2755 struct ip *ip = NULL; 2756 #endif 2757 #if defined(INET6) 2758 struct ip6_hdr *ip6 = NULL; 2759 #endif 2760 2761 evh = mtod(m, struct ether_vlan_header *); 2762 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2763 /* BMV: We should handle nested VLAN tags too. */ 2764 *etype = ntohs(evh->evl_proto); 2765 offset = sizeof(struct ether_vlan_header); 2766 } else { 2767 *etype = ntohs(evh->evl_encap_proto); 2768 offset = sizeof(struct ether_header); 2769 } 2770 2771 switch (*etype) { 2772 #if defined(INET) 2773 case ETHERTYPE_IP: 2774 if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 2775 m = m_pullup(m, offset + sizeof(struct ip)); 2776 if (m == NULL) 2777 return (EINVAL); 2778 } 2779 2780 ip = (struct ip *)(mtod(m, uint8_t *) + offset); 2781 *proto = ip->ip_p; 2782 *start = offset + (ip->ip_hl << 2); 2783 break; 2784 #endif 2785 #if defined(INET6) 2786 case ETHERTYPE_IPV6: 2787 if (__predict_false(m->m_len < 2788 offset + sizeof(struct ip6_hdr))) { 2789 m = m_pullup(m, offset + sizeof(struct ip6_hdr)); 2790 if (m == NULL) 2791 return (EINVAL); 2792 } 2793 2794 ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + offset); 2795 *proto = -1; 2796 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 2797 /* Assert the network stack sent us a valid packet. */ 2798 KASSERT(*start > offset, 2799 ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 2800 *start, offset, *proto)); 2801 break; 2802 #endif 2803 default: 2804 return (EINVAL); 2805 } 2806 2807 #if 0 /* XXX TSO */ 2808 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2809 struct tcphdr *tcp; 2810 2811 if (__predict_false(*proto != IPPROTO_TCP)) { 2812 /* Likely failed to correctly parse the mbuf. */ 2813 return (EINVAL); 2814 } 2815 2816 if (m->m_len < *start + sizeof(struct tcphdr)) { 2817 m = m_pullup(m, *start + sizeof(struct tcphdr)); 2818 if (m == NULL) 2819 return (EINVAL); 2820 } 2821 2822 tcp = (struct tcphdr *)(mtod(m, uint8_t *) + *start); 2823 *start += (tcp->th_off << 2); 2824 2825 txq->vxtxq_stats.vmtxs_tso++; 2826 } else 2827 #endif 2828 txq->vxtxq_stats.vmtxs_csum++; 2829 2830 return (0); 2831 } 2832 2833 static int 2834 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 2835 bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs) 2836 { 2837 struct vmxnet3_txring *txr; 2838 struct mbuf *m; 2839 bus_dma_tag_t tag; 2840 int error; 2841 2842 txr = &txq->vxtxq_cmd_ring; 2843 m = *m0; 2844 tag = txr->vxtxr_txtag; 2845 2846 error = bus_dmamap_load_mbuf_segment(tag, dmap, m, segs, 1, nsegs, 2847 BUS_DMA_NOWAIT); 2848 if (error == 0 || error != EFBIG) 2849 return (error); 2850 2851 m = m_defrag(m, M_NOWAIT); 2852 if (m != NULL) { 2853 *m0 = m; 2854 error = bus_dmamap_load_mbuf_segment(tag, dmap, m, segs, 2855 1, nsegs, BUS_DMA_NOWAIT); 2856 } else 2857 error = ENOBUFS; 2858 2859 if (error) { 2860 m_freem(*m0); 2861 *m0 = NULL; 2862 txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++; 2863 } else 2864 txq->vxtxq_sc->vmx_stats.vmst_defragged++; 2865 2866 return (error); 2867 } 2868 2869 static void 2870 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 2871 { 2872 struct vmxnet3_txring *txr; 2873 2874 txr = &txq->vxtxq_cmd_ring; 2875 bus_dmamap_unload(txr->vxtxr_txtag, dmap); 2876 } 2877 2878 static int 2879 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 2880 { 2881 struct vmxnet3_softc *sc; 2882 struct vmxnet3_txring *txr; 2883 struct vmxnet3_txdesc *txd, *sop; 2884 struct mbuf *m; 2885 bus_dmamap_t dmap; 2886 bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS]; 2887 int i, gen, nsegs, etype, proto, start, error; 2888 2889 sc = txq->vxtxq_sc; 2890 start = 0; 2891 txd = NULL; 2892 txr = &txq->vxtxq_cmd_ring; 2893 dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 2894 2895 error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs); 2896 if (error) 2897 return (error); 2898 2899 m = *m0; 2900 M_ASSERTPKTHDR(m); 2901 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS, 2902 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 2903 2904 if (VMXNET3_TXRING_AVAIL(txr) < nsegs) { 2905 txq->vxtxq_stats.vmtxs_full++; 2906 vmxnet3_txq_unload_mbuf(txq, dmap); 2907 return (ENOSPC); 2908 } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 2909 error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start); 2910 if (error) { 2911 txq->vxtxq_stats.vmtxs_offload_failed++; 2912 vmxnet3_txq_unload_mbuf(txq, dmap); 2913 m_freem(m); 2914 *m0 = NULL; 2915 return (error); 2916 } 2917 } 2918 2919 txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m; 2920 sop = &txr->vxtxr_txd[txr->vxtxr_head]; 2921 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 2922 2923 for (i = 0; i < nsegs; i++) { 2924 txd = &txr->vxtxr_txd[txr->vxtxr_head]; 2925 2926 txd->addr = segs[i].ds_addr; 2927 txd->len = segs[i].ds_len; 2928 txd->gen = gen; 2929 txd->dtype = 0; 2930 txd->offload_mode = VMXNET3_OM_NONE; 2931 txd->offload_pos = 0; 2932 txd->hlen = 0; 2933 txd->eop = 0; 2934 txd->compreq = 0; 2935 txd->vtag_mode = 0; 2936 txd->vtag = 0; 2937 2938 if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 2939 txr->vxtxr_head = 0; 2940 txr->vxtxr_gen ^= 1; 2941 } 2942 gen = txr->vxtxr_gen; 2943 } 2944 txd->eop = 1; 2945 txd->compreq = 1; 2946 2947 if (m->m_flags & M_VLANTAG) { 2948 sop->vtag_mode = 1; 2949 sop->vtag = m->m_pkthdr.ether_vlantag; 2950 } 2951 2952 2953 #if 0 /* XXX TSO */ 2954 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2955 sop->offload_mode = VMXNET3_OM_TSO; 2956 sop->hlen = start; 2957 sop->offload_pos = m->m_pkthdr.tso_segsz; 2958 } else 2959 #endif 2960 if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 2961 VMXNET3_CSUM_OFFLOAD_IPV6)) { 2962 sop->offload_mode = VMXNET3_OM_CSUM; 2963 sop->hlen = start; 2964 sop->offload_pos = start + m->m_pkthdr.csum_data; 2965 } 2966 2967 /* Finally, change the ownership. */ 2968 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 2969 sop->gen ^= 1; 2970 2971 txq->vxtxq_ts->npending += nsegs; 2972 if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 2973 txq->vxtxq_ts->npending = 0; 2974 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), 2975 txr->vxtxr_head); 2976 } 2977 2978 return (0); 2979 } 2980 2981 #ifdef VMXNET3_LEGACY_TX 2982 2983 static void 2984 vmxnet3_start_locked(struct ifnet *ifp) 2985 { 2986 struct vmxnet3_softc *sc; 2987 struct vmxnet3_txqueue *txq; 2988 struct vmxnet3_txring *txr; 2989 struct mbuf *m_head; 2990 int tx, avail; 2991 2992 sc = ifp->if_softc; 2993 txq = &sc->vmx_txq[0]; 2994 txr = &txq->vxtxq_cmd_ring; 2995 tx = 0; 2996 2997 VMXNET3_TXQ_LOCK_ASSERT(txq); 2998 2999 if ((ifp->if_flags & IFF_RUNNING) == 0 || 3000 sc->vmx_link_active == 0) 3001 return; 3002 3003 while (!ifq_is_empty(&ifp->if_snd)) { 3004 if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2) 3005 break; 3006 3007 m_head = ifq_dequeue(&ifp->if_snd); 3008 if (m_head == NULL) 3009 break; 3010 3011 /* Assume worse case if this mbuf is the head of a chain. */ 3012 if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 3013 ifq_prepend(&ifp->if_snd, m_head); 3014 break; 3015 } 3016 3017 if (vmxnet3_txq_encap(txq, &m_head) != 0) { 3018 if (m_head != NULL) 3019 ifq_prepend(&ifp->if_snd, m_head); 3020 break; 3021 } 3022 3023 tx++; 3024 ETHER_BPF_MTAP(ifp, m_head); 3025 } 3026 3027 if (tx > 0) 3028 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 3029 } 3030 3031 static void 3032 vmxnet3_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 3033 { 3034 struct vmxnet3_softc *sc; 3035 struct vmxnet3_txqueue *txq; 3036 3037 sc = ifp->if_softc; 3038 txq = &sc->vmx_txq[0]; 3039 3040 VMXNET3_TXQ_LOCK(txq); 3041 vmxnet3_start_locked(ifp); 3042 VMXNET3_TXQ_UNLOCK(txq); 3043 } 3044 3045 #else /* !VMXNET3_LEGACY_TX */ 3046 3047 static int 3048 vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m) 3049 { 3050 struct vmxnet3_softc *sc; 3051 struct vmxnet3_txring *txr; 3052 struct buf_ring *br; 3053 struct ifnet *ifp; 3054 int tx, avail, error; 3055 3056 sc = txq->vxtxq_sc; 3057 br = txq->vxtxq_br; 3058 ifp = sc->vmx_ifp; 3059 txr = &txq->vxtxq_cmd_ring; 3060 tx = 0; 3061 error = 0; 3062 3063 VMXNET3_TXQ_LOCK_ASSERT(txq); 3064 3065 if ((ifp->if_flags & IFF_RUNNING) == 0 || 3066 sc->vmx_link_active == 0) { 3067 if (m != NULL) 3068 error = drbr_enqueue(ifp, br, m); 3069 return (error); 3070 } 3071 3072 if (m != NULL) { 3073 error = drbr_enqueue(ifp, br, m); 3074 if (error) 3075 return (error); 3076 } 3077 3078 while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) { 3079 m = drbr_peek(ifp, br); 3080 if (m == NULL) 3081 break; 3082 3083 /* Assume worse case if this mbuf is the head of a chain. */ 3084 if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 3085 drbr_putback(ifp, br, m); 3086 break; 3087 } 3088 3089 if (vmxnet3_txq_encap(txq, &m) != 0) { 3090 if (m != NULL) 3091 drbr_putback(ifp, br, m); 3092 else 3093 drbr_advance(ifp, br); 3094 break; 3095 } 3096 drbr_advance(ifp, br); 3097 3098 tx++; 3099 ETHER_BPF_MTAP(ifp, m); 3100 } 3101 3102 if (tx > 0) 3103 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 3104 3105 return (0); 3106 } 3107 3108 static int 3109 vmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m) 3110 { 3111 struct vmxnet3_softc *sc; 3112 struct vmxnet3_txqueue *txq; 3113 int i, ntxq, error; 3114 3115 sc = ifp->if_softc; 3116 ntxq = sc->vmx_ntxqueues; 3117 3118 /* check if flowid is set */ 3119 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 3120 i = m->m_pkthdr.flowid % ntxq; 3121 else 3122 i = curcpu % ntxq; 3123 3124 txq = &sc->vmx_txq[i]; 3125 3126 if (VMXNET3_TXQ_TRYLOCK(txq) != 0) { 3127 error = vmxnet3_txq_mq_start_locked(txq, m); 3128 VMXNET3_TXQ_UNLOCK(txq); 3129 } else { 3130 error = drbr_enqueue(ifp, txq->vxtxq_br, m); 3131 taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask); 3132 } 3133 3134 return (error); 3135 } 3136 3137 static void 3138 vmxnet3_txq_tq_deferred(void *xtxq, int pending) 3139 { 3140 struct vmxnet3_softc *sc; 3141 struct vmxnet3_txqueue *txq; 3142 3143 txq = xtxq; 3144 sc = txq->vxtxq_sc; 3145 3146 VMXNET3_TXQ_LOCK(txq); 3147 if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br)) 3148 vmxnet3_txq_mq_start_locked(txq, NULL); 3149 VMXNET3_TXQ_UNLOCK(txq); 3150 } 3151 3152 #endif /* VMXNET3_LEGACY_TX */ 3153 3154 static void 3155 vmxnet3_txq_start(struct vmxnet3_txqueue *txq) 3156 { 3157 struct vmxnet3_softc *sc; 3158 struct ifnet *ifp; 3159 3160 sc = txq->vxtxq_sc; 3161 ifp = sc->vmx_ifp; 3162 3163 #ifdef VMXNET3_LEGACY_TX 3164 if (!ifq_is_empty(&ifp->if_snd)) 3165 vmxnet3_start_locked(ifp); 3166 #else 3167 if (!drbr_empty(ifp, txq->vxtxq_br)) 3168 vmxnet3_txq_mq_start_locked(txq, NULL); 3169 #endif 3170 } 3171 3172 static void 3173 vmxnet3_tx_start_all(struct vmxnet3_softc *sc) 3174 { 3175 struct vmxnet3_txqueue *txq; 3176 int i; 3177 3178 VMXNET3_CORE_LOCK_ASSERT(sc); 3179 3180 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3181 txq = &sc->vmx_txq[i]; 3182 3183 VMXNET3_TXQ_LOCK(txq); 3184 vmxnet3_txq_start(txq); 3185 VMXNET3_TXQ_UNLOCK(txq); 3186 } 3187 } 3188 3189 static void 3190 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag) 3191 { 3192 struct ifnet *ifp; 3193 int idx, bit; 3194 3195 ifp = sc->vmx_ifp; 3196 idx = (tag >> 5) & 0x7F; 3197 bit = tag & 0x1F; 3198 3199 if (tag == 0 || tag > 4095) 3200 return; 3201 3202 VMXNET3_CORE_LOCK(sc); 3203 3204 /* Update our private VLAN bitvector. */ 3205 if (add) 3206 sc->vmx_vlan_filter[idx] |= (1 << bit); 3207 else 3208 sc->vmx_vlan_filter[idx] &= ~(1 << bit); 3209 3210 #if 0 /* VLAN_HWFILTER */ 3211 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3212 if (add) 3213 sc->vmx_ds->vlan_filter[idx] |= (1 << bit); 3214 else 3215 sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit); 3216 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 3217 } 3218 #endif 3219 3220 VMXNET3_CORE_UNLOCK(sc); 3221 } 3222 3223 static void 3224 vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3225 { 3226 3227 if (ifp->if_softc == arg) 3228 vmxnet3_update_vlan_filter(arg, 1, tag); 3229 } 3230 3231 static void 3232 vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3233 { 3234 3235 if (ifp->if_softc == arg) 3236 vmxnet3_update_vlan_filter(arg, 0, tag); 3237 } 3238 3239 static void 3240 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 3241 { 3242 struct ifnet *ifp; 3243 struct vmxnet3_driver_shared *ds; 3244 struct ifmultiaddr *ifma; 3245 u_int mode; 3246 3247 ifp = sc->vmx_ifp; 3248 ds = sc->vmx_ds; 3249 3250 mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST; 3251 if (ifp->if_flags & IFF_PROMISC) 3252 mode |= VMXNET3_RXMODE_PROMISC; 3253 if (ifp->if_flags & IFF_ALLMULTI) 3254 mode |= VMXNET3_RXMODE_ALLMULTI; 3255 else { 3256 int cnt = 0, overflow = 0; 3257 3258 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3259 if (ifma->ifma_addr->sa_family != AF_LINK) 3260 continue; 3261 else if (cnt == VMXNET3_MULTICAST_MAX) { 3262 overflow = 1; 3263 break; 3264 } 3265 3266 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 3267 &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN); 3268 cnt++; 3269 } 3270 3271 if (overflow != 0) { 3272 cnt = 0; 3273 mode |= VMXNET3_RXMODE_ALLMULTI; 3274 } else if (cnt > 0) 3275 mode |= VMXNET3_RXMODE_MCAST; 3276 ds->mcast_tablelen = cnt * ETHER_ADDR_LEN; 3277 } 3278 3279 ds->rxmode = mode; 3280 3281 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 3282 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 3283 } 3284 3285 static int 3286 vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) 3287 { 3288 struct ifnet *ifp; 3289 3290 ifp = sc->vmx_ifp; 3291 3292 if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) 3293 return (EINVAL); 3294 3295 ifp->if_mtu = mtu; 3296 3297 if (ifp->if_flags & IFF_RUNNING) { 3298 ifp->if_flags &= ~IFF_RUNNING; 3299 vmxnet3_init_locked(sc); 3300 } 3301 3302 return (0); 3303 } 3304 3305 static int 3306 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cred) 3307 { 3308 struct vmxnet3_softc *sc; 3309 struct ifreq *ifr; 3310 int reinit, mask, error; 3311 3312 sc = ifp->if_softc; 3313 ifr = (struct ifreq *) data; 3314 error = 0; 3315 3316 switch (cmd) { 3317 case SIOCSIFMTU: 3318 if (ifp->if_mtu != ifr->ifr_mtu) { 3319 VMXNET3_CORE_LOCK(sc); 3320 error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); 3321 VMXNET3_CORE_UNLOCK(sc); 3322 } 3323 break; 3324 3325 case SIOCSIFFLAGS: 3326 VMXNET3_CORE_LOCK(sc); 3327 if (ifp->if_flags & IFF_UP) { 3328 if ((ifp->if_flags & IFF_RUNNING)) { 3329 if ((ifp->if_flags ^ sc->vmx_if_flags) & 3330 (IFF_PROMISC | IFF_ALLMULTI)) { 3331 vmxnet3_set_rxfilter(sc); 3332 } 3333 } else 3334 vmxnet3_init_locked(sc); 3335 } else { 3336 if (ifp->if_flags & IFF_RUNNING) 3337 vmxnet3_stop(sc); 3338 } 3339 sc->vmx_if_flags = ifp->if_flags; 3340 VMXNET3_CORE_UNLOCK(sc); 3341 break; 3342 3343 case SIOCADDMULTI: 3344 case SIOCDELMULTI: 3345 VMXNET3_CORE_LOCK(sc); 3346 if (ifp->if_flags & IFF_RUNNING) 3347 vmxnet3_set_rxfilter(sc); 3348 VMXNET3_CORE_UNLOCK(sc); 3349 break; 3350 3351 case SIOCSIFMEDIA: 3352 case SIOCGIFMEDIA: 3353 error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd); 3354 break; 3355 3356 case SIOCSIFCAP: 3357 VMXNET3_CORE_LOCK(sc); 3358 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 3359 3360 if (mask & IFCAP_TXCSUM) 3361 ifp->if_capenable ^= IFCAP_TXCSUM; 3362 if (mask & IFCAP_TXCSUM_IPV6) 3363 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 3364 #if 0 /* XXX TSO */ 3365 if (mask & IFCAP_TSO4) 3366 ifp->if_capenable ^= IFCAP_TSO4; 3367 if (mask & IFCAP_TSO6) 3368 ifp->if_capenable ^= IFCAP_TSO6; 3369 #endif 3370 3371 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | /* IFCAP_LRO | */ 3372 IFCAP_VLAN_HWTAGGING /* | IFCAP_VLAN_HWFILTER */)) { 3373 /* Changing these features requires us to reinit. */ 3374 reinit = 1; 3375 3376 if (mask & IFCAP_RXCSUM) 3377 ifp->if_capenable ^= IFCAP_RXCSUM; 3378 if (mask & IFCAP_RXCSUM_IPV6) 3379 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 3380 #if 0 /* XXX LRO */ 3381 if (mask & IFCAP_LRO) 3382 ifp->if_capenable ^= IFCAP_LRO; 3383 #endif 3384 if (mask & IFCAP_VLAN_HWTAGGING) 3385 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 3386 #if 0 /* XXX VLAN_HWFILTER */ 3387 if (mask & IFCAP_VLAN_HWFILTER) 3388 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 3389 #endif 3390 } else 3391 reinit = 0; 3392 3393 #if 0 /* XXX TSO */ 3394 if (mask & IFCAP_VLAN_HWTSO) 3395 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 3396 #endif 3397 3398 if (reinit && (ifp->if_flags & IFF_RUNNING)) { 3399 ifp->if_flags &= ~IFF_RUNNING; 3400 vmxnet3_init_locked(sc); 3401 } else { 3402 vmxnet3_init_hwassist(sc); 3403 } 3404 3405 VMXNET3_CORE_UNLOCK(sc); 3406 #if 0 /* XXX */ 3407 VLAN_CAPABILITIES(ifp); 3408 #endif 3409 break; 3410 3411 default: 3412 error = ether_ioctl(ifp, cmd, data); 3413 break; 3414 } 3415 3416 VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc); 3417 3418 return (error); 3419 } 3420 3421 #ifndef VMXNET3_LEGACY_TX 3422 static void 3423 vmxnet3_qflush(struct ifnet *ifp) 3424 { 3425 struct vmxnet3_softc *sc; 3426 struct vmxnet3_txqueue *txq; 3427 struct mbuf *m; 3428 int i; 3429 3430 sc = ifp->if_softc; 3431 3432 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3433 txq = &sc->vmx_txq[i]; 3434 3435 VMXNET3_TXQ_LOCK(txq); 3436 while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL) 3437 m_freem(m); 3438 VMXNET3_TXQ_UNLOCK(txq); 3439 } 3440 3441 if_qflush(ifp); 3442 } 3443 #endif 3444 3445 static int 3446 vmxnet3_watchdog(struct vmxnet3_txqueue *txq) 3447 { 3448 struct vmxnet3_softc *sc; 3449 3450 sc = txq->vxtxq_sc; 3451 3452 VMXNET3_TXQ_LOCK(txq); 3453 if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 3454 VMXNET3_TXQ_UNLOCK(txq); 3455 return (0); 3456 } 3457 VMXNET3_TXQ_UNLOCK(txq); 3458 3459 if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n", 3460 txq->vxtxq_id); 3461 return (1); 3462 } 3463 3464 static void 3465 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) 3466 { 3467 3468 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 3469 } 3470 3471 static void 3472 vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *txq, 3473 struct vmxnet3_txq_stats *accum) 3474 { 3475 struct vmxnet3_txq_stats *st; 3476 3477 st = &txq->vxtxq_stats; 3478 3479 accum->vmtxs_opackets += st->vmtxs_opackets; 3480 accum->vmtxs_obytes += st->vmtxs_obytes; 3481 accum->vmtxs_omcasts += st->vmtxs_omcasts; 3482 accum->vmtxs_csum += st->vmtxs_csum; 3483 accum->vmtxs_tso += st->vmtxs_tso; 3484 accum->vmtxs_full += st->vmtxs_full; 3485 accum->vmtxs_offload_failed += st->vmtxs_offload_failed; 3486 } 3487 3488 static void 3489 vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *rxq, 3490 struct vmxnet3_rxq_stats *accum) 3491 { 3492 struct vmxnet3_rxq_stats *st; 3493 3494 st = &rxq->vxrxq_stats; 3495 3496 accum->vmrxs_ipackets += st->vmrxs_ipackets; 3497 accum->vmrxs_ibytes += st->vmrxs_ibytes; 3498 accum->vmrxs_iqdrops += st->vmrxs_iqdrops; 3499 accum->vmrxs_ierrors += st->vmrxs_ierrors; 3500 } 3501 3502 static void 3503 vmxnet3_accumulate_stats(struct vmxnet3_softc *sc) 3504 { 3505 struct ifnet *ifp; 3506 struct vmxnet3_statistics *st; 3507 struct vmxnet3_txq_stats txaccum; 3508 struct vmxnet3_rxq_stats rxaccum; 3509 int i; 3510 3511 ifp = sc->vmx_ifp; 3512 st = &sc->vmx_stats; 3513 3514 bzero(&txaccum, sizeof(struct vmxnet3_txq_stats)); 3515 bzero(&rxaccum, sizeof(struct vmxnet3_rxq_stats)); 3516 3517 for (i = 0; i < sc->vmx_ntxqueues; i++) 3518 vmxnet3_txq_accum_stats(&sc->vmx_txq[i], &txaccum); 3519 for (i = 0; i < sc->vmx_nrxqueues; i++) 3520 vmxnet3_rxq_accum_stats(&sc->vmx_rxq[i], &rxaccum); 3521 3522 /* 3523 * With the exception of if_ierrors, these ifnet statistics are 3524 * only updated in the driver, so just set them to our accumulated 3525 * values. if_ierrors is updated in ether_input() for malformed 3526 * frames that we should have already discarded. 3527 */ 3528 ifp->if_ipackets = rxaccum.vmrxs_ipackets; 3529 ifp->if_iqdrops = rxaccum.vmrxs_iqdrops; 3530 ifp->if_ierrors = rxaccum.vmrxs_ierrors; 3531 ifp->if_opackets = txaccum.vmtxs_opackets; 3532 #ifndef VMXNET3_LEGACY_TX 3533 ifp->if_obytes = txaccum.vmtxs_obytes; 3534 ifp->if_omcasts = txaccum.vmtxs_omcasts; 3535 #endif 3536 } 3537 3538 static void 3539 vmxnet3_tick(void *xsc) 3540 { 3541 struct vmxnet3_softc *sc; 3542 struct ifnet *ifp; 3543 int i, timedout; 3544 3545 sc = xsc; 3546 ifp = sc->vmx_ifp; 3547 timedout = 0; 3548 3549 VMXNET3_CORE_LOCK_ASSERT(sc); 3550 3551 vmxnet3_accumulate_stats(sc); 3552 vmxnet3_refresh_host_stats(sc); 3553 3554 for (i = 0; i < sc->vmx_ntxqueues; i++) 3555 timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]); 3556 3557 if (timedout != 0) { 3558 ifp->if_flags &= ~IFF_RUNNING; 3559 vmxnet3_init_locked(sc); 3560 } else 3561 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 3562 } 3563 3564 static int 3565 vmxnet3_link_is_up(struct vmxnet3_softc *sc) 3566 { 3567 uint32_t status; 3568 3569 /* Also update the link speed while here. */ 3570 status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 3571 sc->vmx_link_speed = status >> 16; 3572 return !!(status & 0x1); 3573 } 3574 3575 static void 3576 vmxnet3_link_status(struct vmxnet3_softc *sc) 3577 { 3578 struct ifnet *ifp; 3579 int link; 3580 3581 ifp = sc->vmx_ifp; 3582 link = vmxnet3_link_is_up(sc); 3583 3584 if (link != 0 && sc->vmx_link_active == 0) { 3585 sc->vmx_link_active = 1; 3586 ifp->if_link_state = LINK_STATE_UP; 3587 if_link_state_change(ifp); 3588 } else if (link == 0 && sc->vmx_link_active != 0) { 3589 sc->vmx_link_active = 0; 3590 ifp->if_link_state = LINK_STATE_DOWN; 3591 if_link_state_change(ifp); 3592 } 3593 } 3594 3595 static void 3596 vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 3597 { 3598 struct vmxnet3_softc *sc; 3599 3600 sc = ifp->if_softc; 3601 3602 ifmr->ifm_active = IFM_ETHER | IFM_AUTO; 3603 ifmr->ifm_status = IFM_AVALID; 3604 3605 VMXNET3_CORE_LOCK(sc); 3606 if (vmxnet3_link_is_up(sc) != 0) 3607 ifmr->ifm_status |= IFM_ACTIVE; 3608 else 3609 ifmr->ifm_status |= IFM_NONE; 3610 VMXNET3_CORE_UNLOCK(sc); 3611 } 3612 3613 static int 3614 vmxnet3_media_change(struct ifnet *ifp) 3615 { 3616 3617 /* Ignore. */ 3618 return (0); 3619 } 3620 3621 static void 3622 vmxnet3_set_lladdr(struct vmxnet3_softc *sc) 3623 { 3624 uint32_t ml, mh; 3625 3626 ml = sc->vmx_lladdr[0]; 3627 ml |= sc->vmx_lladdr[1] << 8; 3628 ml |= sc->vmx_lladdr[2] << 16; 3629 ml |= sc->vmx_lladdr[3] << 24; 3630 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 3631 3632 mh = sc->vmx_lladdr[4]; 3633 mh |= sc->vmx_lladdr[5] << 8; 3634 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 3635 } 3636 3637 static void 3638 vmxnet3_get_lladdr(struct vmxnet3_softc *sc) 3639 { 3640 uint32_t ml, mh; 3641 3642 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 3643 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 3644 3645 sc->vmx_lladdr[0] = ml; 3646 sc->vmx_lladdr[1] = ml >> 8; 3647 sc->vmx_lladdr[2] = ml >> 16; 3648 sc->vmx_lladdr[3] = ml >> 24; 3649 sc->vmx_lladdr[4] = mh; 3650 sc->vmx_lladdr[5] = mh >> 8; 3651 } 3652 3653 static void 3654 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq, 3655 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3656 { 3657 struct sysctl_oid *node, *txsnode; 3658 struct sysctl_oid_list *list, *txslist; 3659 struct vmxnet3_txq_stats *stats; 3660 struct UPT1_TxStats *txstats; 3661 char namebuf[16]; 3662 3663 stats = &txq->vxtxq_stats; 3664 txstats = &txq->vxtxq_ts->stats; 3665 3666 ksnprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id); 3667 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3668 NULL, "Transmit Queue"); 3669 txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node); 3670 3671 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, 3672 &stats->vmtxs_opackets, 0, "Transmit packets"); 3673 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, 3674 &stats->vmtxs_obytes, 0, "Transmit bytes"); 3675 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, 3676 &stats->vmtxs_omcasts, 0, "Transmit multicasts"); 3677 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3678 &stats->vmtxs_csum, 0, "Transmit checksum offloaded"); 3679 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, 3680 &stats->vmtxs_tso, 0, "Transmit TCP segmentation offloaded"); 3681 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD, 3682 &stats->vmtxs_full, 0, "Transmit ring full"); 3683 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD, 3684 &stats->vmtxs_offload_failed, 0, "Transmit checksum offload failed"); 3685 3686 /* 3687 * Add statistics reported by the host. These are updated once 3688 * per second. 3689 */ 3690 txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3691 NULL, "Host Statistics"); 3692 txslist = SYSCTL_CHILDREN(txsnode); 3693 #if 0 /* XXX TSO */ 3694 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD, 3695 &txstats->TSO_packets, 0, "TSO packets"); 3696 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD, 3697 &txstats->TSO_bytes, 0, "TSO bytes"); 3698 #endif 3699 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3700 &txstats->ucast_packets, 0, "Unicast packets"); 3701 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3702 &txstats->ucast_bytes, 0, "Unicast bytes"); 3703 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3704 &txstats->mcast_packets, 0, "Multicast packets"); 3705 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3706 &txstats->mcast_bytes, 0, "Multicast bytes"); 3707 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD, 3708 &txstats->error, 0, "Errors"); 3709 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD, 3710 &txstats->discard, 0, "Discards"); 3711 } 3712 3713 static void 3714 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq, 3715 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3716 { 3717 struct sysctl_oid *node, *rxsnode; 3718 struct sysctl_oid_list *list, *rxslist; 3719 struct vmxnet3_rxq_stats *stats; 3720 struct UPT1_RxStats *rxstats; 3721 char namebuf[16]; 3722 3723 stats = &rxq->vxrxq_stats; 3724 rxstats = &rxq->vxrxq_rs->stats; 3725 3726 ksnprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id); 3727 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3728 NULL, "Receive Queue"); 3729 rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node); 3730 3731 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, 3732 &stats->vmrxs_ipackets, 0, "Receive packets"); 3733 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, 3734 &stats->vmrxs_ibytes, 0, "Receive bytes"); 3735 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, 3736 &stats->vmrxs_iqdrops, 0, "Receive drops"); 3737 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, 3738 &stats->vmrxs_ierrors, 0, "Receive errors"); 3739 3740 /* 3741 * Add statistics reported by the host. These are updated once 3742 * per second. 3743 */ 3744 rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3745 NULL, "Host Statistics"); 3746 rxslist = SYSCTL_CHILDREN(rxsnode); 3747 #if 0 /* XXX LRO */ 3748 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD, 3749 &rxstats->LRO_packets, 0, "LRO packets"); 3750 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD, 3751 &rxstats->LRO_bytes, 0, "LRO bytes"); 3752 #endif 3753 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3754 &rxstats->ucast_packets, 0, "Unicast packets"); 3755 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3756 &rxstats->ucast_bytes, 0, "Unicast bytes"); 3757 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3758 &rxstats->mcast_packets, 0, "Multicast packets"); 3759 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3760 &rxstats->mcast_bytes, 0, "Multicast bytes"); 3761 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD, 3762 &rxstats->bcast_packets, 0, "Broadcast packets"); 3763 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD, 3764 &rxstats->bcast_bytes, 0, "Broadcast bytes"); 3765 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD, 3766 &rxstats->nobuffer, 0, "No buffer"); 3767 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD, 3768 &rxstats->error, 0, "Errors"); 3769 } 3770 3771 static void 3772 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc, 3773 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3774 { 3775 struct sysctl_oid *node; 3776 struct sysctl_oid_list *list; 3777 int i; 3778 3779 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3780 struct vmxnet3_txqueue *txq = &sc->vmx_txq[i]; 3781 3782 node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO, 3783 "debug", CTLFLAG_RD, NULL, ""); 3784 list = SYSCTL_CHILDREN(node); 3785 3786 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD, 3787 &txq->vxtxq_cmd_ring.vxtxr_head, 0, ""); 3788 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD, 3789 &txq->vxtxq_cmd_ring.vxtxr_next, 0, ""); 3790 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD, 3791 &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, ""); 3792 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD, 3793 &txq->vxtxq_cmd_ring.vxtxr_gen, 0, ""); 3794 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3795 &txq->vxtxq_comp_ring.vxcr_next, 0, ""); 3796 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3797 &txq->vxtxq_comp_ring.vxcr_ndesc, 0,""); 3798 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3799 &txq->vxtxq_comp_ring.vxcr_gen, 0, ""); 3800 } 3801 3802 for (i = 0; i < sc->vmx_nrxqueues; i++) { 3803 struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i]; 3804 3805 node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO, 3806 "debug", CTLFLAG_RD, NULL, ""); 3807 list = SYSCTL_CHILDREN(node); 3808 3809 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD, 3810 &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, ""); 3811 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD, 3812 &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, ""); 3813 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD, 3814 &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, ""); 3815 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD, 3816 &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, ""); 3817 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD, 3818 &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, ""); 3819 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD, 3820 &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, ""); 3821 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3822 &rxq->vxrxq_comp_ring.vxcr_next, 0, ""); 3823 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3824 &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,""); 3825 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3826 &rxq->vxrxq_comp_ring.vxcr_gen, 0, ""); 3827 } 3828 } 3829 3830 static void 3831 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc, 3832 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3833 { 3834 int i; 3835 3836 for (i = 0; i < sc->vmx_ntxqueues; i++) 3837 vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child); 3838 for (i = 0; i < sc->vmx_nrxqueues; i++) 3839 vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child); 3840 3841 vmxnet3_setup_debug_sysctl(sc, ctx, child); 3842 } 3843 3844 static void 3845 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 3846 { 3847 device_t dev; 3848 struct vmxnet3_statistics *stats; 3849 struct sysctl_ctx_list *ctx; 3850 struct sysctl_oid *tree; 3851 struct sysctl_oid_list *child; 3852 3853 dev = sc->vmx_dev; 3854 ctx = device_get_sysctl_ctx(dev); 3855 tree = device_get_sysctl_tree(dev); 3856 child = SYSCTL_CHILDREN(tree); 3857 3858 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD, 3859 &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues"); 3860 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD, 3861 &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues"); 3862 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD, 3863 &sc->vmx_ntxqueues, 0, "Number of Tx queues"); 3864 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD, 3865 &sc->vmx_nrxqueues, 0, "Number of Rx queues"); 3866 3867 stats = &sc->vmx_stats; 3868 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD, 3869 &stats->vmst_defragged, 0, "Tx mbuf chains defragged"); 3870 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD, 3871 &stats->vmst_defrag_failed, 0, 3872 "Tx mbuf dropped because defrag failed"); 3873 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD, 3874 &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed"); 3875 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD, 3876 &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed"); 3877 3878 vmxnet3_setup_queue_sysctl(sc, ctx, child); 3879 } 3880 3881 static void 3882 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3883 { 3884 3885 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 3886 } 3887 3888 static uint32_t 3889 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 3890 { 3891 3892 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 3893 } 3894 3895 static void 3896 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3897 { 3898 3899 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 3900 } 3901 3902 static void 3903 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3904 { 3905 3906 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 3907 } 3908 3909 static uint32_t 3910 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3911 { 3912 3913 vmxnet3_write_cmd(sc, cmd); 3914 bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0, 3915 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 3916 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 3917 } 3918 3919 static void 3920 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 3921 { 3922 3923 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 3924 } 3925 3926 static void 3927 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 3928 { 3929 3930 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 3931 } 3932 3933 static void 3934 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3935 { 3936 int i; 3937 3938 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3939 for (i = 0; i < sc->vmx_nintrs; i++) 3940 vmxnet3_enable_intr(sc, i); 3941 } 3942 3943 static void 3944 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3945 { 3946 int i; 3947 3948 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3949 for (i = 0; i < sc->vmx_nintrs; i++) 3950 vmxnet3_disable_intr(sc, i); 3951 } 3952 3953 static void 3954 vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3955 { 3956 bus_addr_t *baddr = arg; 3957 3958 if (error == 0) 3959 *baddr = segs->ds_addr; 3960 } 3961 3962 static int 3963 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3964 struct vmxnet3_dma_alloc *dma) 3965 { 3966 device_t dev; 3967 int error; 3968 3969 dev = sc->vmx_dev; 3970 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3971 3972 error = bus_dma_tag_create(bus_get_dma_tag(dev), 3973 align, 0, /* alignment, bounds */ 3974 BUS_SPACE_MAXADDR, /* lowaddr */ 3975 BUS_SPACE_MAXADDR, /* highaddr */ 3976 NULL, NULL, /* filter, filterarg */ 3977 size, /* maxsize */ 3978 1, /* nsegments */ 3979 size, /* maxsegsize */ 3980 BUS_DMA_ALLOCNOW, /* flags */ 3981 &dma->dma_tag); 3982 if (error) { 3983 device_printf(dev, "bus_dma_tag_create failed: %d\n", error); 3984 goto fail; 3985 } 3986 3987 error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 3988 BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); 3989 if (error) { 3990 device_printf(dev, "bus_dmamem_alloc failed: %d\n", error); 3991 goto fail; 3992 } 3993 3994 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3995 size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); 3996 if (error) { 3997 device_printf(dev, "bus_dmamap_load failed: %d\n", error); 3998 goto fail; 3999 } 4000 4001 dma->dma_size = size; 4002 4003 fail: 4004 if (error) 4005 vmxnet3_dma_free(sc, dma); 4006 4007 return (error); 4008 } 4009 4010 static void 4011 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 4012 { 4013 4014 if (dma->dma_tag != NULL) { 4015 if (dma->dma_paddr != 0) { 4016 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 4017 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4018 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 4019 } 4020 4021 if (dma->dma_vaddr != NULL) { 4022 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, 4023 dma->dma_map); 4024 } 4025 4026 bus_dma_tag_destroy(dma->dma_tag); 4027 } 4028 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 4029 } 4030 4031 static int 4032 vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def) 4033 { 4034 char path[64]; 4035 4036 ksnprintf(path, sizeof(path), 4037 "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob); 4038 TUNABLE_INT_FETCH(path, &def); 4039 4040 return (def); 4041 } 4042 4043 #define mb() __asm volatile("mfence" ::: "memory") 4044 #define wmb() __asm volatile("sfence" ::: "memory") 4045 #define rmb() __asm volatile("lfence" ::: "memory") 4046 4047 /* 4048 * Since this is a purely paravirtualized device, we do not have 4049 * to worry about DMA coherency. But at times, we must make sure 4050 * both the compiler and CPU do not reorder memory operations. 4051 */ 4052 static inline void 4053 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 4054 { 4055 4056 switch (type) { 4057 case VMXNET3_BARRIER_RD: 4058 rmb(); 4059 break; 4060 case VMXNET3_BARRIER_WR: 4061 wmb(); 4062 break; 4063 case VMXNET3_BARRIER_RDWR: 4064 mb(); 4065 break; 4066 default: 4067 panic("%s: bad barrier type %d", __func__, type); 4068 } 4069 } 4070