1 /*- 2 * Copyright (c) 2013 Tsubai Masanari 3 * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org> 4 * 5 * Permission to use, copy, modify, and distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 * 17 * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $ 18 * $FreeBSD: head/sys/dev/vmware/vmxnet3/if_vmx.c 318867 2017-05-25 10:49:56Z avg $ 19 */ 20 21 /* Driver for VMware vmxnet3 virtual ethernet devices. */ 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/device.h> 26 #include <sys/eventhandler.h> 27 #include <sys/kernel.h> 28 #include <sys/endian.h> 29 #include <sys/sockio.h> 30 #include <sys/mbuf.h> 31 #include <sys/malloc.h> 32 #include <sys/module.h> 33 #include <sys/socket.h> 34 #include <sys/sysctl.h> 35 #include <sys/taskqueue.h> 36 #include <vm/vm.h> 37 #include <vm/pmap.h> 38 39 #include <net/ethernet.h> 40 #include <net/if.h> 41 #include <net/if_var.h> 42 #include <net/ifq_var.h> 43 #include <net/if_arp.h> 44 #include <net/if_dl.h> 45 #include <net/if_types.h> 46 #include <net/if_media.h> 47 #include <net/vlan/if_vlan_ether.h> 48 #include <net/vlan/if_vlan_var.h> 49 50 #include <net/bpf.h> 51 52 #include <netinet/in_systm.h> 53 #include <netinet/in.h> 54 #include <netinet/ip.h> 55 #include <netinet/ip6.h> 56 #include <netinet6/ip6_var.h> 57 #include <netinet/udp.h> 58 #include <netinet/tcp.h> 59 60 #include <sys/in_cksum.h> 61 62 #include <sys/bus.h> 63 #include <sys/rman.h> 64 65 #include <bus/pci/pcireg.h> 66 #include <bus/pci/pcivar.h> 67 68 #define VMXNET3_LEGACY_TX 1 /* XXX we need this at the moment */ 69 #include "if_vmxreg.h" 70 #include "if_vmxvar.h" 71 72 #include "opt_inet.h" 73 #include "opt_inet6.h" 74 75 #ifdef VMXNET3_FAILPOINTS 76 #include <sys/fail.h> 77 static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0, 78 "vmxnet3 fail points"); 79 #define VMXNET3_FP _debug_fail_point_vmxnet3 80 #endif 81 82 static int vmxnet3_probe(device_t); 83 static int vmxnet3_attach(device_t); 84 static int vmxnet3_detach(device_t); 85 static int vmxnet3_shutdown(device_t); 86 87 static int vmxnet3_alloc_resources(struct vmxnet3_softc *); 88 static void vmxnet3_free_resources(struct vmxnet3_softc *); 89 static int vmxnet3_check_version(struct vmxnet3_softc *); 90 static void vmxnet3_initial_config(struct vmxnet3_softc *); 91 static void vmxnet3_check_multiqueue(struct vmxnet3_softc *); 92 93 #ifdef __FreeBSD__ 94 static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 95 static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 96 #else 97 static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 98 #endif 99 static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 100 static int vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int, 101 struct vmxnet3_interrupt *); 102 static int vmxnet3_alloc_intr_resources(struct vmxnet3_softc *); 103 #ifdef __FreeBSD__ 104 static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 105 #endif 106 static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 107 static int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 108 static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 109 110 static void vmxnet3_free_interrupt(struct vmxnet3_softc *, 111 struct vmxnet3_interrupt *); 112 static void vmxnet3_free_interrupts(struct vmxnet3_softc *); 113 114 #ifndef VMXNET3_LEGACY_TX 115 static int vmxnet3_alloc_taskqueue(struct vmxnet3_softc *); 116 static void vmxnet3_start_taskqueue(struct vmxnet3_softc *); 117 static void vmxnet3_drain_taskqueue(struct vmxnet3_softc *); 118 static void vmxnet3_free_taskqueue(struct vmxnet3_softc *); 119 #endif 120 121 static int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 122 static int vmxnet3_init_txq(struct vmxnet3_softc *, int); 123 static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 124 static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 125 static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 126 static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 127 128 static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 129 static void vmxnet3_free_shared_data(struct vmxnet3_softc *); 130 static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 131 static void vmxnet3_free_txq_data(struct vmxnet3_softc *); 132 static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 133 static void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 134 static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 135 static void vmxnet3_free_queue_data(struct vmxnet3_softc *); 136 static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 137 static void vmxnet3_init_shared_data(struct vmxnet3_softc *); 138 static void vmxnet3_init_hwassist(struct vmxnet3_softc *); 139 static void vmxnet3_reinit_interface(struct vmxnet3_softc *); 140 static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); 141 static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 142 static int vmxnet3_alloc_data(struct vmxnet3_softc *); 143 static void vmxnet3_free_data(struct vmxnet3_softc *); 144 static int vmxnet3_setup_interface(struct vmxnet3_softc *); 145 146 static void vmxnet3_evintr(struct vmxnet3_softc *); 147 static void vmxnet3_txq_eof(struct vmxnet3_txqueue *); 148 static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 149 static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); 150 static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 151 struct vmxnet3_rxring *, int); 152 static void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *); 153 static void vmxnet3_legacy_intr(void *); 154 #ifdef __FreeBSD__ 155 static void vmxnet3_txq_intr(void *); 156 static void vmxnet3_rxq_intr(void *); 157 static void vmxnet3_event_intr(void *); 158 #endif 159 160 static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 161 static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 162 static void vmxnet3_stop(struct vmxnet3_softc *); 163 164 static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 165 static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 166 static int vmxnet3_reinit_queues(struct vmxnet3_softc *); 167 static int vmxnet3_enable_device(struct vmxnet3_softc *); 168 static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 169 static int vmxnet3_reinit(struct vmxnet3_softc *); 170 static void vmxnet3_init_locked(struct vmxnet3_softc *); 171 static void vmxnet3_init(void *); 172 173 static int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *, 174 int *, int *, int *); 175 static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, 176 bus_dmamap_t, bus_dma_segment_t [], int *); 177 static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 178 static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 179 #ifdef VMXNET3_LEGACY_TX 180 static void vmxnet3_start_locked(struct ifnet *); 181 static void vmxnet3_start(struct ifnet *, struct ifaltq_subque *); 182 #else 183 static int vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *, 184 struct mbuf *); 185 static int vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *); 186 static void vmxnet3_txq_tq_deferred(void *, int); 187 #endif 188 static void vmxnet3_txq_start(struct vmxnet3_txqueue *); 189 static void vmxnet3_tx_start_all(struct vmxnet3_softc *); 190 191 static void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int, 192 uint16_t); 193 static void vmxnet3_register_vlan(void *, struct ifnet *, uint16_t); 194 static void vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t); 195 static void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 196 static int vmxnet3_change_mtu(struct vmxnet3_softc *, int); 197 static int vmxnet3_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 198 199 #ifndef VMXNET3_LEGACY_TX 200 static void vmxnet3_qflush(struct ifnet *); 201 #endif 202 203 static int vmxnet3_watchdog(struct vmxnet3_txqueue *); 204 static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); 205 static void vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *, 206 struct vmxnet3_txq_stats *); 207 static void vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *, 208 struct vmxnet3_rxq_stats *); 209 static void vmxnet3_tick(void *); 210 static void vmxnet3_link_status(struct vmxnet3_softc *); 211 static void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); 212 static int vmxnet3_media_change(struct ifnet *); 213 static void vmxnet3_set_lladdr(struct vmxnet3_softc *); 214 static void vmxnet3_get_lladdr(struct vmxnet3_softc *); 215 216 static void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *, 217 struct sysctl_ctx_list *, struct sysctl_oid_list *); 218 static void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *, 219 struct sysctl_ctx_list *, struct sysctl_oid_list *); 220 static void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *, 221 struct sysctl_ctx_list *, struct sysctl_oid_list *); 222 static void vmxnet3_setup_sysctl(struct vmxnet3_softc *); 223 224 static void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t, 225 uint32_t); 226 static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t); 227 static void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t, 228 uint32_t); 229 static void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t); 230 static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t); 231 232 static void vmxnet3_enable_intr(struct vmxnet3_softc *, int); 233 static void vmxnet3_disable_intr(struct vmxnet3_softc *, int); 234 static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 235 static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 236 237 static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, 238 bus_size_t, struct vmxnet3_dma_alloc *); 239 static void vmxnet3_dma_free(struct vmxnet3_softc *, 240 struct vmxnet3_dma_alloc *); 241 static int vmxnet3_tunable_int(struct vmxnet3_softc *, 242 const char *, int); 243 244 typedef enum { 245 VMXNET3_BARRIER_RD, 246 VMXNET3_BARRIER_WR, 247 VMXNET3_BARRIER_RDWR, 248 } vmxnet3_barrier_t; 249 250 static void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t); 251 252 /* Tunables. */ 253 static int vmxnet3_mq_disable = 0; 254 TUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable); 255 static int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES; 256 TUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue); 257 static int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES; 258 TUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue); 259 static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC; 260 TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc); 261 static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC; 262 TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc); 263 264 static device_method_t vmxnet3_methods[] = { 265 /* Device interface. */ 266 DEVMETHOD(device_probe, vmxnet3_probe), 267 DEVMETHOD(device_attach, vmxnet3_attach), 268 DEVMETHOD(device_detach, vmxnet3_detach), 269 DEVMETHOD(device_shutdown, vmxnet3_shutdown), 270 271 DEVMETHOD_END 272 }; 273 274 static driver_t vmxnet3_driver = { 275 "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc) 276 }; 277 278 static devclass_t vmxnet3_devclass; 279 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, NULL, NULL); 280 281 MODULE_DEPEND(vmx, pci, 1, 1, 1); 282 MODULE_DEPEND(vmx, ether, 1, 1, 1); 283 284 #define VMXNET3_VMWARE_VENDOR_ID 0x15AD 285 #define VMXNET3_VMWARE_DEVICE_ID 0x07B0 286 287 static int 288 vmxnet3_probe(device_t dev) 289 { 290 291 if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID && 292 pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) { 293 device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter"); 294 return (BUS_PROBE_DEFAULT); 295 } 296 297 return (ENXIO); 298 } 299 300 static int 301 vmxnet3_attach(device_t dev) 302 { 303 struct vmxnet3_softc *sc; 304 int error; 305 306 sc = device_get_softc(dev); 307 sc->vmx_dev = dev; 308 309 pci_enable_busmaster(dev); 310 311 VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev)); 312 callout_init_lk(&sc->vmx_tick, &sc->vmx_lock); 313 314 vmxnet3_initial_config(sc); 315 316 error = vmxnet3_alloc_resources(sc); 317 if (error) 318 goto fail; 319 320 error = vmxnet3_check_version(sc); 321 if (error) 322 goto fail; 323 324 error = vmxnet3_alloc_rxtx_queues(sc); 325 if (error) 326 goto fail; 327 328 #ifndef VMXNET3_LEGACY_TX 329 error = vmxnet3_alloc_taskqueue(sc); 330 if (error) 331 goto fail; 332 #endif 333 334 error = vmxnet3_alloc_interrupts(sc); 335 if (error) 336 goto fail; 337 338 vmxnet3_check_multiqueue(sc); 339 340 error = vmxnet3_alloc_data(sc); 341 if (error) 342 goto fail; 343 344 error = vmxnet3_setup_interface(sc); 345 if (error) 346 goto fail; 347 348 error = vmxnet3_setup_interrupts(sc); 349 if (error) { 350 ether_ifdetach(sc->vmx_ifp); 351 device_printf(dev, "could not set up interrupt\n"); 352 goto fail; 353 } 354 355 vmxnet3_setup_sysctl(sc); 356 #ifndef VMXNET3_LEGACY_TX 357 vmxnet3_start_taskqueue(sc); 358 #endif 359 360 fail: 361 if (error) 362 vmxnet3_detach(dev); 363 364 return (error); 365 } 366 367 static int 368 vmxnet3_detach(device_t dev) 369 { 370 struct vmxnet3_softc *sc; 371 struct ifnet *ifp; 372 373 sc = device_get_softc(dev); 374 ifp = sc->vmx_ifp; 375 376 if (device_is_attached(dev)) { 377 VMXNET3_CORE_LOCK(sc); 378 vmxnet3_stop(sc); 379 VMXNET3_CORE_UNLOCK(sc); 380 381 callout_terminate(&sc->vmx_tick); 382 #ifndef VMXNET3_LEGACY_TX 383 vmxnet3_drain_taskqueue(sc); 384 #endif 385 386 ether_ifdetach(ifp); 387 } 388 389 if (sc->vmx_vlan_attach != NULL) { 390 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach); 391 sc->vmx_vlan_attach = NULL; 392 } 393 if (sc->vmx_vlan_detach != NULL) { 394 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach); 395 sc->vmx_vlan_detach = NULL; 396 } 397 398 #ifndef VMXNET3_LEGACY_TX 399 vmxnet3_free_taskqueue(sc); 400 #endif 401 vmxnet3_free_interrupts(sc); 402 403 if (ifp != NULL) { 404 if_free(ifp); 405 sc->vmx_ifp = NULL; 406 } 407 408 ifmedia_removeall(&sc->vmx_media); 409 410 vmxnet3_free_data(sc); 411 vmxnet3_free_resources(sc); 412 vmxnet3_free_rxtx_queues(sc); 413 414 VMXNET3_CORE_LOCK_DESTROY(sc); 415 416 return (0); 417 } 418 419 static int 420 vmxnet3_shutdown(device_t dev) 421 { 422 423 return (0); 424 } 425 426 static int 427 vmxnet3_alloc_resources(struct vmxnet3_softc *sc) 428 { 429 device_t dev; 430 int rid; 431 432 dev = sc->vmx_dev; 433 434 rid = PCIR_BAR(0); 435 sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 436 RF_ACTIVE); 437 if (sc->vmx_res0 == NULL) { 438 device_printf(dev, 439 "could not map BAR0 memory\n"); 440 return (ENXIO); 441 } 442 443 sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0); 444 sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0); 445 446 rid = PCIR_BAR(1); 447 sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 448 RF_ACTIVE); 449 if (sc->vmx_res1 == NULL) { 450 device_printf(dev, 451 "could not map BAR1 memory\n"); 452 return (ENXIO); 453 } 454 455 sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1); 456 sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1); 457 458 if (pci_find_extcap(dev, PCIY_MSIX, NULL) == 0) { 459 rid = PCIR_BAR(2); 460 sc->vmx_msix_res = bus_alloc_resource_any(dev, 461 SYS_RES_MEMORY, &rid, RF_ACTIVE); 462 } 463 464 if (sc->vmx_msix_res == NULL) 465 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 466 467 return (0); 468 } 469 470 static void 471 vmxnet3_free_resources(struct vmxnet3_softc *sc) 472 { 473 device_t dev; 474 int rid; 475 476 dev = sc->vmx_dev; 477 478 if (sc->vmx_res0 != NULL) { 479 rid = PCIR_BAR(0); 480 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0); 481 sc->vmx_res0 = NULL; 482 } 483 484 if (sc->vmx_res1 != NULL) { 485 rid = PCIR_BAR(1); 486 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1); 487 sc->vmx_res1 = NULL; 488 } 489 490 if (sc->vmx_msix_res != NULL) { 491 rid = PCIR_BAR(2); 492 bus_release_resource(dev, SYS_RES_MEMORY, rid, 493 sc->vmx_msix_res); 494 sc->vmx_msix_res = NULL; 495 } 496 } 497 498 static int 499 vmxnet3_check_version(struct vmxnet3_softc *sc) 500 { 501 device_t dev; 502 uint32_t version; 503 504 dev = sc->vmx_dev; 505 506 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 507 if ((version & 0x01) == 0) { 508 device_printf(dev, "unsupported hardware version %#x\n", 509 version); 510 return (ENOTSUP); 511 } 512 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 513 514 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 515 if ((version & 0x01) == 0) { 516 device_printf(dev, "unsupported UPT version %#x\n", version); 517 return (ENOTSUP); 518 } 519 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 520 521 return (0); 522 } 523 524 static int 525 trunc_powerof2(int val) 526 { 527 528 return (1U << (fls(val) - 1)); 529 } 530 531 static void 532 vmxnet3_initial_config(struct vmxnet3_softc *sc) 533 { 534 int nqueue, ndesc; 535 536 nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue); 537 if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1) 538 nqueue = VMXNET3_DEF_TX_QUEUES; 539 if (nqueue > ncpus) 540 nqueue = ncpus; 541 sc->vmx_max_ntxqueues = trunc_powerof2(nqueue); 542 543 nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue); 544 if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1) 545 nqueue = VMXNET3_DEF_RX_QUEUES; 546 if (nqueue > ncpus) 547 nqueue = ncpus; 548 sc->vmx_max_nrxqueues = trunc_powerof2(nqueue); 549 550 if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) { 551 sc->vmx_max_nrxqueues = 1; 552 sc->vmx_max_ntxqueues = 1; 553 } 554 555 ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc); 556 if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC) 557 ndesc = VMXNET3_DEF_TX_NDESC; 558 if (ndesc & VMXNET3_MASK_TX_NDESC) 559 ndesc &= ~VMXNET3_MASK_TX_NDESC; 560 sc->vmx_ntxdescs = ndesc; 561 562 ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc); 563 if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC) 564 ndesc = VMXNET3_DEF_RX_NDESC; 565 if (ndesc & VMXNET3_MASK_RX_NDESC) 566 ndesc &= ~VMXNET3_MASK_RX_NDESC; 567 sc->vmx_nrxdescs = ndesc; 568 sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 569 } 570 571 static void 572 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc) 573 { 574 575 if (sc->vmx_intr_type != VMXNET3_IT_MSIX) 576 goto out; 577 578 /* BMV: Just use the maximum configured for now. */ 579 sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; 580 sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; 581 582 if (sc->vmx_nrxqueues > 1) 583 sc->vmx_flags |= VMXNET3_FLAG_RSS; 584 585 return; 586 587 out: 588 sc->vmx_ntxqueues = 1; 589 sc->vmx_nrxqueues = 1; 590 } 591 592 #ifdef __FreeBSD__ 593 static int 594 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 595 { 596 device_t dev; 597 int nmsix, cnt, required; 598 599 dev = sc->vmx_dev; 600 601 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 602 return (1); 603 604 /* Allocate an additional vector for the events interrupt. */ 605 required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1; 606 607 nmsix = pci_msix_count(dev); 608 if (nmsix < required) 609 return (1); 610 611 cnt = required; 612 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 613 sc->vmx_nintrs = required; 614 return (0); 615 } else 616 pci_release_msi(dev); 617 618 /* BMV TODO Fallback to sharing MSIX vectors if possible. */ 619 620 return (1); 621 } 622 623 static int 624 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 625 { 626 device_t dev; 627 int nmsi, cnt, required; 628 629 dev = sc->vmx_dev; 630 required = 1; 631 632 nmsi = pci_msi_count(dev); 633 if (nmsi < required) 634 return (1); 635 636 cnt = required; 637 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) { 638 sc->vmx_nintrs = 1; 639 return (0); 640 } else 641 pci_release_msi(dev); 642 643 return (1); 644 } 645 #else 646 static int 647 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 648 { 649 int irq_flags, rid; 650 int enable = 1; 651 652 sc->vmx_irq_type = pci_alloc_1intr(sc->vmx_dev, enable, &rid, 653 &irq_flags); 654 sc->vmx_irq_flags = irq_flags; 655 sc->vmx_nintrs = 1; 656 return (0); 657 } 658 #endif 659 660 static int 661 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 662 { 663 664 sc->vmx_nintrs = 1; 665 return (0); 666 } 667 668 static int 669 vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags, 670 struct vmxnet3_interrupt *intr) 671 { 672 struct resource *irq; 673 674 irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, 675 sc->vmx_irq_flags); 676 if (irq == NULL) 677 return (ENXIO); 678 679 intr->vmxi_irq = irq; 680 intr->vmxi_rid = rid; 681 682 return (0); 683 } 684 685 static int 686 vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc) 687 { 688 int i, rid, flags, error; 689 690 rid = 0; 691 flags = RF_ACTIVE; 692 693 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) 694 flags |= RF_SHAREABLE; 695 else 696 rid = 1; 697 698 for (i = 0; i < sc->vmx_nintrs; i++, rid++) { 699 error = vmxnet3_alloc_interrupt(sc, rid, flags, 700 &sc->vmx_intrs[i]); 701 if (error) 702 return (error); 703 } 704 705 return (0); 706 } 707 708 #ifdef __FreeBSD__ 709 static int 710 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 711 { 712 device_t dev; 713 struct vmxnet3_txqueue *txq; 714 struct vmxnet3_rxqueue *rxq; 715 struct vmxnet3_interrupt *intr; 716 int i, error; 717 718 dev = sc->vmx_dev; 719 intr = &sc->vmx_intrs[0]; 720 721 for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) { 722 txq = &sc->vmx_txq[i]; 723 error = bus_setup_intr(dev, intr->vmxi_irq, INTR_MPSAFE, 724 vmxnet3_txq_intr, txq, &intr->vmxi_handler, NULL); 725 if (error) 726 return (error); 727 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 728 "tq%d", i); 729 txq->vxtxq_intr_idx = intr->vmxi_rid - 1; 730 } 731 732 for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) { 733 rxq = &sc->vmx_rxq[i]; 734 error = bus_setup_intr(dev, intr->vmxi_irq, INTR_MPSAFE, 735 vmxnet3_rxq_intr, rxq, &intr->vmxi_handler, NULL); 736 if (error) 737 return (error); 738 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 739 "rq%d", i); 740 rxq->vxrxq_intr_idx = intr->vmxi_rid - 1; 741 } 742 743 error = bus_setup_intr(dev, intr->vmxi_irq, INTR_MPSAFE, 744 vmxnet3_event_intr, sc, &intr->vmxi_handler, NULL); 745 if (error) 746 return (error); 747 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, "event"); 748 sc->vmx_event_intr_idx = intr->vmxi_rid - 1; 749 750 return (0); 751 } 752 #endif 753 754 static int 755 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 756 { 757 struct vmxnet3_interrupt *intr; 758 int i, error; 759 760 intr = &sc->vmx_intrs[0]; 761 error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq, 762 INTR_MPSAFE, vmxnet3_legacy_intr, sc, 763 &intr->vmxi_handler, NULL); 764 765 for (i = 0; i < sc->vmx_ntxqueues; i++) 766 sc->vmx_txq[i].vxtxq_intr_idx = 0; 767 for (i = 0; i < sc->vmx_nrxqueues; i++) 768 sc->vmx_rxq[i].vxrxq_intr_idx = 0; 769 sc->vmx_event_intr_idx = 0; 770 771 return (error); 772 } 773 774 static void 775 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 776 { 777 struct vmxnet3_txqueue *txq; 778 struct vmxnet3_txq_shared *txs; 779 struct vmxnet3_rxqueue *rxq; 780 struct vmxnet3_rxq_shared *rxs; 781 int i; 782 783 sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 784 785 for (i = 0; i < sc->vmx_ntxqueues; i++) { 786 txq = &sc->vmx_txq[i]; 787 txs = txq->vxtxq_ts; 788 txs->intr_idx = txq->vxtxq_intr_idx; 789 } 790 791 for (i = 0; i < sc->vmx_nrxqueues; i++) { 792 rxq = &sc->vmx_rxq[i]; 793 rxs = rxq->vxrxq_rs; 794 rxs->intr_idx = rxq->vxrxq_intr_idx; 795 } 796 } 797 798 static int 799 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 800 { 801 int error; 802 803 error = vmxnet3_alloc_intr_resources(sc); 804 if (error) 805 return (error); 806 807 switch (sc->vmx_intr_type) { 808 case VMXNET3_IT_MSIX: 809 #ifdef __FreeBSD__ 810 error = vmxnet3_setup_msix_interrupts(sc); 811 #else 812 device_printf(sc->vmx_dev, "VMXNET3_IT_MSIX unsupported\n"); 813 error = ENXIO; 814 #endif 815 break; 816 case VMXNET3_IT_MSI: 817 case VMXNET3_IT_LEGACY: 818 error = vmxnet3_setup_legacy_interrupt(sc); 819 break; 820 default: 821 panic("%s: invalid interrupt type %d", __func__, 822 sc->vmx_intr_type); 823 } 824 825 if (error == 0) 826 vmxnet3_set_interrupt_idx(sc); 827 828 return (error); 829 } 830 831 #ifdef __FreeBSD__ 832 static int 833 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 834 { 835 device_t dev; 836 uint32_t config; 837 int error; 838 839 dev = sc->vmx_dev; 840 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 841 842 sc->vmx_intr_type = config & 0x03; 843 sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 844 845 switch (sc->vmx_intr_type) { 846 case VMXNET3_IT_AUTO: 847 sc->vmx_intr_type = VMXNET3_IT_MSIX; 848 /* FALLTHROUGH */ 849 case VMXNET3_IT_MSIX: 850 error = vmxnet3_alloc_msix_interrupts(sc); 851 if (error == 0) 852 break; 853 sc->vmx_intr_type = VMXNET3_IT_MSI; 854 /* FALLTHROUGH */ 855 case VMXNET3_IT_MSI: 856 error = vmxnet3_alloc_msi_interrupts(sc); 857 if (error == 0) 858 break; 859 sc->vmx_intr_type = VMXNET3_IT_LEGACY; 860 /* FALLTHROUGH */ 861 case VMXNET3_IT_LEGACY: 862 error = vmxnet3_alloc_legacy_interrupts(sc); 863 if (error == 0) 864 break; 865 /* FALLTHROUGH */ 866 default: 867 sc->vmx_intr_type = -1; 868 device_printf(dev, "cannot allocate any interrupt resources\n"); 869 return (ENXIO); 870 } 871 872 return (error); 873 } 874 #else 875 static int 876 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 877 { 878 device_t dev; 879 uint32_t config; 880 int error; 881 882 dev = sc->vmx_dev; 883 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 884 885 sc->vmx_intr_type = config & 0x03; 886 sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 887 888 switch (sc->vmx_intr_type) { 889 case VMXNET3_IT_AUTO: 890 sc->vmx_intr_type = VMXNET3_IT_MSI; 891 /* FALLTHROUGH */ 892 case VMXNET3_IT_MSI: 893 error = vmxnet3_alloc_msi_interrupts(sc); 894 if (error == 0) 895 break; 896 sc->vmx_intr_type = VMXNET3_IT_LEGACY; 897 case VMXNET3_IT_LEGACY: 898 error = vmxnet3_alloc_legacy_interrupts(sc); 899 if (error == 0) 900 break; 901 /* FALLTHROUGH */ 902 case VMXNET3_IT_MSIX: 903 /* FALLTHROUGH */ 904 default: 905 sc->vmx_intr_type = -1; 906 device_printf(dev, "cannot allocate any interrupt resources\n"); 907 return (ENXIO); 908 } 909 910 return (error); 911 } 912 #endif 913 914 static void 915 vmxnet3_free_interrupt(struct vmxnet3_softc *sc, 916 struct vmxnet3_interrupt *intr) 917 { 918 device_t dev; 919 920 dev = sc->vmx_dev; 921 922 if (intr->vmxi_handler != NULL) { 923 bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler); 924 intr->vmxi_handler = NULL; 925 } 926 927 if (intr->vmxi_irq != NULL) { 928 bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid, 929 intr->vmxi_irq); 930 intr->vmxi_irq = NULL; 931 intr->vmxi_rid = -1; 932 } 933 } 934 935 #ifdef __FreeBSD__ 936 static void 937 vmxnet3_free_interrupts(struct vmxnet3_softc *sc) 938 { 939 int i; 940 941 for (i = 0; i < sc->vmx_nintrs; i++) 942 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); 943 944 if (sc->vmx_intr_type == VMXNET3_IT_MSI || 945 sc->vmx_intr_type == VMXNET3_IT_MSIX) 946 pci_release_msi(sc->vmx_dev); 947 } 948 #else 949 static void 950 vmxnet3_free_interrupts(struct vmxnet3_softc *sc) 951 { 952 int i; 953 954 for (i = 0; i < sc->vmx_nintrs; i++) 955 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); 956 957 if (sc->vmx_irq_type == PCI_INTR_TYPE_MSI) 958 pci_release_msi(sc->vmx_dev); 959 } 960 #endif 961 962 #ifndef VMXNET3_LEGACY_TX 963 static int 964 vmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc) 965 { 966 device_t dev; 967 968 dev = sc->vmx_dev; 969 970 sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT, 971 taskqueue_thread_enqueue, &sc->vmx_tq); 972 if (sc->vmx_tq == NULL) 973 return (ENOMEM); 974 975 return (0); 976 } 977 978 static void 979 vmxnet3_start_taskqueue(struct vmxnet3_softc *sc) 980 { 981 device_t dev; 982 int nthreads, error; 983 984 dev = sc->vmx_dev; 985 986 /* 987 * The taskqueue is typically not frequently used, so a dedicated 988 * thread for each queue is unnecessary. 989 */ 990 nthreads = MAX(1, sc->vmx_ntxqueues / 2); 991 992 /* 993 * Most drivers just ignore the return value - it only fails 994 * with ENOMEM so an error is not likely. It is hard for us 995 * to recover from an error here. 996 */ 997 error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET, 998 "%s taskq", device_get_nameunit(dev)); 999 if (error) 1000 device_printf(dev, "failed to start taskqueue: %d", error); 1001 } 1002 1003 static void 1004 vmxnet3_drain_taskqueue(struct vmxnet3_softc *sc) 1005 { 1006 struct vmxnet3_txqueue *txq; 1007 int i; 1008 1009 if (sc->vmx_tq != NULL) { 1010 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1011 txq = &sc->vmx_txq[i]; 1012 taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask); 1013 } 1014 } 1015 } 1016 1017 static void 1018 vmxnet3_free_taskqueue(struct vmxnet3_softc *sc) 1019 { 1020 if (sc->vmx_tq != NULL) { 1021 taskqueue_free(sc->vmx_tq); 1022 sc->vmx_tq = NULL; 1023 } 1024 } 1025 #endif 1026 1027 static int 1028 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 1029 { 1030 struct vmxnet3_rxqueue *rxq; 1031 struct vmxnet3_rxring *rxr; 1032 int i; 1033 1034 rxq = &sc->vmx_rxq[q]; 1035 1036 ksnprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 1037 device_get_nameunit(sc->vmx_dev), q); 1038 lockinit(&rxq->vxrxq_lock, rxq->vxrxq_name, 0, 0); 1039 1040 rxq->vxrxq_sc = sc; 1041 rxq->vxrxq_id = q; 1042 1043 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1044 rxr = &rxq->vxrxq_cmd_ring[i]; 1045 rxr->vxrxr_rid = i; 1046 rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 1047 rxr->vxrxr_rxbuf = kmalloc(rxr->vxrxr_ndesc * 1048 sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_INTWAIT | M_ZERO); 1049 if (rxr->vxrxr_rxbuf == NULL) 1050 return (ENOMEM); 1051 1052 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 1053 } 1054 1055 return (0); 1056 } 1057 1058 static int 1059 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 1060 { 1061 struct vmxnet3_txqueue *txq; 1062 struct vmxnet3_txring *txr; 1063 1064 txq = &sc->vmx_txq[q]; 1065 txr = &txq->vxtxq_cmd_ring; 1066 1067 ksnprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 1068 device_get_nameunit(sc->vmx_dev), q); 1069 lockinit(&txq->vxtxq_lock, txq->vxtxq_name, 0, 0); 1070 1071 txq->vxtxq_sc = sc; 1072 txq->vxtxq_id = q; 1073 1074 txr->vxtxr_ndesc = sc->vmx_ntxdescs; 1075 txr->vxtxr_txbuf = kmalloc(txr->vxtxr_ndesc * 1076 sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_INTWAIT | M_ZERO); 1077 if (txr->vxtxr_txbuf == NULL) 1078 return (ENOMEM); 1079 1080 txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 1081 1082 #ifndef VMXNET3_LEGACY_TX 1083 TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq); 1084 1085 txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF, 1086 M_NOWAIT, &txq->vxtxq_lock); 1087 if (txq->vxtxq_br == NULL) 1088 return (ENOMEM); 1089 #endif 1090 1091 return (0); 1092 } 1093 1094 static int 1095 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 1096 { 1097 int i, error; 1098 1099 /* 1100 * Only attempt to create multiple queues if MSIX is available. MSIX is 1101 * disabled by default because its apparently broken for devices passed 1102 * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable 1103 * must be set to zero for MSIX. This check prevents us from allocating 1104 * queue structures that we will not use. 1105 */ 1106 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { 1107 sc->vmx_max_nrxqueues = 1; 1108 sc->vmx_max_ntxqueues = 1; 1109 } 1110 1111 sc->vmx_rxq = kmalloc(sizeof(struct vmxnet3_rxqueue) * 1112 sc->vmx_max_nrxqueues, M_DEVBUF, M_INTWAIT | M_ZERO); 1113 sc->vmx_txq = kmalloc(sizeof(struct vmxnet3_txqueue) * 1114 sc->vmx_max_ntxqueues, M_DEVBUF, M_INTWAIT | M_ZERO); 1115 if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL) 1116 return (ENOMEM); 1117 1118 for (i = 0; i < sc->vmx_max_nrxqueues; i++) { 1119 error = vmxnet3_init_rxq(sc, i); 1120 if (error) 1121 return (error); 1122 } 1123 1124 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1125 error = vmxnet3_init_txq(sc, i); 1126 if (error) 1127 return (error); 1128 } 1129 1130 return (0); 1131 } 1132 1133 static void 1134 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 1135 { 1136 struct vmxnet3_rxring *rxr; 1137 int i; 1138 1139 rxq->vxrxq_sc = NULL; 1140 rxq->vxrxq_id = -1; 1141 1142 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1143 rxr = &rxq->vxrxq_cmd_ring[i]; 1144 1145 if (rxr->vxrxr_rxbuf != NULL) { 1146 kfree(rxr->vxrxr_rxbuf, M_DEVBUF); 1147 rxr->vxrxr_rxbuf = NULL; 1148 } 1149 } 1150 1151 #if 0 /* XXX */ 1152 if (mtx_initialized(&rxq->vxrxq_lock) != 0) 1153 #endif 1154 lockuninit(&rxq->vxrxq_lock); 1155 } 1156 1157 static void 1158 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 1159 { 1160 struct vmxnet3_txring *txr; 1161 1162 txr = &txq->vxtxq_cmd_ring; 1163 1164 txq->vxtxq_sc = NULL; 1165 txq->vxtxq_id = -1; 1166 1167 #ifndef VMXNET3_LEGACY_TX 1168 if (txq->vxtxq_br != NULL) { 1169 buf_ring_free(txq->vxtxq_br, M_DEVBUF); 1170 txq->vxtxq_br = NULL; 1171 } 1172 #endif 1173 1174 if (txr->vxtxr_txbuf != NULL) { 1175 kfree(txr->vxtxr_txbuf, M_DEVBUF); 1176 txr->vxtxr_txbuf = NULL; 1177 } 1178 1179 #if 0 /* XXX */ 1180 if (mtx_initialized(&txq->vxtxq_lock) != 0) 1181 #endif 1182 lockuninit(&txq->vxtxq_lock); 1183 } 1184 1185 static void 1186 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 1187 { 1188 int i; 1189 1190 if (sc->vmx_rxq != NULL) { 1191 for (i = 0; i < sc->vmx_max_nrxqueues; i++) 1192 vmxnet3_destroy_rxq(&sc->vmx_rxq[i]); 1193 kfree(sc->vmx_rxq, M_DEVBUF); 1194 sc->vmx_rxq = NULL; 1195 } 1196 1197 if (sc->vmx_txq != NULL) { 1198 for (i = 0; i < sc->vmx_max_ntxqueues; i++) 1199 vmxnet3_destroy_txq(&sc->vmx_txq[i]); 1200 kfree(sc->vmx_txq, M_DEVBUF); 1201 sc->vmx_txq = NULL; 1202 } 1203 } 1204 1205 static int 1206 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 1207 { 1208 device_t dev; 1209 uint8_t *kva; 1210 size_t size; 1211 int i, error; 1212 1213 dev = sc->vmx_dev; 1214 1215 size = sizeof(struct vmxnet3_driver_shared); 1216 error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 1217 if (error) { 1218 device_printf(dev, "cannot alloc shared memory\n"); 1219 return (error); 1220 } 1221 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 1222 1223 size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 1224 sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 1225 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 1226 if (error) { 1227 device_printf(dev, "cannot alloc queue shared memory\n"); 1228 return (error); 1229 } 1230 sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 1231 kva = sc->vmx_qs; 1232 1233 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1234 sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; 1235 kva += sizeof(struct vmxnet3_txq_shared); 1236 } 1237 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1238 sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; 1239 kva += sizeof(struct vmxnet3_rxq_shared); 1240 } 1241 1242 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1243 size = sizeof(struct vmxnet3_rss_shared); 1244 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); 1245 if (error) { 1246 device_printf(dev, "cannot alloc rss shared memory\n"); 1247 return (error); 1248 } 1249 sc->vmx_rss = 1250 (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; 1251 } 1252 1253 return (0); 1254 } 1255 1256 static void 1257 vmxnet3_free_shared_data(struct vmxnet3_softc *sc) 1258 { 1259 1260 if (sc->vmx_rss != NULL) { 1261 vmxnet3_dma_free(sc, &sc->vmx_rss_dma); 1262 sc->vmx_rss = NULL; 1263 } 1264 1265 if (sc->vmx_qs != NULL) { 1266 vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 1267 sc->vmx_qs = NULL; 1268 } 1269 1270 if (sc->vmx_ds != NULL) { 1271 vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 1272 sc->vmx_ds = NULL; 1273 } 1274 } 1275 1276 static int 1277 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 1278 { 1279 device_t dev; 1280 struct vmxnet3_txqueue *txq; 1281 struct vmxnet3_txring *txr; 1282 struct vmxnet3_comp_ring *txc; 1283 size_t descsz, compsz; 1284 int i, q, error; 1285 1286 dev = sc->vmx_dev; 1287 1288 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1289 txq = &sc->vmx_txq[q]; 1290 txr = &txq->vxtxq_cmd_ring; 1291 txc = &txq->vxtxq_comp_ring; 1292 1293 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 1294 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 1295 1296 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1297 1, 0, /* alignment, boundary */ 1298 BUS_SPACE_MAXADDR, /* lowaddr */ 1299 BUS_SPACE_MAXADDR, /* highaddr */ 1300 VMXNET3_TX_MAXSIZE, /* maxsize */ 1301 VMXNET3_TX_MAXSEGS, /* nsegments */ 1302 VMXNET3_TX_MAXSEGSIZE, /* maxsegsize */ 1303 0, /* flags */ 1304 &txr->vxtxr_txtag); 1305 if (error) { 1306 device_printf(dev, 1307 "unable to create Tx buffer tag for queue %d\n", q); 1308 return (error); 1309 } 1310 1311 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 1312 if (error) { 1313 device_printf(dev, "cannot alloc Tx descriptors for " 1314 "queue %d error %d\n", q, error); 1315 return (error); 1316 } 1317 txr->vxtxr_txd = 1318 (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1319 1320 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1321 if (error) { 1322 device_printf(dev, "cannot alloc Tx comp descriptors " 1323 "for queue %d error %d\n", q, error); 1324 return (error); 1325 } 1326 txc->vxcr_u.txcd = 1327 (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1328 1329 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1330 error = bus_dmamap_create(txr->vxtxr_txtag, 0, 1331 &txr->vxtxr_txbuf[i].vtxb_dmamap); 1332 if (error) { 1333 device_printf(dev, "unable to create Tx buf " 1334 "dmamap for queue %d idx %d\n", q, i); 1335 return (error); 1336 } 1337 } 1338 } 1339 1340 return (0); 1341 } 1342 1343 static void 1344 vmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1345 { 1346 device_t dev; 1347 struct vmxnet3_txqueue *txq; 1348 struct vmxnet3_txring *txr; 1349 struct vmxnet3_comp_ring *txc; 1350 struct vmxnet3_txbuf *txb; 1351 int i, q; 1352 1353 dev = sc->vmx_dev; 1354 1355 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1356 txq = &sc->vmx_txq[q]; 1357 txr = &txq->vxtxq_cmd_ring; 1358 txc = &txq->vxtxq_comp_ring; 1359 1360 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1361 txb = &txr->vxtxr_txbuf[i]; 1362 if (txb->vtxb_dmamap != NULL) { 1363 bus_dmamap_destroy(txr->vxtxr_txtag, 1364 txb->vtxb_dmamap); 1365 txb->vtxb_dmamap = NULL; 1366 } 1367 } 1368 1369 if (txc->vxcr_u.txcd != NULL) { 1370 vmxnet3_dma_free(sc, &txc->vxcr_dma); 1371 txc->vxcr_u.txcd = NULL; 1372 } 1373 1374 if (txr->vxtxr_txd != NULL) { 1375 vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1376 txr->vxtxr_txd = NULL; 1377 } 1378 1379 if (txr->vxtxr_txtag != NULL) { 1380 bus_dma_tag_destroy(txr->vxtxr_txtag); 1381 txr->vxtxr_txtag = NULL; 1382 } 1383 } 1384 } 1385 1386 static int 1387 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1388 { 1389 device_t dev; 1390 struct vmxnet3_rxqueue *rxq; 1391 struct vmxnet3_rxring *rxr; 1392 struct vmxnet3_comp_ring *rxc; 1393 int descsz, compsz; 1394 int i, j, q, error; 1395 1396 dev = sc->vmx_dev; 1397 1398 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1399 rxq = &sc->vmx_rxq[q]; 1400 rxc = &rxq->vxrxq_comp_ring; 1401 compsz = 0; 1402 1403 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1404 rxr = &rxq->vxrxq_cmd_ring[i]; 1405 1406 descsz = rxr->vxrxr_ndesc * 1407 sizeof(struct vmxnet3_rxdesc); 1408 compsz += rxr->vxrxr_ndesc * 1409 sizeof(struct vmxnet3_rxcompdesc); 1410 1411 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1412 1, 0, /* alignment, boundary */ 1413 BUS_SPACE_MAXADDR, /* lowaddr */ 1414 BUS_SPACE_MAXADDR, /* highaddr */ 1415 MJUMPAGESIZE, /* maxsize */ 1416 1, /* nsegments */ 1417 MJUMPAGESIZE, /* maxsegsize */ 1418 0, /* flags */ 1419 &rxr->vxrxr_rxtag); 1420 if (error) { 1421 device_printf(dev, 1422 "unable to create Rx buffer tag for " 1423 "queue %d\n", q); 1424 return (error); 1425 } 1426 1427 error = vmxnet3_dma_malloc(sc, descsz, 512, 1428 &rxr->vxrxr_dma); 1429 if (error) { 1430 device_printf(dev, "cannot allocate Rx " 1431 "descriptors for queue %d/%d error %d\n", 1432 i, q, error); 1433 return (error); 1434 } 1435 rxr->vxrxr_rxd = 1436 (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1437 } 1438 1439 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1440 if (error) { 1441 device_printf(dev, "cannot alloc Rx comp descriptors " 1442 "for queue %d error %d\n", q, error); 1443 return (error); 1444 } 1445 rxc->vxcr_u.rxcd = 1446 (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1447 1448 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1449 rxr = &rxq->vxrxq_cmd_ring[i]; 1450 1451 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1452 &rxr->vxrxr_spare_dmap); 1453 if (error) { 1454 device_printf(dev, "unable to create spare " 1455 "dmamap for queue %d/%d error %d\n", 1456 q, i, error); 1457 return (error); 1458 } 1459 1460 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1461 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1462 &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1463 if (error) { 1464 device_printf(dev, "unable to create " 1465 "dmamap for queue %d/%d slot %d " 1466 "error %d\n", 1467 q, i, j, error); 1468 return (error); 1469 } 1470 } 1471 } 1472 } 1473 1474 return (0); 1475 } 1476 1477 static void 1478 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1479 { 1480 device_t dev; 1481 struct vmxnet3_rxqueue *rxq; 1482 struct vmxnet3_rxring *rxr; 1483 struct vmxnet3_comp_ring *rxc; 1484 struct vmxnet3_rxbuf *rxb; 1485 int i, j, q; 1486 1487 dev = sc->vmx_dev; 1488 1489 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1490 rxq = &sc->vmx_rxq[q]; 1491 rxc = &rxq->vxrxq_comp_ring; 1492 1493 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1494 rxr = &rxq->vxrxq_cmd_ring[i]; 1495 1496 if (rxr->vxrxr_spare_dmap != NULL) { 1497 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1498 rxr->vxrxr_spare_dmap); 1499 rxr->vxrxr_spare_dmap = NULL; 1500 } 1501 1502 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1503 rxb = &rxr->vxrxr_rxbuf[j]; 1504 if (rxb->vrxb_dmamap != NULL) { 1505 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1506 rxb->vrxb_dmamap); 1507 rxb->vrxb_dmamap = NULL; 1508 } 1509 } 1510 } 1511 1512 if (rxc->vxcr_u.rxcd != NULL) { 1513 vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1514 rxc->vxcr_u.rxcd = NULL; 1515 } 1516 1517 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1518 rxr = &rxq->vxrxq_cmd_ring[i]; 1519 1520 if (rxr->vxrxr_rxd != NULL) { 1521 vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1522 rxr->vxrxr_rxd = NULL; 1523 } 1524 1525 if (rxr->vxrxr_rxtag != NULL) { 1526 bus_dma_tag_destroy(rxr->vxrxr_rxtag); 1527 rxr->vxrxr_rxtag = NULL; 1528 } 1529 } 1530 } 1531 } 1532 1533 static int 1534 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1535 { 1536 int error; 1537 1538 error = vmxnet3_alloc_txq_data(sc); 1539 if (error) 1540 return (error); 1541 1542 error = vmxnet3_alloc_rxq_data(sc); 1543 if (error) 1544 return (error); 1545 1546 return (0); 1547 } 1548 1549 static void 1550 vmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1551 { 1552 1553 if (sc->vmx_rxq != NULL) 1554 vmxnet3_free_rxq_data(sc); 1555 1556 if (sc->vmx_txq != NULL) 1557 vmxnet3_free_txq_data(sc); 1558 } 1559 1560 static int 1561 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1562 { 1563 int error; 1564 1565 error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1566 32, &sc->vmx_mcast_dma); 1567 if (error) 1568 device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1569 else 1570 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1571 1572 return (error); 1573 } 1574 1575 static void 1576 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1577 { 1578 1579 if (sc->vmx_mcast != NULL) { 1580 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1581 sc->vmx_mcast = NULL; 1582 } 1583 } 1584 1585 static void 1586 vmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1587 { 1588 struct vmxnet3_driver_shared *ds; 1589 struct vmxnet3_txqueue *txq; 1590 struct vmxnet3_txq_shared *txs; 1591 struct vmxnet3_rxqueue *rxq; 1592 struct vmxnet3_rxq_shared *rxs; 1593 int i; 1594 1595 ds = sc->vmx_ds; 1596 1597 /* 1598 * Initialize fields of the shared data that remains the same across 1599 * reinits. Note the shared data is zero'd when allocated. 1600 */ 1601 1602 ds->magic = VMXNET3_REV1_MAGIC; 1603 1604 /* DriverInfo */ 1605 ds->version = VMXNET3_DRIVER_VERSION; 1606 ds->guest = VMXNET3_GOS_FREEBSD | 1607 #ifdef __LP64__ 1608 VMXNET3_GOS_64BIT; 1609 #else 1610 VMXNET3_GOS_32BIT; 1611 #endif 1612 ds->vmxnet3_revision = 1; 1613 ds->upt_version = 1; 1614 1615 /* Misc. conf */ 1616 ds->driver_data = vtophys(sc); 1617 ds->driver_data_len = sizeof(struct vmxnet3_softc); 1618 ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1619 ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1620 ds->nrxsg_max = sc->vmx_max_rxsegs; 1621 1622 /* RSS conf */ 1623 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1624 ds->rss.version = 1; 1625 ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; 1626 ds->rss.len = sc->vmx_rss_dma.dma_size; 1627 } 1628 1629 /* Interrupt control. */ 1630 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1631 ds->nintr = sc->vmx_nintrs; 1632 ds->evintr = sc->vmx_event_intr_idx; 1633 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1634 1635 for (i = 0; i < sc->vmx_nintrs; i++) 1636 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1637 1638 /* Receive filter. */ 1639 ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1640 ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1641 1642 /* Tx queues */ 1643 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1644 txq = &sc->vmx_txq[i]; 1645 txs = txq->vxtxq_ts; 1646 1647 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1648 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1649 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1650 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1651 txs->driver_data = vtophys(txq); 1652 txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1653 } 1654 1655 /* Rx queues */ 1656 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1657 rxq = &sc->vmx_rxq[i]; 1658 rxs = rxq->vxrxq_rs; 1659 1660 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1661 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1662 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1663 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1664 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1665 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1666 rxs->driver_data = vtophys(rxq); 1667 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1668 } 1669 } 1670 1671 static void 1672 vmxnet3_init_hwassist(struct vmxnet3_softc *sc) 1673 { 1674 struct ifnet *ifp = sc->vmx_ifp; 1675 uint64_t hwassist; 1676 1677 hwassist = 0; 1678 if (ifp->if_capenable & IFCAP_TXCSUM) 1679 hwassist |= VMXNET3_CSUM_OFFLOAD; 1680 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 1681 hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6; 1682 #if 0 /* XXX TSO */ 1683 if (ifp->if_capenable & IFCAP_TSO4) 1684 hwassist |= CSUM_IP_TSO; 1685 if (ifp->if_capenable & IFCAP_TSO6) 1686 hwassist |= CSUM_IP6_TSO; 1687 #endif 1688 ifp->if_hwassist = hwassist; 1689 } 1690 1691 static void 1692 vmxnet3_reinit_interface(struct vmxnet3_softc *sc) 1693 { 1694 struct ifnet *ifp; 1695 1696 ifp = sc->vmx_ifp; 1697 1698 /* Use the current MAC address. */ 1699 bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN); 1700 vmxnet3_set_lladdr(sc); 1701 1702 vmxnet3_init_hwassist(sc); 1703 } 1704 1705 static void 1706 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) 1707 { 1708 /* 1709 * Use the same key as the Linux driver until FreeBSD can do 1710 * RSS (presumably Toeplitz) in software. 1711 */ 1712 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 1713 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 1714 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 1715 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 1716 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 1717 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, 1718 }; 1719 1720 struct vmxnet3_driver_shared *ds; 1721 struct vmxnet3_rss_shared *rss; 1722 int i; 1723 1724 ds = sc->vmx_ds; 1725 rss = sc->vmx_rss; 1726 1727 rss->hash_type = 1728 UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | 1729 UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; 1730 rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 1731 rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; 1732 rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 1733 memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); 1734 1735 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) 1736 rss->ind_table[i] = i % sc->vmx_nrxqueues; 1737 } 1738 1739 static void 1740 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1741 { 1742 struct ifnet *ifp; 1743 struct vmxnet3_driver_shared *ds; 1744 1745 ifp = sc->vmx_ifp; 1746 ds = sc->vmx_ds; 1747 1748 ds->mtu = ifp->if_mtu; 1749 ds->ntxqueue = sc->vmx_ntxqueues; 1750 ds->nrxqueue = sc->vmx_nrxqueues; 1751 1752 ds->upt_features = 0; 1753 if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 1754 ds->upt_features |= UPT1_F_CSUM; 1755 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1756 ds->upt_features |= UPT1_F_VLAN; 1757 #if 0 /* XXX LRO */ 1758 if (ifp->if_capenable & IFCAP_LRO) 1759 ds->upt_features |= UPT1_F_LRO; 1760 #endif 1761 1762 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1763 ds->upt_features |= UPT1_F_RSS; 1764 vmxnet3_reinit_rss_shared_data(sc); 1765 } 1766 1767 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1768 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1769 (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1770 } 1771 1772 static int 1773 vmxnet3_alloc_data(struct vmxnet3_softc *sc) 1774 { 1775 int error; 1776 1777 error = vmxnet3_alloc_shared_data(sc); 1778 if (error) 1779 return (error); 1780 1781 error = vmxnet3_alloc_queue_data(sc); 1782 if (error) 1783 return (error); 1784 1785 error = vmxnet3_alloc_mcast_table(sc); 1786 if (error) 1787 return (error); 1788 1789 vmxnet3_init_shared_data(sc); 1790 1791 return (0); 1792 } 1793 1794 static void 1795 vmxnet3_free_data(struct vmxnet3_softc *sc) 1796 { 1797 1798 vmxnet3_free_mcast_table(sc); 1799 vmxnet3_free_queue_data(sc); 1800 vmxnet3_free_shared_data(sc); 1801 } 1802 1803 static int 1804 vmxnet3_setup_interface(struct vmxnet3_softc *sc) 1805 { 1806 device_t dev; 1807 struct ifnet *ifp; 1808 1809 dev = sc->vmx_dev; 1810 1811 ifp = sc->vmx_ifp = if_alloc(IFT_ETHER); 1812 if (ifp == NULL) { 1813 device_printf(dev, "cannot allocate ifnet structure\n"); 1814 return (ENOSPC); 1815 } 1816 1817 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1818 ifp->if_baudrate = IF_Gbps(10ULL); 1819 ifp->if_softc = sc; 1820 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1821 ifp->if_init = vmxnet3_init; 1822 ifp->if_ioctl = vmxnet3_ioctl; 1823 #if 0 /* XXX TSO */ 1824 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1825 ifp->if_hw_tsomaxsegcount = VMXNET3_TX_MAXSEGS; 1826 ifp->if_hw_tsomaxsegsize = VMXNET3_TX_MAXSEGSIZE; 1827 #endif 1828 1829 #ifdef VMXNET3_LEGACY_TX 1830 ifp->if_start = vmxnet3_start; 1831 ifq_set_maxlen(&ifp->if_snd, sc->vmx_ntxdescs - 1); 1832 ifq_set_ready(&ifp->if_snd); 1833 #else 1834 ifp->if_transmit = vmxnet3_txq_mq_start; 1835 ifp->if_qflush = vmxnet3_qflush; 1836 #endif 1837 1838 vmxnet3_get_lladdr(sc); 1839 ether_ifattach(ifp, sc->vmx_lladdr, NULL); 1840 1841 ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; 1842 ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6; 1843 #if 0 /* XXX TSO */ 1844 ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 1845 #endif 1846 ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 1847 IFCAP_VLAN_HWCSUM; 1848 ifp->if_capenable = ifp->if_capabilities; 1849 1850 #if 0 /* XXX LRO / VLAN_HWFILTER */ 1851 /* These capabilities are not enabled by default. */ 1852 ifp->if_capabilities |= /* IFCAP_LRO | */ IFCAP_VLAN_HWFILTER; 1853 #endif 1854 1855 sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 1856 vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 1857 sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config, 1858 vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1859 1860 ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change, 1861 vmxnet3_media_status); 1862 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1863 ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1864 1865 return (0); 1866 } 1867 1868 static void 1869 vmxnet3_evintr(struct vmxnet3_softc *sc) 1870 { 1871 device_t dev; 1872 struct ifnet *ifp; 1873 struct vmxnet3_txq_shared *ts; 1874 struct vmxnet3_rxq_shared *rs; 1875 uint32_t event; 1876 int reset; 1877 1878 dev = sc->vmx_dev; 1879 ifp = sc->vmx_ifp; 1880 reset = 0; 1881 1882 VMXNET3_CORE_LOCK(sc); 1883 1884 /* Clear events. */ 1885 event = sc->vmx_ds->event; 1886 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 1887 1888 if (event & VMXNET3_EVENT_LINK) { 1889 vmxnet3_link_status(sc); 1890 if (sc->vmx_link_active != 0) 1891 vmxnet3_tx_start_all(sc); 1892 } 1893 1894 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 1895 reset = 1; 1896 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 1897 ts = sc->vmx_txq[0].vxtxq_ts; 1898 if (ts->stopped != 0) 1899 device_printf(dev, "Tx queue error %#x\n", ts->error); 1900 rs = sc->vmx_rxq[0].vxrxq_rs; 1901 if (rs->stopped != 0) 1902 device_printf(dev, "Rx queue error %#x\n", rs->error); 1903 device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 1904 } 1905 1906 if (event & VMXNET3_EVENT_DIC) 1907 device_printf(dev, "device implementation change event\n"); 1908 if (event & VMXNET3_EVENT_DEBUG) 1909 device_printf(dev, "debug event\n"); 1910 1911 if (reset != 0) { 1912 ifp->if_flags &= ~IFF_RUNNING; 1913 vmxnet3_init_locked(sc); 1914 } 1915 1916 VMXNET3_CORE_UNLOCK(sc); 1917 } 1918 1919 static void 1920 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq) 1921 { 1922 struct vmxnet3_softc *sc; 1923 struct ifnet *ifp; 1924 struct vmxnet3_txring *txr; 1925 struct vmxnet3_comp_ring *txc; 1926 struct vmxnet3_txcompdesc *txcd; 1927 struct vmxnet3_txbuf *txb; 1928 struct mbuf *m; 1929 u_int sop; 1930 1931 sc = txq->vxtxq_sc; 1932 ifp = sc->vmx_ifp; 1933 txr = &txq->vxtxq_cmd_ring; 1934 txc = &txq->vxtxq_comp_ring; 1935 1936 VMXNET3_TXQ_LOCK_ASSERT(txq); 1937 1938 for (;;) { 1939 txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 1940 if (txcd->gen != txc->vxcr_gen) 1941 break; 1942 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1943 1944 if (++txc->vxcr_next == txc->vxcr_ndesc) { 1945 txc->vxcr_next = 0; 1946 txc->vxcr_gen ^= 1; 1947 } 1948 1949 sop = txr->vxtxr_next; 1950 txb = &txr->vxtxr_txbuf[sop]; 1951 1952 if ((m = txb->vtxb_m) != NULL) { 1953 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 1954 BUS_DMASYNC_POSTWRITE); 1955 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 1956 1957 txq->vxtxq_stats.vmtxs_opackets++; 1958 txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len; 1959 if (m->m_flags & M_MCAST) 1960 txq->vxtxq_stats.vmtxs_omcasts++; 1961 1962 m_freem(m); 1963 txb->vtxb_m = NULL; 1964 } 1965 1966 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 1967 } 1968 1969 if (txr->vxtxr_head == txr->vxtxr_next) 1970 txq->vxtxq_watchdog = 0; 1971 } 1972 1973 static int 1974 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr) 1975 { 1976 struct ifnet *ifp; 1977 struct mbuf *m; 1978 struct vmxnet3_rxdesc *rxd; 1979 struct vmxnet3_rxbuf *rxb; 1980 bus_dma_tag_t tag; 1981 bus_dmamap_t dmap; 1982 bus_dma_segment_t segs[1]; 1983 int idx, clsize, btype, flags, nsegs, error; 1984 1985 ifp = sc->vmx_ifp; 1986 tag = rxr->vxrxr_rxtag; 1987 dmap = rxr->vxrxr_spare_dmap; 1988 idx = rxr->vxrxr_fill; 1989 rxd = &rxr->vxrxr_rxd[idx]; 1990 rxb = &rxr->vxrxr_rxbuf[idx]; 1991 1992 #ifdef VMXNET3_FAILPOINTS 1993 KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS); 1994 if (rxr->vxrxr_rid != 0) 1995 KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS); 1996 #endif 1997 1998 if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) { 1999 flags = M_PKTHDR; 2000 clsize = MCLBYTES; 2001 btype = VMXNET3_BTYPE_HEAD; 2002 } else { 2003 flags = M_PKTHDR; 2004 clsize = MJUMPAGESIZE; 2005 btype = VMXNET3_BTYPE_BODY; 2006 } 2007 2008 m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize); 2009 if (m == NULL) { 2010 sc->vmx_stats.vmst_mgetcl_failed++; 2011 return (ENOBUFS); 2012 } 2013 2014 if (btype == VMXNET3_BTYPE_HEAD) { 2015 m->m_len = m->m_pkthdr.len = clsize; 2016 m_adj(m, ETHER_ALIGN); 2017 } else 2018 m->m_len = clsize; 2019 2020 error = bus_dmamap_load_mbuf_segment(tag, dmap, m, &segs[0], 1, &nsegs, 2021 BUS_DMA_NOWAIT); 2022 if (error) { 2023 m_freem(m); 2024 sc->vmx_stats.vmst_mbuf_load_failed++; 2025 return (error); 2026 } 2027 KASSERT(nsegs == 1, 2028 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 2029 if (btype == VMXNET3_BTYPE_BODY) 2030 m->m_flags &= ~M_PKTHDR; 2031 2032 if (rxb->vrxb_m != NULL) { 2033 bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD); 2034 bus_dmamap_unload(tag, rxb->vrxb_dmamap); 2035 } 2036 2037 rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 2038 rxb->vrxb_dmamap = dmap; 2039 rxb->vrxb_m = m; 2040 2041 rxd->addr = segs[0].ds_addr; 2042 rxd->len = segs[0].ds_len; 2043 rxd->btype = btype; 2044 rxd->gen = rxr->vxrxr_gen; 2045 2046 vmxnet3_rxr_increment_fill(rxr); 2047 return (0); 2048 } 2049 2050 static void 2051 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 2052 struct vmxnet3_rxring *rxr, int idx) 2053 { 2054 struct vmxnet3_rxdesc *rxd; 2055 2056 rxd = &rxr->vxrxr_rxd[idx]; 2057 rxd->gen = rxr->vxrxr_gen; 2058 vmxnet3_rxr_increment_fill(rxr); 2059 } 2060 2061 static void 2062 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 2063 { 2064 struct vmxnet3_softc *sc; 2065 struct vmxnet3_rxring *rxr; 2066 struct vmxnet3_comp_ring *rxc; 2067 struct vmxnet3_rxcompdesc *rxcd; 2068 int idx, eof; 2069 2070 sc = rxq->vxrxq_sc; 2071 rxc = &rxq->vxrxq_comp_ring; 2072 2073 do { 2074 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2075 if (rxcd->gen != rxc->vxcr_gen) 2076 break; /* Not expected. */ 2077 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2078 2079 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2080 rxc->vxcr_next = 0; 2081 rxc->vxcr_gen ^= 1; 2082 } 2083 2084 idx = rxcd->rxd_idx; 2085 eof = rxcd->eop; 2086 if (rxcd->qid < sc->vmx_nrxqueues) 2087 rxr = &rxq->vxrxq_cmd_ring[0]; 2088 else 2089 rxr = &rxq->vxrxq_cmd_ring[1]; 2090 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2091 } while (!eof); 2092 } 2093 2094 static void 2095 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2096 { 2097 2098 if (rxcd->ipv4) { 2099 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 2100 if (rxcd->ipcsum_ok) 2101 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2102 } 2103 2104 if (!rxcd->fragment) { 2105 if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) { 2106 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 2107 CSUM_PSEUDO_HDR; 2108 m->m_pkthdr.csum_data = 0xFFFF; 2109 } 2110 } 2111 } 2112 2113 static void 2114 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 2115 struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2116 { 2117 struct vmxnet3_softc *sc; 2118 struct ifnet *ifp; 2119 2120 sc = rxq->vxrxq_sc; 2121 ifp = sc->vmx_ifp; 2122 2123 if (rxcd->error) { 2124 rxq->vxrxq_stats.vmrxs_ierrors++; 2125 m_freem(m); 2126 return; 2127 } 2128 2129 #if 0 2130 #ifdef notyet 2131 switch (rxcd->rss_type) { 2132 case VMXNET3_RCD_RSS_TYPE_IPV4: 2133 m->m_pkthdr.flowid = rxcd->rss_hash; 2134 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4); 2135 break; 2136 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 2137 m->m_pkthdr.flowid = rxcd->rss_hash; 2138 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4); 2139 break; 2140 case VMXNET3_RCD_RSS_TYPE_IPV6: 2141 m->m_pkthdr.flowid = rxcd->rss_hash; 2142 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6); 2143 break; 2144 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 2145 m->m_pkthdr.flowid = rxcd->rss_hash; 2146 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6); 2147 break; 2148 default: /* VMXNET3_RCD_RSS_TYPE_NONE */ 2149 m->m_pkthdr.flowid = rxq->vxrxq_id; 2150 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2151 break; 2152 } 2153 #else 2154 m->m_pkthdr.flowid = rxq->vxrxq_id; 2155 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2156 #endif 2157 #endif 2158 2159 if (!rxcd->no_csum) 2160 vmxnet3_rx_csum(rxcd, m); 2161 if (rxcd->vlan) { 2162 m->m_flags |= M_VLANTAG; 2163 m->m_pkthdr.ether_vlantag = rxcd->vtag; 2164 } 2165 2166 rxq->vxrxq_stats.vmrxs_ipackets++; 2167 rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len; 2168 2169 VMXNET3_RXQ_UNLOCK(rxq); 2170 (*ifp->if_input)(ifp, m, NULL, -1); 2171 VMXNET3_RXQ_LOCK(rxq); 2172 } 2173 2174 static void 2175 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq) 2176 { 2177 struct vmxnet3_softc *sc; 2178 struct ifnet *ifp; 2179 struct vmxnet3_rxring *rxr; 2180 struct vmxnet3_comp_ring *rxc; 2181 struct vmxnet3_rxdesc *rxd; 2182 struct vmxnet3_rxcompdesc *rxcd; 2183 struct mbuf *m, *m_head, *m_tail; 2184 int idx, length; 2185 2186 sc = rxq->vxrxq_sc; 2187 ifp = sc->vmx_ifp; 2188 rxc = &rxq->vxrxq_comp_ring; 2189 2190 VMXNET3_RXQ_LOCK_ASSERT(rxq); 2191 2192 if ((ifp->if_flags & IFF_RUNNING) == 0) 2193 return; 2194 2195 m_head = rxq->vxrxq_mhead; 2196 rxq->vxrxq_mhead = NULL; 2197 m_tail = rxq->vxrxq_mtail; 2198 rxq->vxrxq_mtail = NULL; 2199 KKASSERT(m_head == NULL || m_tail != NULL); 2200 2201 for (;;) { 2202 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2203 if (rxcd->gen != rxc->vxcr_gen) { 2204 rxq->vxrxq_mhead = m_head; 2205 rxq->vxrxq_mtail = m_tail; 2206 break; 2207 } 2208 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2209 2210 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2211 rxc->vxcr_next = 0; 2212 rxc->vxcr_gen ^= 1; 2213 } 2214 2215 idx = rxcd->rxd_idx; 2216 length = rxcd->len; 2217 if (rxcd->qid < sc->vmx_nrxqueues) 2218 rxr = &rxq->vxrxq_cmd_ring[0]; 2219 else 2220 rxr = &rxq->vxrxq_cmd_ring[1]; 2221 rxd = &rxr->vxrxr_rxd[idx]; 2222 2223 m = rxr->vxrxr_rxbuf[idx].vrxb_m; 2224 KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf", 2225 __func__, rxcd->qid, idx)); 2226 2227 /* 2228 * The host may skip descriptors. We detect this when this 2229 * descriptor does not match the previous fill index. Catch 2230 * up with the host now. 2231 */ 2232 if (__predict_false(rxr->vxrxr_fill != idx)) { 2233 while (rxr->vxrxr_fill != idx) { 2234 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 2235 rxr->vxrxr_gen; 2236 vmxnet3_rxr_increment_fill(rxr); 2237 } 2238 } 2239 2240 if (rxcd->sop) { 2241 KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD, 2242 ("%s: start of frame w/o head buffer", __func__)); 2243 KASSERT(rxr == &rxq->vxrxq_cmd_ring[0], 2244 ("%s: start of frame not in ring 0", __func__)); 2245 KASSERT((idx % sc->vmx_rx_max_chain) == 0, 2246 ("%s: start of frame at unexcepted index %d (%d)", 2247 __func__, idx, sc->vmx_rx_max_chain)); 2248 KASSERT(m_head == NULL, 2249 ("%s: duplicate start of frame?", __func__)); 2250 2251 if (length == 0) { 2252 /* Just ignore this descriptor. */ 2253 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2254 goto nextp; 2255 } 2256 2257 if (vmxnet3_newbuf(sc, rxr) != 0) { 2258 rxq->vxrxq_stats.vmrxs_iqdrops++; 2259 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2260 if (!rxcd->eop) 2261 vmxnet3_rxq_discard_chain(rxq); 2262 goto nextp; 2263 } 2264 2265 m->m_pkthdr.rcvif = ifp; 2266 m->m_pkthdr.len = m->m_len = length; 2267 m->m_pkthdr.csum_flags = 0; 2268 m_head = m_tail = m; 2269 2270 } else { 2271 KASSERT(rxd->btype == VMXNET3_BTYPE_BODY, 2272 ("%s: non start of frame w/o body buffer", __func__)); 2273 2274 if (m_head == NULL && m_tail == NULL) { 2275 /* 2276 * This is a continuation of a packet that we 2277 * started to drop, but could not drop entirely 2278 * because this segment was still owned by the 2279 * host. So, drop the remainder now. 2280 */ 2281 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2282 if (!rxcd->eop) 2283 vmxnet3_rxq_discard_chain(rxq); 2284 goto nextp; 2285 } 2286 2287 KASSERT(m_head != NULL, 2288 ("%s: frame not started?", __func__)); 2289 2290 if (vmxnet3_newbuf(sc, rxr) != 0) { 2291 rxq->vxrxq_stats.vmrxs_iqdrops++; 2292 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2293 if (!rxcd->eop) 2294 vmxnet3_rxq_discard_chain(rxq); 2295 m_freem(m_head); 2296 m_head = m_tail = NULL; 2297 goto nextp; 2298 } 2299 2300 m->m_len = length; 2301 m_head->m_pkthdr.len += length; 2302 m_tail->m_next = m; 2303 m_tail = m; 2304 } 2305 2306 if (rxcd->eop) { 2307 vmxnet3_rxq_input(rxq, rxcd, m_head); 2308 m_head = m_tail = NULL; 2309 2310 /* Must recheck after dropping the Rx lock. */ 2311 if ((ifp->if_flags & IFF_RUNNING) == 0) 2312 break; 2313 } 2314 2315 nextp: 2316 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 2317 int qid = rxcd->qid; 2318 bus_size_t r; 2319 2320 idx = (idx + 1) % rxr->vxrxr_ndesc; 2321 if (qid >= sc->vmx_nrxqueues) { 2322 qid -= sc->vmx_nrxqueues; 2323 r = VMXNET3_BAR0_RXH2(qid); 2324 } else 2325 r = VMXNET3_BAR0_RXH1(qid); 2326 vmxnet3_write_bar0(sc, r, idx); 2327 } 2328 } 2329 } 2330 2331 static void 2332 vmxnet3_legacy_intr(void *xsc) 2333 { 2334 struct vmxnet3_softc *sc; 2335 struct vmxnet3_rxqueue *rxq; 2336 struct vmxnet3_txqueue *txq; 2337 2338 sc = xsc; 2339 rxq = &sc->vmx_rxq[0]; 2340 txq = &sc->vmx_txq[0]; 2341 2342 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 2343 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 2344 return; 2345 } 2346 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2347 vmxnet3_disable_all_intrs(sc); 2348 2349 if (sc->vmx_ds->event != 0) 2350 vmxnet3_evintr(sc); 2351 2352 VMXNET3_RXQ_LOCK(rxq); 2353 vmxnet3_rxq_eof(rxq); 2354 VMXNET3_RXQ_UNLOCK(rxq); 2355 2356 VMXNET3_TXQ_LOCK(txq); 2357 vmxnet3_txq_eof(txq); 2358 vmxnet3_txq_start(txq); 2359 VMXNET3_TXQ_UNLOCK(txq); 2360 2361 vmxnet3_enable_all_intrs(sc); 2362 } 2363 2364 #ifdef __FreeBSD__ 2365 static void 2366 vmxnet3_txq_intr(void *xtxq) 2367 { 2368 struct vmxnet3_softc *sc; 2369 struct vmxnet3_txqueue *txq; 2370 2371 txq = xtxq; 2372 sc = txq->vxtxq_sc; 2373 2374 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2375 vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx); 2376 2377 VMXNET3_TXQ_LOCK(txq); 2378 vmxnet3_txq_eof(txq); 2379 vmxnet3_txq_start(txq); 2380 VMXNET3_TXQ_UNLOCK(txq); 2381 2382 vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx); 2383 } 2384 2385 static void 2386 vmxnet3_rxq_intr(void *xrxq) 2387 { 2388 struct vmxnet3_softc *sc; 2389 struct vmxnet3_rxqueue *rxq; 2390 2391 rxq = xrxq; 2392 sc = rxq->vxrxq_sc; 2393 2394 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2395 vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx); 2396 2397 VMXNET3_RXQ_LOCK(rxq); 2398 vmxnet3_rxq_eof(rxq); 2399 VMXNET3_RXQ_UNLOCK(rxq); 2400 2401 vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx); 2402 } 2403 2404 static void 2405 vmxnet3_event_intr(void *xsc) 2406 { 2407 struct vmxnet3_softc *sc; 2408 2409 sc = xsc; 2410 2411 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2412 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 2413 2414 if (sc->vmx_ds->event != 0) 2415 vmxnet3_evintr(sc); 2416 2417 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2418 } 2419 #endif 2420 2421 static void 2422 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2423 { 2424 struct vmxnet3_txring *txr; 2425 struct vmxnet3_txbuf *txb; 2426 int i; 2427 2428 txr = &txq->vxtxq_cmd_ring; 2429 2430 for (i = 0; i < txr->vxtxr_ndesc; i++) { 2431 txb = &txr->vxtxr_txbuf[i]; 2432 2433 if (txb->vtxb_m == NULL) 2434 continue; 2435 2436 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 2437 BUS_DMASYNC_POSTWRITE); 2438 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 2439 m_freem(txb->vtxb_m); 2440 txb->vtxb_m = NULL; 2441 } 2442 } 2443 2444 static void 2445 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2446 { 2447 struct vmxnet3_rxring *rxr; 2448 struct vmxnet3_rxbuf *rxb; 2449 int i, j; 2450 2451 if (rxq->vxrxq_mhead != NULL) { 2452 m_freem(rxq->vxrxq_mhead); 2453 rxq->vxrxq_mhead = NULL; 2454 rxq->vxrxq_mtail = NULL; 2455 } 2456 2457 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2458 rxr = &rxq->vxrxq_cmd_ring[i]; 2459 2460 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2461 rxb = &rxr->vxrxr_rxbuf[j]; 2462 2463 if (rxb->vrxb_m == NULL) 2464 continue; 2465 2466 bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap, 2467 BUS_DMASYNC_POSTREAD); 2468 bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap); 2469 m_freem(rxb->vrxb_m); 2470 rxb->vrxb_m = NULL; 2471 } 2472 } 2473 } 2474 2475 static void 2476 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2477 { 2478 struct vmxnet3_rxqueue *rxq; 2479 struct vmxnet3_txqueue *txq; 2480 int i; 2481 2482 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2483 rxq = &sc->vmx_rxq[i]; 2484 VMXNET3_RXQ_LOCK(rxq); 2485 VMXNET3_RXQ_UNLOCK(rxq); 2486 } 2487 2488 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2489 txq = &sc->vmx_txq[i]; 2490 VMXNET3_TXQ_LOCK(txq); 2491 VMXNET3_TXQ_UNLOCK(txq); 2492 } 2493 } 2494 2495 static void 2496 vmxnet3_stop(struct vmxnet3_softc *sc) 2497 { 2498 struct ifnet *ifp; 2499 int q; 2500 2501 ifp = sc->vmx_ifp; 2502 VMXNET3_CORE_LOCK_ASSERT(sc); 2503 2504 ifp->if_flags &= ~IFF_RUNNING; 2505 sc->vmx_link_active = 0; 2506 callout_stop(&sc->vmx_tick); 2507 2508 /* Disable interrupts. */ 2509 vmxnet3_disable_all_intrs(sc); 2510 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2511 2512 vmxnet3_stop_rendezvous(sc); 2513 2514 for (q = 0; q < sc->vmx_ntxqueues; q++) 2515 vmxnet3_txstop(sc, &sc->vmx_txq[q]); 2516 for (q = 0; q < sc->vmx_nrxqueues; q++) 2517 vmxnet3_rxstop(sc, &sc->vmx_rxq[q]); 2518 2519 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2520 } 2521 2522 static void 2523 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2524 { 2525 struct vmxnet3_txring *txr; 2526 struct vmxnet3_comp_ring *txc; 2527 2528 txr = &txq->vxtxq_cmd_ring; 2529 txr->vxtxr_head = 0; 2530 txr->vxtxr_next = 0; 2531 txr->vxtxr_gen = VMXNET3_INIT_GEN; 2532 bzero(txr->vxtxr_txd, 2533 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2534 2535 txc = &txq->vxtxq_comp_ring; 2536 txc->vxcr_next = 0; 2537 txc->vxcr_gen = VMXNET3_INIT_GEN; 2538 bzero(txc->vxcr_u.txcd, 2539 txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2540 } 2541 2542 static int 2543 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2544 { 2545 struct ifnet *ifp; 2546 struct vmxnet3_rxring *rxr; 2547 struct vmxnet3_comp_ring *rxc; 2548 int i, populate, idx, frame_size, error; 2549 2550 ifp = sc->vmx_ifp; 2551 frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) + 2552 ifp->if_mtu; 2553 2554 /* 2555 * If the MTU causes us to exceed what a regular sized cluster can 2556 * handle, we allocate a second MJUMPAGESIZE cluster after it in 2557 * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters. 2558 * 2559 * Keep rx_max_chain a divisor of the maximum Rx ring size to make 2560 * our life easier. We do not support changing the ring size after 2561 * the attach. 2562 */ 2563 if (frame_size <= MCLBYTES) 2564 sc->vmx_rx_max_chain = 1; 2565 else 2566 sc->vmx_rx_max_chain = 2; 2567 2568 /* 2569 * Only populate ring 1 if the configuration will take advantage 2570 * of it. That is either when LRO is enabled or the frame size 2571 * exceeds what ring 0 can contain. 2572 */ 2573 #if 0 /* XXX LRO */ 2574 if ((ifp->if_capenable & IFCAP_LRO) == 0 && 2575 #else 2576 if ( 2577 #endif 2578 frame_size <= MCLBYTES + MJUMPAGESIZE) 2579 populate = 1; 2580 else 2581 populate = VMXNET3_RXRINGS_PERQ; 2582 2583 for (i = 0; i < populate; i++) { 2584 rxr = &rxq->vxrxq_cmd_ring[i]; 2585 rxr->vxrxr_fill = 0; 2586 rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2587 bzero(rxr->vxrxr_rxd, 2588 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2589 2590 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2591 error = vmxnet3_newbuf(sc, rxr); 2592 if (error) 2593 return (error); 2594 } 2595 } 2596 2597 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2598 rxr = &rxq->vxrxq_cmd_ring[i]; 2599 rxr->vxrxr_fill = 0; 2600 rxr->vxrxr_gen = 0; 2601 bzero(rxr->vxrxr_rxd, 2602 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2603 } 2604 2605 rxc = &rxq->vxrxq_comp_ring; 2606 rxc->vxcr_next = 0; 2607 rxc->vxcr_gen = VMXNET3_INIT_GEN; 2608 bzero(rxc->vxcr_u.rxcd, 2609 rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2610 2611 return (0); 2612 } 2613 2614 static int 2615 vmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2616 { 2617 device_t dev; 2618 int q, error; 2619 2620 dev = sc->vmx_dev; 2621 2622 for (q = 0; q < sc->vmx_ntxqueues; q++) 2623 vmxnet3_txinit(sc, &sc->vmx_txq[q]); 2624 2625 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2626 error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); 2627 if (error) { 2628 device_printf(dev, "cannot populate Rx queue %d\n", q); 2629 return (error); 2630 } 2631 } 2632 2633 return (0); 2634 } 2635 2636 static int 2637 vmxnet3_enable_device(struct vmxnet3_softc *sc) 2638 { 2639 int q; 2640 2641 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2642 device_printf(sc->vmx_dev, "device enable command failed!\n"); 2643 return (1); 2644 } 2645 2646 /* Reset the Rx queue heads. */ 2647 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2648 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2649 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2650 } 2651 2652 return (0); 2653 } 2654 2655 static void 2656 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2657 { 2658 struct ifnet *ifp; 2659 2660 ifp = sc->vmx_ifp; 2661 2662 vmxnet3_set_rxfilter(sc); 2663 2664 #if 0 /* VLAN_HWFILTER */ 2665 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2666 bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter, 2667 sizeof(sc->vmx_ds->vlan_filter)); 2668 else 2669 #endif 2670 bzero(sc->vmx_ds->vlan_filter, 2671 sizeof(sc->vmx_ds->vlan_filter)); 2672 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2673 } 2674 2675 static int 2676 vmxnet3_reinit(struct vmxnet3_softc *sc) 2677 { 2678 2679 vmxnet3_reinit_interface(sc); 2680 vmxnet3_reinit_shared_data(sc); 2681 2682 if (vmxnet3_reinit_queues(sc) != 0) 2683 return (ENXIO); 2684 2685 if (vmxnet3_enable_device(sc) != 0) 2686 return (ENXIO); 2687 2688 vmxnet3_reinit_rxfilters(sc); 2689 2690 return (0); 2691 } 2692 2693 static void 2694 vmxnet3_init_locked(struct vmxnet3_softc *sc) 2695 { 2696 struct ifnet *ifp; 2697 2698 ifp = sc->vmx_ifp; 2699 2700 if (ifp->if_flags & IFF_RUNNING) 2701 return; 2702 2703 vmxnet3_stop(sc); 2704 2705 if (vmxnet3_reinit(sc) != 0) { 2706 vmxnet3_stop(sc); 2707 return; 2708 } 2709 2710 ifp->if_flags |= IFF_RUNNING; 2711 vmxnet3_link_status(sc); 2712 2713 vmxnet3_enable_all_intrs(sc); 2714 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2715 } 2716 2717 static void 2718 vmxnet3_init(void *xsc) 2719 { 2720 struct vmxnet3_softc *sc; 2721 2722 sc = xsc; 2723 2724 VMXNET3_CORE_LOCK(sc); 2725 vmxnet3_init_locked(sc); 2726 VMXNET3_CORE_UNLOCK(sc); 2727 } 2728 2729 /* 2730 * BMV: Much of this can go away once we finally have offsets in 2731 * the mbuf packet header. Bug andre@. 2732 */ 2733 static int 2734 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, 2735 int *etype, int *proto, int *start) 2736 { 2737 struct ether_vlan_header *evh; 2738 int offset; 2739 #if defined(INET) 2740 struct ip *ip = NULL; 2741 #endif 2742 #if defined(INET6) 2743 struct ip6_hdr *ip6 = NULL; 2744 #endif 2745 2746 evh = mtod(m, struct ether_vlan_header *); 2747 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2748 /* BMV: We should handle nested VLAN tags too. */ 2749 *etype = ntohs(evh->evl_proto); 2750 offset = sizeof(struct ether_vlan_header); 2751 } else { 2752 *etype = ntohs(evh->evl_encap_proto); 2753 offset = sizeof(struct ether_header); 2754 } 2755 2756 switch (*etype) { 2757 #if defined(INET) 2758 case ETHERTYPE_IP: 2759 if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 2760 m = m_pullup(m, offset + sizeof(struct ip)); 2761 if (m == NULL) 2762 return (EINVAL); 2763 } 2764 2765 ip = (struct ip *)(mtod(m, uint8_t *) + offset); 2766 *proto = ip->ip_p; 2767 *start = offset + (ip->ip_hl << 2); 2768 break; 2769 #endif 2770 #if defined(INET6) 2771 case ETHERTYPE_IPV6: 2772 if (__predict_false(m->m_len < 2773 offset + sizeof(struct ip6_hdr))) { 2774 m = m_pullup(m, offset + sizeof(struct ip6_hdr)); 2775 if (m == NULL) 2776 return (EINVAL); 2777 } 2778 2779 ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + offset); 2780 *proto = -1; 2781 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 2782 /* Assert the network stack sent us a valid packet. */ 2783 KASSERT(*start > offset, 2784 ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 2785 *start, offset, *proto)); 2786 break; 2787 #endif 2788 default: 2789 return (EINVAL); 2790 } 2791 2792 #if 0 /* XXX TSO */ 2793 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2794 struct tcphdr *tcp; 2795 2796 if (__predict_false(*proto != IPPROTO_TCP)) { 2797 /* Likely failed to correctly parse the mbuf. */ 2798 return (EINVAL); 2799 } 2800 2801 if (m->m_len < *start + sizeof(struct tcphdr)) { 2802 m = m_pullup(m, *start + sizeof(struct tcphdr)); 2803 if (m == NULL) 2804 return (EINVAL); 2805 } 2806 2807 tcp = (struct tcphdr *)(mtod(m, uint8_t *) + *start); 2808 *start += (tcp->th_off << 2); 2809 2810 txq->vxtxq_stats.vmtxs_tso++; 2811 } else 2812 #endif 2813 txq->vxtxq_stats.vmtxs_csum++; 2814 2815 return (0); 2816 } 2817 2818 static int 2819 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 2820 bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs) 2821 { 2822 struct vmxnet3_txring *txr; 2823 struct mbuf *m; 2824 bus_dma_tag_t tag; 2825 int error; 2826 2827 txr = &txq->vxtxq_cmd_ring; 2828 m = *m0; 2829 tag = txr->vxtxr_txtag; 2830 2831 error = bus_dmamap_load_mbuf_segment(tag, dmap, m, segs, 1, nsegs, 2832 BUS_DMA_NOWAIT); 2833 if (error == 0 || error != EFBIG) 2834 return (error); 2835 2836 m = m_defrag(m, M_NOWAIT); 2837 if (m != NULL) { 2838 *m0 = m; 2839 error = bus_dmamap_load_mbuf_segment(tag, dmap, m, segs, 2840 1, nsegs, BUS_DMA_NOWAIT); 2841 } else 2842 error = ENOBUFS; 2843 2844 if (error) { 2845 m_freem(*m0); 2846 *m0 = NULL; 2847 txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++; 2848 } else 2849 txq->vxtxq_sc->vmx_stats.vmst_defragged++; 2850 2851 return (error); 2852 } 2853 2854 static void 2855 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 2856 { 2857 struct vmxnet3_txring *txr; 2858 2859 txr = &txq->vxtxq_cmd_ring; 2860 bus_dmamap_unload(txr->vxtxr_txtag, dmap); 2861 } 2862 2863 static int 2864 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 2865 { 2866 struct vmxnet3_softc *sc; 2867 struct vmxnet3_txring *txr; 2868 struct vmxnet3_txdesc *txd, *sop; 2869 struct mbuf *m; 2870 bus_dmamap_t dmap; 2871 bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS]; 2872 int i, gen, nsegs, etype, proto, start, error; 2873 2874 sc = txq->vxtxq_sc; 2875 start = 0; 2876 txd = NULL; 2877 txr = &txq->vxtxq_cmd_ring; 2878 dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 2879 2880 error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs); 2881 if (error) 2882 return (error); 2883 2884 m = *m0; 2885 M_ASSERTPKTHDR(m); 2886 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS, 2887 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 2888 2889 if (VMXNET3_TXRING_AVAIL(txr) < nsegs) { 2890 txq->vxtxq_stats.vmtxs_full++; 2891 vmxnet3_txq_unload_mbuf(txq, dmap); 2892 return (ENOSPC); 2893 } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 2894 error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start); 2895 if (error) { 2896 txq->vxtxq_stats.vmtxs_offload_failed++; 2897 vmxnet3_txq_unload_mbuf(txq, dmap); 2898 m_freem(m); 2899 *m0 = NULL; 2900 return (error); 2901 } 2902 } 2903 2904 txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m; 2905 sop = &txr->vxtxr_txd[txr->vxtxr_head]; 2906 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 2907 2908 for (i = 0; i < nsegs; i++) { 2909 txd = &txr->vxtxr_txd[txr->vxtxr_head]; 2910 2911 txd->addr = segs[i].ds_addr; 2912 txd->len = segs[i].ds_len; 2913 txd->gen = gen; 2914 txd->dtype = 0; 2915 txd->offload_mode = VMXNET3_OM_NONE; 2916 txd->offload_pos = 0; 2917 txd->hlen = 0; 2918 txd->eop = 0; 2919 txd->compreq = 0; 2920 txd->vtag_mode = 0; 2921 txd->vtag = 0; 2922 2923 if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 2924 txr->vxtxr_head = 0; 2925 txr->vxtxr_gen ^= 1; 2926 } 2927 gen = txr->vxtxr_gen; 2928 } 2929 txd->eop = 1; 2930 txd->compreq = 1; 2931 2932 if (m->m_flags & M_VLANTAG) { 2933 sop->vtag_mode = 1; 2934 sop->vtag = m->m_pkthdr.ether_vlantag; 2935 } 2936 2937 2938 #if 0 /* XXX TSO */ 2939 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2940 sop->offload_mode = VMXNET3_OM_TSO; 2941 sop->hlen = start; 2942 sop->offload_pos = m->m_pkthdr.tso_segsz; 2943 } else 2944 #endif 2945 if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 2946 VMXNET3_CSUM_OFFLOAD_IPV6)) { 2947 sop->offload_mode = VMXNET3_OM_CSUM; 2948 sop->hlen = start; 2949 sop->offload_pos = start + m->m_pkthdr.csum_data; 2950 } 2951 2952 /* Finally, change the ownership. */ 2953 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 2954 sop->gen ^= 1; 2955 2956 txq->vxtxq_ts->npending += nsegs; 2957 if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 2958 txq->vxtxq_ts->npending = 0; 2959 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), 2960 txr->vxtxr_head); 2961 } 2962 2963 return (0); 2964 } 2965 2966 #ifdef VMXNET3_LEGACY_TX 2967 2968 static void 2969 vmxnet3_start_locked(struct ifnet *ifp) 2970 { 2971 struct vmxnet3_softc *sc; 2972 struct vmxnet3_txqueue *txq; 2973 struct vmxnet3_txring *txr; 2974 struct mbuf *m_head; 2975 int tx, avail; 2976 2977 sc = ifp->if_softc; 2978 txq = &sc->vmx_txq[0]; 2979 txr = &txq->vxtxq_cmd_ring; 2980 tx = 0; 2981 2982 VMXNET3_TXQ_LOCK_ASSERT(txq); 2983 2984 if ((ifp->if_flags & IFF_RUNNING) == 0 || 2985 sc->vmx_link_active == 0) 2986 return; 2987 2988 while (!ifq_is_empty(&ifp->if_snd)) { 2989 if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2) 2990 break; 2991 2992 m_head = ifq_dequeue(&ifp->if_snd); 2993 if (m_head == NULL) 2994 break; 2995 2996 /* Assume worse case if this mbuf is the head of a chain. */ 2997 if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2998 ifq_prepend(&ifp->if_snd, m_head); 2999 break; 3000 } 3001 3002 if (vmxnet3_txq_encap(txq, &m_head) != 0) { 3003 if (m_head != NULL) 3004 ifq_prepend(&ifp->if_snd, m_head); 3005 break; 3006 } 3007 3008 tx++; 3009 ETHER_BPF_MTAP(ifp, m_head); 3010 } 3011 3012 if (tx > 0) 3013 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 3014 } 3015 3016 static void 3017 vmxnet3_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 3018 { 3019 struct vmxnet3_softc *sc; 3020 struct vmxnet3_txqueue *txq; 3021 3022 sc = ifp->if_softc; 3023 txq = &sc->vmx_txq[0]; 3024 3025 VMXNET3_TXQ_LOCK(txq); 3026 vmxnet3_start_locked(ifp); 3027 VMXNET3_TXQ_UNLOCK(txq); 3028 } 3029 3030 #else /* !VMXNET3_LEGACY_TX */ 3031 3032 static int 3033 vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m) 3034 { 3035 struct vmxnet3_softc *sc; 3036 struct vmxnet3_txring *txr; 3037 struct buf_ring *br; 3038 struct ifnet *ifp; 3039 int tx, avail, error; 3040 3041 sc = txq->vxtxq_sc; 3042 br = txq->vxtxq_br; 3043 ifp = sc->vmx_ifp; 3044 txr = &txq->vxtxq_cmd_ring; 3045 tx = 0; 3046 error = 0; 3047 3048 VMXNET3_TXQ_LOCK_ASSERT(txq); 3049 3050 if ((ifp->if_flags & IFF_RUNNING) == 0 || 3051 sc->vmx_link_active == 0) { 3052 if (m != NULL) 3053 error = drbr_enqueue(ifp, br, m); 3054 return (error); 3055 } 3056 3057 if (m != NULL) { 3058 error = drbr_enqueue(ifp, br, m); 3059 if (error) 3060 return (error); 3061 } 3062 3063 while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) { 3064 m = drbr_peek(ifp, br); 3065 if (m == NULL) 3066 break; 3067 3068 /* Assume worse case if this mbuf is the head of a chain. */ 3069 if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 3070 drbr_putback(ifp, br, m); 3071 break; 3072 } 3073 3074 if (vmxnet3_txq_encap(txq, &m) != 0) { 3075 if (m != NULL) 3076 drbr_putback(ifp, br, m); 3077 else 3078 drbr_advance(ifp, br); 3079 break; 3080 } 3081 drbr_advance(ifp, br); 3082 3083 tx++; 3084 ETHER_BPF_MTAP(ifp, m); 3085 } 3086 3087 if (tx > 0) 3088 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 3089 3090 return (0); 3091 } 3092 3093 static int 3094 vmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m) 3095 { 3096 struct vmxnet3_softc *sc; 3097 struct vmxnet3_txqueue *txq; 3098 int i, ntxq, error; 3099 3100 sc = ifp->if_softc; 3101 ntxq = sc->vmx_ntxqueues; 3102 3103 /* check if flowid is set */ 3104 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 3105 i = m->m_pkthdr.flowid % ntxq; 3106 else 3107 i = curcpu % ntxq; 3108 3109 txq = &sc->vmx_txq[i]; 3110 3111 if (VMXNET3_TXQ_TRYLOCK(txq) != 0) { 3112 error = vmxnet3_txq_mq_start_locked(txq, m); 3113 VMXNET3_TXQ_UNLOCK(txq); 3114 } else { 3115 error = drbr_enqueue(ifp, txq->vxtxq_br, m); 3116 taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask); 3117 } 3118 3119 return (error); 3120 } 3121 3122 static void 3123 vmxnet3_txq_tq_deferred(void *xtxq, int pending) 3124 { 3125 struct vmxnet3_softc *sc; 3126 struct vmxnet3_txqueue *txq; 3127 3128 txq = xtxq; 3129 sc = txq->vxtxq_sc; 3130 3131 VMXNET3_TXQ_LOCK(txq); 3132 if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br)) 3133 vmxnet3_txq_mq_start_locked(txq, NULL); 3134 VMXNET3_TXQ_UNLOCK(txq); 3135 } 3136 3137 #endif /* VMXNET3_LEGACY_TX */ 3138 3139 static void 3140 vmxnet3_txq_start(struct vmxnet3_txqueue *txq) 3141 { 3142 struct vmxnet3_softc *sc; 3143 struct ifnet *ifp; 3144 3145 sc = txq->vxtxq_sc; 3146 ifp = sc->vmx_ifp; 3147 3148 #ifdef VMXNET3_LEGACY_TX 3149 if (!ifq_is_empty(&ifp->if_snd)) 3150 vmxnet3_start_locked(ifp); 3151 #else 3152 if (!drbr_empty(ifp, txq->vxtxq_br)) 3153 vmxnet3_txq_mq_start_locked(txq, NULL); 3154 #endif 3155 } 3156 3157 static void 3158 vmxnet3_tx_start_all(struct vmxnet3_softc *sc) 3159 { 3160 struct vmxnet3_txqueue *txq; 3161 int i; 3162 3163 VMXNET3_CORE_LOCK_ASSERT(sc); 3164 3165 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3166 txq = &sc->vmx_txq[i]; 3167 3168 VMXNET3_TXQ_LOCK(txq); 3169 vmxnet3_txq_start(txq); 3170 VMXNET3_TXQ_UNLOCK(txq); 3171 } 3172 } 3173 3174 static void 3175 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag) 3176 { 3177 struct ifnet *ifp; 3178 int idx, bit; 3179 3180 ifp = sc->vmx_ifp; 3181 idx = (tag >> 5) & 0x7F; 3182 bit = tag & 0x1F; 3183 3184 if (tag == 0 || tag > 4095) 3185 return; 3186 3187 VMXNET3_CORE_LOCK(sc); 3188 3189 /* Update our private VLAN bitvector. */ 3190 if (add) 3191 sc->vmx_vlan_filter[idx] |= (1 << bit); 3192 else 3193 sc->vmx_vlan_filter[idx] &= ~(1 << bit); 3194 3195 #if 0 /* VLAN_HWFILTER */ 3196 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3197 if (add) 3198 sc->vmx_ds->vlan_filter[idx] |= (1 << bit); 3199 else 3200 sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit); 3201 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 3202 } 3203 #endif 3204 3205 VMXNET3_CORE_UNLOCK(sc); 3206 } 3207 3208 static void 3209 vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3210 { 3211 3212 if (ifp->if_softc == arg) 3213 vmxnet3_update_vlan_filter(arg, 1, tag); 3214 } 3215 3216 static void 3217 vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3218 { 3219 3220 if (ifp->if_softc == arg) 3221 vmxnet3_update_vlan_filter(arg, 0, tag); 3222 } 3223 3224 static void 3225 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 3226 { 3227 struct ifnet *ifp; 3228 struct vmxnet3_driver_shared *ds; 3229 struct ifmultiaddr *ifma; 3230 u_int mode; 3231 3232 ifp = sc->vmx_ifp; 3233 ds = sc->vmx_ds; 3234 3235 mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST; 3236 if (ifp->if_flags & IFF_PROMISC) 3237 mode |= VMXNET3_RXMODE_PROMISC; 3238 if (ifp->if_flags & IFF_ALLMULTI) 3239 mode |= VMXNET3_RXMODE_ALLMULTI; 3240 else { 3241 int cnt = 0, overflow = 0; 3242 3243 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3244 if (ifma->ifma_addr->sa_family != AF_LINK) 3245 continue; 3246 else if (cnt == VMXNET3_MULTICAST_MAX) { 3247 overflow = 1; 3248 break; 3249 } 3250 3251 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 3252 &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN); 3253 cnt++; 3254 } 3255 3256 if (overflow != 0) { 3257 cnt = 0; 3258 mode |= VMXNET3_RXMODE_ALLMULTI; 3259 } else if (cnt > 0) 3260 mode |= VMXNET3_RXMODE_MCAST; 3261 ds->mcast_tablelen = cnt * ETHER_ADDR_LEN; 3262 } 3263 3264 ds->rxmode = mode; 3265 3266 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 3267 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 3268 } 3269 3270 static int 3271 vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) 3272 { 3273 struct ifnet *ifp; 3274 3275 ifp = sc->vmx_ifp; 3276 3277 if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) 3278 return (EINVAL); 3279 3280 ifp->if_mtu = mtu; 3281 3282 if (ifp->if_flags & IFF_RUNNING) { 3283 ifp->if_flags &= ~IFF_RUNNING; 3284 vmxnet3_init_locked(sc); 3285 } 3286 3287 return (0); 3288 } 3289 3290 static int 3291 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cred) 3292 { 3293 struct vmxnet3_softc *sc; 3294 struct ifreq *ifr; 3295 int reinit, mask, error; 3296 3297 sc = ifp->if_softc; 3298 ifr = (struct ifreq *) data; 3299 error = 0; 3300 3301 switch (cmd) { 3302 case SIOCSIFMTU: 3303 if (ifp->if_mtu != ifr->ifr_mtu) { 3304 VMXNET3_CORE_LOCK(sc); 3305 error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); 3306 VMXNET3_CORE_UNLOCK(sc); 3307 } 3308 break; 3309 3310 case SIOCSIFFLAGS: 3311 VMXNET3_CORE_LOCK(sc); 3312 if (ifp->if_flags & IFF_UP) { 3313 if ((ifp->if_flags & IFF_RUNNING)) { 3314 if ((ifp->if_flags ^ sc->vmx_if_flags) & 3315 (IFF_PROMISC | IFF_ALLMULTI)) { 3316 vmxnet3_set_rxfilter(sc); 3317 } 3318 } else 3319 vmxnet3_init_locked(sc); 3320 } else { 3321 if (ifp->if_flags & IFF_RUNNING) 3322 vmxnet3_stop(sc); 3323 } 3324 sc->vmx_if_flags = ifp->if_flags; 3325 VMXNET3_CORE_UNLOCK(sc); 3326 break; 3327 3328 case SIOCADDMULTI: 3329 case SIOCDELMULTI: 3330 VMXNET3_CORE_LOCK(sc); 3331 if (ifp->if_flags & IFF_RUNNING) 3332 vmxnet3_set_rxfilter(sc); 3333 VMXNET3_CORE_UNLOCK(sc); 3334 break; 3335 3336 case SIOCSIFMEDIA: 3337 case SIOCGIFMEDIA: 3338 error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd); 3339 break; 3340 3341 case SIOCSIFCAP: 3342 VMXNET3_CORE_LOCK(sc); 3343 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 3344 3345 if (mask & IFCAP_TXCSUM) 3346 ifp->if_capenable ^= IFCAP_TXCSUM; 3347 if (mask & IFCAP_TXCSUM_IPV6) 3348 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 3349 #if 0 /* XXX TSO */ 3350 if (mask & IFCAP_TSO4) 3351 ifp->if_capenable ^= IFCAP_TSO4; 3352 if (mask & IFCAP_TSO6) 3353 ifp->if_capenable ^= IFCAP_TSO6; 3354 #endif 3355 3356 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | /* IFCAP_LRO | */ 3357 IFCAP_VLAN_HWTAGGING /* | IFCAP_VLAN_HWFILTER */)) { 3358 /* Changing these features requires us to reinit. */ 3359 reinit = 1; 3360 3361 if (mask & IFCAP_RXCSUM) 3362 ifp->if_capenable ^= IFCAP_RXCSUM; 3363 if (mask & IFCAP_RXCSUM_IPV6) 3364 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 3365 #if 0 /* XXX LRO */ 3366 if (mask & IFCAP_LRO) 3367 ifp->if_capenable ^= IFCAP_LRO; 3368 #endif 3369 if (mask & IFCAP_VLAN_HWTAGGING) 3370 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 3371 #if 0 /* XXX VLAN_HWFILTER */ 3372 if (mask & IFCAP_VLAN_HWFILTER) 3373 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 3374 #endif 3375 } else 3376 reinit = 0; 3377 3378 #if 0 /* XXX TSO */ 3379 if (mask & IFCAP_VLAN_HWTSO) 3380 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 3381 #endif 3382 3383 if (reinit && (ifp->if_flags & IFF_RUNNING)) { 3384 ifp->if_flags &= ~IFF_RUNNING; 3385 vmxnet3_init_locked(sc); 3386 } else { 3387 vmxnet3_init_hwassist(sc); 3388 } 3389 3390 VMXNET3_CORE_UNLOCK(sc); 3391 #if 0 /* XXX */ 3392 VLAN_CAPABILITIES(ifp); 3393 #endif 3394 break; 3395 3396 default: 3397 error = ether_ioctl(ifp, cmd, data); 3398 break; 3399 } 3400 3401 VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc); 3402 3403 return (error); 3404 } 3405 3406 #ifndef VMXNET3_LEGACY_TX 3407 static void 3408 vmxnet3_qflush(struct ifnet *ifp) 3409 { 3410 struct vmxnet3_softc *sc; 3411 struct vmxnet3_txqueue *txq; 3412 struct mbuf *m; 3413 int i; 3414 3415 sc = ifp->if_softc; 3416 3417 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3418 txq = &sc->vmx_txq[i]; 3419 3420 VMXNET3_TXQ_LOCK(txq); 3421 while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL) 3422 m_freem(m); 3423 VMXNET3_TXQ_UNLOCK(txq); 3424 } 3425 3426 if_qflush(ifp); 3427 } 3428 #endif 3429 3430 static int 3431 vmxnet3_watchdog(struct vmxnet3_txqueue *txq) 3432 { 3433 struct vmxnet3_softc *sc; 3434 3435 sc = txq->vxtxq_sc; 3436 3437 VMXNET3_TXQ_LOCK(txq); 3438 if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 3439 VMXNET3_TXQ_UNLOCK(txq); 3440 return (0); 3441 } 3442 VMXNET3_TXQ_UNLOCK(txq); 3443 3444 if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n", 3445 txq->vxtxq_id); 3446 return (1); 3447 } 3448 3449 static void 3450 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) 3451 { 3452 3453 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 3454 } 3455 3456 static void 3457 vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *txq, 3458 struct vmxnet3_txq_stats *accum) 3459 { 3460 struct vmxnet3_txq_stats *st; 3461 3462 st = &txq->vxtxq_stats; 3463 3464 accum->vmtxs_opackets += st->vmtxs_opackets; 3465 accum->vmtxs_obytes += st->vmtxs_obytes; 3466 accum->vmtxs_omcasts += st->vmtxs_omcasts; 3467 accum->vmtxs_csum += st->vmtxs_csum; 3468 accum->vmtxs_tso += st->vmtxs_tso; 3469 accum->vmtxs_full += st->vmtxs_full; 3470 accum->vmtxs_offload_failed += st->vmtxs_offload_failed; 3471 } 3472 3473 static void 3474 vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *rxq, 3475 struct vmxnet3_rxq_stats *accum) 3476 { 3477 struct vmxnet3_rxq_stats *st; 3478 3479 st = &rxq->vxrxq_stats; 3480 3481 accum->vmrxs_ipackets += st->vmrxs_ipackets; 3482 accum->vmrxs_ibytes += st->vmrxs_ibytes; 3483 accum->vmrxs_iqdrops += st->vmrxs_iqdrops; 3484 accum->vmrxs_ierrors += st->vmrxs_ierrors; 3485 } 3486 3487 static void 3488 vmxnet3_accumulate_stats(struct vmxnet3_softc *sc) 3489 { 3490 struct ifnet *ifp; 3491 struct vmxnet3_statistics *st; 3492 struct vmxnet3_txq_stats txaccum; 3493 struct vmxnet3_rxq_stats rxaccum; 3494 int i; 3495 3496 ifp = sc->vmx_ifp; 3497 st = &sc->vmx_stats; 3498 3499 bzero(&txaccum, sizeof(struct vmxnet3_txq_stats)); 3500 bzero(&rxaccum, sizeof(struct vmxnet3_rxq_stats)); 3501 3502 for (i = 0; i < sc->vmx_ntxqueues; i++) 3503 vmxnet3_txq_accum_stats(&sc->vmx_txq[i], &txaccum); 3504 for (i = 0; i < sc->vmx_nrxqueues; i++) 3505 vmxnet3_rxq_accum_stats(&sc->vmx_rxq[i], &rxaccum); 3506 3507 /* 3508 * With the exception of if_ierrors, these ifnet statistics are 3509 * only updated in the driver, so just set them to our accumulated 3510 * values. if_ierrors is updated in ether_input() for malformed 3511 * frames that we should have already discarded. 3512 */ 3513 ifp->if_ipackets = rxaccum.vmrxs_ipackets; 3514 ifp->if_iqdrops = rxaccum.vmrxs_iqdrops; 3515 ifp->if_ierrors = rxaccum.vmrxs_ierrors; 3516 ifp->if_opackets = txaccum.vmtxs_opackets; 3517 #ifndef VMXNET3_LEGACY_TX 3518 ifp->if_obytes = txaccum.vmtxs_obytes; 3519 ifp->if_omcasts = txaccum.vmtxs_omcasts; 3520 #endif 3521 } 3522 3523 static void 3524 vmxnet3_tick(void *xsc) 3525 { 3526 struct vmxnet3_softc *sc; 3527 struct ifnet *ifp; 3528 int i, timedout; 3529 3530 sc = xsc; 3531 ifp = sc->vmx_ifp; 3532 timedout = 0; 3533 3534 VMXNET3_CORE_LOCK_ASSERT(sc); 3535 3536 vmxnet3_accumulate_stats(sc); 3537 vmxnet3_refresh_host_stats(sc); 3538 3539 for (i = 0; i < sc->vmx_ntxqueues; i++) 3540 timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]); 3541 3542 if (timedout != 0) { 3543 ifp->if_flags &= ~IFF_RUNNING; 3544 vmxnet3_init_locked(sc); 3545 } else 3546 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 3547 } 3548 3549 static int 3550 vmxnet3_link_is_up(struct vmxnet3_softc *sc) 3551 { 3552 uint32_t status; 3553 3554 /* Also update the link speed while here. */ 3555 status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 3556 sc->vmx_link_speed = status >> 16; 3557 return !!(status & 0x1); 3558 } 3559 3560 static void 3561 vmxnet3_link_status(struct vmxnet3_softc *sc) 3562 { 3563 struct ifnet *ifp; 3564 int link; 3565 3566 ifp = sc->vmx_ifp; 3567 link = vmxnet3_link_is_up(sc); 3568 3569 if (link != 0 && sc->vmx_link_active == 0) { 3570 sc->vmx_link_active = 1; 3571 ifp->if_link_state = LINK_STATE_UP; 3572 if_link_state_change(ifp); 3573 } else if (link == 0 && sc->vmx_link_active != 0) { 3574 sc->vmx_link_active = 0; 3575 ifp->if_link_state = LINK_STATE_DOWN; 3576 if_link_state_change(ifp); 3577 } 3578 } 3579 3580 static void 3581 vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 3582 { 3583 struct vmxnet3_softc *sc; 3584 3585 sc = ifp->if_softc; 3586 3587 ifmr->ifm_active = IFM_ETHER | IFM_AUTO; 3588 ifmr->ifm_status = IFM_AVALID; 3589 3590 VMXNET3_CORE_LOCK(sc); 3591 if (vmxnet3_link_is_up(sc) != 0) 3592 ifmr->ifm_status |= IFM_ACTIVE; 3593 else 3594 ifmr->ifm_status |= IFM_NONE; 3595 VMXNET3_CORE_UNLOCK(sc); 3596 } 3597 3598 static int 3599 vmxnet3_media_change(struct ifnet *ifp) 3600 { 3601 3602 /* Ignore. */ 3603 return (0); 3604 } 3605 3606 static void 3607 vmxnet3_set_lladdr(struct vmxnet3_softc *sc) 3608 { 3609 uint32_t ml, mh; 3610 3611 ml = sc->vmx_lladdr[0]; 3612 ml |= sc->vmx_lladdr[1] << 8; 3613 ml |= sc->vmx_lladdr[2] << 16; 3614 ml |= sc->vmx_lladdr[3] << 24; 3615 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 3616 3617 mh = sc->vmx_lladdr[4]; 3618 mh |= sc->vmx_lladdr[5] << 8; 3619 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 3620 } 3621 3622 static void 3623 vmxnet3_get_lladdr(struct vmxnet3_softc *sc) 3624 { 3625 uint32_t ml, mh; 3626 3627 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 3628 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 3629 3630 sc->vmx_lladdr[0] = ml; 3631 sc->vmx_lladdr[1] = ml >> 8; 3632 sc->vmx_lladdr[2] = ml >> 16; 3633 sc->vmx_lladdr[3] = ml >> 24; 3634 sc->vmx_lladdr[4] = mh; 3635 sc->vmx_lladdr[5] = mh >> 8; 3636 } 3637 3638 static void 3639 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq, 3640 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3641 { 3642 struct sysctl_oid *node, *txsnode; 3643 struct sysctl_oid_list *list, *txslist; 3644 struct vmxnet3_txq_stats *stats; 3645 struct UPT1_TxStats *txstats; 3646 char namebuf[16]; 3647 3648 stats = &txq->vxtxq_stats; 3649 txstats = &txq->vxtxq_ts->stats; 3650 3651 ksnprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id); 3652 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3653 NULL, "Transmit Queue"); 3654 txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node); 3655 3656 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, 3657 &stats->vmtxs_opackets, 0, "Transmit packets"); 3658 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, 3659 &stats->vmtxs_obytes, 0, "Transmit bytes"); 3660 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, 3661 &stats->vmtxs_omcasts, 0, "Transmit multicasts"); 3662 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3663 &stats->vmtxs_csum, 0, "Transmit checksum offloaded"); 3664 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, 3665 &stats->vmtxs_tso, 0, "Transmit TCP segmentation offloaded"); 3666 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD, 3667 &stats->vmtxs_full, 0, "Transmit ring full"); 3668 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD, 3669 &stats->vmtxs_offload_failed, 0, "Transmit checksum offload failed"); 3670 3671 /* 3672 * Add statistics reported by the host. These are updated once 3673 * per second. 3674 */ 3675 txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3676 NULL, "Host Statistics"); 3677 txslist = SYSCTL_CHILDREN(txsnode); 3678 #if 0 /* XXX TSO */ 3679 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD, 3680 &txstats->TSO_packets, 0, "TSO packets"); 3681 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD, 3682 &txstats->TSO_bytes, 0, "TSO bytes"); 3683 #endif 3684 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3685 &txstats->ucast_packets, 0, "Unicast packets"); 3686 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3687 &txstats->ucast_bytes, 0, "Unicast bytes"); 3688 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3689 &txstats->mcast_packets, 0, "Multicast packets"); 3690 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3691 &txstats->mcast_bytes, 0, "Multicast bytes"); 3692 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD, 3693 &txstats->error, 0, "Errors"); 3694 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD, 3695 &txstats->discard, 0, "Discards"); 3696 } 3697 3698 static void 3699 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq, 3700 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3701 { 3702 struct sysctl_oid *node, *rxsnode; 3703 struct sysctl_oid_list *list, *rxslist; 3704 struct vmxnet3_rxq_stats *stats; 3705 struct UPT1_RxStats *rxstats; 3706 char namebuf[16]; 3707 3708 stats = &rxq->vxrxq_stats; 3709 rxstats = &rxq->vxrxq_rs->stats; 3710 3711 ksnprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id); 3712 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3713 NULL, "Receive Queue"); 3714 rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node); 3715 3716 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, 3717 &stats->vmrxs_ipackets, 0, "Receive packets"); 3718 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, 3719 &stats->vmrxs_ibytes, 0, "Receive bytes"); 3720 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, 3721 &stats->vmrxs_iqdrops, 0, "Receive drops"); 3722 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, 3723 &stats->vmrxs_ierrors, 0, "Receive errors"); 3724 3725 /* 3726 * Add statistics reported by the host. These are updated once 3727 * per second. 3728 */ 3729 rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3730 NULL, "Host Statistics"); 3731 rxslist = SYSCTL_CHILDREN(rxsnode); 3732 #if 0 /* XXX LRO */ 3733 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD, 3734 &rxstats->LRO_packets, 0, "LRO packets"); 3735 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD, 3736 &rxstats->LRO_bytes, 0, "LRO bytes"); 3737 #endif 3738 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3739 &rxstats->ucast_packets, 0, "Unicast packets"); 3740 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3741 &rxstats->ucast_bytes, 0, "Unicast bytes"); 3742 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3743 &rxstats->mcast_packets, 0, "Multicast packets"); 3744 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3745 &rxstats->mcast_bytes, 0, "Multicast bytes"); 3746 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD, 3747 &rxstats->bcast_packets, 0, "Broadcast packets"); 3748 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD, 3749 &rxstats->bcast_bytes, 0, "Broadcast bytes"); 3750 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD, 3751 &rxstats->nobuffer, 0, "No buffer"); 3752 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD, 3753 &rxstats->error, 0, "Errors"); 3754 } 3755 3756 static void 3757 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc, 3758 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3759 { 3760 struct sysctl_oid *node; 3761 struct sysctl_oid_list *list; 3762 int i; 3763 3764 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3765 struct vmxnet3_txqueue *txq = &sc->vmx_txq[i]; 3766 3767 node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO, 3768 "debug", CTLFLAG_RD, NULL, ""); 3769 list = SYSCTL_CHILDREN(node); 3770 3771 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD, 3772 &txq->vxtxq_cmd_ring.vxtxr_head, 0, ""); 3773 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD, 3774 &txq->vxtxq_cmd_ring.vxtxr_next, 0, ""); 3775 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD, 3776 &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, ""); 3777 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD, 3778 &txq->vxtxq_cmd_ring.vxtxr_gen, 0, ""); 3779 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3780 &txq->vxtxq_comp_ring.vxcr_next, 0, ""); 3781 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3782 &txq->vxtxq_comp_ring.vxcr_ndesc, 0,""); 3783 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3784 &txq->vxtxq_comp_ring.vxcr_gen, 0, ""); 3785 } 3786 3787 for (i = 0; i < sc->vmx_nrxqueues; i++) { 3788 struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i]; 3789 3790 node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO, 3791 "debug", CTLFLAG_RD, NULL, ""); 3792 list = SYSCTL_CHILDREN(node); 3793 3794 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD, 3795 &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, ""); 3796 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD, 3797 &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, ""); 3798 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD, 3799 &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, ""); 3800 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD, 3801 &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, ""); 3802 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD, 3803 &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, ""); 3804 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD, 3805 &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, ""); 3806 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3807 &rxq->vxrxq_comp_ring.vxcr_next, 0, ""); 3808 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3809 &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,""); 3810 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3811 &rxq->vxrxq_comp_ring.vxcr_gen, 0, ""); 3812 } 3813 } 3814 3815 static void 3816 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc, 3817 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3818 { 3819 int i; 3820 3821 for (i = 0; i < sc->vmx_ntxqueues; i++) 3822 vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child); 3823 for (i = 0; i < sc->vmx_nrxqueues; i++) 3824 vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child); 3825 3826 vmxnet3_setup_debug_sysctl(sc, ctx, child); 3827 } 3828 3829 static void 3830 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 3831 { 3832 device_t dev; 3833 struct vmxnet3_statistics *stats; 3834 struct sysctl_ctx_list *ctx; 3835 struct sysctl_oid *tree; 3836 struct sysctl_oid_list *child; 3837 3838 dev = sc->vmx_dev; 3839 ctx = device_get_sysctl_ctx(dev); 3840 tree = device_get_sysctl_tree(dev); 3841 child = SYSCTL_CHILDREN(tree); 3842 3843 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD, 3844 &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues"); 3845 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD, 3846 &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues"); 3847 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD, 3848 &sc->vmx_ntxqueues, 0, "Number of Tx queues"); 3849 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD, 3850 &sc->vmx_nrxqueues, 0, "Number of Rx queues"); 3851 3852 stats = &sc->vmx_stats; 3853 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD, 3854 &stats->vmst_defragged, 0, "Tx mbuf chains defragged"); 3855 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD, 3856 &stats->vmst_defrag_failed, 0, 3857 "Tx mbuf dropped because defrag failed"); 3858 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD, 3859 &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed"); 3860 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD, 3861 &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed"); 3862 3863 vmxnet3_setup_queue_sysctl(sc, ctx, child); 3864 } 3865 3866 static void 3867 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3868 { 3869 3870 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 3871 } 3872 3873 static uint32_t 3874 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 3875 { 3876 3877 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 3878 } 3879 3880 static void 3881 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3882 { 3883 3884 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 3885 } 3886 3887 static void 3888 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3889 { 3890 3891 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 3892 } 3893 3894 static uint32_t 3895 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3896 { 3897 3898 vmxnet3_write_cmd(sc, cmd); 3899 bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0, 3900 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 3901 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 3902 } 3903 3904 static void 3905 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 3906 { 3907 3908 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 3909 } 3910 3911 static void 3912 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 3913 { 3914 3915 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 3916 } 3917 3918 static void 3919 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3920 { 3921 int i; 3922 3923 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3924 for (i = 0; i < sc->vmx_nintrs; i++) 3925 vmxnet3_enable_intr(sc, i); 3926 } 3927 3928 static void 3929 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3930 { 3931 int i; 3932 3933 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3934 for (i = 0; i < sc->vmx_nintrs; i++) 3935 vmxnet3_disable_intr(sc, i); 3936 } 3937 3938 static void 3939 vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3940 { 3941 bus_addr_t *baddr = arg; 3942 3943 if (error == 0) 3944 *baddr = segs->ds_addr; 3945 } 3946 3947 static int 3948 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3949 struct vmxnet3_dma_alloc *dma) 3950 { 3951 device_t dev; 3952 int error; 3953 3954 dev = sc->vmx_dev; 3955 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3956 3957 error = bus_dma_tag_create(bus_get_dma_tag(dev), 3958 align, 0, /* alignment, bounds */ 3959 BUS_SPACE_MAXADDR, /* lowaddr */ 3960 BUS_SPACE_MAXADDR, /* highaddr */ 3961 size, /* maxsize */ 3962 1, /* nsegments */ 3963 size, /* maxsegsize */ 3964 BUS_DMA_ALLOCNOW, /* flags */ 3965 &dma->dma_tag); 3966 if (error) { 3967 device_printf(dev, "bus_dma_tag_create failed: %d\n", error); 3968 goto fail; 3969 } 3970 3971 error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 3972 BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); 3973 if (error) { 3974 device_printf(dev, "bus_dmamem_alloc failed: %d\n", error); 3975 goto fail; 3976 } 3977 3978 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3979 size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); 3980 if (error) { 3981 device_printf(dev, "bus_dmamap_load failed: %d\n", error); 3982 goto fail; 3983 } 3984 3985 dma->dma_size = size; 3986 3987 fail: 3988 if (error) 3989 vmxnet3_dma_free(sc, dma); 3990 3991 return (error); 3992 } 3993 3994 static void 3995 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 3996 { 3997 3998 if (dma->dma_tag != NULL) { 3999 if (dma->dma_paddr != 0) { 4000 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 4001 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4002 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 4003 } 4004 4005 if (dma->dma_vaddr != NULL) { 4006 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, 4007 dma->dma_map); 4008 } 4009 4010 bus_dma_tag_destroy(dma->dma_tag); 4011 } 4012 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 4013 } 4014 4015 static int 4016 vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def) 4017 { 4018 char path[64]; 4019 4020 ksnprintf(path, sizeof(path), 4021 "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob); 4022 TUNABLE_INT_FETCH(path, &def); 4023 4024 return (def); 4025 } 4026 4027 #define mb() __asm volatile("mfence" ::: "memory") 4028 #define wmb() __asm volatile("sfence" ::: "memory") 4029 #define rmb() __asm volatile("lfence" ::: "memory") 4030 4031 /* 4032 * Since this is a purely paravirtualized device, we do not have 4033 * to worry about DMA coherency. But at times, we must make sure 4034 * both the compiler and CPU do not reorder memory operations. 4035 */ 4036 static inline void 4037 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 4038 { 4039 4040 switch (type) { 4041 case VMXNET3_BARRIER_RD: 4042 rmb(); 4043 break; 4044 case VMXNET3_BARRIER_WR: 4045 wmb(); 4046 break; 4047 case VMXNET3_BARRIER_RDWR: 4048 mb(); 4049 break; 4050 default: 4051 panic("%s: bad barrier type %d", __func__, type); 4052 } 4053 } 4054