1 /*-
2  * Copyright (c) 2013 Tsubai Masanari
3  * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org>
4  *
5  * Permission to use, copy, modify, and distribute this software for any
6  * purpose with or without fee is hereby granted, provided that the above
7  * copyright notice and this permission notice appear in all copies.
8  *
9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16  *
17  * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $
18  * $FreeBSD: head/sys/dev/vmware/vmxnet3/if_vmx.c 318867 2017-05-25 10:49:56Z avg $
19  */
20 
21 /* Driver for VMware vmxnet3 virtual ethernet devices. */
22 
23 #include <sys/param.h>
24 #include <sys/systm.h>
25 #include <sys/device.h>
26 #include <sys/eventhandler.h>
27 #include <sys/kernel.h>
28 #include <sys/endian.h>
29 #include <sys/sockio.h>
30 #include <sys/mbuf.h>
31 #include <sys/malloc.h>
32 #include <sys/module.h>
33 #include <sys/socket.h>
34 #include <sys/sysctl.h>
35 #include <sys/taskqueue.h>
36 #include <vm/vm.h>
37 #include <vm/pmap.h>
38 
39 #include <net/ethernet.h>
40 #include <net/if.h>
41 #include <net/if_var.h>
42 #include <net/ifq_var.h>
43 #include <net/if_arp.h>
44 #include <net/if_dl.h>
45 #include <net/if_types.h>
46 #include <net/if_media.h>
47 #include <net/vlan/if_vlan_ether.h>
48 #include <net/vlan/if_vlan_var.h>
49 
50 #include <net/bpf.h>
51 
52 #include <netinet/in_systm.h>
53 #include <netinet/in.h>
54 #include <netinet/ip.h>
55 #include <netinet/ip6.h>
56 #include <netinet6/ip6_var.h>
57 #include <netinet/udp.h>
58 #include <netinet/tcp.h>
59 
60 #include <sys/in_cksum.h>
61 
62 #include <sys/bus.h>
63 #include <sys/rman.h>
64 
65 #include <bus/pci/pcireg.h>
66 #include <bus/pci/pcivar.h>
67 
68 #define	VMXNET3_LEGACY_TX 1	/* XXX we need this at the moment */
69 #include "if_vmxreg.h"
70 #include "if_vmxvar.h"
71 
72 #include "opt_inet.h"
73 #include "opt_inet6.h"
74 
75 #ifdef VMXNET3_FAILPOINTS
76 #include <sys/fail.h>
77 static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0,
78     "vmxnet3 fail points");
79 #define VMXNET3_FP	_debug_fail_point_vmxnet3
80 #endif
81 
82 static int	vmxnet3_probe(device_t);
83 static int	vmxnet3_attach(device_t);
84 static int	vmxnet3_detach(device_t);
85 static int	vmxnet3_shutdown(device_t);
86 
87 static int	vmxnet3_alloc_resources(struct vmxnet3_softc *);
88 static void	vmxnet3_free_resources(struct vmxnet3_softc *);
89 static int	vmxnet3_check_version(struct vmxnet3_softc *);
90 static void	vmxnet3_initial_config(struct vmxnet3_softc *);
91 static void	vmxnet3_check_multiqueue(struct vmxnet3_softc *);
92 
93 #ifdef __FreeBSD__
94 static int	vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *);
95 static int	vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *);
96 #else
97 static int	vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *);
98 #endif
99 static int	vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *);
100 static int	vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int,
101 		    struct vmxnet3_interrupt *);
102 static int	vmxnet3_alloc_intr_resources(struct vmxnet3_softc *);
103 #ifdef __FreeBSD__
104 static int	vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *);
105 #endif
106 static int	vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *);
107 static int	vmxnet3_setup_interrupts(struct vmxnet3_softc *);
108 static int	vmxnet3_alloc_interrupts(struct vmxnet3_softc *);
109 
110 static void	vmxnet3_free_interrupt(struct vmxnet3_softc *,
111 		    struct vmxnet3_interrupt *);
112 static void	vmxnet3_free_interrupts(struct vmxnet3_softc *);
113 
114 #ifndef VMXNET3_LEGACY_TX
115 static int	vmxnet3_alloc_taskqueue(struct vmxnet3_softc *);
116 static void	vmxnet3_start_taskqueue(struct vmxnet3_softc *);
117 static void	vmxnet3_drain_taskqueue(struct vmxnet3_softc *);
118 static void	vmxnet3_free_taskqueue(struct vmxnet3_softc *);
119 #endif
120 
121 static int	vmxnet3_init_rxq(struct vmxnet3_softc *, int);
122 static int	vmxnet3_init_txq(struct vmxnet3_softc *, int);
123 static int	vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *);
124 static void	vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *);
125 static void	vmxnet3_destroy_txq(struct vmxnet3_txqueue *);
126 static void	vmxnet3_free_rxtx_queues(struct vmxnet3_softc *);
127 
128 static int	vmxnet3_alloc_shared_data(struct vmxnet3_softc *);
129 static void	vmxnet3_free_shared_data(struct vmxnet3_softc *);
130 static int	vmxnet3_alloc_txq_data(struct vmxnet3_softc *);
131 static void	vmxnet3_free_txq_data(struct vmxnet3_softc *);
132 static int	vmxnet3_alloc_rxq_data(struct vmxnet3_softc *);
133 static void	vmxnet3_free_rxq_data(struct vmxnet3_softc *);
134 static int	vmxnet3_alloc_queue_data(struct vmxnet3_softc *);
135 static void	vmxnet3_free_queue_data(struct vmxnet3_softc *);
136 static int	vmxnet3_alloc_mcast_table(struct vmxnet3_softc *);
137 static void	vmxnet3_init_shared_data(struct vmxnet3_softc *);
138 static void	vmxnet3_init_hwassist(struct vmxnet3_softc *);
139 static void	vmxnet3_reinit_interface(struct vmxnet3_softc *);
140 static void	vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *);
141 static void	vmxnet3_reinit_shared_data(struct vmxnet3_softc *);
142 static int	vmxnet3_alloc_data(struct vmxnet3_softc *);
143 static void	vmxnet3_free_data(struct vmxnet3_softc *);
144 static int	vmxnet3_setup_interface(struct vmxnet3_softc *);
145 
146 static void	vmxnet3_evintr(struct vmxnet3_softc *);
147 static void	vmxnet3_txq_eof(struct vmxnet3_txqueue *);
148 static void	vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *);
149 static int	vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *);
150 static void	vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *,
151 		    struct vmxnet3_rxring *, int);
152 static void	vmxnet3_rxq_eof(struct vmxnet3_rxqueue *);
153 static void	vmxnet3_legacy_intr(void *);
154 #ifdef __FreeBSD__
155 static void	vmxnet3_txq_intr(void *);
156 static void	vmxnet3_rxq_intr(void *);
157 static void	vmxnet3_event_intr(void *);
158 #endif
159 
160 static void	vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
161 static void	vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
162 static void	vmxnet3_stop(struct vmxnet3_softc *);
163 
164 static void	vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
165 static int	vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
166 static int	vmxnet3_reinit_queues(struct vmxnet3_softc *);
167 static int	vmxnet3_enable_device(struct vmxnet3_softc *);
168 static void	vmxnet3_reinit_rxfilters(struct vmxnet3_softc *);
169 static int	vmxnet3_reinit(struct vmxnet3_softc *);
170 static void	vmxnet3_init_locked(struct vmxnet3_softc *);
171 static void	vmxnet3_init(void *);
172 
173 static int	vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *,
174 		    int *, int *, int *);
175 static int	vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **,
176 		    bus_dmamap_t, bus_dma_segment_t [], int *);
177 static void	vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t);
178 static int	vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **);
179 #ifdef VMXNET3_LEGACY_TX
180 static void	vmxnet3_start_locked(struct ifnet *);
181 static void	vmxnet3_start(struct ifnet *, struct ifaltq_subque *);
182 #else
183 static int	vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *,
184 		    struct mbuf *);
185 static int	vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *);
186 static void	vmxnet3_txq_tq_deferred(void *, int);
187 #endif
188 static void	vmxnet3_txq_start(struct vmxnet3_txqueue *);
189 static void	vmxnet3_tx_start_all(struct vmxnet3_softc *);
190 
191 static void	vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int,
192 		    uint16_t);
193 static void	vmxnet3_register_vlan(void *, struct ifnet *, uint16_t);
194 static void	vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t);
195 static void	vmxnet3_set_rxfilter(struct vmxnet3_softc *);
196 static int	vmxnet3_change_mtu(struct vmxnet3_softc *, int);
197 static int	vmxnet3_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
198 
199 #ifndef VMXNET3_LEGACY_TX
200 static void	vmxnet3_qflush(struct ifnet *);
201 #endif
202 
203 static int	vmxnet3_watchdog(struct vmxnet3_txqueue *);
204 static void	vmxnet3_refresh_host_stats(struct vmxnet3_softc *);
205 static void	vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *,
206 		    struct vmxnet3_txq_stats *);
207 static void	vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *,
208 		    struct vmxnet3_rxq_stats *);
209 static void	vmxnet3_tick(void *);
210 static void	vmxnet3_link_status(struct vmxnet3_softc *);
211 static void	vmxnet3_media_status(struct ifnet *, struct ifmediareq *);
212 static int	vmxnet3_media_change(struct ifnet *);
213 static void	vmxnet3_set_lladdr(struct vmxnet3_softc *);
214 static void	vmxnet3_get_lladdr(struct vmxnet3_softc *);
215 
216 static void	vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *,
217 		    struct sysctl_ctx_list *, struct sysctl_oid_list *);
218 static void	vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *,
219 		    struct sysctl_ctx_list *, struct sysctl_oid_list *);
220 static void	vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *,
221 		    struct sysctl_ctx_list *, struct sysctl_oid_list *);
222 static void	vmxnet3_setup_sysctl(struct vmxnet3_softc *);
223 
224 static void	vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t,
225 		    uint32_t);
226 static uint32_t	vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t);
227 static void	vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t,
228 		    uint32_t);
229 static void	vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t);
230 static uint32_t	vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t);
231 
232 static void	vmxnet3_enable_intr(struct vmxnet3_softc *, int);
233 static void	vmxnet3_disable_intr(struct vmxnet3_softc *, int);
234 static void	vmxnet3_enable_all_intrs(struct vmxnet3_softc *);
235 static void	vmxnet3_disable_all_intrs(struct vmxnet3_softc *);
236 
237 static int	vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t,
238 		    bus_size_t, struct vmxnet3_dma_alloc *);
239 static void	vmxnet3_dma_free(struct vmxnet3_softc *,
240 		    struct vmxnet3_dma_alloc *);
241 static int	vmxnet3_tunable_int(struct vmxnet3_softc *,
242 		    const char *, int);
243 
244 typedef enum {
245 	VMXNET3_BARRIER_RD,
246 	VMXNET3_BARRIER_WR,
247 	VMXNET3_BARRIER_RDWR,
248 } vmxnet3_barrier_t;
249 
250 static void	vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t);
251 
252 /* Tunables. */
253 static int vmxnet3_mq_disable = 0;
254 TUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable);
255 static int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES;
256 TUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue);
257 static int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES;
258 TUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue);
259 static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC;
260 TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc);
261 static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC;
262 TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc);
263 
264 static device_method_t vmxnet3_methods[] = {
265 	/* Device interface. */
266 	DEVMETHOD(device_probe,		vmxnet3_probe),
267 	DEVMETHOD(device_attach,	vmxnet3_attach),
268 	DEVMETHOD(device_detach,	vmxnet3_detach),
269 	DEVMETHOD(device_shutdown,	vmxnet3_shutdown),
270 
271 	DEVMETHOD_END
272 };
273 
274 static driver_t vmxnet3_driver = {
275 	"vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc)
276 };
277 
278 static devclass_t vmxnet3_devclass;
279 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0);
280 
281 MODULE_DEPEND(vmx, pci, 1, 1, 1);
282 MODULE_DEPEND(vmx, ether, 1, 1, 1);
283 
284 #define VMXNET3_VMWARE_VENDOR_ID	0x15AD
285 #define VMXNET3_VMWARE_DEVICE_ID	0x07B0
286 
287 static int
288 vmxnet3_probe(device_t dev)
289 {
290 
291 	if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID &&
292 	    pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) {
293 		device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter");
294 		return (BUS_PROBE_DEFAULT);
295 	}
296 
297 	return (ENXIO);
298 }
299 
300 static int
301 vmxnet3_attach(device_t dev)
302 {
303 	struct vmxnet3_softc *sc;
304 	int error;
305 
306 	sc = device_get_softc(dev);
307 	sc->vmx_dev = dev;
308 
309 	pci_enable_busmaster(dev);
310 
311 	VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev));
312 	callout_init_lk(&sc->vmx_tick, &sc->vmx_lock);
313 
314 	vmxnet3_initial_config(sc);
315 
316 	error = vmxnet3_alloc_resources(sc);
317 	if (error)
318 		goto fail;
319 
320 	error = vmxnet3_check_version(sc);
321 	if (error)
322 		goto fail;
323 
324 	error = vmxnet3_alloc_rxtx_queues(sc);
325 	if (error)
326 		goto fail;
327 
328 #ifndef VMXNET3_LEGACY_TX
329 	error = vmxnet3_alloc_taskqueue(sc);
330 	if (error)
331 		goto fail;
332 #endif
333 
334 	error = vmxnet3_alloc_interrupts(sc);
335 	if (error)
336 		goto fail;
337 
338 	vmxnet3_check_multiqueue(sc);
339 
340 	error = vmxnet3_alloc_data(sc);
341 	if (error)
342 		goto fail;
343 
344 	error = vmxnet3_setup_interface(sc);
345 	if (error)
346 		goto fail;
347 
348 	error = vmxnet3_setup_interrupts(sc);
349 	if (error) {
350 		ether_ifdetach(sc->vmx_ifp);
351 		device_printf(dev, "could not set up interrupt\n");
352 		goto fail;
353 	}
354 
355 	vmxnet3_setup_sysctl(sc);
356 #ifndef VMXNET3_LEGACY_TX
357 	vmxnet3_start_taskqueue(sc);
358 #endif
359 
360 fail:
361 	if (error)
362 		vmxnet3_detach(dev);
363 
364 	return (error);
365 }
366 
367 static int
368 vmxnet3_detach(device_t dev)
369 {
370 	struct vmxnet3_softc *sc;
371 	struct ifnet *ifp;
372 
373 	sc = device_get_softc(dev);
374 	ifp = sc->vmx_ifp;
375 
376 	if (device_is_attached(dev)) {
377 		VMXNET3_CORE_LOCK(sc);
378 		vmxnet3_stop(sc);
379 		VMXNET3_CORE_UNLOCK(sc);
380 
381 		callout_stop_sync(&sc->vmx_tick);
382 #ifndef VMXNET3_LEGACY_TX
383 		vmxnet3_drain_taskqueue(sc);
384 #endif
385 
386 		ether_ifdetach(ifp);
387 	}
388 
389 	if (sc->vmx_vlan_attach != NULL) {
390 		EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach);
391 		sc->vmx_vlan_attach = NULL;
392 	}
393 	if (sc->vmx_vlan_detach != NULL) {
394 		EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach);
395 		sc->vmx_vlan_detach = NULL;
396 	}
397 
398 #ifndef VMXNET3_LEGACY_TX
399 	vmxnet3_free_taskqueue(sc);
400 #endif
401 	vmxnet3_free_interrupts(sc);
402 
403 	if (ifp != NULL) {
404 		if_free(ifp);
405 		sc->vmx_ifp = NULL;
406 	}
407 
408 	ifmedia_removeall(&sc->vmx_media);
409 
410 	vmxnet3_free_data(sc);
411 	vmxnet3_free_resources(sc);
412 	vmxnet3_free_rxtx_queues(sc);
413 
414 	VMXNET3_CORE_LOCK_DESTROY(sc);
415 
416 	return (0);
417 }
418 
419 static int
420 vmxnet3_shutdown(device_t dev)
421 {
422 
423 	return (0);
424 }
425 
426 static int
427 vmxnet3_alloc_resources(struct vmxnet3_softc *sc)
428 {
429 	device_t dev;
430 	int rid;
431 
432 	dev = sc->vmx_dev;
433 
434 	rid = PCIR_BAR(0);
435 	sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
436 	    RF_ACTIVE);
437 	if (sc->vmx_res0 == NULL) {
438 		device_printf(dev,
439 		    "could not map BAR0 memory\n");
440 		return (ENXIO);
441 	}
442 
443 	sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0);
444 	sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0);
445 
446 	rid = PCIR_BAR(1);
447 	sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
448 	    RF_ACTIVE);
449 	if (sc->vmx_res1 == NULL) {
450 		device_printf(dev,
451 		    "could not map BAR1 memory\n");
452 		return (ENXIO);
453 	}
454 
455 	sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1);
456 	sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1);
457 
458 	if (pci_find_extcap(dev, PCIY_MSIX, NULL) == 0) {
459 		rid = PCIR_BAR(2);
460 		sc->vmx_msix_res = bus_alloc_resource_any(dev,
461 		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
462 	}
463 
464 	if (sc->vmx_msix_res == NULL)
465 		sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX;
466 
467 	return (0);
468 }
469 
470 static void
471 vmxnet3_free_resources(struct vmxnet3_softc *sc)
472 {
473 	device_t dev;
474 	int rid;
475 
476 	dev = sc->vmx_dev;
477 
478 	if (sc->vmx_res0 != NULL) {
479 		rid = PCIR_BAR(0);
480 		bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0);
481 		sc->vmx_res0 = NULL;
482 	}
483 
484 	if (sc->vmx_res1 != NULL) {
485 		rid = PCIR_BAR(1);
486 		bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1);
487 		sc->vmx_res1 = NULL;
488 	}
489 
490 	if (sc->vmx_msix_res != NULL) {
491 		rid = PCIR_BAR(2);
492 		bus_release_resource(dev, SYS_RES_MEMORY, rid,
493 		    sc->vmx_msix_res);
494 		sc->vmx_msix_res = NULL;
495 	}
496 }
497 
498 static int
499 vmxnet3_check_version(struct vmxnet3_softc *sc)
500 {
501 	device_t dev;
502 	uint32_t version;
503 
504 	dev = sc->vmx_dev;
505 
506 	version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS);
507 	if ((version & 0x01) == 0) {
508 		device_printf(dev, "unsupported hardware version %#x\n",
509 		    version);
510 		return (ENOTSUP);
511 	}
512 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1);
513 
514 	version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS);
515 	if ((version & 0x01) == 0) {
516 		device_printf(dev, "unsupported UPT version %#x\n", version);
517 		return (ENOTSUP);
518 	}
519 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1);
520 
521 	return (0);
522 }
523 
524 static int
525 trunc_powerof2(int val)
526 {
527 
528 	return (1U << (fls(val) - 1));
529 }
530 
531 static void
532 vmxnet3_initial_config(struct vmxnet3_softc *sc)
533 {
534 	int nqueue, ndesc;
535 
536 	nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue);
537 	if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1)
538 		nqueue = VMXNET3_DEF_TX_QUEUES;
539 	if (nqueue > ncpus)
540 		nqueue = ncpus;
541 	sc->vmx_max_ntxqueues = trunc_powerof2(nqueue);
542 
543 	nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue);
544 	if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1)
545 		nqueue = VMXNET3_DEF_RX_QUEUES;
546 	if (nqueue > ncpus)
547 		nqueue = ncpus;
548 	sc->vmx_max_nrxqueues = trunc_powerof2(nqueue);
549 
550 	if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) {
551 		sc->vmx_max_nrxqueues = 1;
552 		sc->vmx_max_ntxqueues = 1;
553 	}
554 
555 	ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc);
556 	if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC)
557 		ndesc = VMXNET3_DEF_TX_NDESC;
558 	if (ndesc & VMXNET3_MASK_TX_NDESC)
559 		ndesc &= ~VMXNET3_MASK_TX_NDESC;
560 	sc->vmx_ntxdescs = ndesc;
561 
562 	ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc);
563 	if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC)
564 		ndesc = VMXNET3_DEF_RX_NDESC;
565 	if (ndesc & VMXNET3_MASK_RX_NDESC)
566 		ndesc &= ~VMXNET3_MASK_RX_NDESC;
567 	sc->vmx_nrxdescs = ndesc;
568 	sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS;
569 }
570 
571 static void
572 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc)
573 {
574 
575 	if (sc->vmx_intr_type != VMXNET3_IT_MSIX)
576 		goto out;
577 
578 	/* BMV: Just use the maximum configured for now. */
579 	sc->vmx_nrxqueues = sc->vmx_max_nrxqueues;
580 	sc->vmx_ntxqueues = sc->vmx_max_ntxqueues;
581 
582 	if (sc->vmx_nrxqueues > 1)
583 		sc->vmx_flags |= VMXNET3_FLAG_RSS;
584 
585 	return;
586 
587 out:
588 	sc->vmx_ntxqueues = 1;
589 	sc->vmx_nrxqueues = 1;
590 }
591 
592 #ifdef __FreeBSD__
593 static int
594 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc)
595 {
596 	device_t dev;
597 	int nmsix, cnt, required;
598 
599 	dev = sc->vmx_dev;
600 
601 	if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX)
602 		return (1);
603 
604 	/* Allocate an additional vector for the events interrupt. */
605 	required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1;
606 
607 	nmsix = pci_msix_count(dev);
608 	if (nmsix < required)
609 		return (1);
610 
611 	cnt = required;
612 	if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
613 		sc->vmx_nintrs = required;
614 		return (0);
615 	} else
616 		pci_release_msi(dev);
617 
618 	/* BMV TODO Fallback to sharing MSIX vectors if possible. */
619 
620 	return (1);
621 }
622 
623 static int
624 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc)
625 {
626 	device_t dev;
627 	int nmsi, cnt, required;
628 
629 	dev = sc->vmx_dev;
630 	required = 1;
631 
632 	nmsi = pci_msi_count(dev);
633 	if (nmsi < required)
634 		return (1);
635 
636 	cnt = required;
637 	if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) {
638 		sc->vmx_nintrs = 1;
639 		return (0);
640 	} else
641 		pci_release_msi(dev);
642 
643 	return (1);
644 }
645 #else
646 static int
647 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc)
648 {
649 	int irq_flags, rid;
650 	int enable = 1;
651 
652 	sc->vmx_irq_type = pci_alloc_1intr(sc->vmx_dev, enable, &rid,
653 	    &irq_flags);
654 	sc->vmx_irq_flags = irq_flags;
655 	sc->vmx_nintrs = 1;
656 	return (0);
657 }
658 #endif
659 
660 static int
661 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc)
662 {
663 
664 	sc->vmx_nintrs = 1;
665 	return (0);
666 }
667 
668 static int
669 vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags,
670     struct vmxnet3_interrupt *intr)
671 {
672 	struct resource *irq;
673 
674 	irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid,
675 	    sc->vmx_irq_flags);
676 	if (irq == NULL)
677 		return (ENXIO);
678 
679 	intr->vmxi_irq = irq;
680 	intr->vmxi_rid = rid;
681 
682 	return (0);
683 }
684 
685 static int
686 vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc)
687 {
688 	int i, rid, flags, error;
689 
690 	rid = 0;
691 	flags = RF_ACTIVE;
692 
693 	if (sc->vmx_intr_type == VMXNET3_IT_LEGACY)
694 		flags |= RF_SHAREABLE;
695 	else
696 		rid = 1;
697 
698 	for (i = 0; i < sc->vmx_nintrs; i++, rid++) {
699 		error = vmxnet3_alloc_interrupt(sc, rid, flags,
700 		    &sc->vmx_intrs[i]);
701 		if (error)
702 			return (error);
703 	}
704 
705 	return (0);
706 }
707 
708 #ifdef __FreeBSD__
709 static int
710 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc)
711 {
712 	device_t dev;
713 	struct vmxnet3_txqueue *txq;
714 	struct vmxnet3_rxqueue *rxq;
715 	struct vmxnet3_interrupt *intr;
716 	int i, error;
717 
718 	dev = sc->vmx_dev;
719 	intr = &sc->vmx_intrs[0];
720 
721 	for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) {
722 		txq = &sc->vmx_txq[i];
723 		error = bus_setup_intr(dev, intr->vmxi_irq, INTR_MPSAFE,
724 		     vmxnet3_txq_intr, txq, &intr->vmxi_handler, NULL);
725 		if (error)
726 			return (error);
727 		bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler,
728 		    "tq%d", i);
729 		txq->vxtxq_intr_idx = intr->vmxi_rid - 1;
730 	}
731 
732 	for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) {
733 		rxq = &sc->vmx_rxq[i];
734 		error = bus_setup_intr(dev, intr->vmxi_irq, INTR_MPSAFE,
735 		    vmxnet3_rxq_intr, rxq, &intr->vmxi_handler, NULL);
736 		if (error)
737 			return (error);
738 		bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler,
739 		    "rq%d", i);
740 		rxq->vxrxq_intr_idx = intr->vmxi_rid - 1;
741 	}
742 
743 	error = bus_setup_intr(dev, intr->vmxi_irq, INTR_MPSAFE,
744 	    vmxnet3_event_intr, sc, &intr->vmxi_handler, NULL);
745 	if (error)
746 		return (error);
747 	bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, "event");
748 	sc->vmx_event_intr_idx = intr->vmxi_rid - 1;
749 
750 	return (0);
751 }
752 #endif
753 
754 static int
755 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc)
756 {
757 	struct vmxnet3_interrupt *intr;
758 	int i, error;
759 
760 	intr = &sc->vmx_intrs[0];
761 	error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq,
762 	    INTR_MPSAFE, vmxnet3_legacy_intr, sc,
763 	    &intr->vmxi_handler, NULL);
764 
765 	for (i = 0; i < sc->vmx_ntxqueues; i++)
766 		sc->vmx_txq[i].vxtxq_intr_idx = 0;
767 	for (i = 0; i < sc->vmx_nrxqueues; i++)
768 		sc->vmx_rxq[i].vxrxq_intr_idx = 0;
769 	sc->vmx_event_intr_idx = 0;
770 
771 	return (error);
772 }
773 
774 static void
775 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc)
776 {
777 	struct vmxnet3_txqueue *txq;
778 	struct vmxnet3_txq_shared *txs;
779 	struct vmxnet3_rxqueue *rxq;
780 	struct vmxnet3_rxq_shared *rxs;
781 	int i;
782 
783 	sc->vmx_ds->evintr = sc->vmx_event_intr_idx;
784 
785 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
786 		txq = &sc->vmx_txq[i];
787 		txs = txq->vxtxq_ts;
788 		txs->intr_idx = txq->vxtxq_intr_idx;
789 	}
790 
791 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
792 		rxq = &sc->vmx_rxq[i];
793 		rxs = rxq->vxrxq_rs;
794 		rxs->intr_idx = rxq->vxrxq_intr_idx;
795 	}
796 }
797 
798 static int
799 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc)
800 {
801 	int error;
802 
803 	error = vmxnet3_alloc_intr_resources(sc);
804 	if (error)
805 		return (error);
806 
807 	switch (sc->vmx_intr_type) {
808 	case VMXNET3_IT_MSIX:
809 #ifdef __FreeBSD__
810 		error = vmxnet3_setup_msix_interrupts(sc);
811 #else
812 		device_printf(sc->vmx_dev, "VMXNET3_IT_MSIX unsupported\n");
813 		error = ENXIO;
814 #endif
815 		break;
816 	case VMXNET3_IT_MSI:
817 	case VMXNET3_IT_LEGACY:
818 		error = vmxnet3_setup_legacy_interrupt(sc);
819 		break;
820 	default:
821 		panic("%s: invalid interrupt type %d", __func__,
822 		    sc->vmx_intr_type);
823 	}
824 
825 	if (error == 0)
826 		vmxnet3_set_interrupt_idx(sc);
827 
828 	return (error);
829 }
830 
831 #ifdef __FreeBSD__
832 static int
833 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc)
834 {
835 	device_t dev;
836 	uint32_t config;
837 	int error;
838 
839 	dev = sc->vmx_dev;
840 	config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG);
841 
842 	sc->vmx_intr_type = config & 0x03;
843 	sc->vmx_intr_mask_mode = (config >> 2) & 0x03;
844 
845 	switch (sc->vmx_intr_type) {
846 	case VMXNET3_IT_AUTO:
847 		sc->vmx_intr_type = VMXNET3_IT_MSIX;
848 		/* FALLTHROUGH */
849 	case VMXNET3_IT_MSIX:
850 		error = vmxnet3_alloc_msix_interrupts(sc);
851 		if (error == 0)
852 			break;
853 		sc->vmx_intr_type = VMXNET3_IT_MSI;
854 		/* FALLTHROUGH */
855 	case VMXNET3_IT_MSI:
856 		error = vmxnet3_alloc_msi_interrupts(sc);
857 		if (error == 0)
858 			break;
859 		sc->vmx_intr_type = VMXNET3_IT_LEGACY;
860 		/* FALLTHROUGH */
861 	case VMXNET3_IT_LEGACY:
862 		error = vmxnet3_alloc_legacy_interrupts(sc);
863 		if (error == 0)
864 			break;
865 		/* FALLTHROUGH */
866 	default:
867 		sc->vmx_intr_type = -1;
868 		device_printf(dev, "cannot allocate any interrupt resources\n");
869 		return (ENXIO);
870 	}
871 
872 	return (error);
873 }
874 #else
875 static int
876 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc)
877 {
878 	device_t dev;
879 	uint32_t config;
880 	int error;
881 
882 	dev = sc->vmx_dev;
883 	config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG);
884 
885 	sc->vmx_intr_type = config & 0x03;
886 	sc->vmx_intr_mask_mode = (config >> 2) & 0x03;
887 
888 	switch (sc->vmx_intr_type) {
889 	case VMXNET3_IT_AUTO:
890 		sc->vmx_intr_type = VMXNET3_IT_MSI;
891 		/* FALLTHROUGH */
892 	case VMXNET3_IT_MSI:
893 		error = vmxnet3_alloc_msi_interrupts(sc);
894 		if (error == 0)
895 			break;
896 		sc->vmx_intr_type = VMXNET3_IT_LEGACY;
897 	case VMXNET3_IT_LEGACY:
898 		error = vmxnet3_alloc_legacy_interrupts(sc);
899 		if (error == 0)
900 			break;
901 		/* FALLTHROUGH */
902 	case VMXNET3_IT_MSIX:
903 		/* FALLTHROUGH */
904 	default:
905 		sc->vmx_intr_type = -1;
906 		device_printf(dev, "cannot allocate any interrupt resources\n");
907 		return (ENXIO);
908 	}
909 
910 	return (error);
911 }
912 #endif
913 
914 static void
915 vmxnet3_free_interrupt(struct vmxnet3_softc *sc,
916     struct vmxnet3_interrupt *intr)
917 {
918 	device_t dev;
919 
920 	dev = sc->vmx_dev;
921 
922 	if (intr->vmxi_handler != NULL) {
923 		bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler);
924 		intr->vmxi_handler = NULL;
925 	}
926 
927 	if (intr->vmxi_irq != NULL) {
928 		bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid,
929 		    intr->vmxi_irq);
930 		intr->vmxi_irq = NULL;
931 		intr->vmxi_rid = -1;
932 	}
933 }
934 
935 #if ___FreeBSD__
936 static void
937 vmxnet3_free_interrupts(struct vmxnet3_softc *sc)
938 {
939 	int i;
940 
941 	for (i = 0; i < sc->vmx_nintrs; i++)
942 		vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]);
943 
944 	if (sc->vmx_intr_type == VMXNET3_IT_MSI ||
945 	    sc->vmx_intr_type == VMXNET3_IT_MSIX)
946 		pci_release_msi(sc->vmx_dev);
947 }
948 #else
949 static void
950 vmxnet3_free_interrupts(struct vmxnet3_softc *sc)
951 {
952 	int i;
953 
954 	for (i = 0; i < sc->vmx_nintrs; i++)
955 		vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]);
956 
957 	if (sc->vmx_irq_type == PCI_INTR_TYPE_MSI)
958 		pci_release_msi(sc->vmx_dev);
959 }
960 #endif
961 
962 #ifndef VMXNET3_LEGACY_TX
963 static int
964 vmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc)
965 {
966 	device_t dev;
967 
968 	dev = sc->vmx_dev;
969 
970 	sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT,
971 	    taskqueue_thread_enqueue, &sc->vmx_tq);
972 	if (sc->vmx_tq == NULL)
973 		return (ENOMEM);
974 
975 	return (0);
976 }
977 
978 static void
979 vmxnet3_start_taskqueue(struct vmxnet3_softc *sc)
980 {
981 	device_t dev;
982 	int nthreads, error;
983 
984 	dev = sc->vmx_dev;
985 
986 	/*
987 	 * The taskqueue is typically not frequently used, so a dedicated
988 	 * thread for each queue is unnecessary.
989 	 */
990 	nthreads = MAX(1, sc->vmx_ntxqueues / 2);
991 
992 	/*
993 	 * Most drivers just ignore the return value - it only fails
994 	 * with ENOMEM so an error is not likely. It is hard for us
995 	 * to recover from an error here.
996 	 */
997 	error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET,
998 	    "%s taskq", device_get_nameunit(dev));
999 	if (error)
1000 		device_printf(dev, "failed to start taskqueue: %d", error);
1001 }
1002 
1003 static void
1004 vmxnet3_drain_taskqueue(struct vmxnet3_softc *sc)
1005 {
1006 	struct vmxnet3_txqueue *txq;
1007 	int i;
1008 
1009 	if (sc->vmx_tq != NULL) {
1010 		for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
1011 			txq = &sc->vmx_txq[i];
1012 			taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask);
1013 		}
1014 	}
1015 }
1016 
1017 static void
1018 vmxnet3_free_taskqueue(struct vmxnet3_softc *sc)
1019 {
1020 	if (sc->vmx_tq != NULL) {
1021 		taskqueue_free(sc->vmx_tq);
1022 		sc->vmx_tq = NULL;
1023 	}
1024 }
1025 #endif
1026 
1027 static int
1028 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q)
1029 {
1030 	struct vmxnet3_rxqueue *rxq;
1031 	struct vmxnet3_rxring *rxr;
1032 	int i;
1033 
1034 	rxq = &sc->vmx_rxq[q];
1035 
1036 	ksnprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d",
1037 	    device_get_nameunit(sc->vmx_dev), q);
1038 	lockinit(&rxq->vxrxq_lock, rxq->vxrxq_name, 0, 0);
1039 
1040 	rxq->vxrxq_sc = sc;
1041 	rxq->vxrxq_id = q;
1042 
1043 	for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1044 		rxr = &rxq->vxrxq_cmd_ring[i];
1045 		rxr->vxrxr_rid = i;
1046 		rxr->vxrxr_ndesc = sc->vmx_nrxdescs;
1047 		rxr->vxrxr_rxbuf = kmalloc(rxr->vxrxr_ndesc *
1048 		    sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_INTWAIT | M_ZERO);
1049 		if (rxr->vxrxr_rxbuf == NULL)
1050 			return (ENOMEM);
1051 
1052 		rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs;
1053 	}
1054 
1055 	return (0);
1056 }
1057 
1058 static int
1059 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q)
1060 {
1061 	struct vmxnet3_txqueue *txq;
1062 	struct vmxnet3_txring *txr;
1063 
1064 	txq = &sc->vmx_txq[q];
1065 	txr = &txq->vxtxq_cmd_ring;
1066 
1067 	ksnprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d",
1068 	    device_get_nameunit(sc->vmx_dev), q);
1069 	lockinit(&txq->vxtxq_lock, txq->vxtxq_name, 0, 0);
1070 
1071 	txq->vxtxq_sc = sc;
1072 	txq->vxtxq_id = q;
1073 
1074 	txr->vxtxr_ndesc = sc->vmx_ntxdescs;
1075 	txr->vxtxr_txbuf = kmalloc(txr->vxtxr_ndesc *
1076 	    sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_INTWAIT | M_ZERO);
1077 	if (txr->vxtxr_txbuf == NULL)
1078 		return (ENOMEM);
1079 
1080 	txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs;
1081 
1082 #ifndef VMXNET3_LEGACY_TX
1083 	TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq);
1084 
1085 	txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF,
1086 	    M_NOWAIT, &txq->vxtxq_lock);
1087 	if (txq->vxtxq_br == NULL)
1088 		return (ENOMEM);
1089 #endif
1090 
1091 	return (0);
1092 }
1093 
1094 static int
1095 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc)
1096 {
1097 	int i, error;
1098 
1099 	/*
1100 	 * Only attempt to create multiple queues if MSIX is available. MSIX is
1101 	 * disabled by default because its apparently broken for devices passed
1102 	 * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable
1103 	 * must be set to zero for MSIX. This check prevents us from allocating
1104 	 * queue structures that we will not use.
1105 	 */
1106 	if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) {
1107 		sc->vmx_max_nrxqueues = 1;
1108 		sc->vmx_max_ntxqueues = 1;
1109 	}
1110 
1111 	sc->vmx_rxq = kmalloc(sizeof(struct vmxnet3_rxqueue) *
1112 	    sc->vmx_max_nrxqueues, M_DEVBUF, M_INTWAIT | M_ZERO);
1113 	sc->vmx_txq = kmalloc(sizeof(struct vmxnet3_txqueue) *
1114 	    sc->vmx_max_ntxqueues, M_DEVBUF, M_INTWAIT | M_ZERO);
1115 	if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL)
1116 		return (ENOMEM);
1117 
1118 	for (i = 0; i < sc->vmx_max_nrxqueues; i++) {
1119 		error = vmxnet3_init_rxq(sc, i);
1120 		if (error)
1121 			return (error);
1122 	}
1123 
1124 	for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
1125 		error = vmxnet3_init_txq(sc, i);
1126 		if (error)
1127 			return (error);
1128 	}
1129 
1130 	return (0);
1131 }
1132 
1133 static void
1134 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq)
1135 {
1136 	struct vmxnet3_rxring *rxr;
1137 	int i;
1138 
1139 	rxq->vxrxq_sc = NULL;
1140 	rxq->vxrxq_id = -1;
1141 
1142 	for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1143 		rxr = &rxq->vxrxq_cmd_ring[i];
1144 
1145 		if (rxr->vxrxr_rxbuf != NULL) {
1146 			kfree(rxr->vxrxr_rxbuf, M_DEVBUF);
1147 			rxr->vxrxr_rxbuf = NULL;
1148 		}
1149 	}
1150 
1151 #if 0 /* XXX */
1152 	if (mtx_initialized(&rxq->vxrxq_lock) != 0)
1153 #endif
1154 		lockuninit(&rxq->vxrxq_lock);
1155 }
1156 
1157 static void
1158 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq)
1159 {
1160 	struct vmxnet3_txring *txr;
1161 
1162 	txr = &txq->vxtxq_cmd_ring;
1163 
1164 	txq->vxtxq_sc = NULL;
1165 	txq->vxtxq_id = -1;
1166 
1167 #ifndef VMXNET3_LEGACY_TX
1168 	if (txq->vxtxq_br != NULL) {
1169 		buf_ring_free(txq->vxtxq_br, M_DEVBUF);
1170 		txq->vxtxq_br = NULL;
1171 	}
1172 #endif
1173 
1174 	if (txr->vxtxr_txbuf != NULL) {
1175 		kfree(txr->vxtxr_txbuf, M_DEVBUF);
1176 		txr->vxtxr_txbuf = NULL;
1177 	}
1178 
1179 #if 0 /* XXX */
1180 	if (mtx_initialized(&txq->vxtxq_lock) != 0)
1181 #endif
1182 		lockuninit(&txq->vxtxq_lock);
1183 }
1184 
1185 static void
1186 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc)
1187 {
1188 	int i;
1189 
1190 	if (sc->vmx_rxq != NULL) {
1191 		for (i = 0; i < sc->vmx_max_nrxqueues; i++)
1192 			vmxnet3_destroy_rxq(&sc->vmx_rxq[i]);
1193 		kfree(sc->vmx_rxq, M_DEVBUF);
1194 		sc->vmx_rxq = NULL;
1195 	}
1196 
1197 	if (sc->vmx_txq != NULL) {
1198 		for (i = 0; i < sc->vmx_max_ntxqueues; i++)
1199 			vmxnet3_destroy_txq(&sc->vmx_txq[i]);
1200 		kfree(sc->vmx_txq, M_DEVBUF);
1201 		sc->vmx_txq = NULL;
1202 	}
1203 }
1204 
1205 static int
1206 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc)
1207 {
1208 	device_t dev;
1209 	uint8_t *kva;
1210 	size_t size;
1211 	int i, error;
1212 
1213 	dev = sc->vmx_dev;
1214 
1215 	size = sizeof(struct vmxnet3_driver_shared);
1216 	error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma);
1217 	if (error) {
1218 		device_printf(dev, "cannot alloc shared memory\n");
1219 		return (error);
1220 	}
1221 	sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr;
1222 
1223 	size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) +
1224 	    sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared);
1225 	error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma);
1226 	if (error) {
1227 		device_printf(dev, "cannot alloc queue shared memory\n");
1228 		return (error);
1229 	}
1230 	sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr;
1231 	kva = sc->vmx_qs;
1232 
1233 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
1234 		sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva;
1235 		kva += sizeof(struct vmxnet3_txq_shared);
1236 	}
1237 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
1238 		sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva;
1239 		kva += sizeof(struct vmxnet3_rxq_shared);
1240 	}
1241 
1242 	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1243 		size = sizeof(struct vmxnet3_rss_shared);
1244 		error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma);
1245 		if (error) {
1246 			device_printf(dev, "cannot alloc rss shared memory\n");
1247 			return (error);
1248 		}
1249 		sc->vmx_rss =
1250 		    (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr;
1251 	}
1252 
1253 	return (0);
1254 }
1255 
1256 static void
1257 vmxnet3_free_shared_data(struct vmxnet3_softc *sc)
1258 {
1259 
1260 	if (sc->vmx_rss != NULL) {
1261 		vmxnet3_dma_free(sc, &sc->vmx_rss_dma);
1262 		sc->vmx_rss = NULL;
1263 	}
1264 
1265 	if (sc->vmx_qs != NULL) {
1266 		vmxnet3_dma_free(sc, &sc->vmx_qs_dma);
1267 		sc->vmx_qs = NULL;
1268 	}
1269 
1270 	if (sc->vmx_ds != NULL) {
1271 		vmxnet3_dma_free(sc, &sc->vmx_ds_dma);
1272 		sc->vmx_ds = NULL;
1273 	}
1274 }
1275 
1276 static int
1277 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc)
1278 {
1279 	device_t dev;
1280 	struct vmxnet3_txqueue *txq;
1281 	struct vmxnet3_txring *txr;
1282 	struct vmxnet3_comp_ring *txc;
1283 	size_t descsz, compsz;
1284 	int i, q, error;
1285 
1286 	dev = sc->vmx_dev;
1287 
1288 	for (q = 0; q < sc->vmx_ntxqueues; q++) {
1289 		txq = &sc->vmx_txq[q];
1290 		txr = &txq->vxtxq_cmd_ring;
1291 		txc = &txq->vxtxq_comp_ring;
1292 
1293 		descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc);
1294 		compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc);
1295 
1296 		error = bus_dma_tag_create(bus_get_dma_tag(dev),
1297 		    1, 0,			/* alignment, boundary */
1298 		    BUS_SPACE_MAXADDR,		/* lowaddr */
1299 		    BUS_SPACE_MAXADDR,		/* highaddr */
1300 		    NULL, NULL,			/* filter, filterarg */
1301 		    VMXNET3_TX_MAXSIZE,		/* maxsize */
1302 		    VMXNET3_TX_MAXSEGS,		/* nsegments */
1303 		    VMXNET3_TX_MAXSEGSIZE,	/* maxsegsize */
1304 		    0,				/* flags */
1305 		    &txr->vxtxr_txtag);
1306 		if (error) {
1307 			device_printf(dev,
1308 			    "unable to create Tx buffer tag for queue %d\n", q);
1309 			return (error);
1310 		}
1311 
1312 		error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma);
1313 		if (error) {
1314 			device_printf(dev, "cannot alloc Tx descriptors for "
1315 			    "queue %d error %d\n", q, error);
1316 			return (error);
1317 		}
1318 		txr->vxtxr_txd =
1319 		    (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr;
1320 
1321 		error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma);
1322 		if (error) {
1323 			device_printf(dev, "cannot alloc Tx comp descriptors "
1324 			   "for queue %d error %d\n", q, error);
1325 			return (error);
1326 		}
1327 		txc->vxcr_u.txcd =
1328 		    (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr;
1329 
1330 		for (i = 0; i < txr->vxtxr_ndesc; i++) {
1331 			error = bus_dmamap_create(txr->vxtxr_txtag, 0,
1332 			    &txr->vxtxr_txbuf[i].vtxb_dmamap);
1333 			if (error) {
1334 				device_printf(dev, "unable to create Tx buf "
1335 				    "dmamap for queue %d idx %d\n", q, i);
1336 				return (error);
1337 			}
1338 		}
1339 	}
1340 
1341 	return (0);
1342 }
1343 
1344 static void
1345 vmxnet3_free_txq_data(struct vmxnet3_softc *sc)
1346 {
1347 	device_t dev;
1348 	struct vmxnet3_txqueue *txq;
1349 	struct vmxnet3_txring *txr;
1350 	struct vmxnet3_comp_ring *txc;
1351 	struct vmxnet3_txbuf *txb;
1352 	int i, q;
1353 
1354 	dev = sc->vmx_dev;
1355 
1356 	for (q = 0; q < sc->vmx_ntxqueues; q++) {
1357 		txq = &sc->vmx_txq[q];
1358 		txr = &txq->vxtxq_cmd_ring;
1359 		txc = &txq->vxtxq_comp_ring;
1360 
1361 		for (i = 0; i < txr->vxtxr_ndesc; i++) {
1362 			txb = &txr->vxtxr_txbuf[i];
1363 			if (txb->vtxb_dmamap != NULL) {
1364 				bus_dmamap_destroy(txr->vxtxr_txtag,
1365 				    txb->vtxb_dmamap);
1366 				txb->vtxb_dmamap = NULL;
1367 			}
1368 		}
1369 
1370 		if (txc->vxcr_u.txcd != NULL) {
1371 			vmxnet3_dma_free(sc, &txc->vxcr_dma);
1372 			txc->vxcr_u.txcd = NULL;
1373 		}
1374 
1375 		if (txr->vxtxr_txd != NULL) {
1376 			vmxnet3_dma_free(sc, &txr->vxtxr_dma);
1377 			txr->vxtxr_txd = NULL;
1378 		}
1379 
1380 		if (txr->vxtxr_txtag != NULL) {
1381 			bus_dma_tag_destroy(txr->vxtxr_txtag);
1382 			txr->vxtxr_txtag = NULL;
1383 		}
1384 	}
1385 }
1386 
1387 static int
1388 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc)
1389 {
1390 	device_t dev;
1391 	struct vmxnet3_rxqueue *rxq;
1392 	struct vmxnet3_rxring *rxr;
1393 	struct vmxnet3_comp_ring *rxc;
1394 	int descsz, compsz;
1395 	int i, j, q, error;
1396 
1397 	dev = sc->vmx_dev;
1398 
1399 	for (q = 0; q < sc->vmx_nrxqueues; q++) {
1400 		rxq = &sc->vmx_rxq[q];
1401 		rxc = &rxq->vxrxq_comp_ring;
1402 		compsz = 0;
1403 
1404 		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1405 			rxr = &rxq->vxrxq_cmd_ring[i];
1406 
1407 			descsz = rxr->vxrxr_ndesc *
1408 			    sizeof(struct vmxnet3_rxdesc);
1409 			compsz += rxr->vxrxr_ndesc *
1410 			    sizeof(struct vmxnet3_rxcompdesc);
1411 
1412 			error = bus_dma_tag_create(bus_get_dma_tag(dev),
1413 			    1, 0,		/* alignment, boundary */
1414 			    BUS_SPACE_MAXADDR,	/* lowaddr */
1415 			    BUS_SPACE_MAXADDR,	/* highaddr */
1416 			    NULL, NULL,		/* filter, filterarg */
1417 			    MJUMPAGESIZE,	/* maxsize */
1418 			    1,			/* nsegments */
1419 			    MJUMPAGESIZE,	/* maxsegsize */
1420 			    0,			/* flags */
1421 			    &rxr->vxrxr_rxtag);
1422 			if (error) {
1423 				device_printf(dev,
1424 				    "unable to create Rx buffer tag for "
1425 				    "queue %d\n", q);
1426 				return (error);
1427 			}
1428 
1429 			error = vmxnet3_dma_malloc(sc, descsz, 512,
1430 			    &rxr->vxrxr_dma);
1431 			if (error) {
1432 				device_printf(dev, "cannot allocate Rx "
1433 				    "descriptors for queue %d/%d error %d\n",
1434 				    i, q, error);
1435 				return (error);
1436 			}
1437 			rxr->vxrxr_rxd =
1438 			    (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr;
1439 		}
1440 
1441 		error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma);
1442 		if (error) {
1443 			device_printf(dev, "cannot alloc Rx comp descriptors "
1444 			    "for queue %d error %d\n", q, error);
1445 			return (error);
1446 		}
1447 		rxc->vxcr_u.rxcd =
1448 		    (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr;
1449 
1450 		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1451 			rxr = &rxq->vxrxq_cmd_ring[i];
1452 
1453 			error = bus_dmamap_create(rxr->vxrxr_rxtag, 0,
1454 			    &rxr->vxrxr_spare_dmap);
1455 			if (error) {
1456 				device_printf(dev, "unable to create spare "
1457 				    "dmamap for queue %d/%d error %d\n",
1458 				    q, i, error);
1459 				return (error);
1460 			}
1461 
1462 			for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1463 				error = bus_dmamap_create(rxr->vxrxr_rxtag, 0,
1464 				    &rxr->vxrxr_rxbuf[j].vrxb_dmamap);
1465 				if (error) {
1466 					device_printf(dev, "unable to create "
1467 					    "dmamap for queue %d/%d slot %d "
1468 					    "error %d\n",
1469 					    q, i, j, error);
1470 					return (error);
1471 				}
1472 			}
1473 		}
1474 	}
1475 
1476 	return (0);
1477 }
1478 
1479 static void
1480 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc)
1481 {
1482 	device_t dev;
1483 	struct vmxnet3_rxqueue *rxq;
1484 	struct vmxnet3_rxring *rxr;
1485 	struct vmxnet3_comp_ring *rxc;
1486 	struct vmxnet3_rxbuf *rxb;
1487 	int i, j, q;
1488 
1489 	dev = sc->vmx_dev;
1490 
1491 	for (q = 0; q < sc->vmx_nrxqueues; q++) {
1492 		rxq = &sc->vmx_rxq[q];
1493 		rxc = &rxq->vxrxq_comp_ring;
1494 
1495 		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1496 			rxr = &rxq->vxrxq_cmd_ring[i];
1497 
1498 			if (rxr->vxrxr_spare_dmap != NULL) {
1499 				bus_dmamap_destroy(rxr->vxrxr_rxtag,
1500 				    rxr->vxrxr_spare_dmap);
1501 				rxr->vxrxr_spare_dmap = NULL;
1502 			}
1503 
1504 			for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1505 				rxb = &rxr->vxrxr_rxbuf[j];
1506 				if (rxb->vrxb_dmamap != NULL) {
1507 					bus_dmamap_destroy(rxr->vxrxr_rxtag,
1508 					    rxb->vrxb_dmamap);
1509 					rxb->vrxb_dmamap = NULL;
1510 				}
1511 			}
1512 		}
1513 
1514 		if (rxc->vxcr_u.rxcd != NULL) {
1515 			vmxnet3_dma_free(sc, &rxc->vxcr_dma);
1516 			rxc->vxcr_u.rxcd = NULL;
1517 		}
1518 
1519 		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1520 			rxr = &rxq->vxrxq_cmd_ring[i];
1521 
1522 			if (rxr->vxrxr_rxd != NULL) {
1523 				vmxnet3_dma_free(sc, &rxr->vxrxr_dma);
1524 				rxr->vxrxr_rxd = NULL;
1525 			}
1526 
1527 			if (rxr->vxrxr_rxtag != NULL) {
1528 				bus_dma_tag_destroy(rxr->vxrxr_rxtag);
1529 				rxr->vxrxr_rxtag = NULL;
1530 			}
1531 		}
1532 	}
1533 }
1534 
1535 static int
1536 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc)
1537 {
1538 	int error;
1539 
1540 	error = vmxnet3_alloc_txq_data(sc);
1541 	if (error)
1542 		return (error);
1543 
1544 	error = vmxnet3_alloc_rxq_data(sc);
1545 	if (error)
1546 		return (error);
1547 
1548 	return (0);
1549 }
1550 
1551 static void
1552 vmxnet3_free_queue_data(struct vmxnet3_softc *sc)
1553 {
1554 
1555 	if (sc->vmx_rxq != NULL)
1556 		vmxnet3_free_rxq_data(sc);
1557 
1558 	if (sc->vmx_txq != NULL)
1559 		vmxnet3_free_txq_data(sc);
1560 }
1561 
1562 static int
1563 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc)
1564 {
1565 	int error;
1566 
1567 	error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN,
1568 	    32, &sc->vmx_mcast_dma);
1569 	if (error)
1570 		device_printf(sc->vmx_dev, "unable to alloc multicast table\n");
1571 	else
1572 		sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr;
1573 
1574 	return (error);
1575 }
1576 
1577 static void
1578 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc)
1579 {
1580 
1581 	if (sc->vmx_mcast != NULL) {
1582 		vmxnet3_dma_free(sc, &sc->vmx_mcast_dma);
1583 		sc->vmx_mcast = NULL;
1584 	}
1585 }
1586 
1587 static void
1588 vmxnet3_init_shared_data(struct vmxnet3_softc *sc)
1589 {
1590 	struct vmxnet3_driver_shared *ds;
1591 	struct vmxnet3_txqueue *txq;
1592 	struct vmxnet3_txq_shared *txs;
1593 	struct vmxnet3_rxqueue *rxq;
1594 	struct vmxnet3_rxq_shared *rxs;
1595 	int i;
1596 
1597 	ds = sc->vmx_ds;
1598 
1599 	/*
1600 	 * Initialize fields of the shared data that remains the same across
1601 	 * reinits. Note the shared data is zero'd when allocated.
1602 	 */
1603 
1604 	ds->magic = VMXNET3_REV1_MAGIC;
1605 
1606 	/* DriverInfo */
1607 	ds->version = VMXNET3_DRIVER_VERSION;
1608 	ds->guest = VMXNET3_GOS_FREEBSD |
1609 #ifdef __LP64__
1610 	    VMXNET3_GOS_64BIT;
1611 #else
1612 	    VMXNET3_GOS_32BIT;
1613 #endif
1614 	ds->vmxnet3_revision = 1;
1615 	ds->upt_version = 1;
1616 
1617 	/* Misc. conf */
1618 	ds->driver_data = vtophys(sc);
1619 	ds->driver_data_len = sizeof(struct vmxnet3_softc);
1620 	ds->queue_shared = sc->vmx_qs_dma.dma_paddr;
1621 	ds->queue_shared_len = sc->vmx_qs_dma.dma_size;
1622 	ds->nrxsg_max = sc->vmx_max_rxsegs;
1623 
1624 	/* RSS conf */
1625 	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1626 		ds->rss.version = 1;
1627 		ds->rss.paddr = sc->vmx_rss_dma.dma_paddr;
1628 		ds->rss.len = sc->vmx_rss_dma.dma_size;
1629 	}
1630 
1631 	/* Interrupt control. */
1632 	ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO;
1633 	ds->nintr = sc->vmx_nintrs;
1634 	ds->evintr = sc->vmx_event_intr_idx;
1635 	ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;
1636 
1637 	for (i = 0; i < sc->vmx_nintrs; i++)
1638 		ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;
1639 
1640 	/* Receive filter. */
1641 	ds->mcast_table = sc->vmx_mcast_dma.dma_paddr;
1642 	ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size;
1643 
1644 	/* Tx queues */
1645 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
1646 		txq = &sc->vmx_txq[i];
1647 		txs = txq->vxtxq_ts;
1648 
1649 		txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr;
1650 		txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc;
1651 		txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr;
1652 		txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc;
1653 		txs->driver_data = vtophys(txq);
1654 		txs->driver_data_len = sizeof(struct vmxnet3_txqueue);
1655 	}
1656 
1657 	/* Rx queues */
1658 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
1659 		rxq = &sc->vmx_rxq[i];
1660 		rxs = rxq->vxrxq_rs;
1661 
1662 		rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr;
1663 		rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc;
1664 		rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr;
1665 		rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc;
1666 		rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr;
1667 		rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc;
1668 		rxs->driver_data = vtophys(rxq);
1669 		rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue);
1670 	}
1671 }
1672 
1673 static void
1674 vmxnet3_init_hwassist(struct vmxnet3_softc *sc)
1675 {
1676 	struct ifnet *ifp = sc->vmx_ifp;
1677 	uint64_t hwassist;
1678 
1679 	hwassist = 0;
1680 	if (ifp->if_capenable & IFCAP_TXCSUM)
1681 		hwassist |= VMXNET3_CSUM_OFFLOAD;
1682 	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1683 		hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6;
1684 #if 0 /* XXX TSO */
1685 	if (ifp->if_capenable & IFCAP_TSO4)
1686 		hwassist |= CSUM_IP_TSO;
1687 	if (ifp->if_capenable & IFCAP_TSO6)
1688 		hwassist |= CSUM_IP6_TSO;
1689 #endif
1690 	ifp->if_hwassist = hwassist;
1691 }
1692 
1693 static void
1694 vmxnet3_reinit_interface(struct vmxnet3_softc *sc)
1695 {
1696 	struct ifnet *ifp;
1697 
1698 	ifp = sc->vmx_ifp;
1699 
1700 	/* Use the current MAC address. */
1701 	bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN);
1702 	vmxnet3_set_lladdr(sc);
1703 
1704 	vmxnet3_init_hwassist(sc);
1705 }
1706 
1707 static void
1708 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc)
1709 {
1710 	/*
1711 	 * Use the same key as the Linux driver until FreeBSD can do
1712 	 * RSS (presumably Toeplitz) in software.
1713 	 */
1714 	static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
1715 	    0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
1716 	    0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
1717 	    0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
1718 	    0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
1719 	    0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
1720 	};
1721 
1722 	struct vmxnet3_driver_shared *ds;
1723 	struct vmxnet3_rss_shared *rss;
1724 	int i;
1725 
1726 	ds = sc->vmx_ds;
1727 	rss = sc->vmx_rss;
1728 
1729 	rss->hash_type =
1730 	    UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 |
1731 	    UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6;
1732 	rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ;
1733 	rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE;
1734 	rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE;
1735 	memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE);
1736 
1737 	for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++)
1738 		rss->ind_table[i] = i % sc->vmx_nrxqueues;
1739 }
1740 
1741 static void
1742 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc)
1743 {
1744 	struct ifnet *ifp;
1745 	struct vmxnet3_driver_shared *ds;
1746 
1747 	ifp = sc->vmx_ifp;
1748 	ds = sc->vmx_ds;
1749 
1750 	ds->mtu = ifp->if_mtu;
1751 	ds->ntxqueue = sc->vmx_ntxqueues;
1752 	ds->nrxqueue = sc->vmx_nrxqueues;
1753 
1754 	ds->upt_features = 0;
1755 	if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
1756 		ds->upt_features |= UPT1_F_CSUM;
1757 	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1758 		ds->upt_features |= UPT1_F_VLAN;
1759 #if 0 /* XXX LRO */
1760 	if (ifp->if_capenable & IFCAP_LRO)
1761 		ds->upt_features |= UPT1_F_LRO;
1762 #endif
1763 
1764 	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1765 		ds->upt_features |= UPT1_F_RSS;
1766 		vmxnet3_reinit_rss_shared_data(sc);
1767 	}
1768 
1769 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr);
1770 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH,
1771 	    (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32);
1772 }
1773 
1774 static int
1775 vmxnet3_alloc_data(struct vmxnet3_softc *sc)
1776 {
1777 	int error;
1778 
1779 	error = vmxnet3_alloc_shared_data(sc);
1780 	if (error)
1781 		return (error);
1782 
1783 	error = vmxnet3_alloc_queue_data(sc);
1784 	if (error)
1785 		return (error);
1786 
1787 	error = vmxnet3_alloc_mcast_table(sc);
1788 	if (error)
1789 		return (error);
1790 
1791 	vmxnet3_init_shared_data(sc);
1792 
1793 	return (0);
1794 }
1795 
1796 static void
1797 vmxnet3_free_data(struct vmxnet3_softc *sc)
1798 {
1799 
1800 	vmxnet3_free_mcast_table(sc);
1801 	vmxnet3_free_queue_data(sc);
1802 	vmxnet3_free_shared_data(sc);
1803 }
1804 
1805 static int
1806 vmxnet3_setup_interface(struct vmxnet3_softc *sc)
1807 {
1808 	device_t dev;
1809 	struct ifnet *ifp;
1810 
1811 	dev = sc->vmx_dev;
1812 
1813 	ifp = sc->vmx_ifp = if_alloc(IFT_ETHER);
1814 	if (ifp == NULL) {
1815 		device_printf(dev, "cannot allocate ifnet structure\n");
1816 		return (ENOSPC);
1817 	}
1818 
1819 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1820 	ifp->if_baudrate = IF_Gbps(10ULL);
1821 	ifp->if_softc = sc;
1822 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1823 	ifp->if_init = vmxnet3_init;
1824 	ifp->if_ioctl = vmxnet3_ioctl;
1825 #if 0 /* XXX TSO */
1826 	ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1827 	ifp->if_hw_tsomaxsegcount = VMXNET3_TX_MAXSEGS;
1828 	ifp->if_hw_tsomaxsegsize = VMXNET3_TX_MAXSEGSIZE;
1829 #endif
1830 
1831 #ifdef VMXNET3_LEGACY_TX
1832 	ifp->if_start = vmxnet3_start;
1833 	ifq_set_maxlen(&ifp->if_snd, sc->vmx_ntxdescs - 1);
1834 	ifq_set_ready(&ifp->if_snd);
1835 #else
1836 	ifp->if_transmit = vmxnet3_txq_mq_start;
1837 	ifp->if_qflush = vmxnet3_qflush;
1838 #endif
1839 
1840 	vmxnet3_get_lladdr(sc);
1841 	ether_ifattach(ifp, sc->vmx_lladdr, NULL);
1842 
1843 	ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM;
1844 	ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6;
1845 #if 0 /* XXX TSO */
1846 	ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6;
1847 #endif
1848 	ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |
1849 	    IFCAP_VLAN_HWCSUM;
1850 	ifp->if_capenable = ifp->if_capabilities;
1851 
1852 #if 0 /* XXX LRO / VLAN_HWFILTER */
1853 	/* These capabilities are not enabled by default. */
1854 	ifp->if_capabilities |= /* IFCAP_LRO | */ IFCAP_VLAN_HWFILTER;
1855 #endif
1856 
1857 	sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
1858 	    vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
1859 	sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config,
1860 	    vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
1861 
1862 	ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change,
1863 	    vmxnet3_media_status);
1864 	ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL);
1865 	ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO);
1866 
1867 	return (0);
1868 }
1869 
1870 static void
1871 vmxnet3_evintr(struct vmxnet3_softc *sc)
1872 {
1873 	device_t dev;
1874 	struct ifnet *ifp;
1875 	struct vmxnet3_txq_shared *ts;
1876 	struct vmxnet3_rxq_shared *rs;
1877 	uint32_t event;
1878 	int reset;
1879 
1880 	dev = sc->vmx_dev;
1881 	ifp = sc->vmx_ifp;
1882 	reset = 0;
1883 
1884 	VMXNET3_CORE_LOCK(sc);
1885 
1886 	/* Clear events. */
1887 	event = sc->vmx_ds->event;
1888 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event);
1889 
1890 	if (event & VMXNET3_EVENT_LINK) {
1891 		vmxnet3_link_status(sc);
1892 		if (sc->vmx_link_active != 0)
1893 			vmxnet3_tx_start_all(sc);
1894 	}
1895 
1896 	if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
1897 		reset = 1;
1898 		vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS);
1899 		ts = sc->vmx_txq[0].vxtxq_ts;
1900 		if (ts->stopped != 0)
1901 			device_printf(dev, "Tx queue error %#x\n", ts->error);
1902 		rs = sc->vmx_rxq[0].vxrxq_rs;
1903 		if (rs->stopped != 0)
1904 			device_printf(dev, "Rx queue error %#x\n", rs->error);
1905 		device_printf(dev, "Rx/Tx queue error event ... resetting\n");
1906 	}
1907 
1908 	if (event & VMXNET3_EVENT_DIC)
1909 		device_printf(dev, "device implementation change event\n");
1910 	if (event & VMXNET3_EVENT_DEBUG)
1911 		device_printf(dev, "debug event\n");
1912 
1913 	if (reset != 0) {
1914 		ifp->if_flags &= ~IFF_RUNNING;
1915 		vmxnet3_init_locked(sc);
1916 	}
1917 
1918 	VMXNET3_CORE_UNLOCK(sc);
1919 }
1920 
1921 static void
1922 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq)
1923 {
1924 	struct vmxnet3_softc *sc;
1925 	struct ifnet *ifp;
1926 	struct vmxnet3_txring *txr;
1927 	struct vmxnet3_comp_ring *txc;
1928 	struct vmxnet3_txcompdesc *txcd;
1929 	struct vmxnet3_txbuf *txb;
1930 	struct mbuf *m;
1931 	u_int sop;
1932 
1933 	sc = txq->vxtxq_sc;
1934 	ifp = sc->vmx_ifp;
1935 	txr = &txq->vxtxq_cmd_ring;
1936 	txc = &txq->vxtxq_comp_ring;
1937 
1938 	VMXNET3_TXQ_LOCK_ASSERT(txq);
1939 
1940 	for (;;) {
1941 		txcd = &txc->vxcr_u.txcd[txc->vxcr_next];
1942 		if (txcd->gen != txc->vxcr_gen)
1943 			break;
1944 		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1945 
1946 		if (++txc->vxcr_next == txc->vxcr_ndesc) {
1947 			txc->vxcr_next = 0;
1948 			txc->vxcr_gen ^= 1;
1949 		}
1950 
1951 		sop = txr->vxtxr_next;
1952 		txb = &txr->vxtxr_txbuf[sop];
1953 
1954 		if ((m = txb->vtxb_m) != NULL) {
1955 			bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap,
1956 			    BUS_DMASYNC_POSTWRITE);
1957 			bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap);
1958 
1959 			txq->vxtxq_stats.vmtxs_opackets++;
1960 			txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len;
1961 			if (m->m_flags & M_MCAST)
1962 				txq->vxtxq_stats.vmtxs_omcasts++;
1963 
1964 			m_freem(m);
1965 			txb->vtxb_m = NULL;
1966 		}
1967 
1968 		txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc;
1969 	}
1970 
1971 	if (txr->vxtxr_head == txr->vxtxr_next)
1972 		txq->vxtxq_watchdog = 0;
1973 }
1974 
1975 static int
1976 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr)
1977 {
1978 	struct ifnet *ifp;
1979 	struct mbuf *m;
1980 	struct vmxnet3_rxdesc *rxd;
1981 	struct vmxnet3_rxbuf *rxb;
1982 	bus_dma_tag_t tag;
1983 	bus_dmamap_t dmap;
1984 	bus_dma_segment_t segs[1];
1985 	int idx, clsize, btype, flags, nsegs, error;
1986 
1987 	ifp = sc->vmx_ifp;
1988 	tag = rxr->vxrxr_rxtag;
1989 	dmap = rxr->vxrxr_spare_dmap;
1990 	idx = rxr->vxrxr_fill;
1991 	rxd = &rxr->vxrxr_rxd[idx];
1992 	rxb = &rxr->vxrxr_rxbuf[idx];
1993 
1994 #ifdef VMXNET3_FAILPOINTS
1995 	KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS);
1996 	if (rxr->vxrxr_rid != 0)
1997 		KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS);
1998 #endif
1999 
2000 	if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) {
2001 		flags = M_PKTHDR;
2002 		clsize = MCLBYTES;
2003 		btype = VMXNET3_BTYPE_HEAD;
2004 	} else {
2005 #if __FreeBSD_version < 902001
2006 		/*
2007 		 * These mbufs will never be used for the start of a frame.
2008 		 * Roughly prior to branching releng/9.2, the load_mbuf_sg()
2009 		 * required the mbuf to always be a packet header. Avoid
2010 		 * unnecessary mbuf initialization in newer versions where
2011 		 * that is not the case.
2012 		 */
2013 		flags = M_PKTHDR;
2014 #else
2015 		flags = 0;
2016 #endif
2017 		clsize = MJUMPAGESIZE;
2018 		btype = VMXNET3_BTYPE_BODY;
2019 	}
2020 
2021 	m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize);
2022 	if (m == NULL) {
2023 		sc->vmx_stats.vmst_mgetcl_failed++;
2024 		return (ENOBUFS);
2025 	}
2026 
2027 	if (btype == VMXNET3_BTYPE_HEAD) {
2028 		m->m_len = m->m_pkthdr.len = clsize;
2029 		m_adj(m, ETHER_ALIGN);
2030 	} else
2031 		m->m_len = clsize;
2032 
2033 	error = bus_dmamap_load_mbuf_segment(tag, dmap, m, &segs[0], 1, &nsegs,
2034 	    BUS_DMA_NOWAIT);
2035 	if (error) {
2036 		m_freem(m);
2037 		sc->vmx_stats.vmst_mbuf_load_failed++;
2038 		return (error);
2039 	}
2040 	KASSERT(nsegs == 1,
2041 	    ("%s: mbuf %p with too many segments %d", __func__, m, nsegs));
2042 #if __FreeBSD_version < 902001
2043 	if (btype == VMXNET3_BTYPE_BODY)
2044 		m->m_flags &= ~M_PKTHDR;
2045 #endif
2046 
2047 	if (rxb->vrxb_m != NULL) {
2048 		bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD);
2049 		bus_dmamap_unload(tag, rxb->vrxb_dmamap);
2050 	}
2051 
2052 	rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap;
2053 	rxb->vrxb_dmamap = dmap;
2054 	rxb->vrxb_m = m;
2055 
2056 	rxd->addr = segs[0].ds_addr;
2057 	rxd->len = segs[0].ds_len;
2058 	rxd->btype = btype;
2059 	rxd->gen = rxr->vxrxr_gen;
2060 
2061 	vmxnet3_rxr_increment_fill(rxr);
2062 	return (0);
2063 }
2064 
2065 static void
2066 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq,
2067     struct vmxnet3_rxring *rxr, int idx)
2068 {
2069 	struct vmxnet3_rxdesc *rxd;
2070 
2071 	rxd = &rxr->vxrxr_rxd[idx];
2072 	rxd->gen = rxr->vxrxr_gen;
2073 	vmxnet3_rxr_increment_fill(rxr);
2074 }
2075 
2076 static void
2077 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq)
2078 {
2079 	struct vmxnet3_softc *sc;
2080 	struct vmxnet3_rxring *rxr;
2081 	struct vmxnet3_comp_ring *rxc;
2082 	struct vmxnet3_rxcompdesc *rxcd;
2083 	int idx, eof;
2084 
2085 	sc = rxq->vxrxq_sc;
2086 	rxc = &rxq->vxrxq_comp_ring;
2087 
2088 	do {
2089 		rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
2090 		if (rxcd->gen != rxc->vxcr_gen)
2091 			break;		/* Not expected. */
2092 		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
2093 
2094 		if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
2095 			rxc->vxcr_next = 0;
2096 			rxc->vxcr_gen ^= 1;
2097 		}
2098 
2099 		idx = rxcd->rxd_idx;
2100 		eof = rxcd->eop;
2101 		if (rxcd->qid < sc->vmx_nrxqueues)
2102 			rxr = &rxq->vxrxq_cmd_ring[0];
2103 		else
2104 			rxr = &rxq->vxrxq_cmd_ring[1];
2105 		vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2106 	} while (!eof);
2107 }
2108 
2109 static void
2110 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
2111 {
2112 
2113 	if (rxcd->ipv4) {
2114 		m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
2115 		if (rxcd->ipcsum_ok)
2116 			m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2117 	}
2118 
2119 	if (!rxcd->fragment) {
2120 		if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) {
2121 			m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
2122 			    CSUM_PSEUDO_HDR;
2123 			m->m_pkthdr.csum_data = 0xFFFF;
2124 		}
2125 	}
2126 }
2127 
2128 static void
2129 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq,
2130     struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
2131 {
2132 	struct vmxnet3_softc *sc;
2133 	struct ifnet *ifp;
2134 
2135 	sc = rxq->vxrxq_sc;
2136 	ifp = sc->vmx_ifp;
2137 
2138 	if (rxcd->error) {
2139 		rxq->vxrxq_stats.vmrxs_ierrors++;
2140 		m_freem(m);
2141 		return;
2142 	}
2143 
2144 #if 0
2145 #ifdef notyet
2146 	switch (rxcd->rss_type) {
2147 	case VMXNET3_RCD_RSS_TYPE_IPV4:
2148 		m->m_pkthdr.flowid = rxcd->rss_hash;
2149 		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4);
2150 		break;
2151 	case VMXNET3_RCD_RSS_TYPE_TCPIPV4:
2152 		m->m_pkthdr.flowid = rxcd->rss_hash;
2153 		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4);
2154 		break;
2155 	case VMXNET3_RCD_RSS_TYPE_IPV6:
2156 		m->m_pkthdr.flowid = rxcd->rss_hash;
2157 		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6);
2158 		break;
2159 	case VMXNET3_RCD_RSS_TYPE_TCPIPV6:
2160 		m->m_pkthdr.flowid = rxcd->rss_hash;
2161 		M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6);
2162 		break;
2163 	default: /* VMXNET3_RCD_RSS_TYPE_NONE */
2164 		m->m_pkthdr.flowid = rxq->vxrxq_id;
2165 		M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
2166 		break;
2167 	}
2168 #else
2169 	m->m_pkthdr.flowid = rxq->vxrxq_id;
2170 	M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
2171 #endif
2172 #endif
2173 
2174 	if (!rxcd->no_csum)
2175 		vmxnet3_rx_csum(rxcd, m);
2176 	if (rxcd->vlan) {
2177 		m->m_flags |= M_VLANTAG;
2178 		m->m_pkthdr.ether_vlantag = rxcd->vtag;
2179 	}
2180 
2181 	rxq->vxrxq_stats.vmrxs_ipackets++;
2182 	rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len;
2183 
2184 	VMXNET3_RXQ_UNLOCK(rxq);
2185 	(*ifp->if_input)(ifp, m, NULL, -1);
2186 	VMXNET3_RXQ_LOCK(rxq);
2187 }
2188 
2189 static void
2190 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq)
2191 {
2192 	struct vmxnet3_softc *sc;
2193 	struct ifnet *ifp;
2194 	struct vmxnet3_rxring *rxr;
2195 	struct vmxnet3_comp_ring *rxc;
2196 	struct vmxnet3_rxdesc *rxd;
2197 	struct vmxnet3_rxcompdesc *rxcd;
2198 	struct mbuf *m, *m_head, *m_tail;
2199 	int idx, length;
2200 
2201 	sc = rxq->vxrxq_sc;
2202 	ifp = sc->vmx_ifp;
2203 	rxc = &rxq->vxrxq_comp_ring;
2204 
2205 	VMXNET3_RXQ_LOCK_ASSERT(rxq);
2206 
2207 	if ((ifp->if_flags & IFF_RUNNING) == 0)
2208 		return;
2209 
2210 	m_head = rxq->vxrxq_mhead;
2211 	rxq->vxrxq_mhead = NULL;
2212 	m_tail = rxq->vxrxq_mtail;
2213 	rxq->vxrxq_mtail = NULL;
2214 	KKASSERT(m_head == NULL || m_tail != NULL);
2215 
2216 	for (;;) {
2217 		rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
2218 		if (rxcd->gen != rxc->vxcr_gen) {
2219 			rxq->vxrxq_mhead = m_head;
2220 			rxq->vxrxq_mtail = m_tail;
2221 			break;
2222 		}
2223 		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
2224 
2225 		if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
2226 			rxc->vxcr_next = 0;
2227 			rxc->vxcr_gen ^= 1;
2228 		}
2229 
2230 		idx = rxcd->rxd_idx;
2231 		length = rxcd->len;
2232 		if (rxcd->qid < sc->vmx_nrxqueues)
2233 			rxr = &rxq->vxrxq_cmd_ring[0];
2234 		else
2235 			rxr = &rxq->vxrxq_cmd_ring[1];
2236 		rxd = &rxr->vxrxr_rxd[idx];
2237 
2238 		m = rxr->vxrxr_rxbuf[idx].vrxb_m;
2239 		KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf",
2240 		    __func__, rxcd->qid, idx));
2241 
2242 		/*
2243 		 * The host may skip descriptors. We detect this when this
2244 		 * descriptor does not match the previous fill index. Catch
2245 		 * up with the host now.
2246 		 */
2247 		if (__predict_false(rxr->vxrxr_fill != idx)) {
2248 			while (rxr->vxrxr_fill != idx) {
2249 				rxr->vxrxr_rxd[rxr->vxrxr_fill].gen =
2250 				    rxr->vxrxr_gen;
2251 				vmxnet3_rxr_increment_fill(rxr);
2252 			}
2253 		}
2254 
2255 		if (rxcd->sop) {
2256 			KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD,
2257 			    ("%s: start of frame w/o head buffer", __func__));
2258 			KASSERT(rxr == &rxq->vxrxq_cmd_ring[0],
2259 			    ("%s: start of frame not in ring 0", __func__));
2260 			KASSERT((idx % sc->vmx_rx_max_chain) == 0,
2261 			    ("%s: start of frame at unexcepted index %d (%d)",
2262 			     __func__, idx, sc->vmx_rx_max_chain));
2263 			KASSERT(m_head == NULL,
2264 			    ("%s: duplicate start of frame?", __func__));
2265 
2266 			if (length == 0) {
2267 				/* Just ignore this descriptor. */
2268 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2269 				goto nextp;
2270 			}
2271 
2272 			if (vmxnet3_newbuf(sc, rxr) != 0) {
2273 				rxq->vxrxq_stats.vmrxs_iqdrops++;
2274 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2275 				if (!rxcd->eop)
2276 					vmxnet3_rxq_discard_chain(rxq);
2277 				goto nextp;
2278 			}
2279 
2280 			m->m_pkthdr.rcvif = ifp;
2281 			m->m_pkthdr.len = m->m_len = length;
2282 			m->m_pkthdr.csum_flags = 0;
2283 			m_head = m_tail = m;
2284 
2285 		} else {
2286 			KASSERT(rxd->btype == VMXNET3_BTYPE_BODY,
2287 			    ("%s: non start of frame w/o body buffer", __func__));
2288 
2289 			if (m_head == NULL && m_tail == NULL) {
2290 				/*
2291 				 * This is a continuation of a packet that we
2292 				 * started to drop, but could not drop entirely
2293 				 * because this segment was still owned by the
2294 				 * host.  So, drop the remainder now.
2295 				 */
2296 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2297 				if (!rxcd->eop)
2298 					vmxnet3_rxq_discard_chain(rxq);
2299 				goto nextp;
2300 			}
2301 
2302 			KASSERT(m_head != NULL,
2303 			    ("%s: frame not started?", __func__));
2304 
2305 			if (vmxnet3_newbuf(sc, rxr) != 0) {
2306 				rxq->vxrxq_stats.vmrxs_iqdrops++;
2307 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2308 				if (!rxcd->eop)
2309 					vmxnet3_rxq_discard_chain(rxq);
2310 				m_freem(m_head);
2311 				m_head = m_tail = NULL;
2312 				goto nextp;
2313 			}
2314 
2315 			m->m_len = length;
2316 			m_head->m_pkthdr.len += length;
2317 			m_tail->m_next = m;
2318 			m_tail = m;
2319 		}
2320 
2321 		if (rxcd->eop) {
2322 			vmxnet3_rxq_input(rxq, rxcd, m_head);
2323 			m_head = m_tail = NULL;
2324 
2325 			/* Must recheck after dropping the Rx lock. */
2326 			if ((ifp->if_flags & IFF_RUNNING) == 0)
2327 				break;
2328 		}
2329 
2330 nextp:
2331 		if (__predict_false(rxq->vxrxq_rs->update_rxhead)) {
2332 			int qid = rxcd->qid;
2333 			bus_size_t r;
2334 
2335 			idx = (idx + 1) % rxr->vxrxr_ndesc;
2336 			if (qid >= sc->vmx_nrxqueues) {
2337 				qid -= sc->vmx_nrxqueues;
2338 				r = VMXNET3_BAR0_RXH2(qid);
2339 			} else
2340 				r = VMXNET3_BAR0_RXH1(qid);
2341 			vmxnet3_write_bar0(sc, r, idx);
2342 		}
2343 	}
2344 }
2345 
2346 static void
2347 vmxnet3_legacy_intr(void *xsc)
2348 {
2349 	struct vmxnet3_softc *sc;
2350 	struct vmxnet3_rxqueue *rxq;
2351 	struct vmxnet3_txqueue *txq;
2352 
2353 	sc = xsc;
2354 	rxq = &sc->vmx_rxq[0];
2355 	txq = &sc->vmx_txq[0];
2356 
2357 	if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) {
2358 		if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0)
2359 			return;
2360 	}
2361 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2362 		vmxnet3_disable_all_intrs(sc);
2363 
2364 	if (sc->vmx_ds->event != 0)
2365 		vmxnet3_evintr(sc);
2366 
2367 	VMXNET3_RXQ_LOCK(rxq);
2368 	vmxnet3_rxq_eof(rxq);
2369 	VMXNET3_RXQ_UNLOCK(rxq);
2370 
2371 	VMXNET3_TXQ_LOCK(txq);
2372 	vmxnet3_txq_eof(txq);
2373 	vmxnet3_txq_start(txq);
2374 	VMXNET3_TXQ_UNLOCK(txq);
2375 
2376 	vmxnet3_enable_all_intrs(sc);
2377 }
2378 
2379 #ifdef __FreeBSD__
2380 static void
2381 vmxnet3_txq_intr(void *xtxq)
2382 {
2383 	struct vmxnet3_softc *sc;
2384 	struct vmxnet3_txqueue *txq;
2385 
2386 	txq = xtxq;
2387 	sc = txq->vxtxq_sc;
2388 
2389 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2390 		vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx);
2391 
2392 	VMXNET3_TXQ_LOCK(txq);
2393 	vmxnet3_txq_eof(txq);
2394 	vmxnet3_txq_start(txq);
2395 	VMXNET3_TXQ_UNLOCK(txq);
2396 
2397 	vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx);
2398 }
2399 
2400 static void
2401 vmxnet3_rxq_intr(void *xrxq)
2402 {
2403 	struct vmxnet3_softc *sc;
2404 	struct vmxnet3_rxqueue *rxq;
2405 
2406 	rxq = xrxq;
2407 	sc = rxq->vxrxq_sc;
2408 
2409 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2410 		vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx);
2411 
2412 	VMXNET3_RXQ_LOCK(rxq);
2413 	vmxnet3_rxq_eof(rxq);
2414 	VMXNET3_RXQ_UNLOCK(rxq);
2415 
2416 	vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx);
2417 }
2418 
2419 static void
2420 vmxnet3_event_intr(void *xsc)
2421 {
2422 	struct vmxnet3_softc *sc;
2423 
2424 	sc = xsc;
2425 
2426 	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2427 		vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx);
2428 
2429 	if (sc->vmx_ds->event != 0)
2430 		vmxnet3_evintr(sc);
2431 
2432 	vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx);
2433 }
2434 #endif
2435 
2436 static void
2437 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2438 {
2439 	struct vmxnet3_txring *txr;
2440 	struct vmxnet3_txbuf *txb;
2441 	int i;
2442 
2443 	txr = &txq->vxtxq_cmd_ring;
2444 
2445 	for (i = 0; i < txr->vxtxr_ndesc; i++) {
2446 		txb = &txr->vxtxr_txbuf[i];
2447 
2448 		if (txb->vtxb_m == NULL)
2449 			continue;
2450 
2451 		bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap,
2452 		    BUS_DMASYNC_POSTWRITE);
2453 		bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap);
2454 		m_freem(txb->vtxb_m);
2455 		txb->vtxb_m = NULL;
2456 	}
2457 }
2458 
2459 static void
2460 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2461 {
2462 	struct vmxnet3_rxring *rxr;
2463 	struct vmxnet3_rxbuf *rxb;
2464 	int i, j;
2465 
2466 	if (rxq->vxrxq_mhead != NULL) {
2467 		m_freem(rxq->vxrxq_mhead);
2468 		rxq->vxrxq_mhead = NULL;
2469 		rxq->vxrxq_mtail = NULL;
2470 	}
2471 
2472 	for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
2473 		rxr = &rxq->vxrxq_cmd_ring[i];
2474 
2475 		for (j = 0; j < rxr->vxrxr_ndesc; j++) {
2476 			rxb = &rxr->vxrxr_rxbuf[j];
2477 
2478 			if (rxb->vrxb_m == NULL)
2479 				continue;
2480 
2481 			bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap,
2482 			    BUS_DMASYNC_POSTREAD);
2483 			bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap);
2484 			m_freem(rxb->vrxb_m);
2485 			rxb->vrxb_m = NULL;
2486 		}
2487 	}
2488 }
2489 
2490 static void
2491 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc)
2492 {
2493 	struct vmxnet3_rxqueue *rxq;
2494 	struct vmxnet3_txqueue *txq;
2495 	int i;
2496 
2497 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
2498 		rxq = &sc->vmx_rxq[i];
2499 		VMXNET3_RXQ_LOCK(rxq);
2500 		VMXNET3_RXQ_UNLOCK(rxq);
2501 	}
2502 
2503 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
2504 		txq = &sc->vmx_txq[i];
2505 		VMXNET3_TXQ_LOCK(txq);
2506 		VMXNET3_TXQ_UNLOCK(txq);
2507 	}
2508 }
2509 
2510 static void
2511 vmxnet3_stop(struct vmxnet3_softc *sc)
2512 {
2513 	struct ifnet *ifp;
2514 	int q;
2515 
2516 	ifp = sc->vmx_ifp;
2517 	VMXNET3_CORE_LOCK_ASSERT(sc);
2518 
2519 	ifp->if_flags &= ~IFF_RUNNING;
2520 	sc->vmx_link_active = 0;
2521 	callout_stop(&sc->vmx_tick);
2522 
2523 	/* Disable interrupts. */
2524 	vmxnet3_disable_all_intrs(sc);
2525 	vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE);
2526 
2527 	vmxnet3_stop_rendezvous(sc);
2528 
2529 	for (q = 0; q < sc->vmx_ntxqueues; q++)
2530 		vmxnet3_txstop(sc, &sc->vmx_txq[q]);
2531 	for (q = 0; q < sc->vmx_nrxqueues; q++)
2532 		vmxnet3_rxstop(sc, &sc->vmx_rxq[q]);
2533 
2534 	vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET);
2535 }
2536 
2537 static void
2538 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2539 {
2540 	struct vmxnet3_txring *txr;
2541 	struct vmxnet3_comp_ring *txc;
2542 
2543 	txr = &txq->vxtxq_cmd_ring;
2544 	txr->vxtxr_head = 0;
2545 	txr->vxtxr_next = 0;
2546 	txr->vxtxr_gen = VMXNET3_INIT_GEN;
2547 	bzero(txr->vxtxr_txd,
2548 	    txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc));
2549 
2550 	txc = &txq->vxtxq_comp_ring;
2551 	txc->vxcr_next = 0;
2552 	txc->vxcr_gen = VMXNET3_INIT_GEN;
2553 	bzero(txc->vxcr_u.txcd,
2554 	    txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc));
2555 }
2556 
2557 static int
2558 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2559 {
2560 	struct ifnet *ifp;
2561 	struct vmxnet3_rxring *rxr;
2562 	struct vmxnet3_comp_ring *rxc;
2563 	int i, populate, idx, frame_size, error;
2564 
2565 	ifp = sc->vmx_ifp;
2566 	frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) +
2567 	    ifp->if_mtu;
2568 
2569 	/*
2570 	 * If the MTU causes us to exceed what a regular sized cluster can
2571 	 * handle, we allocate a second MJUMPAGESIZE cluster after it in
2572 	 * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters.
2573 	 *
2574 	 * Keep rx_max_chain a divisor of the maximum Rx ring size to make
2575 	 * our life easier. We do not support changing the ring size after
2576 	 * the attach.
2577 	 */
2578 	if (frame_size <= MCLBYTES)
2579 		sc->vmx_rx_max_chain = 1;
2580 	else
2581 		sc->vmx_rx_max_chain = 2;
2582 
2583 	/*
2584 	 * Only populate ring 1 if the configuration will take advantage
2585 	 * of it. That is either when LRO is enabled or the frame size
2586 	 * exceeds what ring 0 can contain.
2587 	 */
2588 #if 0 /* XXX LRO */
2589 	if ((ifp->if_capenable & IFCAP_LRO) == 0 &&
2590 #else
2591 	if (
2592 #endif
2593 	    frame_size <= MCLBYTES + MJUMPAGESIZE)
2594 		populate = 1;
2595 	else
2596 		populate = VMXNET3_RXRINGS_PERQ;
2597 
2598 	for (i = 0; i < populate; i++) {
2599 		rxr = &rxq->vxrxq_cmd_ring[i];
2600 		rxr->vxrxr_fill = 0;
2601 		rxr->vxrxr_gen = VMXNET3_INIT_GEN;
2602 		bzero(rxr->vxrxr_rxd,
2603 		    rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2604 
2605 		for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) {
2606 			error = vmxnet3_newbuf(sc, rxr);
2607 			if (error)
2608 				return (error);
2609 		}
2610 	}
2611 
2612 	for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {
2613 		rxr = &rxq->vxrxq_cmd_ring[i];
2614 		rxr->vxrxr_fill = 0;
2615 		rxr->vxrxr_gen = 0;
2616 		bzero(rxr->vxrxr_rxd,
2617 		    rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2618 	}
2619 
2620 	rxc = &rxq->vxrxq_comp_ring;
2621 	rxc->vxcr_next = 0;
2622 	rxc->vxcr_gen = VMXNET3_INIT_GEN;
2623 	bzero(rxc->vxcr_u.rxcd,
2624 	    rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc));
2625 
2626 	return (0);
2627 }
2628 
2629 static int
2630 vmxnet3_reinit_queues(struct vmxnet3_softc *sc)
2631 {
2632 	device_t dev;
2633 	int q, error;
2634 
2635 	dev = sc->vmx_dev;
2636 
2637 	for (q = 0; q < sc->vmx_ntxqueues; q++)
2638 		vmxnet3_txinit(sc, &sc->vmx_txq[q]);
2639 
2640 	for (q = 0; q < sc->vmx_nrxqueues; q++) {
2641 		error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]);
2642 		if (error) {
2643 			device_printf(dev, "cannot populate Rx queue %d\n", q);
2644 			return (error);
2645 		}
2646 	}
2647 
2648 	return (0);
2649 }
2650 
2651 static int
2652 vmxnet3_enable_device(struct vmxnet3_softc *sc)
2653 {
2654 	int q;
2655 
2656 	if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) {
2657 		device_printf(sc->vmx_dev, "device enable command failed!\n");
2658 		return (1);
2659 	}
2660 
2661 	/* Reset the Rx queue heads. */
2662 	for (q = 0; q < sc->vmx_nrxqueues; q++) {
2663 		vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0);
2664 		vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0);
2665 	}
2666 
2667 	return (0);
2668 }
2669 
2670 static void
2671 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc)
2672 {
2673 	struct ifnet *ifp;
2674 
2675 	ifp = sc->vmx_ifp;
2676 
2677 	vmxnet3_set_rxfilter(sc);
2678 
2679 #if 0 /* VLAN_HWFILTER */
2680 	if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2681 		bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter,
2682 		    sizeof(sc->vmx_ds->vlan_filter));
2683 	else
2684 #endif
2685 		bzero(sc->vmx_ds->vlan_filter,
2686 		    sizeof(sc->vmx_ds->vlan_filter));
2687 	vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
2688 }
2689 
2690 static int
2691 vmxnet3_reinit(struct vmxnet3_softc *sc)
2692 {
2693 
2694 	vmxnet3_reinit_interface(sc);
2695 	vmxnet3_reinit_shared_data(sc);
2696 
2697 	if (vmxnet3_reinit_queues(sc) != 0)
2698 		return (ENXIO);
2699 
2700 	if (vmxnet3_enable_device(sc) != 0)
2701 		return (ENXIO);
2702 
2703 	vmxnet3_reinit_rxfilters(sc);
2704 
2705 	return (0);
2706 }
2707 
2708 static void
2709 vmxnet3_init_locked(struct vmxnet3_softc *sc)
2710 {
2711 	struct ifnet *ifp;
2712 
2713 	ifp = sc->vmx_ifp;
2714 
2715 	if (ifp->if_flags & IFF_RUNNING)
2716 		return;
2717 
2718 	vmxnet3_stop(sc);
2719 
2720 	if (vmxnet3_reinit(sc) != 0) {
2721 		vmxnet3_stop(sc);
2722 		return;
2723 	}
2724 
2725 	ifp->if_flags |= IFF_RUNNING;
2726 	vmxnet3_link_status(sc);
2727 
2728 	vmxnet3_enable_all_intrs(sc);
2729 	callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
2730 }
2731 
2732 static void
2733 vmxnet3_init(void *xsc)
2734 {
2735 	struct vmxnet3_softc *sc;
2736 
2737 	sc = xsc;
2738 
2739 	VMXNET3_CORE_LOCK(sc);
2740 	vmxnet3_init_locked(sc);
2741 	VMXNET3_CORE_UNLOCK(sc);
2742 }
2743 
2744 /*
2745  * BMV: Much of this can go away once we finally have offsets in
2746  * the mbuf packet header. Bug andre@.
2747  */
2748 static int
2749 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m,
2750     int *etype, int *proto, int *start)
2751 {
2752 	struct ether_vlan_header *evh;
2753 	int offset;
2754 #if defined(INET)
2755 	struct ip *ip = NULL;
2756 #endif
2757 #if defined(INET6)
2758 	struct ip6_hdr *ip6 = NULL;
2759 #endif
2760 
2761 	evh = mtod(m, struct ether_vlan_header *);
2762 	if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2763 		/* BMV: We should handle nested VLAN tags too. */
2764 		*etype = ntohs(evh->evl_proto);
2765 		offset = sizeof(struct ether_vlan_header);
2766 	} else {
2767 		*etype = ntohs(evh->evl_encap_proto);
2768 		offset = sizeof(struct ether_header);
2769 	}
2770 
2771 	switch (*etype) {
2772 #if defined(INET)
2773 	case ETHERTYPE_IP:
2774 		if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
2775 			m = m_pullup(m, offset + sizeof(struct ip));
2776 			if (m == NULL)
2777 				return (EINVAL);
2778 		}
2779 
2780 		ip = (struct ip *)(mtod(m, uint8_t *) + offset);
2781 		*proto = ip->ip_p;
2782 		*start = offset + (ip->ip_hl << 2);
2783 		break;
2784 #endif
2785 #if defined(INET6)
2786 	case ETHERTYPE_IPV6:
2787 		if (__predict_false(m->m_len <
2788 		    offset + sizeof(struct ip6_hdr))) {
2789 			m = m_pullup(m, offset + sizeof(struct ip6_hdr));
2790 			if (m == NULL)
2791 				return (EINVAL);
2792 		}
2793 
2794 		ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + offset);
2795 		*proto = -1;
2796 		*start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
2797 		/* Assert the network stack sent us a valid packet. */
2798 		KASSERT(*start > offset,
2799 		    ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
2800 		    *start, offset, *proto));
2801 		break;
2802 #endif
2803 	default:
2804 		return (EINVAL);
2805 	}
2806 
2807 #if 0 /* XXX TSO */
2808 	if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2809 		struct tcphdr *tcp;
2810 
2811 		if (__predict_false(*proto != IPPROTO_TCP)) {
2812 			/* Likely failed to correctly parse the mbuf. */
2813 			return (EINVAL);
2814 		}
2815 
2816 		if (m->m_len < *start + sizeof(struct tcphdr)) {
2817 			m = m_pullup(m, *start + sizeof(struct tcphdr));
2818 			if (m == NULL)
2819 				return (EINVAL);
2820 		}
2821 
2822 		tcp = (struct tcphdr *)(mtod(m, uint8_t *) + *start);
2823 		*start += (tcp->th_off << 2);
2824 
2825 		txq->vxtxq_stats.vmtxs_tso++;
2826 	} else
2827 #endif
2828 		txq->vxtxq_stats.vmtxs_csum++;
2829 
2830 	return (0);
2831 }
2832 
2833 static int
2834 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0,
2835     bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs)
2836 {
2837 	struct vmxnet3_txring *txr;
2838 	struct mbuf *m;
2839 	bus_dma_tag_t tag;
2840 	int error;
2841 
2842 	txr = &txq->vxtxq_cmd_ring;
2843 	m = *m0;
2844 	tag = txr->vxtxr_txtag;
2845 
2846 	error = bus_dmamap_load_mbuf_segment(tag, dmap, m, segs, 1, nsegs,
2847 	    BUS_DMA_NOWAIT);
2848 	if (error == 0 || error != EFBIG)
2849 		return (error);
2850 
2851 	m = m_defrag(m, M_NOWAIT);
2852 	if (m != NULL) {
2853 		*m0 = m;
2854 		error = bus_dmamap_load_mbuf_segment(tag, dmap, m, segs,
2855 		    1, nsegs, BUS_DMA_NOWAIT);
2856 	} else
2857 		error = ENOBUFS;
2858 
2859 	if (error) {
2860 		m_freem(*m0);
2861 		*m0 = NULL;
2862 		txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++;
2863 	} else
2864 		txq->vxtxq_sc->vmx_stats.vmst_defragged++;
2865 
2866 	return (error);
2867 }
2868 
2869 static void
2870 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap)
2871 {
2872 	struct vmxnet3_txring *txr;
2873 
2874 	txr = &txq->vxtxq_cmd_ring;
2875 	bus_dmamap_unload(txr->vxtxr_txtag, dmap);
2876 }
2877 
2878 static int
2879 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0)
2880 {
2881 	struct vmxnet3_softc *sc;
2882 	struct vmxnet3_txring *txr;
2883 	struct vmxnet3_txdesc *txd, *sop;
2884 	struct mbuf *m;
2885 	bus_dmamap_t dmap;
2886 	bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS];
2887 	int i, gen, nsegs, etype, proto, start, error;
2888 
2889 	sc = txq->vxtxq_sc;
2890 	start = 0;
2891 	txd = NULL;
2892 	txr = &txq->vxtxq_cmd_ring;
2893 	dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap;
2894 
2895 	error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs);
2896 	if (error)
2897 		return (error);
2898 
2899 	m = *m0;
2900 	M_ASSERTPKTHDR(m);
2901 	KASSERT(nsegs <= VMXNET3_TX_MAXSEGS,
2902 	    ("%s: mbuf %p with too many segments %d", __func__, m, nsegs));
2903 
2904 	if (VMXNET3_TXRING_AVAIL(txr) < nsegs) {
2905 		txq->vxtxq_stats.vmtxs_full++;
2906 		vmxnet3_txq_unload_mbuf(txq, dmap);
2907 		return (ENOSPC);
2908 	} else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) {
2909 		error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start);
2910 		if (error) {
2911 			txq->vxtxq_stats.vmtxs_offload_failed++;
2912 			vmxnet3_txq_unload_mbuf(txq, dmap);
2913 			m_freem(m);
2914 			*m0 = NULL;
2915 			return (error);
2916 		}
2917 	}
2918 
2919 	txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m;
2920 	sop = &txr->vxtxr_txd[txr->vxtxr_head];
2921 	gen = txr->vxtxr_gen ^ 1;	/* Owned by cpu (yet) */
2922 
2923 	for (i = 0; i < nsegs; i++) {
2924 		txd = &txr->vxtxr_txd[txr->vxtxr_head];
2925 
2926 		txd->addr = segs[i].ds_addr;
2927 		txd->len = segs[i].ds_len;
2928 		txd->gen = gen;
2929 		txd->dtype = 0;
2930 		txd->offload_mode = VMXNET3_OM_NONE;
2931 		txd->offload_pos = 0;
2932 		txd->hlen = 0;
2933 		txd->eop = 0;
2934 		txd->compreq = 0;
2935 		txd->vtag_mode = 0;
2936 		txd->vtag = 0;
2937 
2938 		if (++txr->vxtxr_head == txr->vxtxr_ndesc) {
2939 			txr->vxtxr_head = 0;
2940 			txr->vxtxr_gen ^= 1;
2941 		}
2942 		gen = txr->vxtxr_gen;
2943 	}
2944 	txd->eop = 1;
2945 	txd->compreq = 1;
2946 
2947 	if (m->m_flags & M_VLANTAG) {
2948 		sop->vtag_mode = 1;
2949 		sop->vtag = m->m_pkthdr.ether_vlantag;
2950 	}
2951 
2952 
2953 #if 0 /* XXX TSO */
2954 	if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2955 		sop->offload_mode = VMXNET3_OM_TSO;
2956 		sop->hlen = start;
2957 		sop->offload_pos = m->m_pkthdr.tso_segsz;
2958 	} else
2959 #endif
2960 	if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD |
2961 	    VMXNET3_CSUM_OFFLOAD_IPV6))	{
2962 		sop->offload_mode = VMXNET3_OM_CSUM;
2963 		sop->hlen = start;
2964 		sop->offload_pos = start + m->m_pkthdr.csum_data;
2965 	}
2966 
2967 	/* Finally, change the ownership. */
2968 	vmxnet3_barrier(sc, VMXNET3_BARRIER_WR);
2969 	sop->gen ^= 1;
2970 
2971 	txq->vxtxq_ts->npending += nsegs;
2972 	if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) {
2973 		txq->vxtxq_ts->npending = 0;
2974 		vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id),
2975 		    txr->vxtxr_head);
2976 	}
2977 
2978 	return (0);
2979 }
2980 
2981 #ifdef VMXNET3_LEGACY_TX
2982 
2983 static void
2984 vmxnet3_start_locked(struct ifnet *ifp)
2985 {
2986 	struct vmxnet3_softc *sc;
2987 	struct vmxnet3_txqueue *txq;
2988 	struct vmxnet3_txring *txr;
2989 	struct mbuf *m_head;
2990 	int tx, avail;
2991 
2992 	sc = ifp->if_softc;
2993 	txq = &sc->vmx_txq[0];
2994 	txr = &txq->vxtxq_cmd_ring;
2995 	tx = 0;
2996 
2997 	VMXNET3_TXQ_LOCK_ASSERT(txq);
2998 
2999 	if ((ifp->if_flags & IFF_RUNNING) == 0 ||
3000 	    sc->vmx_link_active == 0)
3001 		return;
3002 
3003 	while (!ifq_is_empty(&ifp->if_snd)) {
3004 		if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2)
3005 			break;
3006 
3007 		m_head = ifq_dequeue(&ifp->if_snd);
3008 		if (m_head == NULL)
3009 			break;
3010 
3011 		/* Assume worse case if this mbuf is the head of a chain. */
3012 		if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) {
3013 			ifq_prepend(&ifp->if_snd, m_head);
3014 			break;
3015 		}
3016 
3017 		if (vmxnet3_txq_encap(txq, &m_head) != 0) {
3018 			if (m_head != NULL)
3019 				ifq_prepend(&ifp->if_snd, m_head);
3020 			break;
3021 		}
3022 
3023 		tx++;
3024 		ETHER_BPF_MTAP(ifp, m_head);
3025 	}
3026 
3027 	if (tx > 0)
3028 		txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
3029 }
3030 
3031 static void
3032 vmxnet3_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
3033 {
3034 	struct vmxnet3_softc *sc;
3035 	struct vmxnet3_txqueue *txq;
3036 
3037 	sc = ifp->if_softc;
3038 	txq = &sc->vmx_txq[0];
3039 
3040 	VMXNET3_TXQ_LOCK(txq);
3041 	vmxnet3_start_locked(ifp);
3042 	VMXNET3_TXQ_UNLOCK(txq);
3043 }
3044 
3045 #else /* !VMXNET3_LEGACY_TX */
3046 
3047 static int
3048 vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m)
3049 {
3050 	struct vmxnet3_softc *sc;
3051 	struct vmxnet3_txring *txr;
3052 	struct buf_ring *br;
3053 	struct ifnet *ifp;
3054 	int tx, avail, error;
3055 
3056 	sc = txq->vxtxq_sc;
3057 	br = txq->vxtxq_br;
3058 	ifp = sc->vmx_ifp;
3059 	txr = &txq->vxtxq_cmd_ring;
3060 	tx = 0;
3061 	error = 0;
3062 
3063 	VMXNET3_TXQ_LOCK_ASSERT(txq);
3064 
3065 	if ((ifp->if_flags & IFF_RUNNING) == 0 ||
3066 	    sc->vmx_link_active == 0) {
3067 		if (m != NULL)
3068 			error = drbr_enqueue(ifp, br, m);
3069 		return (error);
3070 	}
3071 
3072 	if (m != NULL) {
3073 		error = drbr_enqueue(ifp, br, m);
3074 		if (error)
3075 			return (error);
3076 	}
3077 
3078 	while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) {
3079 		m = drbr_peek(ifp, br);
3080 		if (m == NULL)
3081 			break;
3082 
3083 		/* Assume worse case if this mbuf is the head of a chain. */
3084 		if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) {
3085 			drbr_putback(ifp, br, m);
3086 			break;
3087 		}
3088 
3089 		if (vmxnet3_txq_encap(txq, &m) != 0) {
3090 			if (m != NULL)
3091 				drbr_putback(ifp, br, m);
3092 			else
3093 				drbr_advance(ifp, br);
3094 			break;
3095 		}
3096 		drbr_advance(ifp, br);
3097 
3098 		tx++;
3099 		ETHER_BPF_MTAP(ifp, m);
3100 	}
3101 
3102 	if (tx > 0)
3103 		txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
3104 
3105 	return (0);
3106 }
3107 
3108 static int
3109 vmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m)
3110 {
3111 	struct vmxnet3_softc *sc;
3112 	struct vmxnet3_txqueue *txq;
3113 	int i, ntxq, error;
3114 
3115 	sc = ifp->if_softc;
3116 	ntxq = sc->vmx_ntxqueues;
3117 
3118 	/* check if flowid is set */
3119 	if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
3120 		i = m->m_pkthdr.flowid % ntxq;
3121 	else
3122 		i = curcpu % ntxq;
3123 
3124 	txq = &sc->vmx_txq[i];
3125 
3126 	if (VMXNET3_TXQ_TRYLOCK(txq) != 0) {
3127 		error = vmxnet3_txq_mq_start_locked(txq, m);
3128 		VMXNET3_TXQ_UNLOCK(txq);
3129 	} else {
3130 		error = drbr_enqueue(ifp, txq->vxtxq_br, m);
3131 		taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask);
3132 	}
3133 
3134 	return (error);
3135 }
3136 
3137 static void
3138 vmxnet3_txq_tq_deferred(void *xtxq, int pending)
3139 {
3140 	struct vmxnet3_softc *sc;
3141 	struct vmxnet3_txqueue *txq;
3142 
3143 	txq = xtxq;
3144 	sc = txq->vxtxq_sc;
3145 
3146 	VMXNET3_TXQ_LOCK(txq);
3147 	if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br))
3148 		vmxnet3_txq_mq_start_locked(txq, NULL);
3149 	VMXNET3_TXQ_UNLOCK(txq);
3150 }
3151 
3152 #endif /* VMXNET3_LEGACY_TX */
3153 
3154 static void
3155 vmxnet3_txq_start(struct vmxnet3_txqueue *txq)
3156 {
3157 	struct vmxnet3_softc *sc;
3158 	struct ifnet *ifp;
3159 
3160 	sc = txq->vxtxq_sc;
3161 	ifp = sc->vmx_ifp;
3162 
3163 #ifdef VMXNET3_LEGACY_TX
3164 	if (!ifq_is_empty(&ifp->if_snd))
3165 		vmxnet3_start_locked(ifp);
3166 #else
3167 	if (!drbr_empty(ifp, txq->vxtxq_br))
3168 		vmxnet3_txq_mq_start_locked(txq, NULL);
3169 #endif
3170 }
3171 
3172 static void
3173 vmxnet3_tx_start_all(struct vmxnet3_softc *sc)
3174 {
3175 	struct vmxnet3_txqueue *txq;
3176 	int i;
3177 
3178 	VMXNET3_CORE_LOCK_ASSERT(sc);
3179 
3180 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
3181 		txq = &sc->vmx_txq[i];
3182 
3183 		VMXNET3_TXQ_LOCK(txq);
3184 		vmxnet3_txq_start(txq);
3185 		VMXNET3_TXQ_UNLOCK(txq);
3186 	}
3187 }
3188 
3189 static void
3190 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag)
3191 {
3192 	struct ifnet *ifp;
3193 	int idx, bit;
3194 
3195 	ifp = sc->vmx_ifp;
3196 	idx = (tag >> 5) & 0x7F;
3197 	bit = tag & 0x1F;
3198 
3199 	if (tag == 0 || tag > 4095)
3200 		return;
3201 
3202 	VMXNET3_CORE_LOCK(sc);
3203 
3204 	/* Update our private VLAN bitvector. */
3205 	if (add)
3206 		sc->vmx_vlan_filter[idx] |= (1 << bit);
3207 	else
3208 		sc->vmx_vlan_filter[idx] &= ~(1 << bit);
3209 
3210 #if 0 /* VLAN_HWFILTER */
3211 	if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
3212 		if (add)
3213 			sc->vmx_ds->vlan_filter[idx] |= (1 << bit);
3214 		else
3215 			sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit);
3216 		vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
3217 	}
3218 #endif
3219 
3220 	VMXNET3_CORE_UNLOCK(sc);
3221 }
3222 
3223 static void
3224 vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
3225 {
3226 
3227 	if (ifp->if_softc == arg)
3228 		vmxnet3_update_vlan_filter(arg, 1, tag);
3229 }
3230 
3231 static void
3232 vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
3233 {
3234 
3235 	if (ifp->if_softc == arg)
3236 		vmxnet3_update_vlan_filter(arg, 0, tag);
3237 }
3238 
3239 static void
3240 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc)
3241 {
3242 	struct ifnet *ifp;
3243 	struct vmxnet3_driver_shared *ds;
3244 	struct ifmultiaddr *ifma;
3245 	u_int mode;
3246 
3247 	ifp = sc->vmx_ifp;
3248 	ds = sc->vmx_ds;
3249 
3250 	mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST;
3251 	if (ifp->if_flags & IFF_PROMISC)
3252 		mode |= VMXNET3_RXMODE_PROMISC;
3253 	if (ifp->if_flags & IFF_ALLMULTI)
3254 		mode |= VMXNET3_RXMODE_ALLMULTI;
3255 	else {
3256 		int cnt = 0, overflow = 0;
3257 
3258 		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3259 			if (ifma->ifma_addr->sa_family != AF_LINK)
3260 				continue;
3261 			else if (cnt == VMXNET3_MULTICAST_MAX) {
3262 				overflow = 1;
3263 				break;
3264 			}
3265 
3266 			bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
3267 			   &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN);
3268 			cnt++;
3269 		}
3270 
3271 		if (overflow != 0) {
3272 			cnt = 0;
3273 			mode |= VMXNET3_RXMODE_ALLMULTI;
3274 		} else if (cnt > 0)
3275 			mode |= VMXNET3_RXMODE_MCAST;
3276 		ds->mcast_tablelen = cnt * ETHER_ADDR_LEN;
3277 	}
3278 
3279 	ds->rxmode = mode;
3280 
3281 	vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER);
3282 	vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE);
3283 }
3284 
3285 static int
3286 vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu)
3287 {
3288 	struct ifnet *ifp;
3289 
3290 	ifp = sc->vmx_ifp;
3291 
3292 	if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU)
3293 		return (EINVAL);
3294 
3295 	ifp->if_mtu = mtu;
3296 
3297 	if (ifp->if_flags & IFF_RUNNING) {
3298 		ifp->if_flags &= ~IFF_RUNNING;
3299 		vmxnet3_init_locked(sc);
3300 	}
3301 
3302 	return (0);
3303 }
3304 
3305 static int
3306 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cred)
3307 {
3308 	struct vmxnet3_softc *sc;
3309 	struct ifreq *ifr;
3310 	int reinit, mask, error;
3311 
3312 	sc = ifp->if_softc;
3313 	ifr = (struct ifreq *) data;
3314 	error = 0;
3315 
3316 	switch (cmd) {
3317 	case SIOCSIFMTU:
3318 		if (ifp->if_mtu != ifr->ifr_mtu) {
3319 			VMXNET3_CORE_LOCK(sc);
3320 			error = vmxnet3_change_mtu(sc, ifr->ifr_mtu);
3321 			VMXNET3_CORE_UNLOCK(sc);
3322 		}
3323 		break;
3324 
3325 	case SIOCSIFFLAGS:
3326 		VMXNET3_CORE_LOCK(sc);
3327 		if (ifp->if_flags & IFF_UP) {
3328 			if ((ifp->if_flags & IFF_RUNNING)) {
3329 				if ((ifp->if_flags ^ sc->vmx_if_flags) &
3330 				    (IFF_PROMISC | IFF_ALLMULTI)) {
3331 					vmxnet3_set_rxfilter(sc);
3332 				}
3333 			} else
3334 				vmxnet3_init_locked(sc);
3335 		} else {
3336 			if (ifp->if_flags & IFF_RUNNING)
3337 				vmxnet3_stop(sc);
3338 		}
3339 		sc->vmx_if_flags = ifp->if_flags;
3340 		VMXNET3_CORE_UNLOCK(sc);
3341 		break;
3342 
3343 	case SIOCADDMULTI:
3344 	case SIOCDELMULTI:
3345 		VMXNET3_CORE_LOCK(sc);
3346 		if (ifp->if_flags & IFF_RUNNING)
3347 			vmxnet3_set_rxfilter(sc);
3348 		VMXNET3_CORE_UNLOCK(sc);
3349 		break;
3350 
3351 	case SIOCSIFMEDIA:
3352 	case SIOCGIFMEDIA:
3353 		error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd);
3354 		break;
3355 
3356 	case SIOCSIFCAP:
3357 		VMXNET3_CORE_LOCK(sc);
3358 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3359 
3360 		if (mask & IFCAP_TXCSUM)
3361 			ifp->if_capenable ^= IFCAP_TXCSUM;
3362 		if (mask & IFCAP_TXCSUM_IPV6)
3363 			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
3364 #if 0 /* XXX TSO */
3365 		if (mask & IFCAP_TSO4)
3366 			ifp->if_capenable ^= IFCAP_TSO4;
3367 		if (mask & IFCAP_TSO6)
3368 			ifp->if_capenable ^= IFCAP_TSO6;
3369 #endif
3370 
3371 		if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | /* IFCAP_LRO | */
3372 		    IFCAP_VLAN_HWTAGGING /* | IFCAP_VLAN_HWFILTER */)) {
3373 			/* Changing these features requires us to reinit. */
3374 			reinit = 1;
3375 
3376 			if (mask & IFCAP_RXCSUM)
3377 				ifp->if_capenable ^= IFCAP_RXCSUM;
3378 			if (mask & IFCAP_RXCSUM_IPV6)
3379 				ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
3380 #if 0 /* XXX LRO */
3381 			if (mask & IFCAP_LRO)
3382 				ifp->if_capenable ^= IFCAP_LRO;
3383 #endif
3384 			if (mask & IFCAP_VLAN_HWTAGGING)
3385 				ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3386 #if 0 /* XXX VLAN_HWFILTER */
3387 			if (mask & IFCAP_VLAN_HWFILTER)
3388 				ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
3389 #endif
3390 		} else
3391 			reinit = 0;
3392 
3393 #if 0 /* XXX TSO */
3394 		if (mask & IFCAP_VLAN_HWTSO)
3395 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
3396 #endif
3397 
3398 		if (reinit && (ifp->if_flags & IFF_RUNNING)) {
3399 			ifp->if_flags &= ~IFF_RUNNING;
3400 			vmxnet3_init_locked(sc);
3401 		} else {
3402 			vmxnet3_init_hwassist(sc);
3403 		}
3404 
3405 		VMXNET3_CORE_UNLOCK(sc);
3406 #if 0 /* XXX */
3407 		VLAN_CAPABILITIES(ifp);
3408 #endif
3409 		break;
3410 
3411 	default:
3412 		error = ether_ioctl(ifp, cmd, data);
3413 		break;
3414 	}
3415 
3416 	VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc);
3417 
3418 	return (error);
3419 }
3420 
3421 #ifndef VMXNET3_LEGACY_TX
3422 static void
3423 vmxnet3_qflush(struct ifnet *ifp)
3424 {
3425 	struct vmxnet3_softc *sc;
3426 	struct vmxnet3_txqueue *txq;
3427 	struct mbuf *m;
3428 	int i;
3429 
3430 	sc = ifp->if_softc;
3431 
3432 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
3433 		txq = &sc->vmx_txq[i];
3434 
3435 		VMXNET3_TXQ_LOCK(txq);
3436 		while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL)
3437 			m_freem(m);
3438 		VMXNET3_TXQ_UNLOCK(txq);
3439 	}
3440 
3441 	if_qflush(ifp);
3442 }
3443 #endif
3444 
3445 static int
3446 vmxnet3_watchdog(struct vmxnet3_txqueue *txq)
3447 {
3448 	struct vmxnet3_softc *sc;
3449 
3450 	sc = txq->vxtxq_sc;
3451 
3452 	VMXNET3_TXQ_LOCK(txq);
3453 	if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) {
3454 		VMXNET3_TXQ_UNLOCK(txq);
3455 		return (0);
3456 	}
3457 	VMXNET3_TXQ_UNLOCK(txq);
3458 
3459 	if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n",
3460 	    txq->vxtxq_id);
3461 	return (1);
3462 }
3463 
3464 static void
3465 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc)
3466 {
3467 
3468 	vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS);
3469 }
3470 
3471 static void
3472 vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *txq,
3473     struct vmxnet3_txq_stats *accum)
3474 {
3475 	struct vmxnet3_txq_stats *st;
3476 
3477 	st = &txq->vxtxq_stats;
3478 
3479 	accum->vmtxs_opackets += st->vmtxs_opackets;
3480 	accum->vmtxs_obytes += st->vmtxs_obytes;
3481 	accum->vmtxs_omcasts += st->vmtxs_omcasts;
3482 	accum->vmtxs_csum += st->vmtxs_csum;
3483 	accum->vmtxs_tso += st->vmtxs_tso;
3484 	accum->vmtxs_full += st->vmtxs_full;
3485 	accum->vmtxs_offload_failed += st->vmtxs_offload_failed;
3486 }
3487 
3488 static void
3489 vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *rxq,
3490     struct vmxnet3_rxq_stats *accum)
3491 {
3492 	struct vmxnet3_rxq_stats *st;
3493 
3494 	st = &rxq->vxrxq_stats;
3495 
3496 	accum->vmrxs_ipackets += st->vmrxs_ipackets;
3497 	accum->vmrxs_ibytes += st->vmrxs_ibytes;
3498 	accum->vmrxs_iqdrops += st->vmrxs_iqdrops;
3499 	accum->vmrxs_ierrors += st->vmrxs_ierrors;
3500 }
3501 
3502 static void
3503 vmxnet3_accumulate_stats(struct vmxnet3_softc *sc)
3504 {
3505 	struct ifnet *ifp;
3506 	struct vmxnet3_statistics *st;
3507 	struct vmxnet3_txq_stats txaccum;
3508 	struct vmxnet3_rxq_stats rxaccum;
3509 	int i;
3510 
3511 	ifp = sc->vmx_ifp;
3512 	st = &sc->vmx_stats;
3513 
3514 	bzero(&txaccum, sizeof(struct vmxnet3_txq_stats));
3515 	bzero(&rxaccum, sizeof(struct vmxnet3_rxq_stats));
3516 
3517 	for (i = 0; i < sc->vmx_ntxqueues; i++)
3518 		vmxnet3_txq_accum_stats(&sc->vmx_txq[i], &txaccum);
3519 	for (i = 0; i < sc->vmx_nrxqueues; i++)
3520 		vmxnet3_rxq_accum_stats(&sc->vmx_rxq[i], &rxaccum);
3521 
3522 	/*
3523 	 * With the exception of if_ierrors, these ifnet statistics are
3524 	 * only updated in the driver, so just set them to our accumulated
3525 	 * values. if_ierrors is updated in ether_input() for malformed
3526 	 * frames that we should have already discarded.
3527 	 */
3528 	ifp->if_ipackets = rxaccum.vmrxs_ipackets;
3529 	ifp->if_iqdrops = rxaccum.vmrxs_iqdrops;
3530 	ifp->if_ierrors = rxaccum.vmrxs_ierrors;
3531 	ifp->if_opackets = txaccum.vmtxs_opackets;
3532 #ifndef VMXNET3_LEGACY_TX
3533 	ifp->if_obytes = txaccum.vmtxs_obytes;
3534 	ifp->if_omcasts = txaccum.vmtxs_omcasts;
3535 #endif
3536 }
3537 
3538 static void
3539 vmxnet3_tick(void *xsc)
3540 {
3541 	struct vmxnet3_softc *sc;
3542 	struct ifnet *ifp;
3543 	int i, timedout;
3544 
3545 	sc = xsc;
3546 	ifp = sc->vmx_ifp;
3547 	timedout = 0;
3548 
3549 	VMXNET3_CORE_LOCK_ASSERT(sc);
3550 
3551 	vmxnet3_accumulate_stats(sc);
3552 	vmxnet3_refresh_host_stats(sc);
3553 
3554 	for (i = 0; i < sc->vmx_ntxqueues; i++)
3555 		timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]);
3556 
3557 	if (timedout != 0) {
3558 		ifp->if_flags &= ~IFF_RUNNING;
3559 		vmxnet3_init_locked(sc);
3560 	} else
3561 		callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
3562 }
3563 
3564 static int
3565 vmxnet3_link_is_up(struct vmxnet3_softc *sc)
3566 {
3567 	uint32_t status;
3568 
3569 	/* Also update the link speed while here. */
3570 	status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK);
3571 	sc->vmx_link_speed = status >> 16;
3572 	return !!(status & 0x1);
3573 }
3574 
3575 static void
3576 vmxnet3_link_status(struct vmxnet3_softc *sc)
3577 {
3578 	struct ifnet *ifp;
3579 	int link;
3580 
3581 	ifp = sc->vmx_ifp;
3582 	link = vmxnet3_link_is_up(sc);
3583 
3584 	if (link != 0 && sc->vmx_link_active == 0) {
3585 		sc->vmx_link_active = 1;
3586 		ifp->if_link_state = LINK_STATE_UP;
3587 		if_link_state_change(ifp);
3588 	} else if (link == 0 && sc->vmx_link_active != 0) {
3589 		sc->vmx_link_active = 0;
3590 		ifp->if_link_state = LINK_STATE_DOWN;
3591 		if_link_state_change(ifp);
3592 	}
3593 }
3594 
3595 static void
3596 vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
3597 {
3598 	struct vmxnet3_softc *sc;
3599 
3600 	sc = ifp->if_softc;
3601 
3602 	ifmr->ifm_active = IFM_ETHER | IFM_AUTO;
3603 	ifmr->ifm_status = IFM_AVALID;
3604 
3605 	VMXNET3_CORE_LOCK(sc);
3606 	if (vmxnet3_link_is_up(sc) != 0)
3607 		ifmr->ifm_status |= IFM_ACTIVE;
3608 	else
3609 		ifmr->ifm_status |= IFM_NONE;
3610 	VMXNET3_CORE_UNLOCK(sc);
3611 }
3612 
3613 static int
3614 vmxnet3_media_change(struct ifnet *ifp)
3615 {
3616 
3617 	/* Ignore. */
3618 	return (0);
3619 }
3620 
3621 static void
3622 vmxnet3_set_lladdr(struct vmxnet3_softc *sc)
3623 {
3624 	uint32_t ml, mh;
3625 
3626 	ml  = sc->vmx_lladdr[0];
3627 	ml |= sc->vmx_lladdr[1] << 8;
3628 	ml |= sc->vmx_lladdr[2] << 16;
3629 	ml |= sc->vmx_lladdr[3] << 24;
3630 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml);
3631 
3632 	mh  = sc->vmx_lladdr[4];
3633 	mh |= sc->vmx_lladdr[5] << 8;
3634 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh);
3635 }
3636 
3637 static void
3638 vmxnet3_get_lladdr(struct vmxnet3_softc *sc)
3639 {
3640 	uint32_t ml, mh;
3641 
3642 	ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL);
3643 	mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH);
3644 
3645 	sc->vmx_lladdr[0] = ml;
3646 	sc->vmx_lladdr[1] = ml >> 8;
3647 	sc->vmx_lladdr[2] = ml >> 16;
3648 	sc->vmx_lladdr[3] = ml >> 24;
3649 	sc->vmx_lladdr[4] = mh;
3650 	sc->vmx_lladdr[5] = mh >> 8;
3651 }
3652 
3653 static void
3654 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq,
3655     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3656 {
3657 	struct sysctl_oid *node, *txsnode;
3658 	struct sysctl_oid_list *list, *txslist;
3659 	struct vmxnet3_txq_stats *stats;
3660 	struct UPT1_TxStats *txstats;
3661 	char namebuf[16];
3662 
3663 	stats = &txq->vxtxq_stats;
3664 	txstats = &txq->vxtxq_ts->stats;
3665 
3666 	ksnprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id);
3667 	node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD,
3668 	    NULL, "Transmit Queue");
3669 	txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node);
3670 
3671 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD,
3672 	    &stats->vmtxs_opackets, 0, "Transmit packets");
3673 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD,
3674 	    &stats->vmtxs_obytes, 0, "Transmit bytes");
3675 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD,
3676 	    &stats->vmtxs_omcasts, 0, "Transmit multicasts");
3677 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD,
3678 	    &stats->vmtxs_csum, 0, "Transmit checksum offloaded");
3679 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD,
3680 	    &stats->vmtxs_tso, 0, "Transmit TCP segmentation offloaded");
3681 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD,
3682 	    &stats->vmtxs_full, 0, "Transmit ring full");
3683 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD,
3684 	    &stats->vmtxs_offload_failed, 0, "Transmit checksum offload failed");
3685 
3686 	/*
3687 	 * Add statistics reported by the host. These are updated once
3688 	 * per second.
3689 	 */
3690 	txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD,
3691 	    NULL, "Host Statistics");
3692 	txslist = SYSCTL_CHILDREN(txsnode);
3693 #if 0 /* XXX TSO */
3694 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD,
3695 	    &txstats->TSO_packets, 0, "TSO packets");
3696 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD,
3697 	    &txstats->TSO_bytes, 0, "TSO bytes");
3698 #endif
3699 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
3700 	    &txstats->ucast_packets, 0, "Unicast packets");
3701 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
3702 	    &txstats->ucast_bytes, 0, "Unicast bytes");
3703 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
3704 	    &txstats->mcast_packets, 0, "Multicast packets");
3705 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
3706 	    &txstats->mcast_bytes, 0, "Multicast bytes");
3707 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD,
3708 	    &txstats->error, 0, "Errors");
3709 	SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD,
3710 	    &txstats->discard, 0, "Discards");
3711 }
3712 
3713 static void
3714 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq,
3715     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3716 {
3717 	struct sysctl_oid *node, *rxsnode;
3718 	struct sysctl_oid_list *list, *rxslist;
3719 	struct vmxnet3_rxq_stats *stats;
3720 	struct UPT1_RxStats *rxstats;
3721 	char namebuf[16];
3722 
3723 	stats = &rxq->vxrxq_stats;
3724 	rxstats = &rxq->vxrxq_rs->stats;
3725 
3726 	ksnprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id);
3727 	node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD,
3728 	    NULL, "Receive Queue");
3729 	rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node);
3730 
3731 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD,
3732 	    &stats->vmrxs_ipackets, 0, "Receive packets");
3733 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD,
3734 	    &stats->vmrxs_ibytes, 0, "Receive bytes");
3735 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD,
3736 	    &stats->vmrxs_iqdrops, 0, "Receive drops");
3737 	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD,
3738 	    &stats->vmrxs_ierrors, 0, "Receive errors");
3739 
3740 	/*
3741 	 * Add statistics reported by the host. These are updated once
3742 	 * per second.
3743 	 */
3744 	rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD,
3745 	    NULL, "Host Statistics");
3746 	rxslist = SYSCTL_CHILDREN(rxsnode);
3747 #if 0 /* XXX LRO */
3748 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD,
3749 	    &rxstats->LRO_packets, 0, "LRO packets");
3750 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD,
3751 	    &rxstats->LRO_bytes, 0, "LRO bytes");
3752 #endif
3753 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
3754 	    &rxstats->ucast_packets, 0, "Unicast packets");
3755 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
3756 	    &rxstats->ucast_bytes, 0, "Unicast bytes");
3757 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
3758 	    &rxstats->mcast_packets, 0, "Multicast packets");
3759 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
3760 	    &rxstats->mcast_bytes, 0, "Multicast bytes");
3761 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD,
3762 	    &rxstats->bcast_packets, 0, "Broadcast packets");
3763 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD,
3764 	    &rxstats->bcast_bytes, 0, "Broadcast bytes");
3765 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD,
3766 	    &rxstats->nobuffer, 0, "No buffer");
3767 	SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD,
3768 	    &rxstats->error, 0, "Errors");
3769 }
3770 
3771 static void
3772 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc,
3773     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3774 {
3775 	struct sysctl_oid *node;
3776 	struct sysctl_oid_list *list;
3777 	int i;
3778 
3779 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
3780 		struct vmxnet3_txqueue *txq = &sc->vmx_txq[i];
3781 
3782 		node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO,
3783 		    "debug", CTLFLAG_RD, NULL, "");
3784 		list = SYSCTL_CHILDREN(node);
3785 
3786 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD,
3787 		    &txq->vxtxq_cmd_ring.vxtxr_head, 0, "");
3788 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD,
3789 		    &txq->vxtxq_cmd_ring.vxtxr_next, 0, "");
3790 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD,
3791 		    &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, "");
3792 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD,
3793 		    &txq->vxtxq_cmd_ring.vxtxr_gen, 0, "");
3794 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
3795 		    &txq->vxtxq_comp_ring.vxcr_next, 0, "");
3796 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
3797 		    &txq->vxtxq_comp_ring.vxcr_ndesc, 0,"");
3798 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
3799 		    &txq->vxtxq_comp_ring.vxcr_gen, 0, "");
3800 	}
3801 
3802 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
3803 		struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i];
3804 
3805 		node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO,
3806 		    "debug", CTLFLAG_RD, NULL, "");
3807 		list = SYSCTL_CHILDREN(node);
3808 
3809 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD,
3810 		    &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, "");
3811 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD,
3812 		    &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, "");
3813 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD,
3814 		    &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, "");
3815 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD,
3816 		    &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, "");
3817 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD,
3818 		    &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, "");
3819 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD,
3820 		    &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, "");
3821 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
3822 		    &rxq->vxrxq_comp_ring.vxcr_next, 0, "");
3823 		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
3824 		    &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,"");
3825 		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
3826 		    &rxq->vxrxq_comp_ring.vxcr_gen, 0, "");
3827 	}
3828 }
3829 
3830 static void
3831 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc,
3832     struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
3833 {
3834 	int i;
3835 
3836 	for (i = 0; i < sc->vmx_ntxqueues; i++)
3837 		vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child);
3838 	for (i = 0; i < sc->vmx_nrxqueues; i++)
3839 		vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child);
3840 
3841 	vmxnet3_setup_debug_sysctl(sc, ctx, child);
3842 }
3843 
3844 static void
3845 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc)
3846 {
3847 	device_t dev;
3848 	struct vmxnet3_statistics *stats;
3849 	struct sysctl_ctx_list *ctx;
3850 	struct sysctl_oid *tree;
3851 	struct sysctl_oid_list *child;
3852 
3853 	dev = sc->vmx_dev;
3854 	ctx = device_get_sysctl_ctx(dev);
3855 	tree = device_get_sysctl_tree(dev);
3856 	child = SYSCTL_CHILDREN(tree);
3857 
3858 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD,
3859 	    &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues");
3860 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD,
3861 	    &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues");
3862 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD,
3863 	    &sc->vmx_ntxqueues, 0, "Number of Tx queues");
3864 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD,
3865 	    &sc->vmx_nrxqueues, 0, "Number of Rx queues");
3866 
3867 	stats = &sc->vmx_stats;
3868 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD,
3869 	    &stats->vmst_defragged, 0, "Tx mbuf chains defragged");
3870 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD,
3871 	    &stats->vmst_defrag_failed, 0,
3872 	    "Tx mbuf dropped because defrag failed");
3873 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD,
3874 	    &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed");
3875 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD,
3876 	    &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed");
3877 
3878 	vmxnet3_setup_queue_sysctl(sc, ctx, child);
3879 }
3880 
3881 static void
3882 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
3883 {
3884 
3885 	bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v);
3886 }
3887 
3888 static uint32_t
3889 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r)
3890 {
3891 
3892 	return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r));
3893 }
3894 
3895 static void
3896 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
3897 {
3898 
3899 	bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v);
3900 }
3901 
3902 static void
3903 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
3904 {
3905 
3906 	vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd);
3907 }
3908 
3909 static uint32_t
3910 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
3911 {
3912 
3913 	vmxnet3_write_cmd(sc, cmd);
3914 	bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0,
3915 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
3916 	return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD));
3917 }
3918 
3919 static void
3920 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq)
3921 {
3922 
3923 	vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0);
3924 }
3925 
3926 static void
3927 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq)
3928 {
3929 
3930 	vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1);
3931 }
3932 
3933 static void
3934 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc)
3935 {
3936 	int i;
3937 
3938 	sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL;
3939 	for (i = 0; i < sc->vmx_nintrs; i++)
3940 		vmxnet3_enable_intr(sc, i);
3941 }
3942 
3943 static void
3944 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc)
3945 {
3946 	int i;
3947 
3948 	sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL;
3949 	for (i = 0; i < sc->vmx_nintrs; i++)
3950 		vmxnet3_disable_intr(sc, i);
3951 }
3952 
3953 static void
3954 vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3955 {
3956 	bus_addr_t *baddr = arg;
3957 
3958 	if (error == 0)
3959 		*baddr = segs->ds_addr;
3960 }
3961 
3962 static int
3963 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align,
3964     struct vmxnet3_dma_alloc *dma)
3965 {
3966 	device_t dev;
3967 	int error;
3968 
3969 	dev = sc->vmx_dev;
3970 	bzero(dma, sizeof(struct vmxnet3_dma_alloc));
3971 
3972 	error = bus_dma_tag_create(bus_get_dma_tag(dev),
3973 	    align, 0,		/* alignment, bounds */
3974 	    BUS_SPACE_MAXADDR,	/* lowaddr */
3975 	    BUS_SPACE_MAXADDR,	/* highaddr */
3976 	    NULL, NULL,		/* filter, filterarg */
3977 	    size,		/* maxsize */
3978 	    1,			/* nsegments */
3979 	    size,		/* maxsegsize */
3980 	    BUS_DMA_ALLOCNOW,	/* flags */
3981 	    &dma->dma_tag);
3982 	if (error) {
3983 		device_printf(dev, "bus_dma_tag_create failed: %d\n", error);
3984 		goto fail;
3985 	}
3986 
3987 	error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
3988 	    BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map);
3989 	if (error) {
3990 		device_printf(dev, "bus_dmamem_alloc failed: %d\n", error);
3991 		goto fail;
3992 	}
3993 
3994 	error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3995 	    size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT);
3996 	if (error) {
3997 		device_printf(dev, "bus_dmamap_load failed: %d\n", error);
3998 		goto fail;
3999 	}
4000 
4001 	dma->dma_size = size;
4002 
4003 fail:
4004 	if (error)
4005 		vmxnet3_dma_free(sc, dma);
4006 
4007 	return (error);
4008 }
4009 
4010 static void
4011 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma)
4012 {
4013 
4014 	if (dma->dma_tag != NULL) {
4015 		if (dma->dma_paddr != 0) {
4016 			bus_dmamap_sync(dma->dma_tag, dma->dma_map,
4017 			    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4018 			bus_dmamap_unload(dma->dma_tag, dma->dma_map);
4019 		}
4020 
4021 		if (dma->dma_vaddr != NULL) {
4022 			bus_dmamem_free(dma->dma_tag, dma->dma_vaddr,
4023 			    dma->dma_map);
4024 		}
4025 
4026 		bus_dma_tag_destroy(dma->dma_tag);
4027 	}
4028 	bzero(dma, sizeof(struct vmxnet3_dma_alloc));
4029 }
4030 
4031 static int
4032 vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def)
4033 {
4034 	char path[64];
4035 
4036 	ksnprintf(path, sizeof(path),
4037 	    "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob);
4038 	TUNABLE_INT_FETCH(path, &def);
4039 
4040 	return (def);
4041 }
4042 
4043 #define mb()	__asm volatile("mfence" ::: "memory")
4044 #define wmb()	__asm volatile("sfence" ::: "memory")
4045 #define rmb()	__asm volatile("lfence" ::: "memory")
4046 
4047 /*
4048  * Since this is a purely paravirtualized device, we do not have
4049  * to worry about DMA coherency. But at times, we must make sure
4050  * both the compiler and CPU do not reorder memory operations.
4051  */
4052 static inline void
4053 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type)
4054 {
4055 
4056 	switch (type) {
4057 	case VMXNET3_BARRIER_RD:
4058 		rmb();
4059 		break;
4060 	case VMXNET3_BARRIER_WR:
4061 		wmb();
4062 		break;
4063 	case VMXNET3_BARRIER_RDWR:
4064 		mb();
4065 		break;
4066 	default:
4067 		panic("%s: bad barrier type %d", __func__, type);
4068 	}
4069 }
4070