1*63d58f04Sclaudio /* $OpenBSD: vnet.c,v 1.67 2023/03/09 10:29:04 claudio Exp $ */
2ba14339aSkettenis /*
390354c4fSkettenis * Copyright (c) 2009, 2015 Mark Kettenis
4ba14339aSkettenis *
5ba14339aSkettenis * Permission to use, copy, modify, and distribute this software for any
6ba14339aSkettenis * purpose with or without fee is hereby granted, provided that the above
7ba14339aSkettenis * copyright notice and this permission notice appear in all copies.
8ba14339aSkettenis *
9ba14339aSkettenis * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10ba14339aSkettenis * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11ba14339aSkettenis * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12ba14339aSkettenis * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13ba14339aSkettenis * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14ba14339aSkettenis * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15ba14339aSkettenis * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16ba14339aSkettenis */
17ba14339aSkettenis
18ba14339aSkettenis #include "bpfilter.h"
19ba14339aSkettenis
20ba14339aSkettenis #include <sys/param.h>
2190354c4fSkettenis #include <sys/atomic.h>
22ba14339aSkettenis #include <sys/device.h>
23ba14339aSkettenis #include <sys/malloc.h>
2490ac40a5Sdlg #include <sys/pool.h>
25ba14339aSkettenis #include <sys/mbuf.h>
26ba14339aSkettenis #include <sys/socket.h>
27ba14339aSkettenis #include <sys/sockio.h>
28ba14339aSkettenis #include <sys/systm.h>
29d7091787Skettenis #include <sys/timeout.h>
30ba14339aSkettenis
31ba14339aSkettenis #include <machine/autoconf.h>
32ba14339aSkettenis #include <machine/hypervisor.h>
33ba14339aSkettenis #include <machine/openfirm.h>
34ba14339aSkettenis
35ba14339aSkettenis #include <net/if.h>
36ba14339aSkettenis #include <net/if_media.h>
37ba14339aSkettenis
38ba14339aSkettenis #include <netinet/in.h>
39ba14339aSkettenis #include <netinet/if_ether.h>
40ba14339aSkettenis
41ba14339aSkettenis #if NBPFILTER > 0
42ba14339aSkettenis #include <net/bpf.h>
43ba14339aSkettenis #endif
44ba14339aSkettenis
458716a13cSmpi #include <uvm/uvm_extern.h>
46ba14339aSkettenis
47ba14339aSkettenis #include <sparc64/dev/cbusvar.h>
48d9889caeSkettenis #include <sparc64/dev/ldcvar.h>
49ac50164fSkettenis #include <sparc64/dev/viovar.h>
50ba14339aSkettenis
51c736a730Skettenis #ifdef VNET_DEBUG
52c736a730Skettenis #define DPRINTF(x) printf x
53c736a730Skettenis #else
54c736a730Skettenis #define DPRINTF(x)
55c736a730Skettenis #endif
56c736a730Skettenis
57*63d58f04Sclaudio #define VNET_BUF_SIZE 2048
58*63d58f04Sclaudio
59*63d58f04Sclaudio /* 128 * 64 = 8192 which is a full page */
60*63d58f04Sclaudio #define VNET_TX_ENTRIES 128
61*63d58f04Sclaudio #define VNET_RX_ENTRIES 128
62ba14339aSkettenis
63ba14339aSkettenis struct vnet_attr_info {
64ba14339aSkettenis struct vio_msg_tag tag;
65ba14339aSkettenis uint8_t xfer_mode;
66ba14339aSkettenis uint8_t addr_type;
67ba14339aSkettenis uint16_t ack_freq;
681ad61d19Skettenis uint32_t _reserved1;
69ba14339aSkettenis uint64_t addr;
70ba14339aSkettenis uint64_t mtu;
711ad61d19Skettenis uint64_t _reserved2[3];
72ba14339aSkettenis };
73ba14339aSkettenis
74ac50164fSkettenis /* Address types. */
75ac50164fSkettenis #define VNET_ADDR_ETHERMAC 0x01
76ac50164fSkettenis
77ac50164fSkettenis /* Sub-Type envelopes. */
78ac50164fSkettenis #define VNET_MCAST_INFO 0x0101
79ac50164fSkettenis
80ac50164fSkettenis #define VNET_NUM_MCAST 7
81ba14339aSkettenis
82a0c88015Skettenis struct vnet_mcast_info {
83a0c88015Skettenis struct vio_msg_tag tag;
84a0c88015Skettenis uint8_t set;
85a0c88015Skettenis uint8_t count;
86ac50164fSkettenis uint8_t mcast_addr[VNET_NUM_MCAST][ETHER_ADDR_LEN];
87a0c88015Skettenis uint32_t _reserved;
88a0c88015Skettenis };
89a0c88015Skettenis
90ba14339aSkettenis struct vnet_desc {
91ba14339aSkettenis struct vio_dring_hdr hdr;
92ba14339aSkettenis uint32_t nbytes;
93ba14339aSkettenis uint32_t ncookies;
94ba14339aSkettenis struct ldc_cookie cookie[2];
95ba14339aSkettenis };
96ba14339aSkettenis
9708c8e8d2Skettenis struct vnet_desc_msg {
9808c8e8d2Skettenis struct vio_msg_tag tag;
9908c8e8d2Skettenis uint64_t seq_no;
10008c8e8d2Skettenis uint64_t desc_handle;
10108c8e8d2Skettenis uint32_t nbytes;
10208c8e8d2Skettenis uint32_t ncookies;
10308c8e8d2Skettenis struct ldc_cookie cookie[1];
10408c8e8d2Skettenis };
10508c8e8d2Skettenis
106ba14339aSkettenis struct vnet_dring {
107ba14339aSkettenis bus_dmamap_t vd_map;
108ba14339aSkettenis bus_dma_segment_t vd_seg;
109ba14339aSkettenis struct vnet_desc *vd_desc;
110ba14339aSkettenis int vd_nentries;
111ba14339aSkettenis };
112ba14339aSkettenis
113ba14339aSkettenis struct vnet_dring *vnet_dring_alloc(bus_dma_tag_t, int);
114ba14339aSkettenis void vnet_dring_free(bus_dma_tag_t, struct vnet_dring *);
115ba14339aSkettenis
116ba14339aSkettenis /*
117ba14339aSkettenis * For now, we only support vNet 1.0.
118ba14339aSkettenis */
119ba14339aSkettenis #define VNET_MAJOR 1
120ba14339aSkettenis #define VNET_MINOR 0
121ba14339aSkettenis
122c736a730Skettenis /*
123c736a730Skettenis * The vNet protocol wants the IP header to be 64-bit aligned, so
124c736a730Skettenis * define out own variant of ETHER_ALIGN.
125c736a730Skettenis */
126c736a730Skettenis #define VNET_ETHER_ALIGN 6
127c736a730Skettenis
128ba14339aSkettenis struct vnet_soft_desc {
129ba14339aSkettenis int vsd_map_idx;
130ba14339aSkettenis caddr_t vsd_buf;
131ba14339aSkettenis };
132ba14339aSkettenis
133ba14339aSkettenis struct vnet_softc {
134ba14339aSkettenis struct device sc_dv;
135ba14339aSkettenis bus_space_tag_t sc_bustag;
136ba14339aSkettenis bus_dma_tag_t sc_dmatag;
137ba14339aSkettenis
138b3a497edSkettenis uint64_t sc_tx_ino;
139b3a497edSkettenis uint64_t sc_rx_ino;
140ba14339aSkettenis void *sc_tx_ih;
141ba14339aSkettenis void *sc_rx_ih;
142ba14339aSkettenis
143d9889caeSkettenis struct ldc_conn sc_lc;
144ba14339aSkettenis
1456d2008c7Skettenis uint16_t sc_vio_state;
1466d2008c7Skettenis #define VIO_SND_VER_INFO 0x0001
1476d2008c7Skettenis #define VIO_ACK_VER_INFO 0x0002
1486d2008c7Skettenis #define VIO_RCV_VER_INFO 0x0004
1496d2008c7Skettenis #define VIO_SND_ATTR_INFO 0x0008
1506d2008c7Skettenis #define VIO_ACK_ATTR_INFO 0x0010
1516d2008c7Skettenis #define VIO_RCV_ATTR_INFO 0x0020
1526d2008c7Skettenis #define VIO_SND_DRING_REG 0x0040
1536d2008c7Skettenis #define VIO_ACK_DRING_REG 0x0080
1546d2008c7Skettenis #define VIO_RCV_DRING_REG 0x0100
1556d2008c7Skettenis #define VIO_SND_RDX 0x0200
1566d2008c7Skettenis #define VIO_ACK_RDX 0x0400
1576d2008c7Skettenis #define VIO_RCV_RDX 0x0800
15808c8e8d2Skettenis
159d7091787Skettenis struct timeout sc_handshake_to;
160d7091787Skettenis
16108c8e8d2Skettenis uint8_t sc_xfer_mode;
162ba14339aSkettenis
163ba14339aSkettenis uint32_t sc_local_sid;
164ba14339aSkettenis uint64_t sc_dring_ident;
165ba14339aSkettenis uint64_t sc_seq_no;
166ba14339aSkettenis
16790354c4fSkettenis u_int sc_tx_prod;
16890354c4fSkettenis u_int sc_tx_cons;
169ba14339aSkettenis
17090354c4fSkettenis u_int sc_peer_state;
171a212e6c2Skettenis
172ba14339aSkettenis struct ldc_map *sc_lm;
173ba14339aSkettenis struct vnet_dring *sc_vd;
174ba14339aSkettenis struct vnet_soft_desc *sc_vsd;
1751373cec8Sstsp #define VNET_NUM_SOFT_DESC 128
176ba14339aSkettenis
177ba14339aSkettenis size_t sc_peer_desc_size;
178ba14339aSkettenis struct ldc_cookie sc_peer_dring_cookie;
179ba14339aSkettenis int sc_peer_dring_nentries;
180ba14339aSkettenis
181ba14339aSkettenis struct pool sc_pool;
182ba14339aSkettenis
183ba14339aSkettenis struct arpcom sc_ac;
184ba14339aSkettenis struct ifmedia sc_media;
185ba14339aSkettenis };
186ba14339aSkettenis
187ba14339aSkettenis int vnet_match(struct device *, void *, void *);
188ba14339aSkettenis void vnet_attach(struct device *, struct device *, void *);
189ba14339aSkettenis
190eb7eaf8dSmpi const struct cfattach vnet_ca = {
191ba14339aSkettenis sizeof(struct vnet_softc), vnet_match, vnet_attach
192ba14339aSkettenis };
193ba14339aSkettenis
194ba14339aSkettenis struct cfdriver vnet_cd = {
195ba14339aSkettenis NULL, "vnet", DV_IFNET
196ba14339aSkettenis };
197ba14339aSkettenis
198ba14339aSkettenis int vnet_tx_intr(void *);
199ba14339aSkettenis int vnet_rx_intr(void *);
200d7091787Skettenis void vnet_handshake(void *);
201ba14339aSkettenis
202d9889caeSkettenis void vio_rx_data(struct ldc_conn *, struct ldc_pkt *);
203ba14339aSkettenis void vnet_rx_vio_ctrl(struct vnet_softc *, struct vio_msg *);
204ba14339aSkettenis void vnet_rx_vio_ver_info(struct vnet_softc *, struct vio_msg_tag *);
205ba14339aSkettenis void vnet_rx_vio_attr_info(struct vnet_softc *, struct vio_msg_tag *);
206ba14339aSkettenis void vnet_rx_vio_dring_reg(struct vnet_softc *, struct vio_msg_tag *);
207ba14339aSkettenis void vnet_rx_vio_rdx(struct vnet_softc *sc, struct vio_msg_tag *);
208ba14339aSkettenis void vnet_rx_vio_data(struct vnet_softc *sc, struct vio_msg *);
20908c8e8d2Skettenis void vnet_rx_vio_desc_data(struct vnet_softc *sc, struct vio_msg_tag *);
210ba14339aSkettenis void vnet_rx_vio_dring_data(struct vnet_softc *sc, struct vio_msg_tag *);
211ba14339aSkettenis
212a7acb8c5Skettenis void vnet_ldc_reset(struct ldc_conn *);
213a7acb8c5Skettenis void vnet_ldc_start(struct ldc_conn *);
214ba14339aSkettenis
2153d4b6a30Skettenis void vnet_sendmsg(struct vnet_softc *, void *, size_t);
216a7acb8c5Skettenis void vnet_send_ver_info(struct vnet_softc *, uint16_t, uint16_t);
217ba14339aSkettenis void vnet_send_attr_info(struct vnet_softc *);
218ba14339aSkettenis void vnet_send_dring_reg(struct vnet_softc *);
219ba14339aSkettenis void vio_send_rdx(struct vnet_softc *);
220a212e6c2Skettenis void vnet_send_dring_data(struct vnet_softc *, uint32_t);
221ba14339aSkettenis
222ba14339aSkettenis void vnet_start(struct ifnet *);
22308c8e8d2Skettenis void vnet_start_desc(struct ifnet *);
224ba14339aSkettenis int vnet_ioctl(struct ifnet *, u_long, caddr_t);
225ba14339aSkettenis void vnet_watchdog(struct ifnet *);
226ba14339aSkettenis
227ba14339aSkettenis int vnet_media_change(struct ifnet *);
228ba14339aSkettenis void vnet_media_status(struct ifnet *, struct ifmediareq *);
229ba14339aSkettenis
23039c67f9aSkettenis void vnet_link_state(struct vnet_softc *sc);
23139c67f9aSkettenis
232a0c88015Skettenis void vnet_setmulti(struct vnet_softc *, int);
233a0c88015Skettenis
234ba14339aSkettenis void vnet_init(struct ifnet *);
235ba14339aSkettenis void vnet_stop(struct ifnet *);
236ba14339aSkettenis
237ba14339aSkettenis int
vnet_match(struct device * parent,void * match,void * aux)238ba14339aSkettenis vnet_match(struct device *parent, void *match, void *aux)
239ba14339aSkettenis {
240ba14339aSkettenis struct cbus_attach_args *ca = aux;
241ba14339aSkettenis
242ba14339aSkettenis if (strcmp(ca->ca_name, "network") == 0)
243ba14339aSkettenis return (1);
244ba14339aSkettenis
245ba14339aSkettenis return (0);
246ba14339aSkettenis }
247ba14339aSkettenis
248ba14339aSkettenis void
vnet_attach(struct device * parent,struct device * self,void * aux)249ba14339aSkettenis vnet_attach(struct device *parent, struct device *self, void *aux)
250ba14339aSkettenis {
251ba14339aSkettenis struct vnet_softc *sc = (struct vnet_softc *)self;
252ba14339aSkettenis struct cbus_attach_args *ca = aux;
253d9889caeSkettenis struct ldc_conn *lc;
254ba14339aSkettenis struct ifnet *ifp;
255ba14339aSkettenis
256ba14339aSkettenis sc->sc_bustag = ca->ca_bustag;
257ba14339aSkettenis sc->sc_dmatag = ca->ca_dmatag;
258b3a497edSkettenis sc->sc_tx_ino = ca->ca_tx_ino;
259b3a497edSkettenis sc->sc_rx_ino = ca->ca_rx_ino;
260ba14339aSkettenis
261b3a497edSkettenis printf(": ivec 0x%llx, 0x%llx", sc->sc_tx_ino, sc->sc_rx_ino);
262ba14339aSkettenis
263ba14339aSkettenis /*
264ba14339aSkettenis * Un-configure queues before registering interrupt handlers,
265ba14339aSkettenis * such that we dont get any stale LDC packets or events.
266ba14339aSkettenis */
267d9889caeSkettenis hv_ldc_tx_qconf(ca->ca_id, 0, 0);
268d9889caeSkettenis hv_ldc_rx_qconf(ca->ca_id, 0, 0);
269ba14339aSkettenis
270b3a497edSkettenis sc->sc_tx_ih = bus_intr_establish(ca->ca_bustag, sc->sc_tx_ino,
2718dd7c4a8Skettenis IPL_NET, BUS_INTR_ESTABLISH_MPSAFE, vnet_tx_intr,
2728dd7c4a8Skettenis sc, sc->sc_dv.dv_xname);
273b3a497edSkettenis sc->sc_rx_ih = bus_intr_establish(ca->ca_bustag, sc->sc_rx_ino,
2748dd7c4a8Skettenis IPL_NET, BUS_INTR_ESTABLISH_MPSAFE, vnet_rx_intr,
2758dd7c4a8Skettenis sc, sc->sc_dv.dv_xname);
276ba14339aSkettenis if (sc->sc_tx_ih == NULL || sc->sc_rx_ih == NULL) {
277ba14339aSkettenis printf(", can't establish interrupt\n");
278ba14339aSkettenis return;
279ba14339aSkettenis }
280ba14339aSkettenis
281d9889caeSkettenis lc = &sc->sc_lc;
282d9889caeSkettenis lc->lc_id = ca->ca_id;
283d9889caeSkettenis lc->lc_sc = sc;
284a7acb8c5Skettenis lc->lc_reset = vnet_ldc_reset;
285a7acb8c5Skettenis lc->lc_start = vnet_ldc_start;
286d9889caeSkettenis lc->lc_rx_data = vio_rx_data;
287d9889caeSkettenis
288d7091787Skettenis timeout_set(&sc->sc_handshake_to, vnet_handshake, sc);
28990354c4fSkettenis sc->sc_peer_state = VIO_DP_STOPPED;
290d7091787Skettenis
291d9889caeSkettenis lc->lc_txq = ldc_queue_alloc(sc->sc_dmatag, VNET_TX_ENTRIES);
292d9889caeSkettenis if (lc->lc_txq == NULL) {
293ba14339aSkettenis printf(", can't allocate tx queue\n");
294ba14339aSkettenis return;
295ba14339aSkettenis }
296ba14339aSkettenis
297d9889caeSkettenis lc->lc_rxq = ldc_queue_alloc(sc->sc_dmatag, VNET_RX_ENTRIES);
298d9889caeSkettenis if (lc->lc_rxq == NULL) {
299ba14339aSkettenis printf(", can't allocate rx queue\n");
300ba14339aSkettenis goto free_txqueue;
301ba14339aSkettenis }
302ba14339aSkettenis
30315ed68a5Skettenis if (OF_getprop(ca->ca_node, "local-mac-address",
30415ed68a5Skettenis sc->sc_ac.ac_enaddr, ETHER_ADDR_LEN) > 0)
30515ed68a5Skettenis printf(", address %s", ether_sprintf(sc->sc_ac.ac_enaddr));
30615ed68a5Skettenis
307ba14339aSkettenis /*
308ba14339aSkettenis * Each interface gets its own pool.
309ba14339aSkettenis */
310*63d58f04Sclaudio pool_init(&sc->sc_pool, VNET_BUF_SIZE, 0, IPL_NET, 0,
311*63d58f04Sclaudio sc->sc_dv.dv_xname, NULL);
312ba14339aSkettenis
313ba14339aSkettenis ifp = &sc->sc_ac.ac_if;
314ba14339aSkettenis ifp->if_softc = sc;
315ba14339aSkettenis ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3169464f18aSkettenis ifp->if_link_state = LINK_STATE_DOWN;
317ba14339aSkettenis ifp->if_ioctl = vnet_ioctl;
318ba14339aSkettenis ifp->if_start = vnet_start;
319ba14339aSkettenis ifp->if_watchdog = vnet_watchdog;
320ba14339aSkettenis strlcpy(ifp->if_xname, sc->sc_dv.dv_xname, IFNAMSIZ);
321ba14339aSkettenis
322ba14339aSkettenis ifmedia_init(&sc->sc_media, 0, vnet_media_change, vnet_media_status);
323ba14339aSkettenis ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
324ba14339aSkettenis ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
325ba14339aSkettenis
326ba14339aSkettenis if_attach(ifp);
327ba14339aSkettenis ether_ifattach(ifp);
328ba14339aSkettenis
32915ed68a5Skettenis printf("\n");
330ba14339aSkettenis return;
331ba14339aSkettenis
332ba14339aSkettenis free_txqueue:
333d9889caeSkettenis ldc_queue_free(sc->sc_dmatag, lc->lc_txq);
334ba14339aSkettenis }
335ba14339aSkettenis
336ba14339aSkettenis int
vnet_tx_intr(void * arg)337ba14339aSkettenis vnet_tx_intr(void *arg)
338ba14339aSkettenis {
339ba14339aSkettenis struct vnet_softc *sc = arg;
340d9889caeSkettenis struct ldc_conn *lc = &sc->sc_lc;
341ba14339aSkettenis uint64_t tx_head, tx_tail, tx_state;
342ba14339aSkettenis
343d9889caeSkettenis hv_ldc_tx_get_state(lc->lc_id, &tx_head, &tx_tail, &tx_state);
344d9889caeSkettenis if (tx_state != lc->lc_tx_state) {
345ba14339aSkettenis switch (tx_state) {
346ba14339aSkettenis case LDC_CHANNEL_DOWN:
3478f296756Sstsp DPRINTF(("%s: Tx link down\n", __func__));
348ba14339aSkettenis break;
349ba14339aSkettenis case LDC_CHANNEL_UP:
3508f296756Sstsp DPRINTF(("%s: Tx link up\n", __func__));
351ba14339aSkettenis break;
352ba14339aSkettenis case LDC_CHANNEL_RESET:
3538f296756Sstsp DPRINTF(("%s: Tx link reset\n", __func__));
354ba14339aSkettenis break;
355ba14339aSkettenis }
356d9889caeSkettenis lc->lc_tx_state = tx_state;
357ba14339aSkettenis }
358ba14339aSkettenis
359ba14339aSkettenis return (1);
360ba14339aSkettenis }
361ba14339aSkettenis
362ba14339aSkettenis int
vnet_rx_intr(void * arg)363ba14339aSkettenis vnet_rx_intr(void *arg)
364ba14339aSkettenis {
365ba14339aSkettenis struct vnet_softc *sc = arg;
366d9889caeSkettenis struct ldc_conn *lc = &sc->sc_lc;
367ba14339aSkettenis uint64_t rx_head, rx_tail, rx_state;
368d9889caeSkettenis struct ldc_pkt *lp;
369ba14339aSkettenis int err;
370ba14339aSkettenis
371d9889caeSkettenis err = hv_ldc_rx_get_state(lc->lc_id, &rx_head, &rx_tail, &rx_state);
372ba14339aSkettenis if (err == H_EINVAL)
373ba14339aSkettenis return (0);
374ba14339aSkettenis if (err != H_EOK) {
375ba14339aSkettenis printf("hv_ldc_rx_get_state %d\n", err);
376ba14339aSkettenis return (0);
377ba14339aSkettenis }
378ba14339aSkettenis
379d9889caeSkettenis if (rx_state != lc->lc_rx_state) {
380ba14339aSkettenis switch (rx_state) {
381ba14339aSkettenis case LDC_CHANNEL_DOWN:
3828f296756Sstsp DPRINTF(("%s: Rx link down\n", __func__));
383cd52ad88Skettenis lc->lc_tx_seqid = 0;
384cd52ad88Skettenis lc->lc_state = 0;
385cd52ad88Skettenis lc->lc_reset(lc);
386566caca0Sstsp if (rx_head == rx_tail)
387566caca0Sstsp break;
388566caca0Sstsp /* Discard and ack pending I/O. */
389566caca0Sstsp DPRINTF(("setting rx qhead to %lld\n", rx_tail));
390566caca0Sstsp err = hv_ldc_rx_set_qhead(lc->lc_id, rx_tail);
391566caca0Sstsp if (err == H_EOK)
392566caca0Sstsp break;
393566caca0Sstsp printf("%s: hv_ldc_rx_set_qhead %d\n", __func__, err);
394ba14339aSkettenis break;
395ba14339aSkettenis case LDC_CHANNEL_UP:
3968f296756Sstsp DPRINTF(("%s: Rx link up\n", __func__));
397d7091787Skettenis timeout_add_msec(&sc->sc_handshake_to, 500);
398ba14339aSkettenis break;
399ba14339aSkettenis case LDC_CHANNEL_RESET:
4008f296756Sstsp DPRINTF(("%s: Rx link reset\n", __func__));
401cd52ad88Skettenis lc->lc_tx_seqid = 0;
402cd52ad88Skettenis lc->lc_state = 0;
403cd52ad88Skettenis lc->lc_reset(lc);
4049464f18aSkettenis timeout_add_msec(&sc->sc_handshake_to, 500);
405566caca0Sstsp if (rx_head == rx_tail)
406566caca0Sstsp break;
407566caca0Sstsp /* Discard and ack pending I/O. */
408566caca0Sstsp DPRINTF(("setting rx qhead to %lld\n", rx_tail));
409566caca0Sstsp err = hv_ldc_rx_set_qhead(lc->lc_id, rx_tail);
410566caca0Sstsp if (err == H_EOK)
411566caca0Sstsp break;
412566caca0Sstsp printf("%s: hv_ldc_rx_set_qhead %d\n", __func__, err);
413ba14339aSkettenis break;
414ba14339aSkettenis }
415d9889caeSkettenis lc->lc_rx_state = rx_state;
416ba14339aSkettenis return (1);
417ba14339aSkettenis }
418ba14339aSkettenis
419d7091787Skettenis if (rx_head == rx_tail)
420d7091787Skettenis return (0);
421d7091787Skettenis
422d9889caeSkettenis lp = (struct ldc_pkt *)(lc->lc_rxq->lq_va + rx_head);
423d9889caeSkettenis switch (lp->type) {
424ba14339aSkettenis case LDC_CTRL:
425d9889caeSkettenis ldc_rx_ctrl(lc, lp);
426ba14339aSkettenis break;
427ba14339aSkettenis
428ba14339aSkettenis case LDC_DATA:
429d9889caeSkettenis ldc_rx_data(lc, lp);
430ba14339aSkettenis break;
431ba14339aSkettenis
432ba14339aSkettenis default:
433d9889caeSkettenis DPRINTF(("%0x02/%0x02/%0x02\n", lp->type, lp->stype,
434d9889caeSkettenis lp->ctrl));
435d9889caeSkettenis ldc_reset(lc);
436ba14339aSkettenis break;
437ba14339aSkettenis }
438ba14339aSkettenis
439d9889caeSkettenis if (lc->lc_state == 0)
440ba14339aSkettenis return (1);
441ba14339aSkettenis
442d9889caeSkettenis rx_head += sizeof(*lp);
443d9889caeSkettenis rx_head &= ((lc->lc_rxq->lq_nentries * sizeof(*lp)) - 1);
444d9889caeSkettenis err = hv_ldc_rx_set_qhead(lc->lc_id, rx_head);
445ba14339aSkettenis if (err != H_EOK)
446ba14339aSkettenis printf("%s: hv_ldc_rx_set_qhead %d\n", __func__, err);
447ba14339aSkettenis
448ba14339aSkettenis return (1);
449ba14339aSkettenis }
450ba14339aSkettenis
451ba14339aSkettenis void
vnet_handshake(void * arg)452d7091787Skettenis vnet_handshake(void *arg)
453d7091787Skettenis {
454d7091787Skettenis struct vnet_softc *sc = arg;
455d7091787Skettenis
456d7091787Skettenis ldc_send_vers(&sc->sc_lc);
457d7091787Skettenis }
458d7091787Skettenis
459d7091787Skettenis void
vio_rx_data(struct ldc_conn * lc,struct ldc_pkt * lp)460d9889caeSkettenis vio_rx_data(struct ldc_conn *lc, struct ldc_pkt *lp)
461ba14339aSkettenis {
462d9889caeSkettenis struct vio_msg *vm = (struct vio_msg *)lp;
463ba14339aSkettenis
464ba14339aSkettenis switch (vm->type) {
465ba14339aSkettenis case VIO_TYPE_CTRL:
466d9889caeSkettenis if ((lp->env & LDC_FRAG_START) == 0 &&
467d9889caeSkettenis (lp->env & LDC_FRAG_STOP) == 0)
468ba14339aSkettenis return;
469d9889caeSkettenis vnet_rx_vio_ctrl(lc->lc_sc, vm);
470ba14339aSkettenis break;
471ba14339aSkettenis
472ba14339aSkettenis case VIO_TYPE_DATA:
473d9889caeSkettenis if((lp->env & LDC_FRAG_START) == 0)
474ba14339aSkettenis return;
475d9889caeSkettenis vnet_rx_vio_data(lc->lc_sc, vm);
476ba14339aSkettenis break;
477ba14339aSkettenis
478ba14339aSkettenis default:
479c736a730Skettenis DPRINTF(("Unhandled packet type 0x%02x\n", vm->type));
480d9889caeSkettenis ldc_reset(lc);
481ba14339aSkettenis break;
482ba14339aSkettenis }
483ba14339aSkettenis }
484ba14339aSkettenis
485ba14339aSkettenis void
vnet_rx_vio_ctrl(struct vnet_softc * sc,struct vio_msg * vm)486ba14339aSkettenis vnet_rx_vio_ctrl(struct vnet_softc *sc, struct vio_msg *vm)
487ba14339aSkettenis {
488ba14339aSkettenis struct vio_msg_tag *tag = (struct vio_msg_tag *)&vm->type;
489ba14339aSkettenis
490ba14339aSkettenis switch (tag->stype_env) {
491ba14339aSkettenis case VIO_VER_INFO:
492ba14339aSkettenis vnet_rx_vio_ver_info(sc, tag);
493ba14339aSkettenis break;
494ba14339aSkettenis case VIO_ATTR_INFO:
495ba14339aSkettenis vnet_rx_vio_attr_info(sc, tag);
496ba14339aSkettenis break;
497ba14339aSkettenis case VIO_DRING_REG:
498ba14339aSkettenis vnet_rx_vio_dring_reg(sc, tag);
499ba14339aSkettenis break;
500ba14339aSkettenis case VIO_RDX:
501ba14339aSkettenis vnet_rx_vio_rdx(sc, tag);
502ba14339aSkettenis break;
503ba14339aSkettenis default:
504c736a730Skettenis DPRINTF(("CTRL/0x%02x/0x%04x\n", tag->stype, tag->stype_env));
505ba14339aSkettenis break;
506ba14339aSkettenis }
507ba14339aSkettenis }
508ba14339aSkettenis
509ba14339aSkettenis void
vnet_rx_vio_ver_info(struct vnet_softc * sc,struct vio_msg_tag * tag)510ba14339aSkettenis vnet_rx_vio_ver_info(struct vnet_softc *sc, struct vio_msg_tag *tag)
511ba14339aSkettenis {
512ba14339aSkettenis struct vio_ver_info *vi = (struct vio_ver_info *)tag;
513ba14339aSkettenis
514ba14339aSkettenis switch (vi->tag.stype) {
515ba14339aSkettenis case VIO_SUBTYPE_INFO:
516c736a730Skettenis DPRINTF(("CTRL/INFO/VER_INFO\n"));
517ba14339aSkettenis
518ba14339aSkettenis /* Make sure we're talking to a virtual network device. */
519ba14339aSkettenis if (vi->dev_class != VDEV_NETWORK &&
520ba14339aSkettenis vi->dev_class != VDEV_NETWORK_SWITCH) {
521ba14339aSkettenis /* Huh, we're not talking to a network device? */
522ba14339aSkettenis printf("Not a network device\n");
523ba14339aSkettenis vi->tag.stype = VIO_SUBTYPE_NACK;
5243d4b6a30Skettenis vnet_sendmsg(sc, vi, sizeof(*vi));
525ba14339aSkettenis return;
526ba14339aSkettenis }
527ba14339aSkettenis
528ba14339aSkettenis if (vi->major != VNET_MAJOR) {
529ba14339aSkettenis vi->tag.stype = VIO_SUBTYPE_NACK;
530ba14339aSkettenis vi->major = VNET_MAJOR;
531ba14339aSkettenis vi->minor = VNET_MINOR;
5323d4b6a30Skettenis vnet_sendmsg(sc, vi, sizeof(*vi));
533ba14339aSkettenis return;
534ba14339aSkettenis }
535ba14339aSkettenis
536ba14339aSkettenis vi->tag.stype = VIO_SUBTYPE_ACK;
537ba14339aSkettenis vi->tag.sid = sc->sc_local_sid;
538ba14339aSkettenis vi->minor = VNET_MINOR;
5393d4b6a30Skettenis vnet_sendmsg(sc, vi, sizeof(*vi));
540a7acb8c5Skettenis sc->sc_vio_state |= VIO_RCV_VER_INFO;
541ba14339aSkettenis break;
542ba14339aSkettenis
543ba14339aSkettenis case VIO_SUBTYPE_ACK:
544c736a730Skettenis DPRINTF(("CTRL/ACK/VER_INFO\n"));
5456d2008c7Skettenis if (!ISSET(sc->sc_vio_state, VIO_SND_VER_INFO)) {
5466d2008c7Skettenis ldc_reset(&sc->sc_lc);
5476d2008c7Skettenis break;
5486d2008c7Skettenis }
549a7acb8c5Skettenis sc->sc_vio_state |= VIO_ACK_VER_INFO;
550ba14339aSkettenis break;
551ba14339aSkettenis
552ba14339aSkettenis default:
553c736a730Skettenis DPRINTF(("CTRL/0x%02x/VER_INFO\n", vi->tag.stype));
554ba14339aSkettenis break;
555ba14339aSkettenis }
556a7acb8c5Skettenis
557a7acb8c5Skettenis if (ISSET(sc->sc_vio_state, VIO_RCV_VER_INFO) &&
558a7acb8c5Skettenis ISSET(sc->sc_vio_state, VIO_ACK_VER_INFO))
559a7acb8c5Skettenis vnet_send_attr_info(sc);
560ba14339aSkettenis }
561ba14339aSkettenis
562ba14339aSkettenis void
vnet_rx_vio_attr_info(struct vnet_softc * sc,struct vio_msg_tag * tag)563ba14339aSkettenis vnet_rx_vio_attr_info(struct vnet_softc *sc, struct vio_msg_tag *tag)
564ba14339aSkettenis {
565ba14339aSkettenis struct vnet_attr_info *ai = (struct vnet_attr_info *)tag;
566ba14339aSkettenis
567ba14339aSkettenis switch (ai->tag.stype) {
568ba14339aSkettenis case VIO_SUBTYPE_INFO:
569c736a730Skettenis DPRINTF(("CTRL/INFO/ATTR_INFO\n"));
57008c8e8d2Skettenis sc->sc_xfer_mode = ai->xfer_mode;
571ba14339aSkettenis
572ba14339aSkettenis ai->tag.stype = VIO_SUBTYPE_ACK;
573ba14339aSkettenis ai->tag.sid = sc->sc_local_sid;
5743d4b6a30Skettenis vnet_sendmsg(sc, ai, sizeof(*ai));
575a7acb8c5Skettenis sc->sc_vio_state |= VIO_RCV_ATTR_INFO;
576ba14339aSkettenis break;
577ba14339aSkettenis
578ba14339aSkettenis case VIO_SUBTYPE_ACK:
579c736a730Skettenis DPRINTF(("CTRL/ACK/ATTR_INFO\n"));
5806d2008c7Skettenis if (!ISSET(sc->sc_vio_state, VIO_SND_ATTR_INFO)) {
5816d2008c7Skettenis ldc_reset(&sc->sc_lc);
5826d2008c7Skettenis break;
5836d2008c7Skettenis }
584a7acb8c5Skettenis sc->sc_vio_state |= VIO_ACK_ATTR_INFO;
585ba14339aSkettenis break;
586ba14339aSkettenis
587ba14339aSkettenis default:
58841635321Skettenis DPRINTF(("CTRL/0x%02x/ATTR_INFO\n", ai->tag.stype));
589ba14339aSkettenis break;
590ba14339aSkettenis }
591a7acb8c5Skettenis
592a7acb8c5Skettenis if (ISSET(sc->sc_vio_state, VIO_RCV_ATTR_INFO) &&
59308c8e8d2Skettenis ISSET(sc->sc_vio_state, VIO_ACK_ATTR_INFO)) {
59408c8e8d2Skettenis if (sc->sc_xfer_mode == VIO_DRING_MODE)
595a7acb8c5Skettenis vnet_send_dring_reg(sc);
59608c8e8d2Skettenis else
59708c8e8d2Skettenis vio_send_rdx(sc);
59808c8e8d2Skettenis }
599ba14339aSkettenis }
600ba14339aSkettenis
601ba14339aSkettenis void
vnet_rx_vio_dring_reg(struct vnet_softc * sc,struct vio_msg_tag * tag)602ba14339aSkettenis vnet_rx_vio_dring_reg(struct vnet_softc *sc, struct vio_msg_tag *tag)
603ba14339aSkettenis {
604ba14339aSkettenis struct vio_dring_reg *dr = (struct vio_dring_reg *)tag;
605ba14339aSkettenis
606ba14339aSkettenis switch (dr->tag.stype) {
607ba14339aSkettenis case VIO_SUBTYPE_INFO:
608c736a730Skettenis DPRINTF(("CTRL/INFO/DRING_REG\n"));
609ba14339aSkettenis
610ba14339aSkettenis sc->sc_peer_dring_nentries = dr->num_descriptors;
611ba14339aSkettenis sc->sc_peer_desc_size = dr->descriptor_size;
612ba14339aSkettenis sc->sc_peer_dring_cookie = dr->cookie[0];
613ba14339aSkettenis
614ba14339aSkettenis dr->tag.stype = VIO_SUBTYPE_ACK;
615ba14339aSkettenis dr->tag.sid = sc->sc_local_sid;
6163d4b6a30Skettenis vnet_sendmsg(sc, dr, sizeof(*dr));
617a7acb8c5Skettenis sc->sc_vio_state |= VIO_RCV_DRING_REG;
618ba14339aSkettenis break;
619ba14339aSkettenis
620ba14339aSkettenis case VIO_SUBTYPE_ACK:
621c736a730Skettenis DPRINTF(("CTRL/ACK/DRING_REG\n"));
6226d2008c7Skettenis if (!ISSET(sc->sc_vio_state, VIO_SND_DRING_REG)) {
6236d2008c7Skettenis ldc_reset(&sc->sc_lc);
6246d2008c7Skettenis break;
6256d2008c7Skettenis }
626ba14339aSkettenis
627ba14339aSkettenis sc->sc_dring_ident = dr->dring_ident;
628ba14339aSkettenis sc->sc_seq_no = 1;
629ba14339aSkettenis
630a7acb8c5Skettenis sc->sc_vio_state |= VIO_ACK_DRING_REG;
631ba14339aSkettenis break;
632ba14339aSkettenis
633ba14339aSkettenis default:
634c736a730Skettenis DPRINTF(("CTRL/0x%02x/DRING_REG\n", dr->tag.stype));
635ba14339aSkettenis break;
636ba14339aSkettenis }
637a7acb8c5Skettenis
638a7acb8c5Skettenis if (ISSET(sc->sc_vio_state, VIO_RCV_DRING_REG) &&
639a7acb8c5Skettenis ISSET(sc->sc_vio_state, VIO_ACK_DRING_REG))
640a7acb8c5Skettenis vio_send_rdx(sc);
641ba14339aSkettenis }
642ba14339aSkettenis
643ba14339aSkettenis void
vnet_rx_vio_rdx(struct vnet_softc * sc,struct vio_msg_tag * tag)644ba14339aSkettenis vnet_rx_vio_rdx(struct vnet_softc *sc, struct vio_msg_tag *tag)
645ba14339aSkettenis {
646a0c88015Skettenis struct ifnet *ifp = &sc->sc_ac.ac_if;
647a0c88015Skettenis
648ba14339aSkettenis switch(tag->stype) {
649ba14339aSkettenis case VIO_SUBTYPE_INFO:
650c736a730Skettenis DPRINTF(("CTRL/INFO/RDX\n"));
651ba14339aSkettenis
652ba14339aSkettenis tag->stype = VIO_SUBTYPE_ACK;
653ba14339aSkettenis tag->sid = sc->sc_local_sid;
6543d4b6a30Skettenis vnet_sendmsg(sc, tag, sizeof(*tag));
655a7acb8c5Skettenis sc->sc_vio_state |= VIO_RCV_RDX;
656ba14339aSkettenis break;
657ba14339aSkettenis
658ba14339aSkettenis case VIO_SUBTYPE_ACK:
659c736a730Skettenis DPRINTF(("CTRL/ACK/RDX\n"));
6606d2008c7Skettenis if (!ISSET(sc->sc_vio_state, VIO_SND_RDX)) {
6616d2008c7Skettenis ldc_reset(&sc->sc_lc);
6626d2008c7Skettenis break;
6636d2008c7Skettenis }
664a7acb8c5Skettenis sc->sc_vio_state |= VIO_ACK_RDX;
665ba14339aSkettenis break;
666ba14339aSkettenis
667ba14339aSkettenis default:
668c736a730Skettenis DPRINTF(("CTRL/0x%02x/RDX (VIO)\n", tag->stype));
669ba14339aSkettenis break;
670ba14339aSkettenis }
671a7acb8c5Skettenis
67208c8e8d2Skettenis if (ISSET(sc->sc_vio_state, VIO_RCV_RDX) &&
67308c8e8d2Skettenis ISSET(sc->sc_vio_state, VIO_ACK_RDX)) {
674a7acb8c5Skettenis /* Link is up! */
675a7acb8c5Skettenis vnet_link_state(sc);
676a7acb8c5Skettenis
677a7acb8c5Skettenis /* Configure multicast now that we can. */
678a7acb8c5Skettenis vnet_setmulti(sc, 1);
6799464f18aSkettenis
6808dd7c4a8Skettenis KERNEL_LOCK();
681de6cd8fbSdlg ifq_clr_oactive(&ifp->if_snd);
682a7acb8c5Skettenis vnet_start(ifp);
6838dd7c4a8Skettenis KERNEL_UNLOCK();
684a7acb8c5Skettenis }
685ba14339aSkettenis }
686ba14339aSkettenis
687ba14339aSkettenis void
vnet_rx_vio_data(struct vnet_softc * sc,struct vio_msg * vm)688ba14339aSkettenis vnet_rx_vio_data(struct vnet_softc *sc, struct vio_msg *vm)
689ba14339aSkettenis {
690ba14339aSkettenis struct vio_msg_tag *tag = (struct vio_msg_tag *)&vm->type;
691ba14339aSkettenis
69208c8e8d2Skettenis if (!ISSET(sc->sc_vio_state, VIO_RCV_RDX) ||
69308c8e8d2Skettenis !ISSET(sc->sc_vio_state, VIO_ACK_RDX)) {
694c736a730Skettenis DPRINTF(("Spurious DATA/0x%02x/0x%04x\n", tag->stype,
695c736a730Skettenis tag->stype_env));
696ba14339aSkettenis return;
697ba14339aSkettenis }
698ba14339aSkettenis
699ba14339aSkettenis switch(tag->stype_env) {
70008c8e8d2Skettenis case VIO_DESC_DATA:
70108c8e8d2Skettenis vnet_rx_vio_desc_data(sc, tag);
70208c8e8d2Skettenis break;
70308c8e8d2Skettenis
704ba14339aSkettenis case VIO_DRING_DATA:
705ba14339aSkettenis vnet_rx_vio_dring_data(sc, tag);
706ba14339aSkettenis break;
707ba14339aSkettenis
708ba14339aSkettenis default:
709c736a730Skettenis DPRINTF(("DATA/0x%02x/0x%04x\n", tag->stype, tag->stype_env));
710ba14339aSkettenis break;
711ba14339aSkettenis }
712ba14339aSkettenis }
713ba14339aSkettenis
714ba14339aSkettenis void
vnet_rx_vio_desc_data(struct vnet_softc * sc,struct vio_msg_tag * tag)71508c8e8d2Skettenis vnet_rx_vio_desc_data(struct vnet_softc *sc, struct vio_msg_tag *tag)
71608c8e8d2Skettenis {
71708c8e8d2Skettenis struct vnet_desc_msg *dm = (struct vnet_desc_msg *)tag;
71808c8e8d2Skettenis struct ldc_conn *lc = &sc->sc_lc;
71908c8e8d2Skettenis struct ldc_map *map = sc->sc_lm;
72008c8e8d2Skettenis struct ifnet *ifp = &sc->sc_ac.ac_if;
721afad3ffeSmpi struct mbuf_list ml = MBUF_LIST_INITIALIZER();
72208c8e8d2Skettenis struct mbuf *m;
72308c8e8d2Skettenis caddr_t buf;
72408c8e8d2Skettenis paddr_t pa;
72508c8e8d2Skettenis psize_t nbytes;
72690354c4fSkettenis u_int cons;
72708c8e8d2Skettenis int err;
72808c8e8d2Skettenis
72908c8e8d2Skettenis switch(tag->stype) {
73008c8e8d2Skettenis case VIO_SUBTYPE_INFO:
731*63d58f04Sclaudio nbytes = roundup(dm->nbytes, 8);
732*63d58f04Sclaudio if (nbytes > VNET_BUF_SIZE) {
73308c8e8d2Skettenis ifp->if_ierrors++;
73408c8e8d2Skettenis goto skip;
73508c8e8d2Skettenis }
73608c8e8d2Skettenis
737*63d58f04Sclaudio buf = pool_get(&sc->sc_pool, PR_NOWAIT|PR_ZERO);
738*63d58f04Sclaudio if (buf == NULL) {
739e8c944bcSkettenis ifp->if_ierrors++;
740e8c944bcSkettenis goto skip;
741e8c944bcSkettenis }
742e8c944bcSkettenis
74308c8e8d2Skettenis pmap_extract(pmap_kernel(), (vaddr_t)buf, &pa);
74408c8e8d2Skettenis err = hv_ldc_copy(lc->lc_id, LDC_COPY_IN,
74508c8e8d2Skettenis dm->cookie[0].addr, pa, nbytes, &nbytes);
74608c8e8d2Skettenis if (err != H_EOK) {
74708c8e8d2Skettenis pool_put(&sc->sc_pool, buf);
74808c8e8d2Skettenis ifp->if_ierrors++;
74908c8e8d2Skettenis goto skip;
75008c8e8d2Skettenis }
75108c8e8d2Skettenis
75208c8e8d2Skettenis /* Stupid OBP doesn't align properly. */
753c35f95b8Smpi m = m_devget(buf, dm->nbytes, ETHER_ALIGN);
75408c8e8d2Skettenis pool_put(&sc->sc_pool, buf);
75508c8e8d2Skettenis if (m == NULL) {
75608c8e8d2Skettenis ifp->if_ierrors++;
75708c8e8d2Skettenis goto skip;
75808c8e8d2Skettenis }
75908c8e8d2Skettenis
76008c8e8d2Skettenis /* Pass it on. */
761afad3ffeSmpi ml_enqueue(&ml, m);
762afad3ffeSmpi if_input(ifp, &ml);
76308c8e8d2Skettenis
76408c8e8d2Skettenis skip:
76508c8e8d2Skettenis dm->tag.stype = VIO_SUBTYPE_ACK;
76608c8e8d2Skettenis dm->tag.sid = sc->sc_local_sid;
7673d4b6a30Skettenis vnet_sendmsg(sc, dm, sizeof(*dm));
76808c8e8d2Skettenis break;
76908c8e8d2Skettenis
77008c8e8d2Skettenis case VIO_SUBTYPE_ACK:
77108c8e8d2Skettenis DPRINTF(("DATA/ACK/DESC_DATA\n"));
77208c8e8d2Skettenis
77308c8e8d2Skettenis if (dm->desc_handle != sc->sc_tx_cons) {
774871b955fSkn printf("%s: out of order\n", __func__);
77508c8e8d2Skettenis return;
77608c8e8d2Skettenis }
77708c8e8d2Skettenis
77890354c4fSkettenis cons = sc->sc_tx_cons & (sc->sc_vd->vd_nentries - 1);
77908c8e8d2Skettenis
78090354c4fSkettenis map->lm_slot[sc->sc_vsd[cons].vsd_map_idx].entry = 0;
78190354c4fSkettenis atomic_dec_int(&map->lm_count);
78290354c4fSkettenis
78390354c4fSkettenis pool_put(&sc->sc_pool, sc->sc_vsd[cons].vsd_buf);
7848678061cSkettenis sc->sc_vsd[cons].vsd_buf = NULL;
78508c8e8d2Skettenis
78608c8e8d2Skettenis sc->sc_tx_cons++;
78708c8e8d2Skettenis break;
78808c8e8d2Skettenis
78908c8e8d2Skettenis case VIO_SUBTYPE_NACK:
79008c8e8d2Skettenis DPRINTF(("DATA/NACK/DESC_DATA\n"));
79108c8e8d2Skettenis break;
79208c8e8d2Skettenis
79308c8e8d2Skettenis default:
79408c8e8d2Skettenis DPRINTF(("DATA/0x%02x/DESC_DATA\n", tag->stype));
79508c8e8d2Skettenis break;
79608c8e8d2Skettenis }
79708c8e8d2Skettenis }
79808c8e8d2Skettenis
79908c8e8d2Skettenis void
vnet_rx_vio_dring_data(struct vnet_softc * sc,struct vio_msg_tag * tag)800ba14339aSkettenis vnet_rx_vio_dring_data(struct vnet_softc *sc, struct vio_msg_tag *tag)
801ba14339aSkettenis {
802ac50164fSkettenis struct vio_dring_msg *dm = (struct vio_dring_msg *)tag;
803d9889caeSkettenis struct ldc_conn *lc = &sc->sc_lc;
804ba14339aSkettenis struct ifnet *ifp = &sc->sc_ac.ac_if;
805ba14339aSkettenis struct mbuf *m;
806ba14339aSkettenis paddr_t pa;
807ba14339aSkettenis psize_t nbytes;
808ba14339aSkettenis int err;
809ba14339aSkettenis
810ba14339aSkettenis switch(tag->stype) {
811ba14339aSkettenis case VIO_SUBTYPE_INFO:
812ba14339aSkettenis {
813ba14339aSkettenis struct vnet_desc desc;
814ba14339aSkettenis uint64_t cookie;
815ba14339aSkettenis paddr_t desc_pa;
816ba14339aSkettenis int idx, ack_end_idx = -1;
817afad3ffeSmpi struct mbuf_list ml = MBUF_LIST_INITIALIZER();
818ba14339aSkettenis
819ba14339aSkettenis idx = dm->start_idx;
820ba14339aSkettenis for (;;) {
821ba14339aSkettenis cookie = sc->sc_peer_dring_cookie.addr;
822ba14339aSkettenis cookie += idx * sc->sc_peer_desc_size;
823ba14339aSkettenis nbytes = sc->sc_peer_desc_size;
824ba14339aSkettenis pmap_extract(pmap_kernel(), (vaddr_t)&desc, &desc_pa);
825d9889caeSkettenis err = hv_ldc_copy(lc->lc_id, LDC_COPY_IN, cookie,
826ba14339aSkettenis desc_pa, nbytes, &nbytes);
827ba14339aSkettenis if (err != H_EOK) {
828*63d58f04Sclaudio printf("hv_ldc_copy in %d\n", err);
829ba14339aSkettenis break;
830ba14339aSkettenis }
831ba14339aSkettenis
832ba14339aSkettenis if (desc.hdr.dstate != VIO_DESC_READY)
833ba14339aSkettenis break;
834ba14339aSkettenis
835e8c944bcSkettenis if (desc.nbytes > (ETHER_MAX_LEN - ETHER_CRC_LEN)) {
836e8c944bcSkettenis ifp->if_ierrors++;
837e8c944bcSkettenis goto skip;
838e8c944bcSkettenis }
839e8c944bcSkettenis
840471f2571Sjan m = MCLGETL(NULL, M_DONTWAIT, desc.nbytes);
841415ac77aSderaadt if (!m)
842ba14339aSkettenis break;
843ba14339aSkettenis m->m_len = m->m_pkthdr.len = desc.nbytes;
844c736a730Skettenis nbytes = roundup(desc.nbytes + VNET_ETHER_ALIGN, 8);
845ba14339aSkettenis
846ba14339aSkettenis pmap_extract(pmap_kernel(), (vaddr_t)m->m_data, &pa);
847d9889caeSkettenis err = hv_ldc_copy(lc->lc_id, LDC_COPY_IN,
848ba14339aSkettenis desc.cookie[0].addr, pa, nbytes, &nbytes);
849ba14339aSkettenis if (err != H_EOK) {
850ba14339aSkettenis m_freem(m);
851ba14339aSkettenis goto skip;
852ba14339aSkettenis }
853c736a730Skettenis m->m_data += VNET_ETHER_ALIGN;
854ba14339aSkettenis
855afad3ffeSmpi ml_enqueue(&ml, m);
856ba14339aSkettenis
857ba14339aSkettenis skip:
858ba14339aSkettenis desc.hdr.dstate = VIO_DESC_DONE;
859ba14339aSkettenis nbytes = sc->sc_peer_desc_size;
860d9889caeSkettenis err = hv_ldc_copy(lc->lc_id, LDC_COPY_OUT, cookie,
861ba14339aSkettenis desc_pa, nbytes, &nbytes);
862ba14339aSkettenis if (err != H_EOK)
863*63d58f04Sclaudio printf("hv_ldc_copy out %d\n", err);
864ba14339aSkettenis
865ba14339aSkettenis ack_end_idx = idx;
866ba14339aSkettenis if (++idx == sc->sc_peer_dring_nentries)
867ba14339aSkettenis idx = 0;
868ba14339aSkettenis }
869ba14339aSkettenis
870afad3ffeSmpi if_input(ifp, &ml);
871afad3ffeSmpi
872ba14339aSkettenis if (ack_end_idx == -1) {
873ba14339aSkettenis dm->tag.stype = VIO_SUBTYPE_NACK;
874ba14339aSkettenis } else {
875ba14339aSkettenis dm->tag.stype = VIO_SUBTYPE_ACK;
876ba14339aSkettenis dm->end_idx = ack_end_idx;
877ba14339aSkettenis }
878ba14339aSkettenis dm->tag.sid = sc->sc_local_sid;
879ba14339aSkettenis dm->proc_state = VIO_DP_STOPPED;
8803d4b6a30Skettenis vnet_sendmsg(sc, dm, sizeof(*dm));
881ba14339aSkettenis break;
882ba14339aSkettenis }
883ba14339aSkettenis
884ba14339aSkettenis case VIO_SUBTYPE_ACK:
885ba14339aSkettenis {
886ba14339aSkettenis struct ldc_map *map = sc->sc_lm;
88790354c4fSkettenis u_int cons, count;
888ba14339aSkettenis
889a212e6c2Skettenis sc->sc_peer_state = dm->proc_state;
890a212e6c2Skettenis
89190354c4fSkettenis cons = sc->sc_tx_cons & (sc->sc_vd->vd_nentries - 1);
892ba14339aSkettenis while (sc->sc_vd->vd_desc[cons].hdr.dstate == VIO_DESC_DONE) {
893ba14339aSkettenis map->lm_slot[sc->sc_vsd[cons].vsd_map_idx].entry = 0;
89490354c4fSkettenis atomic_dec_int(&map->lm_count);
895ba14339aSkettenis
896ba14339aSkettenis pool_put(&sc->sc_pool, sc->sc_vsd[cons].vsd_buf);
8978678061cSkettenis sc->sc_vsd[cons].vsd_buf = NULL;
898ba14339aSkettenis
89990354c4fSkettenis sc->sc_vd->vd_desc[cons].hdr.dstate = VIO_DESC_FREE;
90090354c4fSkettenis sc->sc_tx_cons++;
90190354c4fSkettenis cons = sc->sc_tx_cons & (sc->sc_vd->vd_nentries - 1);
902ba14339aSkettenis }
903a8c5aadfSkettenis
904*63d58f04Sclaudio KERNEL_LOCK();
90590354c4fSkettenis count = sc->sc_tx_prod - sc->sc_tx_cons;
90690354c4fSkettenis if (count > 0 && sc->sc_peer_state != VIO_DP_ACTIVE)
90790354c4fSkettenis vnet_send_dring_data(sc, cons);
908a212e6c2Skettenis
90990354c4fSkettenis if (count < (sc->sc_vd->vd_nentries - 1))
910de6cd8fbSdlg ifq_clr_oactive(&ifp->if_snd);
91190354c4fSkettenis if (count == 0)
912199381a1Skettenis ifp->if_timer = 0;
913a8c5aadfSkettenis
914a8c5aadfSkettenis vnet_start(ifp);
9158dd7c4a8Skettenis KERNEL_UNLOCK();
916ba14339aSkettenis break;
917ba14339aSkettenis }
918ba14339aSkettenis
919ba14339aSkettenis case VIO_SUBTYPE_NACK:
920c736a730Skettenis DPRINTF(("DATA/NACK/DRING_DATA\n"));
92190354c4fSkettenis sc->sc_peer_state = VIO_DP_STOPPED;
922ba14339aSkettenis break;
923ba14339aSkettenis
924ba14339aSkettenis default:
925c736a730Skettenis DPRINTF(("DATA/0x%02x/DRING_DATA\n", tag->stype));
926ba14339aSkettenis break;
927ba14339aSkettenis }
928ba14339aSkettenis }
929ba14339aSkettenis
930ba14339aSkettenis void
vnet_ldc_reset(struct ldc_conn * lc)931a7acb8c5Skettenis vnet_ldc_reset(struct ldc_conn *lc)
932ba14339aSkettenis {
933d9889caeSkettenis struct vnet_softc *sc = lc->lc_sc;
9349464f18aSkettenis int i;
935ba14339aSkettenis
936d7091787Skettenis timeout_del(&sc->sc_handshake_to);
93790354c4fSkettenis sc->sc_tx_prod = sc->sc_tx_cons = 0;
93890354c4fSkettenis sc->sc_peer_state = VIO_DP_STOPPED;
939ba14339aSkettenis sc->sc_vio_state = 0;
94039c67f9aSkettenis vnet_link_state(sc);
9419464f18aSkettenis
9429464f18aSkettenis sc->sc_lm->lm_next = 1;
9439464f18aSkettenis sc->sc_lm->lm_count = 1;
9449464f18aSkettenis for (i = 1; i < sc->sc_lm->lm_nentries; i++)
9459464f18aSkettenis sc->sc_lm->lm_slot[i].entry = 0;
9469464f18aSkettenis
9478678061cSkettenis for (i = 0; i < sc->sc_vd->vd_nentries; i++) {
9488678061cSkettenis if (sc->sc_vsd[i].vsd_buf) {
9498678061cSkettenis pool_put(&sc->sc_pool, sc->sc_vsd[i].vsd_buf);
9508678061cSkettenis sc->sc_vsd[i].vsd_buf = NULL;
9518678061cSkettenis }
9529464f18aSkettenis sc->sc_vd->vd_desc[i].hdr.dstate = VIO_DESC_FREE;
953ba14339aSkettenis }
9548678061cSkettenis }
955ba14339aSkettenis
956ba14339aSkettenis void
vnet_ldc_start(struct ldc_conn * lc)957a7acb8c5Skettenis vnet_ldc_start(struct ldc_conn *lc)
958a7acb8c5Skettenis {
959a7acb8c5Skettenis struct vnet_softc *sc = lc->lc_sc;
960a7acb8c5Skettenis
961d7091787Skettenis timeout_del(&sc->sc_handshake_to);
962a7acb8c5Skettenis vnet_send_ver_info(sc, VNET_MAJOR, VNET_MINOR);
963a7acb8c5Skettenis }
964a7acb8c5Skettenis
965a7acb8c5Skettenis void
vnet_sendmsg(struct vnet_softc * sc,void * msg,size_t len)9663d4b6a30Skettenis vnet_sendmsg(struct vnet_softc *sc, void *msg, size_t len)
967ba14339aSkettenis {
968d9889caeSkettenis struct ldc_conn *lc = &sc->sc_lc;
969ba14339aSkettenis int err;
970ba14339aSkettenis
9713d4b6a30Skettenis err = ldc_send_unreliable(lc, msg, len);
9723d4b6a30Skettenis if (err)
9733d4b6a30Skettenis printf("%s: ldc_send_unreliable: %d\n", __func__, err);
974ba14339aSkettenis }
975ba14339aSkettenis
976ba14339aSkettenis void
vnet_send_ver_info(struct vnet_softc * sc,uint16_t major,uint16_t minor)977a7acb8c5Skettenis vnet_send_ver_info(struct vnet_softc *sc, uint16_t major, uint16_t minor)
978ba14339aSkettenis {
979ba14339aSkettenis struct vio_ver_info vi;
980ba14339aSkettenis
981ba14339aSkettenis bzero(&vi, sizeof(vi));
982ba14339aSkettenis vi.tag.type = VIO_TYPE_CTRL;
983ba14339aSkettenis vi.tag.stype = VIO_SUBTYPE_INFO;
984ba14339aSkettenis vi.tag.stype_env = VIO_VER_INFO;
985ba14339aSkettenis vi.tag.sid = sc->sc_local_sid;
986ba14339aSkettenis vi.major = major;
987ba14339aSkettenis vi.minor = minor;
988ba14339aSkettenis vi.dev_class = VDEV_NETWORK;
9893d4b6a30Skettenis vnet_sendmsg(sc, &vi, sizeof(vi));
9906d2008c7Skettenis
9916d2008c7Skettenis sc->sc_vio_state |= VIO_SND_VER_INFO;
992ba14339aSkettenis }
993ba14339aSkettenis
994ba14339aSkettenis void
vnet_send_attr_info(struct vnet_softc * sc)995ba14339aSkettenis vnet_send_attr_info(struct vnet_softc *sc)
996ba14339aSkettenis {
997ba14339aSkettenis struct vnet_attr_info ai;
998ba14339aSkettenis int i;
999ba14339aSkettenis
1000ba14339aSkettenis bzero(&ai, sizeof(ai));
1001ba14339aSkettenis ai.tag.type = VIO_TYPE_CTRL;
1002ba14339aSkettenis ai.tag.stype = VIO_SUBTYPE_INFO;
1003ba14339aSkettenis ai.tag.stype_env = VIO_ATTR_INFO;
1004ba14339aSkettenis ai.tag.sid = sc->sc_local_sid;
1005ba14339aSkettenis ai.xfer_mode = VIO_DRING_MODE;
1006ba14339aSkettenis ai.addr_type = VNET_ADDR_ETHERMAC;
1007c663a3ceSkettenis ai.ack_freq = 0;
1008ba14339aSkettenis ai.addr = 0;
1009ba14339aSkettenis for (i = 0; i < ETHER_ADDR_LEN; i++) {
1010ba14339aSkettenis ai.addr <<= 8;
1011ba14339aSkettenis ai.addr |= sc->sc_ac.ac_enaddr[i];
1012ba14339aSkettenis }
1013ba14339aSkettenis ai.mtu = ETHER_MAX_LEN - ETHER_CRC_LEN;
10143d4b6a30Skettenis vnet_sendmsg(sc, &ai, sizeof(ai));
10156d2008c7Skettenis
10166d2008c7Skettenis sc->sc_vio_state |= VIO_SND_ATTR_INFO;
1017ba14339aSkettenis }
1018ba14339aSkettenis
1019ba14339aSkettenis void
vnet_send_dring_reg(struct vnet_softc * sc)1020ba14339aSkettenis vnet_send_dring_reg(struct vnet_softc *sc)
1021ba14339aSkettenis {
1022ba14339aSkettenis struct vio_dring_reg dr;
1023ba14339aSkettenis
1024ba14339aSkettenis bzero(&dr, sizeof(dr));
1025ba14339aSkettenis dr.tag.type = VIO_TYPE_CTRL;
1026ba14339aSkettenis dr.tag.stype = VIO_SUBTYPE_INFO;
1027ba14339aSkettenis dr.tag.stype_env = VIO_DRING_REG;
1028ba14339aSkettenis dr.tag.sid = sc->sc_local_sid;
1029ba14339aSkettenis dr.dring_ident = 0;
1030ba14339aSkettenis dr.num_descriptors = sc->sc_vd->vd_nentries;
1031ba14339aSkettenis dr.descriptor_size = sizeof(struct vnet_desc);
1032ba14339aSkettenis dr.options = VIO_TX_RING;
1033ba14339aSkettenis dr.ncookies = 1;
1034ba14339aSkettenis dr.cookie[0].addr = 0;
1035ba14339aSkettenis dr.cookie[0].size = PAGE_SIZE;
10363d4b6a30Skettenis vnet_sendmsg(sc, &dr, sizeof(dr));
10376d2008c7Skettenis
10386d2008c7Skettenis sc->sc_vio_state |= VIO_SND_DRING_REG;
1039ba14339aSkettenis };
1040ba14339aSkettenis
1041ba14339aSkettenis void
vio_send_rdx(struct vnet_softc * sc)1042ba14339aSkettenis vio_send_rdx(struct vnet_softc *sc)
1043ba14339aSkettenis {
1044ba14339aSkettenis struct vio_msg_tag tag;
1045ba14339aSkettenis
1046ba14339aSkettenis tag.type = VIO_TYPE_CTRL;
1047ba14339aSkettenis tag.stype = VIO_SUBTYPE_INFO;
1048ba14339aSkettenis tag.stype_env = VIO_RDX;
1049ba14339aSkettenis tag.sid = sc->sc_local_sid;
10503d4b6a30Skettenis vnet_sendmsg(sc, &tag, sizeof(tag));
10516d2008c7Skettenis
10526d2008c7Skettenis sc->sc_vio_state |= VIO_SND_RDX;
1053ba14339aSkettenis }
1054ba14339aSkettenis
1055ba14339aSkettenis void
vnet_send_dring_data(struct vnet_softc * sc,uint32_t start_idx)1056a212e6c2Skettenis vnet_send_dring_data(struct vnet_softc *sc, uint32_t start_idx)
1057a212e6c2Skettenis {
1058a212e6c2Skettenis struct vio_dring_msg dm;
105990354c4fSkettenis u_int peer_state;
106090354c4fSkettenis
106190354c4fSkettenis peer_state = atomic_swap_uint(&sc->sc_peer_state, VIO_DP_ACTIVE);
106290354c4fSkettenis if (peer_state == VIO_DP_ACTIVE)
106390354c4fSkettenis return;
1064a212e6c2Skettenis
1065a212e6c2Skettenis bzero(&dm, sizeof(dm));
1066a212e6c2Skettenis dm.tag.type = VIO_TYPE_DATA;
1067a212e6c2Skettenis dm.tag.stype = VIO_SUBTYPE_INFO;
1068a212e6c2Skettenis dm.tag.stype_env = VIO_DRING_DATA;
1069a212e6c2Skettenis dm.tag.sid = sc->sc_local_sid;
1070a212e6c2Skettenis dm.seq_no = sc->sc_seq_no++;
1071a212e6c2Skettenis dm.dring_ident = sc->sc_dring_ident;
1072a212e6c2Skettenis dm.start_idx = start_idx;
1073a212e6c2Skettenis dm.end_idx = -1;
10743d4b6a30Skettenis vnet_sendmsg(sc, &dm, sizeof(dm));
1075a212e6c2Skettenis }
1076a212e6c2Skettenis
1077a212e6c2Skettenis void
vnet_start(struct ifnet * ifp)1078ba14339aSkettenis vnet_start(struct ifnet *ifp)
1079ba14339aSkettenis {
108039c67f9aSkettenis struct vnet_softc *sc = ifp->if_softc;
1081f1f394b1Skettenis struct ldc_conn *lc = &sc->sc_lc;
1082a8c5aadfSkettenis struct ldc_map *map = sc->sc_lm;
1083ba14339aSkettenis struct mbuf *m;
1084ba14339aSkettenis paddr_t pa;
1085ba14339aSkettenis caddr_t buf;
1086f1f394b1Skettenis uint64_t tx_head, tx_tail, tx_state;
108790354c4fSkettenis u_int start, prod, count;
108890354c4fSkettenis int err;
1089ba14339aSkettenis
1090de6cd8fbSdlg if (!(ifp->if_flags & IFF_RUNNING) || ifq_is_oactive(&ifp->if_snd))
1091ba14339aSkettenis return;
1092ba14339aSkettenis
10930cae21bdSpatrick if (ifq_empty(&ifp->if_snd))
1094ba14339aSkettenis return;
1095ba14339aSkettenis
109639c67f9aSkettenis /*
109739c67f9aSkettenis * We cannot transmit packets until a VIO connection has been
109839c67f9aSkettenis * established.
109939c67f9aSkettenis */
110008c8e8d2Skettenis if (!ISSET(sc->sc_vio_state, VIO_RCV_RDX) ||
110108c8e8d2Skettenis !ISSET(sc->sc_vio_state, VIO_ACK_RDX))
110239c67f9aSkettenis return;
1103ba14339aSkettenis
1104f1f394b1Skettenis /*
1105f1f394b1Skettenis * Make sure there is room in the LDC transmit queue to send a
1106f1f394b1Skettenis * DRING_DATA message.
1107f1f394b1Skettenis */
1108f1f394b1Skettenis err = hv_ldc_tx_get_state(lc->lc_id, &tx_head, &tx_tail, &tx_state);
1109f1f394b1Skettenis if (err != H_EOK)
1110f1f394b1Skettenis return;
1111f1f394b1Skettenis tx_tail += sizeof(struct ldc_pkt);
1112f1f394b1Skettenis tx_tail &= ((lc->lc_txq->lq_nentries * sizeof(struct ldc_pkt)) - 1);
1113f1f394b1Skettenis if (tx_tail == tx_head) {
1114de6cd8fbSdlg ifq_set_oactive(&ifp->if_snd);
1115f1f394b1Skettenis return;
1116f1f394b1Skettenis }
1117f1f394b1Skettenis
1118c663a3ceSkettenis if (sc->sc_xfer_mode == VIO_DESC_MODE) {
1119c663a3ceSkettenis vnet_start_desc(ifp);
1120c663a3ceSkettenis return;
1121c663a3ceSkettenis }
1122c663a3ceSkettenis
112390354c4fSkettenis start = prod = sc->sc_tx_prod & (sc->sc_vd->vd_nentries - 1);
112490354c4fSkettenis while (sc->sc_vd->vd_desc[prod].hdr.dstate == VIO_DESC_FREE) {
112590354c4fSkettenis count = sc->sc_tx_prod - sc->sc_tx_cons;
112690354c4fSkettenis if (count >= (sc->sc_vd->vd_nentries - 1) ||
1127a8c5aadfSkettenis map->lm_count >= map->lm_nentries) {
1128de6cd8fbSdlg ifq_set_oactive(&ifp->if_snd);
1129ba14339aSkettenis break;
1130a8c5aadfSkettenis }
1131a8c5aadfSkettenis
1132a8c5aadfSkettenis buf = pool_get(&sc->sc_pool, PR_NOWAIT|PR_ZERO);
1133a8c5aadfSkettenis if (buf == NULL) {
1134de6cd8fbSdlg ifq_set_oactive(&ifp->if_snd);
1135a8c5aadfSkettenis break;
1136a8c5aadfSkettenis }
1137d7421602Skettenis
113863bcfa73Spatrick m = ifq_dequeue(&ifp->if_snd);
1139d7421602Skettenis if (m == NULL) {
1140d7421602Skettenis pool_put(&sc->sc_pool, buf);
1141d7421602Skettenis break;
1142d7421602Skettenis }
1143d7421602Skettenis
1144*63d58f04Sclaudio if (m->m_pkthdr.len > VNET_BUF_SIZE - VNET_ETHER_ALIGN) {
1145*63d58f04Sclaudio ifp->if_oerrors++;
1146*63d58f04Sclaudio pool_put(&sc->sc_pool, buf);
1147*63d58f04Sclaudio m_freem(m);
1148*63d58f04Sclaudio break;
1149*63d58f04Sclaudio }
1150c736a730Skettenis m_copydata(m, 0, m->m_pkthdr.len, buf + VNET_ETHER_ALIGN);
1151ba14339aSkettenis
1152ba14339aSkettenis #if NBPFILTER > 0
1153ba14339aSkettenis /*
1154ba14339aSkettenis * If BPF is listening on this interface, let it see the
1155ba14339aSkettenis * packet before we commit it to the wire.
1156ba14339aSkettenis */
1157ba14339aSkettenis if (ifp->if_bpf)
1158ba14339aSkettenis bpf_mtap(ifp->if_bpf, m, BPF_DIRECTION_OUT);
1159ba14339aSkettenis #endif
1160ba14339aSkettenis
1161ba14339aSkettenis pmap_extract(pmap_kernel(), (vaddr_t)buf, &pa);
1162ba14339aSkettenis KASSERT((pa & ~PAGE_MASK) == (pa & LDC_MTE_RA_MASK));
1163ba14339aSkettenis while (map->lm_slot[map->lm_next].entry != 0) {
1164ba14339aSkettenis map->lm_next++;
1165ba14339aSkettenis map->lm_next &= (map->lm_nentries - 1);
1166ba14339aSkettenis }
1167ba14339aSkettenis map->lm_slot[map->lm_next].entry = (pa & LDC_MTE_RA_MASK);
1168ba14339aSkettenis map->lm_slot[map->lm_next].entry |= LDC_MTE_CPR;
116990354c4fSkettenis atomic_inc_int(&map->lm_count);
1170ba14339aSkettenis
117190354c4fSkettenis sc->sc_vd->vd_desc[prod].nbytes = max(m->m_pkthdr.len, 60);
117290354c4fSkettenis sc->sc_vd->vd_desc[prod].ncookies = 1;
117390354c4fSkettenis sc->sc_vd->vd_desc[prod].cookie[0].addr =
1174ba14339aSkettenis map->lm_next << PAGE_SHIFT | (pa & PAGE_MASK);
1175*63d58f04Sclaudio sc->sc_vd->vd_desc[prod].cookie[0].size = VNET_BUF_SIZE;
1176*63d58f04Sclaudio if (prod != start)
117790354c4fSkettenis sc->sc_vd->vd_desc[prod].hdr.dstate = VIO_DESC_READY;
1178ba14339aSkettenis
117990354c4fSkettenis sc->sc_vsd[prod].vsd_map_idx = map->lm_next;
118090354c4fSkettenis sc->sc_vsd[prod].vsd_buf = buf;
1181ba14339aSkettenis
118290354c4fSkettenis sc->sc_tx_prod++;
118390354c4fSkettenis prod = sc->sc_tx_prod & (sc->sc_vd->vd_nentries - 1);
1184ba14339aSkettenis
1185ba14339aSkettenis m_freem(m);
1186ba14339aSkettenis }
1187ba14339aSkettenis
118890354c4fSkettenis
1189*63d58f04Sclaudio if (start != prod) {
1190*63d58f04Sclaudio membar_producer();
1191*63d58f04Sclaudio sc->sc_vd->vd_desc[start].hdr.dstate = VIO_DESC_READY;
1192*63d58f04Sclaudio if (sc->sc_peer_state != VIO_DP_ACTIVE) {
119390354c4fSkettenis vnet_send_dring_data(sc, start);
1194*63d58f04Sclaudio ifp->if_timer = 10;
1195*63d58f04Sclaudio }
1196ba14339aSkettenis }
1197ba14339aSkettenis }
1198ba14339aSkettenis
119908c8e8d2Skettenis void
vnet_start_desc(struct ifnet * ifp)120008c8e8d2Skettenis vnet_start_desc(struct ifnet *ifp)
120108c8e8d2Skettenis {
120208c8e8d2Skettenis struct vnet_softc *sc = ifp->if_softc;
120308c8e8d2Skettenis struct ldc_map *map = sc->sc_lm;
120408c8e8d2Skettenis struct vnet_desc_msg dm;
120508c8e8d2Skettenis struct mbuf *m;
120608c8e8d2Skettenis paddr_t pa;
120708c8e8d2Skettenis caddr_t buf;
120890354c4fSkettenis u_int prod, count;
120908c8e8d2Skettenis
121008c8e8d2Skettenis for (;;) {
121190354c4fSkettenis count = sc->sc_tx_prod - sc->sc_tx_cons;
121290354c4fSkettenis if (count >= (sc->sc_vd->vd_nentries - 1) ||
121308c8e8d2Skettenis map->lm_count >= map->lm_nentries) {
1214de6cd8fbSdlg ifq_set_oactive(&ifp->if_snd);
121508c8e8d2Skettenis return;
121608c8e8d2Skettenis }
121708c8e8d2Skettenis
121808c8e8d2Skettenis buf = pool_get(&sc->sc_pool, PR_NOWAIT|PR_ZERO);
121908c8e8d2Skettenis if (buf == NULL) {
1220de6cd8fbSdlg ifq_set_oactive(&ifp->if_snd);
122108c8e8d2Skettenis return;
122208c8e8d2Skettenis }
1223d7421602Skettenis
122463bcfa73Spatrick m = ifq_dequeue(&ifp->if_snd);
1225d7421602Skettenis if (m == NULL) {
1226d7421602Skettenis pool_put(&sc->sc_pool, buf);
1227d7421602Skettenis return;
1228d7421602Skettenis }
1229d7421602Skettenis
123008c8e8d2Skettenis m_copydata(m, 0, m->m_pkthdr.len, buf);
123108c8e8d2Skettenis
123208c8e8d2Skettenis #if NBPFILTER > 0
123308c8e8d2Skettenis /*
123408c8e8d2Skettenis * If BPF is listening on this interface, let it see the
123508c8e8d2Skettenis * packet before we commit it to the wire.
123608c8e8d2Skettenis */
123708c8e8d2Skettenis if (ifp->if_bpf)
123808c8e8d2Skettenis bpf_mtap(ifp->if_bpf, m, BPF_DIRECTION_OUT);
123908c8e8d2Skettenis #endif
124008c8e8d2Skettenis
124108c8e8d2Skettenis pmap_extract(pmap_kernel(), (vaddr_t)buf, &pa);
124208c8e8d2Skettenis KASSERT((pa & ~PAGE_MASK) == (pa & LDC_MTE_RA_MASK));
124308c8e8d2Skettenis while (map->lm_slot[map->lm_next].entry != 0) {
124408c8e8d2Skettenis map->lm_next++;
124508c8e8d2Skettenis map->lm_next &= (map->lm_nentries - 1);
124608c8e8d2Skettenis }
124708c8e8d2Skettenis map->lm_slot[map->lm_next].entry = (pa & LDC_MTE_RA_MASK);
124808c8e8d2Skettenis map->lm_slot[map->lm_next].entry |= LDC_MTE_CPR;
124990354c4fSkettenis atomic_inc_int(&map->lm_count);
125008c8e8d2Skettenis
125190354c4fSkettenis prod = sc->sc_tx_prod & (sc->sc_vd->vd_nentries - 1);
125290354c4fSkettenis sc->sc_vsd[prod].vsd_map_idx = map->lm_next;
125390354c4fSkettenis sc->sc_vsd[prod].vsd_buf = buf;
125408c8e8d2Skettenis
125508c8e8d2Skettenis bzero(&dm, sizeof(dm));
125608c8e8d2Skettenis dm.tag.type = VIO_TYPE_DATA;
125708c8e8d2Skettenis dm.tag.stype = VIO_SUBTYPE_INFO;
125808c8e8d2Skettenis dm.tag.stype_env = VIO_DESC_DATA;
125908c8e8d2Skettenis dm.tag.sid = sc->sc_local_sid;
126008c8e8d2Skettenis dm.seq_no = sc->sc_seq_no++;
126108c8e8d2Skettenis dm.desc_handle = sc->sc_tx_prod;
126208c8e8d2Skettenis dm.nbytes = max(m->m_pkthdr.len, 60);
126308c8e8d2Skettenis dm.ncookies = 1;
126408c8e8d2Skettenis dm.cookie[0].addr =
126508c8e8d2Skettenis map->lm_next << PAGE_SHIFT | (pa & PAGE_MASK);
1266*63d58f04Sclaudio dm.cookie[0].size = VNET_BUF_SIZE;
12673d4b6a30Skettenis vnet_sendmsg(sc, &dm, sizeof(dm));
126808c8e8d2Skettenis
126908c8e8d2Skettenis sc->sc_tx_prod++;
127008c8e8d2Skettenis sc->sc_tx_prod &= (sc->sc_vd->vd_nentries - 1);
127108c8e8d2Skettenis
127208c8e8d2Skettenis m_freem(m);
127308c8e8d2Skettenis }
127408c8e8d2Skettenis }
127508c8e8d2Skettenis
1276ba14339aSkettenis int
vnet_ioctl(struct ifnet * ifp,u_long cmd,caddr_t data)1277ba14339aSkettenis vnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
1278ba14339aSkettenis {
1279ba14339aSkettenis struct vnet_softc *sc = ifp->if_softc;
1280ba14339aSkettenis struct ifreq *ifr = (struct ifreq *)data;
1281ba14339aSkettenis int s, error = 0;
1282ba14339aSkettenis
1283ba14339aSkettenis s = splnet();
1284ba14339aSkettenis
1285ba14339aSkettenis switch (cmd) {
1286ba14339aSkettenis case SIOCSIFADDR:
1287ba14339aSkettenis ifp->if_flags |= IFF_UP;
1288ba14339aSkettenis /* FALLTHROUGH */
1289ba14339aSkettenis case SIOCSIFFLAGS:
1290ba14339aSkettenis if (ifp->if_flags & IFF_UP) {
1291ba14339aSkettenis if ((ifp->if_flags & IFF_RUNNING) == 0)
1292ba14339aSkettenis vnet_init(ifp);
1293ba14339aSkettenis } else {
1294ba14339aSkettenis if (ifp->if_flags & IFF_RUNNING)
1295ba14339aSkettenis vnet_stop(ifp);
1296ba14339aSkettenis }
1297ba14339aSkettenis break;
1298ba14339aSkettenis
1299ba14339aSkettenis case SIOCGIFMEDIA:
1300ba14339aSkettenis case SIOCSIFMEDIA:
1301ba14339aSkettenis error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd);
1302ba14339aSkettenis break;
1303ba14339aSkettenis
1304a0c88015Skettenis case SIOCADDMULTI:
1305a0c88015Skettenis case SIOCDELMULTI:
1306a0c88015Skettenis /*
1307a0c88015Skettenis * XXX Removing all multicast addresses and adding
1308a0c88015Skettenis * most of them back, is somewhat retarded.
1309a0c88015Skettenis */
1310a0c88015Skettenis vnet_setmulti(sc, 0);
1311a0c88015Skettenis error = ether_ioctl(ifp, &sc->sc_ac, cmd, data);
1312a0c88015Skettenis vnet_setmulti(sc, 1);
1313a0c88015Skettenis if (error == ENETRESET)
1314a0c88015Skettenis error = 0;
1315a0c88015Skettenis break;
1316a0c88015Skettenis
1317ba14339aSkettenis default:
1318ba14339aSkettenis error = ether_ioctl(ifp, &sc->sc_ac, cmd, data);
1319ba14339aSkettenis }
1320ba14339aSkettenis
1321ba14339aSkettenis splx(s);
1322ba14339aSkettenis return (error);
1323ba14339aSkettenis }
1324ba14339aSkettenis
1325ba14339aSkettenis void
vnet_watchdog(struct ifnet * ifp)1326ba14339aSkettenis vnet_watchdog(struct ifnet *ifp)
1327ba14339aSkettenis {
1328199381a1Skettenis struct vnet_softc *sc = ifp->if_softc;
1329199381a1Skettenis
1330199381a1Skettenis printf("%s: watchdog timeout\n", sc->sc_dv.dv_xname);
1331ba14339aSkettenis }
1332ba14339aSkettenis
1333ba14339aSkettenis int
vnet_media_change(struct ifnet * ifp)1334ba14339aSkettenis vnet_media_change(struct ifnet *ifp)
1335ba14339aSkettenis {
1336ba14339aSkettenis return (0);
1337ba14339aSkettenis }
1338ba14339aSkettenis
1339ba14339aSkettenis void
vnet_media_status(struct ifnet * ifp,struct ifmediareq * imr)1340ba14339aSkettenis vnet_media_status(struct ifnet *ifp, struct ifmediareq *imr)
1341ba14339aSkettenis {
1342ba14339aSkettenis imr->ifm_active = IFM_ETHER | IFM_AUTO;
1343ba14339aSkettenis imr->ifm_status = IFM_AVALID;
134439c67f9aSkettenis
134539c67f9aSkettenis if (LINK_STATE_IS_UP(ifp->if_link_state) &&
134639c67f9aSkettenis ifp->if_flags & IFF_UP)
134739c67f9aSkettenis imr->ifm_status |= IFM_ACTIVE;
134839c67f9aSkettenis }
134939c67f9aSkettenis
135039c67f9aSkettenis void
vnet_link_state(struct vnet_softc * sc)135139c67f9aSkettenis vnet_link_state(struct vnet_softc *sc)
135239c67f9aSkettenis {
135339c67f9aSkettenis struct ifnet *ifp = &sc->sc_ac.ac_if;
135439c67f9aSkettenis int link_state = LINK_STATE_DOWN;
135539c67f9aSkettenis
13568dd7c4a8Skettenis KERNEL_LOCK();
135708c8e8d2Skettenis if (ISSET(sc->sc_vio_state, VIO_RCV_RDX) &&
135808c8e8d2Skettenis ISSET(sc->sc_vio_state, VIO_ACK_RDX))
135939c67f9aSkettenis link_state = LINK_STATE_FULL_DUPLEX;
136039c67f9aSkettenis if (ifp->if_link_state != link_state) {
136139c67f9aSkettenis ifp->if_link_state = link_state;
136239c67f9aSkettenis if_link_state_change(ifp);
136339c67f9aSkettenis }
13648dd7c4a8Skettenis KERNEL_UNLOCK();
1365ba14339aSkettenis }
1366ba14339aSkettenis
1367ba14339aSkettenis void
vnet_setmulti(struct vnet_softc * sc,int set)1368a0c88015Skettenis vnet_setmulti(struct vnet_softc *sc, int set)
1369a0c88015Skettenis {
1370a0c88015Skettenis struct arpcom *ac = &sc->sc_ac;
1371a0c88015Skettenis struct ether_multi *enm;
1372a0c88015Skettenis struct ether_multistep step;
1373a0c88015Skettenis struct vnet_mcast_info mi;
1374a0c88015Skettenis int count = 0;
1375a0c88015Skettenis
137608c8e8d2Skettenis if (!ISSET(sc->sc_vio_state, VIO_RCV_RDX) ||
137708c8e8d2Skettenis !ISSET(sc->sc_vio_state, VIO_ACK_RDX))
1378a0c88015Skettenis return;
1379a0c88015Skettenis
1380a0c88015Skettenis bzero(&mi, sizeof(mi));
1381a0c88015Skettenis mi.tag.type = VIO_TYPE_CTRL;
1382a0c88015Skettenis mi.tag.stype = VIO_SUBTYPE_INFO;
1383a0c88015Skettenis mi.tag.stype_env = VNET_MCAST_INFO;
1384a0c88015Skettenis mi.tag.sid = sc->sc_local_sid;
1385a0c88015Skettenis mi.set = set ? 1 : 0;
13868dd7c4a8Skettenis KERNEL_LOCK();
1387a0c88015Skettenis ETHER_FIRST_MULTI(step, ac, enm);
1388a0c88015Skettenis while (enm != NULL) {
1389a0c88015Skettenis /* XXX What about multicast ranges? */
1390a0c88015Skettenis bcopy(enm->enm_addrlo, mi.mcast_addr[count], ETHER_ADDR_LEN);
1391a0c88015Skettenis ETHER_NEXT_MULTI(step, enm);
1392a0c88015Skettenis
1393a0c88015Skettenis count++;
1394a0c88015Skettenis if (count < VNET_NUM_MCAST)
1395a0c88015Skettenis continue;
1396a0c88015Skettenis
1397a0c88015Skettenis mi.count = VNET_NUM_MCAST;
13983d4b6a30Skettenis vnet_sendmsg(sc, &mi, sizeof(mi));
1399a0c88015Skettenis count = 0;
1400a0c88015Skettenis }
1401a0c88015Skettenis
1402a0c88015Skettenis if (count > 0) {
1403a0c88015Skettenis mi.count = count;
14043d4b6a30Skettenis vnet_sendmsg(sc, &mi, sizeof(mi));
1405a0c88015Skettenis }
14068dd7c4a8Skettenis KERNEL_UNLOCK();
1407a0c88015Skettenis }
1408a0c88015Skettenis
1409a0c88015Skettenis void
vnet_init(struct ifnet * ifp)1410ba14339aSkettenis vnet_init(struct ifnet *ifp)
1411ba14339aSkettenis {
1412ba14339aSkettenis struct vnet_softc *sc = ifp->if_softc;
1413d9889caeSkettenis struct ldc_conn *lc = &sc->sc_lc;
1414ba14339aSkettenis int err;
1415ba14339aSkettenis
1416ba14339aSkettenis sc->sc_lm = ldc_map_alloc(sc->sc_dmatag, 2048);
1417ba14339aSkettenis if (sc->sc_lm == NULL)
1418ba14339aSkettenis return;
1419ba14339aSkettenis
1420d9889caeSkettenis err = hv_ldc_set_map_table(lc->lc_id,
1421ba14339aSkettenis sc->sc_lm->lm_map->dm_segs[0].ds_addr, sc->sc_lm->lm_nentries);
1422ba14339aSkettenis if (err != H_EOK) {
1423ba14339aSkettenis printf("hv_ldc_set_map_table %d\n", err);
1424ba14339aSkettenis return;
1425ba14339aSkettenis }
1426ba14339aSkettenis
14271373cec8Sstsp sc->sc_vd = vnet_dring_alloc(sc->sc_dmatag, VNET_NUM_SOFT_DESC);
1428ba14339aSkettenis if (sc->sc_vd == NULL)
1429ba14339aSkettenis return;
14301373cec8Sstsp sc->sc_vsd = malloc(VNET_NUM_SOFT_DESC * sizeof(*sc->sc_vsd), M_DEVBUF,
14318678061cSkettenis M_NOWAIT|M_ZERO);
1432ba14339aSkettenis if (sc->sc_vsd == NULL)
1433ba14339aSkettenis return;
1434ba14339aSkettenis
1435ba14339aSkettenis sc->sc_lm->lm_slot[0].entry = sc->sc_vd->vd_map->dm_segs[0].ds_addr;
1436ba14339aSkettenis sc->sc_lm->lm_slot[0].entry &= LDC_MTE_RA_MASK;
1437ba14339aSkettenis sc->sc_lm->lm_slot[0].entry |= LDC_MTE_CPR | LDC_MTE_CPW;
1438ba14339aSkettenis sc->sc_lm->lm_next = 1;
1439ba14339aSkettenis sc->sc_lm->lm_count = 1;
1440ba14339aSkettenis
1441d9889caeSkettenis err = hv_ldc_tx_qconf(lc->lc_id,
1442d9889caeSkettenis lc->lc_txq->lq_map->dm_segs[0].ds_addr, lc->lc_txq->lq_nentries);
1443ba14339aSkettenis if (err != H_EOK)
1444ba14339aSkettenis printf("hv_ldc_tx_qconf %d\n", err);
1445ba14339aSkettenis
1446d9889caeSkettenis err = hv_ldc_rx_qconf(lc->lc_id,
1447d9889caeSkettenis lc->lc_rxq->lq_map->dm_segs[0].ds_addr, lc->lc_rxq->lq_nentries);
1448ba14339aSkettenis if (err != H_EOK)
1449ba14339aSkettenis printf("hv_ldc_rx_qconf %d\n", err);
1450ba14339aSkettenis
1451b3a497edSkettenis cbus_intr_setenabled(sc->sc_bustag, sc->sc_tx_ino, INTR_ENABLED);
1452b3a497edSkettenis cbus_intr_setenabled(sc->sc_bustag, sc->sc_rx_ino, INTR_ENABLED);
1453b7bd479dSkettenis
1454d9889caeSkettenis ldc_send_vers(lc);
1455ba14339aSkettenis
145639c67f9aSkettenis ifp->if_flags |= IFF_RUNNING;
1457ba14339aSkettenis }
1458ba14339aSkettenis
1459ba14339aSkettenis void
vnet_stop(struct ifnet * ifp)1460ba14339aSkettenis vnet_stop(struct ifnet *ifp)
1461ba14339aSkettenis {
1462ba14339aSkettenis struct vnet_softc *sc = ifp->if_softc;
1463d9889caeSkettenis struct ldc_conn *lc = &sc->sc_lc;
1464ba14339aSkettenis
1465de6cd8fbSdlg ifp->if_flags &= ~IFF_RUNNING;
1466de6cd8fbSdlg ifq_clr_oactive(&ifp->if_snd);
1467199381a1Skettenis ifp->if_timer = 0;
1468ba14339aSkettenis
1469b3a497edSkettenis cbus_intr_setenabled(sc->sc_bustag, sc->sc_tx_ino, INTR_DISABLED);
1470b3a497edSkettenis cbus_intr_setenabled(sc->sc_bustag, sc->sc_rx_ino, INTR_DISABLED);
1471b7bd479dSkettenis
147211a28cd8Skettenis intr_barrier(sc->sc_tx_ih);
147311a28cd8Skettenis intr_barrier(sc->sc_rx_ih);
147411a28cd8Skettenis
1475d9889caeSkettenis hv_ldc_tx_qconf(lc->lc_id, 0, 0);
1476d9889caeSkettenis hv_ldc_rx_qconf(lc->lc_id, 0, 0);
14779464f18aSkettenis lc->lc_tx_seqid = 0;
14789464f18aSkettenis lc->lc_state = 0;
1479d9889caeSkettenis lc->lc_tx_state = lc->lc_rx_state = LDC_CHANNEL_DOWN;
14809464f18aSkettenis vnet_ldc_reset(lc);
1481ba14339aSkettenis
14821373cec8Sstsp free(sc->sc_vsd, M_DEVBUF, VNET_NUM_SOFT_DESC * sizeof(*sc->sc_vsd));
14831373cec8Sstsp
1484ba14339aSkettenis vnet_dring_free(sc->sc_dmatag, sc->sc_vd);
1485ba14339aSkettenis
1486d9889caeSkettenis hv_ldc_set_map_table(lc->lc_id, 0, 0);
1487ba14339aSkettenis ldc_map_free(sc->sc_dmatag, sc->sc_lm);
1488ba14339aSkettenis }
1489ba14339aSkettenis
1490ba14339aSkettenis struct vnet_dring *
vnet_dring_alloc(bus_dma_tag_t t,int nentries)1491ba14339aSkettenis vnet_dring_alloc(bus_dma_tag_t t, int nentries)
1492ba14339aSkettenis {
1493ba14339aSkettenis struct vnet_dring *vd;
1494ba14339aSkettenis bus_size_t size;
1495ba14339aSkettenis caddr_t va;
1496ba14339aSkettenis int nsegs;
1497ba14339aSkettenis int i;
1498ba14339aSkettenis
1499ba14339aSkettenis vd = malloc(sizeof(struct vnet_dring), M_DEVBUF, M_NOWAIT);
1500ba14339aSkettenis if (vd == NULL)
1501ba14339aSkettenis return NULL;
1502ba14339aSkettenis
1503ba14339aSkettenis size = roundup(nentries * sizeof(struct vnet_desc), PAGE_SIZE);
1504ba14339aSkettenis
1505ba14339aSkettenis if (bus_dmamap_create(t, size, 1, size, 0,
1506ba14339aSkettenis BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &vd->vd_map) != 0)
1507ba14339aSkettenis return (NULL);
1508ba14339aSkettenis
1509ba14339aSkettenis if (bus_dmamem_alloc(t, size, PAGE_SIZE, 0, &vd->vd_seg, 1,
1510ba14339aSkettenis &nsegs, BUS_DMA_NOWAIT) != 0)
1511ba14339aSkettenis goto destroy;
1512ba14339aSkettenis
1513ba14339aSkettenis if (bus_dmamem_map(t, &vd->vd_seg, 1, size, &va,
1514ba14339aSkettenis BUS_DMA_NOWAIT) != 0)
1515ba14339aSkettenis goto free;
1516ba14339aSkettenis
1517ba14339aSkettenis if (bus_dmamap_load(t, vd->vd_map, va, size, NULL,
1518ba14339aSkettenis BUS_DMA_NOWAIT) != 0)
1519ba14339aSkettenis goto unmap;
1520ba14339aSkettenis
1521ba14339aSkettenis vd->vd_desc = (struct vnet_desc *)va;
1522ba14339aSkettenis vd->vd_nentries = nentries;
1523ba14339aSkettenis bzero(vd->vd_desc, nentries * sizeof(struct vnet_desc));
1524ba14339aSkettenis for (i = 0; i < vd->vd_nentries; i++)
1525ba14339aSkettenis vd->vd_desc[i].hdr.dstate = VIO_DESC_FREE;
1526ba14339aSkettenis return (vd);
1527ba14339aSkettenis
1528ba14339aSkettenis unmap:
1529ba14339aSkettenis bus_dmamem_unmap(t, va, size);
1530ba14339aSkettenis free:
1531ba14339aSkettenis bus_dmamem_free(t, &vd->vd_seg, 1);
1532ba14339aSkettenis destroy:
1533ba14339aSkettenis bus_dmamap_destroy(t, vd->vd_map);
1534ba14339aSkettenis
1535ba14339aSkettenis return (NULL);
1536ba14339aSkettenis }
1537ba14339aSkettenis
1538ba14339aSkettenis void
vnet_dring_free(bus_dma_tag_t t,struct vnet_dring * vd)1539ba14339aSkettenis vnet_dring_free(bus_dma_tag_t t, struct vnet_dring *vd)
1540ba14339aSkettenis {
1541ba14339aSkettenis bus_size_t size;
1542ba14339aSkettenis
1543ba14339aSkettenis size = vd->vd_nentries * sizeof(struct vnet_desc);
1544ba14339aSkettenis size = roundup(size, PAGE_SIZE);
1545ba14339aSkettenis
1546ba14339aSkettenis bus_dmamap_unload(t, vd->vd_map);
1547ba14339aSkettenis bus_dmamem_unmap(t, (caddr_t)vd->vd_desc, size);
1548ba14339aSkettenis bus_dmamem_free(t, &vd->vd_seg, 1);
1549ba14339aSkettenis bus_dmamap_destroy(t, vd->vd_map);
1550c1a1ba6cSkn free(vd, M_DEVBUF, sizeof(*vd));
1551ba14339aSkettenis }
1552