111447b59SVenkatesh Srinivas /*-
211447b59SVenkatesh Srinivas  * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
311447b59SVenkatesh Srinivas  * All rights reserved.
411447b59SVenkatesh Srinivas  *
511447b59SVenkatesh Srinivas  * Redistribution and use in source and binary forms, with or without
611447b59SVenkatesh Srinivas  * modification, are permitted provided that the following conditions
711447b59SVenkatesh Srinivas  * are met:
811447b59SVenkatesh Srinivas  * 1. Redistributions of source code must retain the above copyright
911447b59SVenkatesh Srinivas  *    notice unmodified, this list of conditions, and the following
1011447b59SVenkatesh Srinivas  *    disclaimer.
1111447b59SVenkatesh Srinivas  * 2. Redistributions in binary form must reproduce the above copyright
1211447b59SVenkatesh Srinivas  *    notice, this list of conditions and the following disclaimer in the
1311447b59SVenkatesh Srinivas  *    documentation and/or other materials provided with the distribution.
1411447b59SVenkatesh Srinivas  *
1511447b59SVenkatesh Srinivas  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
1611447b59SVenkatesh Srinivas  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
1711447b59SVenkatesh Srinivas  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
1811447b59SVenkatesh Srinivas  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
1911447b59SVenkatesh Srinivas  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
2011447b59SVenkatesh Srinivas  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
2111447b59SVenkatesh Srinivas  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
2211447b59SVenkatesh Srinivas  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
2311447b59SVenkatesh Srinivas  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
2411447b59SVenkatesh Srinivas  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2511447b59SVenkatesh Srinivas  *
2611447b59SVenkatesh Srinivas  * $FreeBSD: src/sys/dev/virtio/virtqueue.c,v 1.2 2012/04/14 05:48:04 grehan Exp $
2711447b59SVenkatesh Srinivas  */
2811447b59SVenkatesh Srinivas 
2911447b59SVenkatesh Srinivas /*
3011447b59SVenkatesh Srinivas  * Implements the virtqueue interface as basically described
3111447b59SVenkatesh Srinivas  * in the original VirtIO paper.
3211447b59SVenkatesh Srinivas  */
3311447b59SVenkatesh Srinivas 
3411447b59SVenkatesh Srinivas #include <sys/param.h>
3511447b59SVenkatesh Srinivas #include <sys/systm.h>
3611447b59SVenkatesh Srinivas #include <sys/kernel.h>
3711447b59SVenkatesh Srinivas #include <sys/malloc.h>
3811447b59SVenkatesh Srinivas #include <sys/sglist.h>
392f1382caSVenkatesh Srinivas #include <sys/serialize.h>
4011447b59SVenkatesh Srinivas #include <vm/vm.h>
4111447b59SVenkatesh Srinivas #include <vm/pmap.h>
4211447b59SVenkatesh Srinivas 
4311447b59SVenkatesh Srinivas #include <machine/cpu.h>
4411447b59SVenkatesh Srinivas #include <machine/atomic.h>
4511447b59SVenkatesh Srinivas #include <sys/bus.h>
4611447b59SVenkatesh Srinivas #include <sys/rman.h>
4711447b59SVenkatesh Srinivas 
4811447b59SVenkatesh Srinivas #include "virtio.h"
4911447b59SVenkatesh Srinivas #include "virtqueue.h"
5011447b59SVenkatesh Srinivas #include "virtio_ring.h"
5111447b59SVenkatesh Srinivas 
5211447b59SVenkatesh Srinivas #include "virtio_bus_if.h"
5311447b59SVenkatesh Srinivas 
5411447b59SVenkatesh Srinivas struct virtqueue {
5511447b59SVenkatesh Srinivas 	device_t		 vq_dev;
5611447b59SVenkatesh Srinivas 	char			 vq_name[VIRTQUEUE_MAX_NAME_SZ];
5711447b59SVenkatesh Srinivas 	uint16_t		 vq_queue_index;
5811447b59SVenkatesh Srinivas 	uint16_t		 vq_nentries;
5911447b59SVenkatesh Srinivas 	uint32_t		 vq_flags;
6011447b59SVenkatesh Srinivas 
61cb6e1f75SImre Vadász #define	VIRTQUEUE_FLAG_INDIRECT  0x0001
6211447b59SVenkatesh Srinivas #define	VIRTQUEUE_FLAG_EVENT_IDX 0x0002
6311447b59SVenkatesh Srinivas 
6411447b59SVenkatesh Srinivas 	int			 vq_alignment;
6511447b59SVenkatesh Srinivas 	int			 vq_ring_size;
6611447b59SVenkatesh Srinivas 	void			*vq_ring_mem;
67cb6e1f75SImre Vadász 	int			 vq_max_indirect_size;
68cb6e1f75SImre Vadász 	int			 vq_indirect_mem_size;
6911447b59SVenkatesh Srinivas 
7011447b59SVenkatesh Srinivas 	struct vring		 vq_ring;
7111447b59SVenkatesh Srinivas 	uint16_t		 vq_free_cnt;
7211447b59SVenkatesh Srinivas 	uint16_t		 vq_queued_cnt;
7311447b59SVenkatesh Srinivas 	/*
7411447b59SVenkatesh Srinivas 	 * Head of the free chain in the descriptor table. If
7511447b59SVenkatesh Srinivas 	 * there are no free descriptors, this will be set to
7611447b59SVenkatesh Srinivas 	 * VQ_RING_DESC_CHAIN_END.
7711447b59SVenkatesh Srinivas 	 */
7811447b59SVenkatesh Srinivas 	uint16_t		 vq_desc_head_idx;
7911447b59SVenkatesh Srinivas 	/*
8011447b59SVenkatesh Srinivas 	 * Last consumed descriptor in the used table,
8111447b59SVenkatesh Srinivas 	 * trails vq_ring.used->idx.
8211447b59SVenkatesh Srinivas 	 */
8311447b59SVenkatesh Srinivas 	uint16_t		 vq_used_cons_idx;
8411447b59SVenkatesh Srinivas 
8511447b59SVenkatesh Srinivas 	struct vq_desc_extra {
8611447b59SVenkatesh Srinivas 		void		  *cookie;
87cb6e1f75SImre Vadász 		struct vring_desc *indirect;
88cb6e1f75SImre Vadász 		vm_paddr_t         indirect_paddr;
8911447b59SVenkatesh Srinivas 		uint16_t	   ndescs;
9011447b59SVenkatesh Srinivas 	} vq_descx[0];
9111447b59SVenkatesh Srinivas };
9211447b59SVenkatesh Srinivas 
9311447b59SVenkatesh Srinivas /*
9411447b59SVenkatesh Srinivas  * The maximum virtqueue size is 2^15. Use that value as the end of
9511447b59SVenkatesh Srinivas  * descriptor chain terminator since it will never be a valid index
9611447b59SVenkatesh Srinivas  * in the descriptor table. This is used to verify we are correctly
9711447b59SVenkatesh Srinivas  * handling vq_free_cnt.
9811447b59SVenkatesh Srinivas  */
9911447b59SVenkatesh Srinivas #define VQ_RING_DESC_CHAIN_END 32768
10011447b59SVenkatesh Srinivas 
10111447b59SVenkatesh Srinivas #define VQASSERT(_vq, _exp, _msg, ...)				\
10211447b59SVenkatesh Srinivas     KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name,	\
10311447b59SVenkatesh Srinivas 	##__VA_ARGS__))
10411447b59SVenkatesh Srinivas 
10511447b59SVenkatesh Srinivas #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx)			\
10611447b59SVenkatesh Srinivas     VQASSERT((_vq), (_idx) < (_vq)->vq_nentries,		\
10711447b59SVenkatesh Srinivas 	"invalid ring index: %d, max: %d", (_idx),		\
10811447b59SVenkatesh Srinivas 	(_vq)->vq_nentries)
10911447b59SVenkatesh Srinivas 
11011447b59SVenkatesh Srinivas #define VQ_RING_ASSERT_CHAIN_TERM(_vq)				\
11111447b59SVenkatesh Srinivas     VQASSERT((_vq), (_vq)->vq_desc_head_idx ==			\
11211447b59SVenkatesh Srinivas 	VQ_RING_DESC_CHAIN_END,	"full ring terminated "		\
11311447b59SVenkatesh Srinivas 	"incorrectly: head idx: %d", (_vq)->vq_desc_head_idx)
11411447b59SVenkatesh Srinivas 
115cb6e1f75SImre Vadász static int	virtqueue_init_indirect(struct virtqueue *vq, int);
116cb6e1f75SImre Vadász static void	virtqueue_free_indirect(struct virtqueue *vq);
117cb6e1f75SImre Vadász static void	virtqueue_init_indirect_list(struct virtqueue *,
118cb6e1f75SImre Vadász 		    struct vring_desc *);
119cb6e1f75SImre Vadász 
12011447b59SVenkatesh Srinivas static void	vq_ring_init(struct virtqueue *);
12111447b59SVenkatesh Srinivas static void	vq_ring_update_avail(struct virtqueue *, uint16_t);
12211447b59SVenkatesh Srinivas static uint16_t	vq_ring_enqueue_segments(struct virtqueue *,
12311447b59SVenkatesh Srinivas 		    struct vring_desc *, uint16_t, struct sglist *, int, int);
124cb6e1f75SImre Vadász static int	vq_ring_use_indirect(struct virtqueue *, int);
125cb6e1f75SImre Vadász static void	vq_ring_enqueue_indirect(struct virtqueue *, void *,
126cb6e1f75SImre Vadász 		    struct sglist *, int, int);
12711447b59SVenkatesh Srinivas static int	vq_ring_must_notify_host(struct virtqueue *);
12811447b59SVenkatesh Srinivas static void	vq_ring_notify_host(struct virtqueue *);
12911447b59SVenkatesh Srinivas static void	vq_ring_free_chain(struct virtqueue *, uint16_t);
13011447b59SVenkatesh Srinivas 
13111447b59SVenkatesh Srinivas uint64_t
virtqueue_filter_features(uint64_t features)13211447b59SVenkatesh Srinivas virtqueue_filter_features(uint64_t features)
13311447b59SVenkatesh Srinivas {
13411447b59SVenkatesh Srinivas 	uint64_t mask;
13511447b59SVenkatesh Srinivas 
13611447b59SVenkatesh Srinivas 	mask = (1 << VIRTIO_TRANSPORT_F_START) - 1;
137cb6e1f75SImre Vadász 	mask |= VIRTIO_RING_F_INDIRECT_DESC;
13811447b59SVenkatesh Srinivas 	mask |= VIRTIO_RING_F_EVENT_IDX;
13911447b59SVenkatesh Srinivas 
14011447b59SVenkatesh Srinivas 	return (features & mask);
14111447b59SVenkatesh Srinivas }
14211447b59SVenkatesh Srinivas 
14311447b59SVenkatesh Srinivas int
virtqueue_alloc(device_t dev,uint16_t queue,uint16_t size,int align,vm_paddr_t highaddr,struct vq_alloc_info * info,struct virtqueue ** vqp)14411447b59SVenkatesh Srinivas virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align,
14511447b59SVenkatesh Srinivas     vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp)
14611447b59SVenkatesh Srinivas {
14711447b59SVenkatesh Srinivas 	struct virtqueue *vq;
14811447b59SVenkatesh Srinivas 	int error;
14911447b59SVenkatesh Srinivas 
15011447b59SVenkatesh Srinivas 	*vqp = NULL;
15111447b59SVenkatesh Srinivas 	error = 0;
15211447b59SVenkatesh Srinivas 
15311447b59SVenkatesh Srinivas 	if (size == 0) {
15411447b59SVenkatesh Srinivas 		device_printf(dev,
15511447b59SVenkatesh Srinivas 		    "virtqueue %d (%s) does not exist (size is zero)\n",
15611447b59SVenkatesh Srinivas 		    queue, info->vqai_name);
15711447b59SVenkatesh Srinivas 		return (ENODEV);
15811447b59SVenkatesh Srinivas 	} else if (!powerof2(size)) {
15911447b59SVenkatesh Srinivas 		device_printf(dev,
16011447b59SVenkatesh Srinivas 		    "virtqueue %d (%s) size is not a power of 2: %d\n",
16111447b59SVenkatesh Srinivas 		    queue, info->vqai_name, size);
16211447b59SVenkatesh Srinivas 		return (ENXIO);
163cb6e1f75SImre Vadász 	} else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) {
164cb6e1f75SImre Vadász 		device_printf(dev, "virtqueue %d (%s) requested too many "
165cb6e1f75SImre Vadász 		    "indirect descriptors: %d, max %d\n",
166cb6e1f75SImre Vadász 		    queue, info->vqai_name, info->vqai_maxindirsz,
167cb6e1f75SImre Vadász 		    VIRTIO_MAX_INDIRECT);
168cb6e1f75SImre Vadász 		return (EINVAL);
16911447b59SVenkatesh Srinivas 	}
17011447b59SVenkatesh Srinivas 
17111447b59SVenkatesh Srinivas 	vq = kmalloc(sizeof(struct virtqueue) +
172eb55c32fSMatthew Dillon 	    size * sizeof(struct vq_desc_extra), M_DEVBUF, M_INTWAIT | M_ZERO);
17311447b59SVenkatesh Srinivas 	if (vq == NULL) {
17411447b59SVenkatesh Srinivas 		device_printf(dev, "cannot allocate virtqueue\n");
17511447b59SVenkatesh Srinivas 		return (ENOMEM);
17611447b59SVenkatesh Srinivas 	}
17711447b59SVenkatesh Srinivas 
17811447b59SVenkatesh Srinivas 	vq->vq_dev = dev;
17911447b59SVenkatesh Srinivas 	strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
18011447b59SVenkatesh Srinivas 	vq->vq_queue_index = queue;
18111447b59SVenkatesh Srinivas 	vq->vq_alignment = align;
18211447b59SVenkatesh Srinivas 	vq->vq_nentries = size;
18311447b59SVenkatesh Srinivas 	vq->vq_free_cnt = size;
18411447b59SVenkatesh Srinivas 
18511447b59SVenkatesh Srinivas 	if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
18611447b59SVenkatesh Srinivas 		vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;
18711447b59SVenkatesh Srinivas 
188cb6e1f75SImre Vadász 	if (info->vqai_maxindirsz > 1) {
189cb6e1f75SImre Vadász 		error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
190cb6e1f75SImre Vadász 		if (error)
191cb6e1f75SImre Vadász 			goto fail;
192cb6e1f75SImre Vadász 	}
193cb6e1f75SImre Vadász 
19411447b59SVenkatesh Srinivas 	vq->vq_ring_size = round_page(vring_size(size, align));
19511447b59SVenkatesh Srinivas 	vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,
196ba08e4d1SMatthew Dillon 	    M_WAITOK | M_ZERO, 0, highaddr, PAGE_SIZE, 0);
19711447b59SVenkatesh Srinivas 	if (vq->vq_ring_mem == NULL) {
19811447b59SVenkatesh Srinivas 		device_printf(dev,
19911447b59SVenkatesh Srinivas 		    "cannot allocate memory for virtqueue ring\n");
20011447b59SVenkatesh Srinivas 		error = ENOMEM;
20111447b59SVenkatesh Srinivas 		goto fail;
20211447b59SVenkatesh Srinivas 	}
20311447b59SVenkatesh Srinivas 
20411447b59SVenkatesh Srinivas 	vq_ring_init(vq);
20511447b59SVenkatesh Srinivas 	virtqueue_disable_intr(vq);
20611447b59SVenkatesh Srinivas 
20711447b59SVenkatesh Srinivas 	*vqp = vq;
20811447b59SVenkatesh Srinivas 
20911447b59SVenkatesh Srinivas fail:
21011447b59SVenkatesh Srinivas 	if (error)
21111447b59SVenkatesh Srinivas 		virtqueue_free(vq);
21211447b59SVenkatesh Srinivas 
21311447b59SVenkatesh Srinivas 	return (error);
21411447b59SVenkatesh Srinivas }
21511447b59SVenkatesh Srinivas 
216cb6e1f75SImre Vadász static int
virtqueue_init_indirect(struct virtqueue * vq,int indirect_size)217cb6e1f75SImre Vadász virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
218cb6e1f75SImre Vadász {
219cb6e1f75SImre Vadász 	device_t dev;
220cb6e1f75SImre Vadász 	struct vq_desc_extra *dxp;
221cb6e1f75SImre Vadász 	int i, size;
222cb6e1f75SImre Vadász 
223cb6e1f75SImre Vadász 	dev = vq->vq_dev;
224cb6e1f75SImre Vadász 
225cb6e1f75SImre Vadász 	if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) {
226cb6e1f75SImre Vadász 		/*
227cb6e1f75SImre Vadász 		 * Indirect descriptors requested by the driver but not
228cb6e1f75SImre Vadász 		 * negotiated. Return zero to keep the initialization
229cb6e1f75SImre Vadász 		 * going: we'll run fine without.
230cb6e1f75SImre Vadász 		 */
231cb6e1f75SImre Vadász 		if (bootverbose)
232cb6e1f75SImre Vadász 			device_printf(dev, "virtqueue %d (%s) requested "
233cb6e1f75SImre Vadász 			    "indirect descriptors but not negotiated\n",
234cb6e1f75SImre Vadász 			    vq->vq_queue_index, vq->vq_name);
235cb6e1f75SImre Vadász 		return (0);
236cb6e1f75SImre Vadász 	}
237cb6e1f75SImre Vadász 
238cb6e1f75SImre Vadász 	size = indirect_size * sizeof(struct vring_desc);
239cb6e1f75SImre Vadász 	vq->vq_max_indirect_size = indirect_size;
240cb6e1f75SImre Vadász 	vq->vq_indirect_mem_size = size;
241cb6e1f75SImre Vadász 	vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT;
242cb6e1f75SImre Vadász 
243cb6e1f75SImre Vadász 	for (i = 0; i < vq->vq_nentries; i++) {
244cb6e1f75SImre Vadász 		dxp = &vq->vq_descx[i];
245cb6e1f75SImre Vadász 
246ba08e4d1SMatthew Dillon 		dxp->indirect = contigmalloc(size, M_DEVBUF, M_WAITOK,
247eb55c32fSMatthew Dillon 		    0, BUS_SPACE_MAXADDR, 16, 0);
248cb6e1f75SImre Vadász 		if (dxp->indirect == NULL) {
249cb6e1f75SImre Vadász 			device_printf(dev, "cannot allocate indirect list\n");
250cb6e1f75SImre Vadász 			return (ENOMEM);
251cb6e1f75SImre Vadász 		}
252cb6e1f75SImre Vadász 
253cb6e1f75SImre Vadász 		dxp->indirect_paddr = vtophys(dxp->indirect);
254cb6e1f75SImre Vadász 		virtqueue_init_indirect_list(vq, dxp->indirect);
255cb6e1f75SImre Vadász 	}
256cb6e1f75SImre Vadász 
257cb6e1f75SImre Vadász 	return (0);
258cb6e1f75SImre Vadász }
259cb6e1f75SImre Vadász 
260cb6e1f75SImre Vadász static void
virtqueue_free_indirect(struct virtqueue * vq)261cb6e1f75SImre Vadász virtqueue_free_indirect(struct virtqueue *vq)
262cb6e1f75SImre Vadász {
263cb6e1f75SImre Vadász 	struct vq_desc_extra *dxp;
264cb6e1f75SImre Vadász 	int i;
265cb6e1f75SImre Vadász 
266cb6e1f75SImre Vadász 	for (i = 0; i < vq->vq_nentries; i++) {
267cb6e1f75SImre Vadász 		dxp = &vq->vq_descx[i];
268cb6e1f75SImre Vadász 
269cb6e1f75SImre Vadász 		if (dxp->indirect == NULL)
270cb6e1f75SImre Vadász 			break;
271cb6e1f75SImre Vadász 
272cb6e1f75SImre Vadász 		contigfree(dxp->indirect, vq->vq_indirect_mem_size, M_DEVBUF);
273cb6e1f75SImre Vadász 		dxp->indirect = NULL;
274cb6e1f75SImre Vadász 		dxp->indirect_paddr = 0;
275cb6e1f75SImre Vadász 	}
276cb6e1f75SImre Vadász 
277cb6e1f75SImre Vadász 	vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT;
278cb6e1f75SImre Vadász 	vq->vq_indirect_mem_size = 0;
279cb6e1f75SImre Vadász }
280cb6e1f75SImre Vadász 
281cb6e1f75SImre Vadász static void
virtqueue_init_indirect_list(struct virtqueue * vq,struct vring_desc * indirect)282cb6e1f75SImre Vadász virtqueue_init_indirect_list(struct virtqueue *vq,
283cb6e1f75SImre Vadász     struct vring_desc *indirect)
284cb6e1f75SImre Vadász {
285cb6e1f75SImre Vadász 	int i;
286cb6e1f75SImre Vadász 
287cb6e1f75SImre Vadász 	bzero(indirect, vq->vq_indirect_mem_size);
288cb6e1f75SImre Vadász 
289cb6e1f75SImre Vadász 	for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
290cb6e1f75SImre Vadász 		indirect[i].next = i + 1;
291cb6e1f75SImre Vadász 	indirect[i].next = VQ_RING_DESC_CHAIN_END;
292cb6e1f75SImre Vadász }
293cb6e1f75SImre Vadász 
29411447b59SVenkatesh Srinivas int
virtqueue_reinit(struct virtqueue * vq,uint16_t size)29511447b59SVenkatesh Srinivas virtqueue_reinit(struct virtqueue *vq, uint16_t size)
29611447b59SVenkatesh Srinivas {
29711447b59SVenkatesh Srinivas 	struct vq_desc_extra *dxp;
29811447b59SVenkatesh Srinivas 	int i;
29911447b59SVenkatesh Srinivas 
30011447b59SVenkatesh Srinivas 	if (vq->vq_nentries != size) {
30111447b59SVenkatesh Srinivas 		device_printf(vq->vq_dev,
30211447b59SVenkatesh Srinivas 		    "%s: '%s' changed size; old=%hu, new=%hu\n",
30311447b59SVenkatesh Srinivas 		    __func__, vq->vq_name, vq->vq_nentries, size);
30411447b59SVenkatesh Srinivas 		return (EINVAL);
30511447b59SVenkatesh Srinivas 	}
30611447b59SVenkatesh Srinivas 
30711447b59SVenkatesh Srinivas 	/* Warn if the virtqueue was not properly cleaned up. */
30811447b59SVenkatesh Srinivas 	if (vq->vq_free_cnt != vq->vq_nentries) {
30911447b59SVenkatesh Srinivas 		device_printf(vq->vq_dev,
31011447b59SVenkatesh Srinivas 		    "%s: warning, '%s' virtqueue not empty, "
31111447b59SVenkatesh Srinivas 		    "leaking %d entries\n", __func__, vq->vq_name,
31211447b59SVenkatesh Srinivas 		    vq->vq_nentries - vq->vq_free_cnt);
31311447b59SVenkatesh Srinivas 	}
31411447b59SVenkatesh Srinivas 
31511447b59SVenkatesh Srinivas 	vq->vq_desc_head_idx = 0;
31611447b59SVenkatesh Srinivas 	vq->vq_used_cons_idx = 0;
31711447b59SVenkatesh Srinivas 	vq->vq_queued_cnt = 0;
31811447b59SVenkatesh Srinivas 	vq->vq_free_cnt = vq->vq_nentries;
31911447b59SVenkatesh Srinivas 
32011447b59SVenkatesh Srinivas 	/* To be safe, reset all our allocated memory. */
32111447b59SVenkatesh Srinivas 	bzero(vq->vq_ring_mem, vq->vq_ring_size);
32211447b59SVenkatesh Srinivas 	for (i = 0; i < vq->vq_nentries; i++) {
32311447b59SVenkatesh Srinivas 		dxp = &vq->vq_descx[i];
32411447b59SVenkatesh Srinivas 		dxp->cookie = NULL;
32511447b59SVenkatesh Srinivas 		dxp->ndescs = 0;
326cb6e1f75SImre Vadász 		if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
327cb6e1f75SImre Vadász 			virtqueue_init_indirect_list(vq, dxp->indirect);
32811447b59SVenkatesh Srinivas 	}
32911447b59SVenkatesh Srinivas 
33011447b59SVenkatesh Srinivas 	vq_ring_init(vq);
33111447b59SVenkatesh Srinivas 	virtqueue_disable_intr(vq);
33211447b59SVenkatesh Srinivas 
33311447b59SVenkatesh Srinivas 	return (0);
33411447b59SVenkatesh Srinivas }
33511447b59SVenkatesh Srinivas 
33611447b59SVenkatesh Srinivas void
virtqueue_free(struct virtqueue * vq)33711447b59SVenkatesh Srinivas virtqueue_free(struct virtqueue *vq)
33811447b59SVenkatesh Srinivas {
33911447b59SVenkatesh Srinivas 
34011447b59SVenkatesh Srinivas 	if (vq->vq_free_cnt != vq->vq_nentries) {
34111447b59SVenkatesh Srinivas 		device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, "
34211447b59SVenkatesh Srinivas 		    "leaking %d entries\n", vq->vq_name,
34311447b59SVenkatesh Srinivas 		    vq->vq_nentries - vq->vq_free_cnt);
34411447b59SVenkatesh Srinivas 	}
34511447b59SVenkatesh Srinivas 
346cb6e1f75SImre Vadász 	if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
347cb6e1f75SImre Vadász 		virtqueue_free_indirect(vq);
348cb6e1f75SImre Vadász 
34911447b59SVenkatesh Srinivas 	if (vq->vq_ring_mem != NULL) {
35011447b59SVenkatesh Srinivas 		contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF);
35111447b59SVenkatesh Srinivas 		vq->vq_ring_size = 0;
35211447b59SVenkatesh Srinivas 		vq->vq_ring_mem = NULL;
35311447b59SVenkatesh Srinivas 	}
35411447b59SVenkatesh Srinivas 
35511447b59SVenkatesh Srinivas 	kfree(vq, M_DEVBUF);
35611447b59SVenkatesh Srinivas }
35711447b59SVenkatesh Srinivas 
35811447b59SVenkatesh Srinivas vm_paddr_t
virtqueue_paddr(struct virtqueue * vq)35911447b59SVenkatesh Srinivas virtqueue_paddr(struct virtqueue *vq)
36011447b59SVenkatesh Srinivas {
36111447b59SVenkatesh Srinivas 	return (vtophys(vq->vq_ring_mem));
36211447b59SVenkatesh Srinivas }
36311447b59SVenkatesh Srinivas 
36411447b59SVenkatesh Srinivas int
virtqueue_size(struct virtqueue * vq)36511447b59SVenkatesh Srinivas virtqueue_size(struct virtqueue *vq)
36611447b59SVenkatesh Srinivas {
36711447b59SVenkatesh Srinivas 	return (vq->vq_nentries);
36811447b59SVenkatesh Srinivas }
36911447b59SVenkatesh Srinivas 
37011447b59SVenkatesh Srinivas int
virtqueue_empty(struct virtqueue * vq)37111447b59SVenkatesh Srinivas virtqueue_empty(struct virtqueue *vq)
37211447b59SVenkatesh Srinivas {
37311447b59SVenkatesh Srinivas 
37411447b59SVenkatesh Srinivas 	return (vq->vq_nentries == vq->vq_free_cnt);
37511447b59SVenkatesh Srinivas }
37611447b59SVenkatesh Srinivas 
37711447b59SVenkatesh Srinivas int
virtqueue_full(struct virtqueue * vq)37811447b59SVenkatesh Srinivas virtqueue_full(struct virtqueue *vq)
37911447b59SVenkatesh Srinivas {
38011447b59SVenkatesh Srinivas 
38111447b59SVenkatesh Srinivas 	return (vq->vq_free_cnt == 0);
38211447b59SVenkatesh Srinivas }
38311447b59SVenkatesh Srinivas 
38411447b59SVenkatesh Srinivas void
virtqueue_notify(struct virtqueue * vq,lwkt_serialize_t interlock)3852f1382caSVenkatesh Srinivas virtqueue_notify(struct virtqueue *vq, lwkt_serialize_t interlock)
38611447b59SVenkatesh Srinivas {
38711447b59SVenkatesh Srinivas 	/* Ensure updated avail->idx is visible to host. */
38811447b59SVenkatesh Srinivas 	cpu_mfence();
38911447b59SVenkatesh Srinivas 
39011447b59SVenkatesh Srinivas 	if (vq_ring_must_notify_host(vq)) {
39158db6936SImre Vadasz 		if (interlock != NULL)
3922f1382caSVenkatesh Srinivas 			lwkt_serialize_exit(interlock);
39311447b59SVenkatesh Srinivas 		vq_ring_notify_host(vq);
39458db6936SImre Vadasz 		if (interlock != NULL)
3952f1382caSVenkatesh Srinivas 			lwkt_serialize_enter(interlock);
39611447b59SVenkatesh Srinivas 	}
39711447b59SVenkatesh Srinivas 	vq->vq_queued_cnt = 0;
39811447b59SVenkatesh Srinivas }
39911447b59SVenkatesh Srinivas 
40011447b59SVenkatesh Srinivas int
virtqueue_nused(struct virtqueue * vq)40111447b59SVenkatesh Srinivas virtqueue_nused(struct virtqueue *vq)
40211447b59SVenkatesh Srinivas {
40311447b59SVenkatesh Srinivas 	uint16_t used_idx, nused;
40411447b59SVenkatesh Srinivas 
40511447b59SVenkatesh Srinivas 	used_idx = vq->vq_ring.used->idx;
40611447b59SVenkatesh Srinivas 	nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
40711447b59SVenkatesh Srinivas 	VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
40811447b59SVenkatesh Srinivas 
40911447b59SVenkatesh Srinivas 	return (nused);
41011447b59SVenkatesh Srinivas }
41111447b59SVenkatesh Srinivas 
41211447b59SVenkatesh Srinivas int
virtqueue_pending(struct virtqueue * vq)413*9d96478cSImre Vadász virtqueue_pending(struct virtqueue *vq)
41411447b59SVenkatesh Srinivas {
415*9d96478cSImre Vadász 	return (vq->vq_used_cons_idx != vq->vq_ring.used->idx);
41611447b59SVenkatesh Srinivas }
41711447b59SVenkatesh Srinivas 
41811447b59SVenkatesh Srinivas /*
41911447b59SVenkatesh Srinivas  * Enable interrupts on a given virtqueue. Returns 1 if there are
42011447b59SVenkatesh Srinivas  * additional entries to process on the virtqueue after we return.
42111447b59SVenkatesh Srinivas  */
42211447b59SVenkatesh Srinivas int
virtqueue_enable_intr(struct virtqueue * vq)42311447b59SVenkatesh Srinivas virtqueue_enable_intr(struct virtqueue *vq)
42411447b59SVenkatesh Srinivas {
42511447b59SVenkatesh Srinivas 	/*
42611447b59SVenkatesh Srinivas 	 * Enable interrupts, making sure we get the latest
42711447b59SVenkatesh Srinivas 	 * index of what's already been consumed.
42811447b59SVenkatesh Srinivas 	 */
42911447b59SVenkatesh Srinivas 	vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
43011447b59SVenkatesh Srinivas 	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
43111447b59SVenkatesh Srinivas 		vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx;
43211447b59SVenkatesh Srinivas 	} else {
43311447b59SVenkatesh Srinivas 	       vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
43411447b59SVenkatesh Srinivas 	}
43511447b59SVenkatesh Srinivas 
43611447b59SVenkatesh Srinivas 	cpu_mfence();
43711447b59SVenkatesh Srinivas 
43811447b59SVenkatesh Srinivas 	/*
43911447b59SVenkatesh Srinivas 	 * Additional items may have been consumed in the time between
44011447b59SVenkatesh Srinivas 	 * since we last checked and enabled interrupts above. Let our
44111447b59SVenkatesh Srinivas 	 * caller know so it processes the new entries.
44211447b59SVenkatesh Srinivas 	 */
44311447b59SVenkatesh Srinivas 	if (vq->vq_used_cons_idx != vq->vq_ring.used->idx)
44411447b59SVenkatesh Srinivas 		return (1);
44511447b59SVenkatesh Srinivas 
44611447b59SVenkatesh Srinivas 	return (0);
44711447b59SVenkatesh Srinivas }
44811447b59SVenkatesh Srinivas 
44911447b59SVenkatesh Srinivas int
virtqueue_postpone_intr(struct virtqueue * vq)45011447b59SVenkatesh Srinivas virtqueue_postpone_intr(struct virtqueue *vq)
45111447b59SVenkatesh Srinivas {
45211447b59SVenkatesh Srinivas 	uint16_t ndesc;
45311447b59SVenkatesh Srinivas 
45411447b59SVenkatesh Srinivas 	/*
45511447b59SVenkatesh Srinivas 	 * Postpone until at least half of the available descriptors
45611447b59SVenkatesh Srinivas 	 * have been consumed.
45711447b59SVenkatesh Srinivas 	 *
45811447b59SVenkatesh Srinivas 	 * XXX Adaptive factor? (Linux uses 3/4)
45911447b59SVenkatesh Srinivas 	 */
46011447b59SVenkatesh Srinivas 	ndesc = (uint16_t)(vq->vq_ring.avail->idx - vq->vq_used_cons_idx) / 2;
46111447b59SVenkatesh Srinivas 
46211447b59SVenkatesh Srinivas 	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
46311447b59SVenkatesh Srinivas 		vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc;
46411447b59SVenkatesh Srinivas 	else
46511447b59SVenkatesh Srinivas 		vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
46611447b59SVenkatesh Srinivas 
46711447b59SVenkatesh Srinivas 	cpu_mfence();
46811447b59SVenkatesh Srinivas 
46911447b59SVenkatesh Srinivas 	/*
47011447b59SVenkatesh Srinivas 	 * Enough items may have already been consumed to meet our
47111447b59SVenkatesh Srinivas 	 * threshold since we last checked. Let our caller know so
47211447b59SVenkatesh Srinivas 	 * it processes the new entries.
47311447b59SVenkatesh Srinivas 	 */
47411447b59SVenkatesh Srinivas 	if (virtqueue_nused(vq) > ndesc)
47511447b59SVenkatesh Srinivas 		return (1);
47611447b59SVenkatesh Srinivas 
47711447b59SVenkatesh Srinivas 	return (0);
47811447b59SVenkatesh Srinivas }
47911447b59SVenkatesh Srinivas 
48011447b59SVenkatesh Srinivas void
virtqueue_disable_intr(struct virtqueue * vq)48111447b59SVenkatesh Srinivas virtqueue_disable_intr(struct virtqueue *vq)
48211447b59SVenkatesh Srinivas {
48311447b59SVenkatesh Srinivas 	/*
48411447b59SVenkatesh Srinivas 	 * Note this is only considered a hint to the host.
48511447b59SVenkatesh Srinivas 	 */
48611447b59SVenkatesh Srinivas 	if ((vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) == 0)
48711447b59SVenkatesh Srinivas 		vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
48811447b59SVenkatesh Srinivas }
48911447b59SVenkatesh Srinivas 
49011447b59SVenkatesh Srinivas int
virtqueue_enqueue(struct virtqueue * vq,void * cookie,struct sglist * sg,int readable,int writable)49111447b59SVenkatesh Srinivas virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
49211447b59SVenkatesh Srinivas     int readable, int writable)
49311447b59SVenkatesh Srinivas {
49411447b59SVenkatesh Srinivas 	struct vq_desc_extra *dxp;
49511447b59SVenkatesh Srinivas 	int needed;
49611447b59SVenkatesh Srinivas 	uint16_t head_idx, idx;
49711447b59SVenkatesh Srinivas 
49811447b59SVenkatesh Srinivas 	needed = readable + writable;
49911447b59SVenkatesh Srinivas 
50011447b59SVenkatesh Srinivas 	VQASSERT(vq, cookie != NULL, "enqueuing with no cookie");
50111447b59SVenkatesh Srinivas 	VQASSERT(vq, needed == sg->sg_nseg,
50211447b59SVenkatesh Srinivas 	    "segment count mismatch, %d, %d", needed, sg->sg_nseg);
503cb6e1f75SImre Vadász 	VQASSERT(vq,
504cb6e1f75SImre Vadász 	    needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size,
505cb6e1f75SImre Vadász 	    "too many segments to enqueue: %d, %d/%d", needed,
506cb6e1f75SImre Vadász 	    vq->vq_nentries, vq->vq_max_indirect_size);
50711447b59SVenkatesh Srinivas 
50811447b59SVenkatesh Srinivas 	if (needed < 1)
50911447b59SVenkatesh Srinivas 		return (EINVAL);
51011447b59SVenkatesh Srinivas 	if (vq->vq_free_cnt == 0)
51111447b59SVenkatesh Srinivas 		return (ENOSPC);
512cb6e1f75SImre Vadász 
513cb6e1f75SImre Vadász 	if (vq_ring_use_indirect(vq, needed)) {
514cb6e1f75SImre Vadász 		vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable);
515cb6e1f75SImre Vadász 		return (0);
516cb6e1f75SImre Vadász 	} else if (vq->vq_free_cnt < needed)
51711447b59SVenkatesh Srinivas 		return (EMSGSIZE);
51811447b59SVenkatesh Srinivas 
51911447b59SVenkatesh Srinivas 	head_idx = vq->vq_desc_head_idx;
52011447b59SVenkatesh Srinivas 	VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
52111447b59SVenkatesh Srinivas 	dxp = &vq->vq_descx[head_idx];
52211447b59SVenkatesh Srinivas 
52311447b59SVenkatesh Srinivas 	VQASSERT(vq, dxp->cookie == NULL,
52411447b59SVenkatesh Srinivas 	    "cookie already exists for index %d", head_idx);
52511447b59SVenkatesh Srinivas 	dxp->cookie = cookie;
52611447b59SVenkatesh Srinivas 	dxp->ndescs = needed;
52711447b59SVenkatesh Srinivas 
52811447b59SVenkatesh Srinivas 	idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
52911447b59SVenkatesh Srinivas 	    sg, readable, writable);
53011447b59SVenkatesh Srinivas 
53111447b59SVenkatesh Srinivas 	vq->vq_desc_head_idx = idx;
53211447b59SVenkatesh Srinivas 	vq->vq_free_cnt -= needed;
53311447b59SVenkatesh Srinivas 	if (vq->vq_free_cnt == 0)
53411447b59SVenkatesh Srinivas 		VQ_RING_ASSERT_CHAIN_TERM(vq);
53511447b59SVenkatesh Srinivas 	else
53611447b59SVenkatesh Srinivas 		VQ_RING_ASSERT_VALID_IDX(vq, idx);
53711447b59SVenkatesh Srinivas 
53811447b59SVenkatesh Srinivas 	vq_ring_update_avail(vq, head_idx);
53911447b59SVenkatesh Srinivas 
54011447b59SVenkatesh Srinivas 	return (0);
54111447b59SVenkatesh Srinivas }
54211447b59SVenkatesh Srinivas 
54311447b59SVenkatesh Srinivas void *
virtqueue_dequeue(struct virtqueue * vq,uint32_t * len)54411447b59SVenkatesh Srinivas virtqueue_dequeue(struct virtqueue *vq, uint32_t *len)
54511447b59SVenkatesh Srinivas {
54611447b59SVenkatesh Srinivas 	struct vring_used_elem *uep;
54711447b59SVenkatesh Srinivas 	void *cookie;
54811447b59SVenkatesh Srinivas 	uint16_t used_idx, desc_idx;
54911447b59SVenkatesh Srinivas 
55011447b59SVenkatesh Srinivas 	if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
55111447b59SVenkatesh Srinivas 		return (NULL);
55211447b59SVenkatesh Srinivas 
55311447b59SVenkatesh Srinivas 	used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
55411447b59SVenkatesh Srinivas 	uep = &vq->vq_ring.used->ring[used_idx];
55511447b59SVenkatesh Srinivas 
556dfa40836SImre Vadász 	cpu_lfence();
55711447b59SVenkatesh Srinivas 	desc_idx = (uint16_t) uep->id;
55811447b59SVenkatesh Srinivas 	if (len != NULL)
55911447b59SVenkatesh Srinivas 		*len = uep->len;
56011447b59SVenkatesh Srinivas 
56111447b59SVenkatesh Srinivas 	vq_ring_free_chain(vq, desc_idx);
56211447b59SVenkatesh Srinivas 
56311447b59SVenkatesh Srinivas 	cookie = vq->vq_descx[desc_idx].cookie;
56411447b59SVenkatesh Srinivas 	VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx);
56511447b59SVenkatesh Srinivas 	vq->vq_descx[desc_idx].cookie = NULL;
56611447b59SVenkatesh Srinivas 
56711447b59SVenkatesh Srinivas 	return (cookie);
56811447b59SVenkatesh Srinivas }
56911447b59SVenkatesh Srinivas 
57011447b59SVenkatesh Srinivas void *
virtqueue_poll(struct virtqueue * vq,uint32_t * len)57111447b59SVenkatesh Srinivas virtqueue_poll(struct virtqueue *vq, uint32_t *len)
57211447b59SVenkatesh Srinivas {
57311447b59SVenkatesh Srinivas 	void *cookie;
57411447b59SVenkatesh Srinivas 
57511447b59SVenkatesh Srinivas 	/* We only poll the virtqueue when dumping to virtio-blk */
57611447b59SVenkatesh Srinivas 	while ((cookie = virtqueue_dequeue(vq, len)) == NULL)
57711447b59SVenkatesh Srinivas 		;
57811447b59SVenkatesh Srinivas 
57911447b59SVenkatesh Srinivas 	return (cookie);
58011447b59SVenkatesh Srinivas }
58111447b59SVenkatesh Srinivas 
58211447b59SVenkatesh Srinivas void *
virtqueue_drain(struct virtqueue * vq,int * last)58311447b59SVenkatesh Srinivas virtqueue_drain(struct virtqueue *vq, int *last)
58411447b59SVenkatesh Srinivas {
58511447b59SVenkatesh Srinivas 	void *cookie;
58611447b59SVenkatesh Srinivas 	int idx;
58711447b59SVenkatesh Srinivas 
58811447b59SVenkatesh Srinivas 	cookie = NULL;
58911447b59SVenkatesh Srinivas 	idx = *last;
59011447b59SVenkatesh Srinivas 
59111447b59SVenkatesh Srinivas 	while (idx < vq->vq_nentries && cookie == NULL) {
59211447b59SVenkatesh Srinivas 		if ((cookie = vq->vq_descx[idx].cookie) != NULL) {
59311447b59SVenkatesh Srinivas 			vq->vq_descx[idx].cookie = NULL;
59411447b59SVenkatesh Srinivas 			/* Free chain to keep free count consistent. */
59511447b59SVenkatesh Srinivas 			vq_ring_free_chain(vq, idx);
59611447b59SVenkatesh Srinivas 		}
59711447b59SVenkatesh Srinivas 		idx++;
59811447b59SVenkatesh Srinivas 	}
59911447b59SVenkatesh Srinivas 
60011447b59SVenkatesh Srinivas 	*last = idx;
60111447b59SVenkatesh Srinivas 
60211447b59SVenkatesh Srinivas 	return (cookie);
60311447b59SVenkatesh Srinivas }
60411447b59SVenkatesh Srinivas 
60511447b59SVenkatesh Srinivas void
virtqueue_dump(struct virtqueue * vq)60611447b59SVenkatesh Srinivas virtqueue_dump(struct virtqueue *vq)
60711447b59SVenkatesh Srinivas {
60811447b59SVenkatesh Srinivas 
60911447b59SVenkatesh Srinivas 	if (vq == NULL)
61011447b59SVenkatesh Srinivas 		return;
61111447b59SVenkatesh Srinivas 
61211447b59SVenkatesh Srinivas 	kprintf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
61311447b59SVenkatesh Srinivas 	    "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
61411447b59SVenkatesh Srinivas 	    "used.idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
61511447b59SVenkatesh Srinivas 	    vq->vq_name, vq->vq_nentries, vq->vq_free_cnt,
61611447b59SVenkatesh Srinivas 	    virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx,
61711447b59SVenkatesh Srinivas 	    vq->vq_ring.avail->idx, vq->vq_used_cons_idx,
61811447b59SVenkatesh Srinivas 	    vq->vq_ring.used->idx, vq->vq_ring.avail->flags,
61911447b59SVenkatesh Srinivas 	    vq->vq_ring.used->flags);
62011447b59SVenkatesh Srinivas }
62111447b59SVenkatesh Srinivas 
62211447b59SVenkatesh Srinivas static void
vq_ring_init(struct virtqueue * vq)62311447b59SVenkatesh Srinivas vq_ring_init(struct virtqueue *vq)
62411447b59SVenkatesh Srinivas {
62511447b59SVenkatesh Srinivas 	struct vring *vr;
62611447b59SVenkatesh Srinivas 	char *ring_mem;
62711447b59SVenkatesh Srinivas 	int i, size;
62811447b59SVenkatesh Srinivas 
62911447b59SVenkatesh Srinivas 	ring_mem = vq->vq_ring_mem;
63011447b59SVenkatesh Srinivas 	size = vq->vq_nentries;
63111447b59SVenkatesh Srinivas 	vr = &vq->vq_ring;
63211447b59SVenkatesh Srinivas 
63311447b59SVenkatesh Srinivas 	vring_init(vr, size, ring_mem, vq->vq_alignment);
63411447b59SVenkatesh Srinivas 
63511447b59SVenkatesh Srinivas 	for (i = 0; i < size - 1; i++)
63611447b59SVenkatesh Srinivas 		vr->desc[i].next = i + 1;
63711447b59SVenkatesh Srinivas 	vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
63811447b59SVenkatesh Srinivas }
63911447b59SVenkatesh Srinivas 
64011447b59SVenkatesh Srinivas static void
vq_ring_update_avail(struct virtqueue * vq,uint16_t desc_idx)64111447b59SVenkatesh Srinivas vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
64211447b59SVenkatesh Srinivas {
64311447b59SVenkatesh Srinivas 	uint16_t avail_idx;
64411447b59SVenkatesh Srinivas 
64511447b59SVenkatesh Srinivas 	/*
64611447b59SVenkatesh Srinivas 	 * Place the head of the descriptor chain into the next slot and make
64711447b59SVenkatesh Srinivas 	 * it usable to the host. The chain is made available now rather than
64811447b59SVenkatesh Srinivas 	 * deferring to virtqueue_notify() in the hopes that if the host is
64911447b59SVenkatesh Srinivas 	 * currently running on another CPU, we can keep it processing the new
65011447b59SVenkatesh Srinivas 	 * descriptor.
65111447b59SVenkatesh Srinivas 	 */
65211447b59SVenkatesh Srinivas 	avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1);
65311447b59SVenkatesh Srinivas 	vq->vq_ring.avail->ring[avail_idx] = desc_idx;
65411447b59SVenkatesh Srinivas 
655dfa40836SImre Vadász 	cpu_sfence();
65611447b59SVenkatesh Srinivas 	vq->vq_ring.avail->idx++;
65711447b59SVenkatesh Srinivas 
65811447b59SVenkatesh Srinivas 	/* Keep pending count until virtqueue_notify() for debugging. */
65911447b59SVenkatesh Srinivas 	vq->vq_queued_cnt++;
66011447b59SVenkatesh Srinivas }
66111447b59SVenkatesh Srinivas 
66211447b59SVenkatesh Srinivas static uint16_t
vq_ring_enqueue_segments(struct virtqueue * vq,struct vring_desc * desc,uint16_t head_idx,struct sglist * sg,int readable,int writable)66311447b59SVenkatesh Srinivas vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
66411447b59SVenkatesh Srinivas     uint16_t head_idx, struct sglist *sg, int readable, int writable)
66511447b59SVenkatesh Srinivas {
66611447b59SVenkatesh Srinivas 	struct sglist_seg *seg;
66711447b59SVenkatesh Srinivas 	struct vring_desc *dp;
66811447b59SVenkatesh Srinivas 	int i, needed;
66911447b59SVenkatesh Srinivas 	uint16_t idx;
67011447b59SVenkatesh Srinivas 
67111447b59SVenkatesh Srinivas 	needed = readable + writable;
67211447b59SVenkatesh Srinivas 
67311447b59SVenkatesh Srinivas 	for (i = 0, idx = head_idx, seg = sg->sg_segs;
67411447b59SVenkatesh Srinivas 	     i < needed;
67511447b59SVenkatesh Srinivas 	     i++, idx = dp->next, seg++) {
67611447b59SVenkatesh Srinivas 		VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
67711447b59SVenkatesh Srinivas 		    "premature end of free desc chain");
67811447b59SVenkatesh Srinivas 
67911447b59SVenkatesh Srinivas 		dp = &desc[idx];
68011447b59SVenkatesh Srinivas 		dp->addr = seg->ss_paddr;
68111447b59SVenkatesh Srinivas 		dp->len = seg->ss_len;
68211447b59SVenkatesh Srinivas 		dp->flags = 0;
68311447b59SVenkatesh Srinivas 
68411447b59SVenkatesh Srinivas 		if (i < needed - 1)
68511447b59SVenkatesh Srinivas 			dp->flags |= VRING_DESC_F_NEXT;
68611447b59SVenkatesh Srinivas 		if (i >= readable)
68711447b59SVenkatesh Srinivas 			dp->flags |= VRING_DESC_F_WRITE;
68811447b59SVenkatesh Srinivas 	}
68911447b59SVenkatesh Srinivas 
69011447b59SVenkatesh Srinivas 	return (idx);
69111447b59SVenkatesh Srinivas }
69211447b59SVenkatesh Srinivas 
69311447b59SVenkatesh Srinivas static int
vq_ring_use_indirect(struct virtqueue * vq,int needed)694cb6e1f75SImre Vadász vq_ring_use_indirect(struct virtqueue *vq, int needed)
695cb6e1f75SImre Vadász {
696cb6e1f75SImre Vadász 
697cb6e1f75SImre Vadász 	if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0)
698cb6e1f75SImre Vadász 		return (0);
699cb6e1f75SImre Vadász 
700cb6e1f75SImre Vadász 	if (vq->vq_max_indirect_size < needed)
701cb6e1f75SImre Vadász 		return (0);
702cb6e1f75SImre Vadász 
703cb6e1f75SImre Vadász 	if (needed < 2)
704cb6e1f75SImre Vadász 		return (0);
705cb6e1f75SImre Vadász 
706cb6e1f75SImre Vadász 	return (1);
707cb6e1f75SImre Vadász }
708cb6e1f75SImre Vadász 
709cb6e1f75SImre Vadász static void
vq_ring_enqueue_indirect(struct virtqueue * vq,void * cookie,struct sglist * sg,int readable,int writable)710cb6e1f75SImre Vadász vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
711cb6e1f75SImre Vadász     struct sglist *sg, int readable, int writable)
712cb6e1f75SImre Vadász {
713cb6e1f75SImre Vadász 	struct vring_desc *dp;
714cb6e1f75SImre Vadász 	struct vq_desc_extra *dxp;
715cb6e1f75SImre Vadász 	int needed;
716cb6e1f75SImre Vadász 	uint16_t head_idx;
717cb6e1f75SImre Vadász 
718cb6e1f75SImre Vadász 	needed = readable + writable;
719cb6e1f75SImre Vadász 	VQASSERT(vq, needed <= vq->vq_max_indirect_size,
720cb6e1f75SImre Vadász 	    "enqueuing too many indirect descriptors");
721cb6e1f75SImre Vadász 
722cb6e1f75SImre Vadász 	head_idx = vq->vq_desc_head_idx;
723cb6e1f75SImre Vadász 	VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
724cb6e1f75SImre Vadász 	dp = &vq->vq_ring.desc[head_idx];
725cb6e1f75SImre Vadász 	dxp = &vq->vq_descx[head_idx];
726cb6e1f75SImre Vadász 
727cb6e1f75SImre Vadász 	VQASSERT(vq, dxp->cookie == NULL,
728cb6e1f75SImre Vadász 	    "cookie already exists for index %d", head_idx);
729cb6e1f75SImre Vadász 	dxp->cookie = cookie;
730cb6e1f75SImre Vadász 	dxp->ndescs = 1;
731cb6e1f75SImre Vadász 
732cb6e1f75SImre Vadász 	dp->addr = dxp->indirect_paddr;
733cb6e1f75SImre Vadász 	dp->len = needed * sizeof(struct vring_desc);
734cb6e1f75SImre Vadász 	dp->flags = VRING_DESC_F_INDIRECT;
735cb6e1f75SImre Vadász 
736cb6e1f75SImre Vadász 	vq_ring_enqueue_segments(vq, dxp->indirect, 0,
737cb6e1f75SImre Vadász 	    sg, readable, writable);
738cb6e1f75SImre Vadász 
739cb6e1f75SImre Vadász 	vq->vq_desc_head_idx = dp->next;
740cb6e1f75SImre Vadász 	vq->vq_free_cnt--;
741cb6e1f75SImre Vadász 	if (vq->vq_free_cnt == 0)
742cb6e1f75SImre Vadász 		VQ_RING_ASSERT_CHAIN_TERM(vq);
743cb6e1f75SImre Vadász 	else
744cb6e1f75SImre Vadász 		VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
745cb6e1f75SImre Vadász 
746cb6e1f75SImre Vadász 	vq_ring_update_avail(vq, head_idx);
747cb6e1f75SImre Vadász }
748cb6e1f75SImre Vadász 
749cb6e1f75SImre Vadász static int
vq_ring_must_notify_host(struct virtqueue * vq)75011447b59SVenkatesh Srinivas vq_ring_must_notify_host(struct virtqueue *vq)
75111447b59SVenkatesh Srinivas {
75211447b59SVenkatesh Srinivas 	uint16_t new_idx, prev_idx, event_idx;
75311447b59SVenkatesh Srinivas 
75411447b59SVenkatesh Srinivas 	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
75511447b59SVenkatesh Srinivas 		new_idx = vq->vq_ring.avail->idx;
75611447b59SVenkatesh Srinivas 		prev_idx = new_idx - vq->vq_queued_cnt;
75711447b59SVenkatesh Srinivas 		event_idx = vring_avail_event(&vq->vq_ring);
75811447b59SVenkatesh Srinivas 
75911447b59SVenkatesh Srinivas 		return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
76011447b59SVenkatesh Srinivas 	}
76111447b59SVenkatesh Srinivas 
76211447b59SVenkatesh Srinivas 	return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0);
76311447b59SVenkatesh Srinivas }
76411447b59SVenkatesh Srinivas 
76511447b59SVenkatesh Srinivas static void
vq_ring_notify_host(struct virtqueue * vq)76611447b59SVenkatesh Srinivas vq_ring_notify_host(struct virtqueue *vq)
76711447b59SVenkatesh Srinivas {
76811447b59SVenkatesh Srinivas 	VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index);
76911447b59SVenkatesh Srinivas }
77011447b59SVenkatesh Srinivas 
77111447b59SVenkatesh Srinivas static void
vq_ring_free_chain(struct virtqueue * vq,uint16_t desc_idx)77211447b59SVenkatesh Srinivas vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
77311447b59SVenkatesh Srinivas {
77411447b59SVenkatesh Srinivas 	struct vring_desc *dp;
77511447b59SVenkatesh Srinivas 	struct vq_desc_extra *dxp;
77611447b59SVenkatesh Srinivas 
77711447b59SVenkatesh Srinivas 	VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
77811447b59SVenkatesh Srinivas 	dp = &vq->vq_ring.desc[desc_idx];
77911447b59SVenkatesh Srinivas 	dxp = &vq->vq_descx[desc_idx];
78011447b59SVenkatesh Srinivas 
78111447b59SVenkatesh Srinivas 	if (vq->vq_free_cnt == 0)
78211447b59SVenkatesh Srinivas 		VQ_RING_ASSERT_CHAIN_TERM(vq);
78311447b59SVenkatesh Srinivas 
78411447b59SVenkatesh Srinivas 	vq->vq_free_cnt += dxp->ndescs;
78511447b59SVenkatesh Srinivas 	dxp->ndescs--;
78611447b59SVenkatesh Srinivas 
787cb6e1f75SImre Vadász 	if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
78811447b59SVenkatesh Srinivas 		while (dp->flags & VRING_DESC_F_NEXT) {
78911447b59SVenkatesh Srinivas 			VQ_RING_ASSERT_VALID_IDX(vq, dp->next);
79011447b59SVenkatesh Srinivas 			dp = &vq->vq_ring.desc[dp->next];
79111447b59SVenkatesh Srinivas 			dxp->ndescs--;
79211447b59SVenkatesh Srinivas 		}
793cb6e1f75SImre Vadász 	}
79411447b59SVenkatesh Srinivas 	VQASSERT(vq, dxp->ndescs == 0, "failed to free entire desc chain");
79511447b59SVenkatesh Srinivas 
79611447b59SVenkatesh Srinivas 	/*
79711447b59SVenkatesh Srinivas 	 * We must append the existing free chain, if any, to the end of
79811447b59SVenkatesh Srinivas 	 * newly freed chain. If the virtqueue was completely used, then
79911447b59SVenkatesh Srinivas 	 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
80011447b59SVenkatesh Srinivas 	 */
80111447b59SVenkatesh Srinivas 	dp->next = vq->vq_desc_head_idx;
80211447b59SVenkatesh Srinivas 	vq->vq_desc_head_idx = desc_idx;
80311447b59SVenkatesh Srinivas }
804