111447b59SVenkatesh Srinivas /*-
211447b59SVenkatesh Srinivas * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
311447b59SVenkatesh Srinivas * All rights reserved.
411447b59SVenkatesh Srinivas *
511447b59SVenkatesh Srinivas * Redistribution and use in source and binary forms, with or without
611447b59SVenkatesh Srinivas * modification, are permitted provided that the following conditions
711447b59SVenkatesh Srinivas * are met:
811447b59SVenkatesh Srinivas * 1. Redistributions of source code must retain the above copyright
911447b59SVenkatesh Srinivas * notice unmodified, this list of conditions, and the following
1011447b59SVenkatesh Srinivas * disclaimer.
1111447b59SVenkatesh Srinivas * 2. Redistributions in binary form must reproduce the above copyright
1211447b59SVenkatesh Srinivas * notice, this list of conditions and the following disclaimer in the
1311447b59SVenkatesh Srinivas * documentation and/or other materials provided with the distribution.
1411447b59SVenkatesh Srinivas *
1511447b59SVenkatesh Srinivas * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
1611447b59SVenkatesh Srinivas * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
1711447b59SVenkatesh Srinivas * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
1811447b59SVenkatesh Srinivas * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
1911447b59SVenkatesh Srinivas * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
2011447b59SVenkatesh Srinivas * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
2111447b59SVenkatesh Srinivas * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
2211447b59SVenkatesh Srinivas * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
2311447b59SVenkatesh Srinivas * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
2411447b59SVenkatesh Srinivas * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2511447b59SVenkatesh Srinivas *
2611447b59SVenkatesh Srinivas * $FreeBSD: src/sys/dev/virtio/virtqueue.c,v 1.2 2012/04/14 05:48:04 grehan Exp $
2711447b59SVenkatesh Srinivas */
2811447b59SVenkatesh Srinivas
2911447b59SVenkatesh Srinivas /*
3011447b59SVenkatesh Srinivas * Implements the virtqueue interface as basically described
3111447b59SVenkatesh Srinivas * in the original VirtIO paper.
3211447b59SVenkatesh Srinivas */
3311447b59SVenkatesh Srinivas
3411447b59SVenkatesh Srinivas #include <sys/param.h>
3511447b59SVenkatesh Srinivas #include <sys/systm.h>
3611447b59SVenkatesh Srinivas #include <sys/kernel.h>
3711447b59SVenkatesh Srinivas #include <sys/malloc.h>
3811447b59SVenkatesh Srinivas #include <sys/sglist.h>
392f1382caSVenkatesh Srinivas #include <sys/serialize.h>
4011447b59SVenkatesh Srinivas #include <vm/vm.h>
4111447b59SVenkatesh Srinivas #include <vm/pmap.h>
4211447b59SVenkatesh Srinivas
4311447b59SVenkatesh Srinivas #include <machine/cpu.h>
4411447b59SVenkatesh Srinivas #include <machine/atomic.h>
4511447b59SVenkatesh Srinivas #include <sys/bus.h>
4611447b59SVenkatesh Srinivas #include <sys/rman.h>
4711447b59SVenkatesh Srinivas
4811447b59SVenkatesh Srinivas #include "virtio.h"
4911447b59SVenkatesh Srinivas #include "virtqueue.h"
5011447b59SVenkatesh Srinivas #include "virtio_ring.h"
5111447b59SVenkatesh Srinivas
5211447b59SVenkatesh Srinivas #include "virtio_bus_if.h"
5311447b59SVenkatesh Srinivas
5411447b59SVenkatesh Srinivas struct virtqueue {
5511447b59SVenkatesh Srinivas device_t vq_dev;
5611447b59SVenkatesh Srinivas char vq_name[VIRTQUEUE_MAX_NAME_SZ];
5711447b59SVenkatesh Srinivas uint16_t vq_queue_index;
5811447b59SVenkatesh Srinivas uint16_t vq_nentries;
5911447b59SVenkatesh Srinivas uint32_t vq_flags;
6011447b59SVenkatesh Srinivas
61cb6e1f75SImre Vadász #define VIRTQUEUE_FLAG_INDIRECT 0x0001
6211447b59SVenkatesh Srinivas #define VIRTQUEUE_FLAG_EVENT_IDX 0x0002
6311447b59SVenkatesh Srinivas
6411447b59SVenkatesh Srinivas int vq_alignment;
6511447b59SVenkatesh Srinivas int vq_ring_size;
6611447b59SVenkatesh Srinivas void *vq_ring_mem;
67cb6e1f75SImre Vadász int vq_max_indirect_size;
68cb6e1f75SImre Vadász int vq_indirect_mem_size;
6911447b59SVenkatesh Srinivas
7011447b59SVenkatesh Srinivas struct vring vq_ring;
7111447b59SVenkatesh Srinivas uint16_t vq_free_cnt;
7211447b59SVenkatesh Srinivas uint16_t vq_queued_cnt;
7311447b59SVenkatesh Srinivas /*
7411447b59SVenkatesh Srinivas * Head of the free chain in the descriptor table. If
7511447b59SVenkatesh Srinivas * there are no free descriptors, this will be set to
7611447b59SVenkatesh Srinivas * VQ_RING_DESC_CHAIN_END.
7711447b59SVenkatesh Srinivas */
7811447b59SVenkatesh Srinivas uint16_t vq_desc_head_idx;
7911447b59SVenkatesh Srinivas /*
8011447b59SVenkatesh Srinivas * Last consumed descriptor in the used table,
8111447b59SVenkatesh Srinivas * trails vq_ring.used->idx.
8211447b59SVenkatesh Srinivas */
8311447b59SVenkatesh Srinivas uint16_t vq_used_cons_idx;
8411447b59SVenkatesh Srinivas
8511447b59SVenkatesh Srinivas struct vq_desc_extra {
8611447b59SVenkatesh Srinivas void *cookie;
87cb6e1f75SImre Vadász struct vring_desc *indirect;
88cb6e1f75SImre Vadász vm_paddr_t indirect_paddr;
8911447b59SVenkatesh Srinivas uint16_t ndescs;
9011447b59SVenkatesh Srinivas } vq_descx[0];
9111447b59SVenkatesh Srinivas };
9211447b59SVenkatesh Srinivas
9311447b59SVenkatesh Srinivas /*
9411447b59SVenkatesh Srinivas * The maximum virtqueue size is 2^15. Use that value as the end of
9511447b59SVenkatesh Srinivas * descriptor chain terminator since it will never be a valid index
9611447b59SVenkatesh Srinivas * in the descriptor table. This is used to verify we are correctly
9711447b59SVenkatesh Srinivas * handling vq_free_cnt.
9811447b59SVenkatesh Srinivas */
9911447b59SVenkatesh Srinivas #define VQ_RING_DESC_CHAIN_END 32768
10011447b59SVenkatesh Srinivas
10111447b59SVenkatesh Srinivas #define VQASSERT(_vq, _exp, _msg, ...) \
10211447b59SVenkatesh Srinivas KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name, \
10311447b59SVenkatesh Srinivas ##__VA_ARGS__))
10411447b59SVenkatesh Srinivas
10511447b59SVenkatesh Srinivas #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx) \
10611447b59SVenkatesh Srinivas VQASSERT((_vq), (_idx) < (_vq)->vq_nentries, \
10711447b59SVenkatesh Srinivas "invalid ring index: %d, max: %d", (_idx), \
10811447b59SVenkatesh Srinivas (_vq)->vq_nentries)
10911447b59SVenkatesh Srinivas
11011447b59SVenkatesh Srinivas #define VQ_RING_ASSERT_CHAIN_TERM(_vq) \
11111447b59SVenkatesh Srinivas VQASSERT((_vq), (_vq)->vq_desc_head_idx == \
11211447b59SVenkatesh Srinivas VQ_RING_DESC_CHAIN_END, "full ring terminated " \
11311447b59SVenkatesh Srinivas "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx)
11411447b59SVenkatesh Srinivas
115cb6e1f75SImre Vadász static int virtqueue_init_indirect(struct virtqueue *vq, int);
116cb6e1f75SImre Vadász static void virtqueue_free_indirect(struct virtqueue *vq);
117cb6e1f75SImre Vadász static void virtqueue_init_indirect_list(struct virtqueue *,
118cb6e1f75SImre Vadász struct vring_desc *);
119cb6e1f75SImre Vadász
12011447b59SVenkatesh Srinivas static void vq_ring_init(struct virtqueue *);
12111447b59SVenkatesh Srinivas static void vq_ring_update_avail(struct virtqueue *, uint16_t);
12211447b59SVenkatesh Srinivas static uint16_t vq_ring_enqueue_segments(struct virtqueue *,
12311447b59SVenkatesh Srinivas struct vring_desc *, uint16_t, struct sglist *, int, int);
124cb6e1f75SImre Vadász static int vq_ring_use_indirect(struct virtqueue *, int);
125cb6e1f75SImre Vadász static void vq_ring_enqueue_indirect(struct virtqueue *, void *,
126cb6e1f75SImre Vadász struct sglist *, int, int);
12711447b59SVenkatesh Srinivas static int vq_ring_must_notify_host(struct virtqueue *);
12811447b59SVenkatesh Srinivas static void vq_ring_notify_host(struct virtqueue *);
12911447b59SVenkatesh Srinivas static void vq_ring_free_chain(struct virtqueue *, uint16_t);
13011447b59SVenkatesh Srinivas
13111447b59SVenkatesh Srinivas uint64_t
virtqueue_filter_features(uint64_t features)13211447b59SVenkatesh Srinivas virtqueue_filter_features(uint64_t features)
13311447b59SVenkatesh Srinivas {
13411447b59SVenkatesh Srinivas uint64_t mask;
13511447b59SVenkatesh Srinivas
13611447b59SVenkatesh Srinivas mask = (1 << VIRTIO_TRANSPORT_F_START) - 1;
137cb6e1f75SImre Vadász mask |= VIRTIO_RING_F_INDIRECT_DESC;
13811447b59SVenkatesh Srinivas mask |= VIRTIO_RING_F_EVENT_IDX;
13911447b59SVenkatesh Srinivas
14011447b59SVenkatesh Srinivas return (features & mask);
14111447b59SVenkatesh Srinivas }
14211447b59SVenkatesh Srinivas
14311447b59SVenkatesh Srinivas int
virtqueue_alloc(device_t dev,uint16_t queue,uint16_t size,int align,vm_paddr_t highaddr,struct vq_alloc_info * info,struct virtqueue ** vqp)14411447b59SVenkatesh Srinivas virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align,
14511447b59SVenkatesh Srinivas vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp)
14611447b59SVenkatesh Srinivas {
14711447b59SVenkatesh Srinivas struct virtqueue *vq;
14811447b59SVenkatesh Srinivas int error;
14911447b59SVenkatesh Srinivas
15011447b59SVenkatesh Srinivas *vqp = NULL;
15111447b59SVenkatesh Srinivas error = 0;
15211447b59SVenkatesh Srinivas
15311447b59SVenkatesh Srinivas if (size == 0) {
15411447b59SVenkatesh Srinivas device_printf(dev,
15511447b59SVenkatesh Srinivas "virtqueue %d (%s) does not exist (size is zero)\n",
15611447b59SVenkatesh Srinivas queue, info->vqai_name);
15711447b59SVenkatesh Srinivas return (ENODEV);
15811447b59SVenkatesh Srinivas } else if (!powerof2(size)) {
15911447b59SVenkatesh Srinivas device_printf(dev,
16011447b59SVenkatesh Srinivas "virtqueue %d (%s) size is not a power of 2: %d\n",
16111447b59SVenkatesh Srinivas queue, info->vqai_name, size);
16211447b59SVenkatesh Srinivas return (ENXIO);
163cb6e1f75SImre Vadász } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) {
164cb6e1f75SImre Vadász device_printf(dev, "virtqueue %d (%s) requested too many "
165cb6e1f75SImre Vadász "indirect descriptors: %d, max %d\n",
166cb6e1f75SImre Vadász queue, info->vqai_name, info->vqai_maxindirsz,
167cb6e1f75SImre Vadász VIRTIO_MAX_INDIRECT);
168cb6e1f75SImre Vadász return (EINVAL);
16911447b59SVenkatesh Srinivas }
17011447b59SVenkatesh Srinivas
17111447b59SVenkatesh Srinivas vq = kmalloc(sizeof(struct virtqueue) +
172eb55c32fSMatthew Dillon size * sizeof(struct vq_desc_extra), M_DEVBUF, M_INTWAIT | M_ZERO);
17311447b59SVenkatesh Srinivas if (vq == NULL) {
17411447b59SVenkatesh Srinivas device_printf(dev, "cannot allocate virtqueue\n");
17511447b59SVenkatesh Srinivas return (ENOMEM);
17611447b59SVenkatesh Srinivas }
17711447b59SVenkatesh Srinivas
17811447b59SVenkatesh Srinivas vq->vq_dev = dev;
17911447b59SVenkatesh Srinivas strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
18011447b59SVenkatesh Srinivas vq->vq_queue_index = queue;
18111447b59SVenkatesh Srinivas vq->vq_alignment = align;
18211447b59SVenkatesh Srinivas vq->vq_nentries = size;
18311447b59SVenkatesh Srinivas vq->vq_free_cnt = size;
18411447b59SVenkatesh Srinivas
18511447b59SVenkatesh Srinivas if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
18611447b59SVenkatesh Srinivas vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;
18711447b59SVenkatesh Srinivas
188cb6e1f75SImre Vadász if (info->vqai_maxindirsz > 1) {
189cb6e1f75SImre Vadász error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
190cb6e1f75SImre Vadász if (error)
191cb6e1f75SImre Vadász goto fail;
192cb6e1f75SImre Vadász }
193cb6e1f75SImre Vadász
19411447b59SVenkatesh Srinivas vq->vq_ring_size = round_page(vring_size(size, align));
19511447b59SVenkatesh Srinivas vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,
196ba08e4d1SMatthew Dillon M_WAITOK | M_ZERO, 0, highaddr, PAGE_SIZE, 0);
19711447b59SVenkatesh Srinivas if (vq->vq_ring_mem == NULL) {
19811447b59SVenkatesh Srinivas device_printf(dev,
19911447b59SVenkatesh Srinivas "cannot allocate memory for virtqueue ring\n");
20011447b59SVenkatesh Srinivas error = ENOMEM;
20111447b59SVenkatesh Srinivas goto fail;
20211447b59SVenkatesh Srinivas }
20311447b59SVenkatesh Srinivas
20411447b59SVenkatesh Srinivas vq_ring_init(vq);
20511447b59SVenkatesh Srinivas virtqueue_disable_intr(vq);
20611447b59SVenkatesh Srinivas
20711447b59SVenkatesh Srinivas *vqp = vq;
20811447b59SVenkatesh Srinivas
20911447b59SVenkatesh Srinivas fail:
21011447b59SVenkatesh Srinivas if (error)
21111447b59SVenkatesh Srinivas virtqueue_free(vq);
21211447b59SVenkatesh Srinivas
21311447b59SVenkatesh Srinivas return (error);
21411447b59SVenkatesh Srinivas }
21511447b59SVenkatesh Srinivas
216cb6e1f75SImre Vadász static int
virtqueue_init_indirect(struct virtqueue * vq,int indirect_size)217cb6e1f75SImre Vadász virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
218cb6e1f75SImre Vadász {
219cb6e1f75SImre Vadász device_t dev;
220cb6e1f75SImre Vadász struct vq_desc_extra *dxp;
221cb6e1f75SImre Vadász int i, size;
222cb6e1f75SImre Vadász
223cb6e1f75SImre Vadász dev = vq->vq_dev;
224cb6e1f75SImre Vadász
225cb6e1f75SImre Vadász if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) {
226cb6e1f75SImre Vadász /*
227cb6e1f75SImre Vadász * Indirect descriptors requested by the driver but not
228cb6e1f75SImre Vadász * negotiated. Return zero to keep the initialization
229cb6e1f75SImre Vadász * going: we'll run fine without.
230cb6e1f75SImre Vadász */
231cb6e1f75SImre Vadász if (bootverbose)
232cb6e1f75SImre Vadász device_printf(dev, "virtqueue %d (%s) requested "
233cb6e1f75SImre Vadász "indirect descriptors but not negotiated\n",
234cb6e1f75SImre Vadász vq->vq_queue_index, vq->vq_name);
235cb6e1f75SImre Vadász return (0);
236cb6e1f75SImre Vadász }
237cb6e1f75SImre Vadász
238cb6e1f75SImre Vadász size = indirect_size * sizeof(struct vring_desc);
239cb6e1f75SImre Vadász vq->vq_max_indirect_size = indirect_size;
240cb6e1f75SImre Vadász vq->vq_indirect_mem_size = size;
241cb6e1f75SImre Vadász vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT;
242cb6e1f75SImre Vadász
243cb6e1f75SImre Vadász for (i = 0; i < vq->vq_nentries; i++) {
244cb6e1f75SImre Vadász dxp = &vq->vq_descx[i];
245cb6e1f75SImre Vadász
246ba08e4d1SMatthew Dillon dxp->indirect = contigmalloc(size, M_DEVBUF, M_WAITOK,
247eb55c32fSMatthew Dillon 0, BUS_SPACE_MAXADDR, 16, 0);
248cb6e1f75SImre Vadász if (dxp->indirect == NULL) {
249cb6e1f75SImre Vadász device_printf(dev, "cannot allocate indirect list\n");
250cb6e1f75SImre Vadász return (ENOMEM);
251cb6e1f75SImre Vadász }
252cb6e1f75SImre Vadász
253cb6e1f75SImre Vadász dxp->indirect_paddr = vtophys(dxp->indirect);
254cb6e1f75SImre Vadász virtqueue_init_indirect_list(vq, dxp->indirect);
255cb6e1f75SImre Vadász }
256cb6e1f75SImre Vadász
257cb6e1f75SImre Vadász return (0);
258cb6e1f75SImre Vadász }
259cb6e1f75SImre Vadász
260cb6e1f75SImre Vadász static void
virtqueue_free_indirect(struct virtqueue * vq)261cb6e1f75SImre Vadász virtqueue_free_indirect(struct virtqueue *vq)
262cb6e1f75SImre Vadász {
263cb6e1f75SImre Vadász struct vq_desc_extra *dxp;
264cb6e1f75SImre Vadász int i;
265cb6e1f75SImre Vadász
266cb6e1f75SImre Vadász for (i = 0; i < vq->vq_nentries; i++) {
267cb6e1f75SImre Vadász dxp = &vq->vq_descx[i];
268cb6e1f75SImre Vadász
269cb6e1f75SImre Vadász if (dxp->indirect == NULL)
270cb6e1f75SImre Vadász break;
271cb6e1f75SImre Vadász
272cb6e1f75SImre Vadász contigfree(dxp->indirect, vq->vq_indirect_mem_size, M_DEVBUF);
273cb6e1f75SImre Vadász dxp->indirect = NULL;
274cb6e1f75SImre Vadász dxp->indirect_paddr = 0;
275cb6e1f75SImre Vadász }
276cb6e1f75SImre Vadász
277cb6e1f75SImre Vadász vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT;
278cb6e1f75SImre Vadász vq->vq_indirect_mem_size = 0;
279cb6e1f75SImre Vadász }
280cb6e1f75SImre Vadász
281cb6e1f75SImre Vadász static void
virtqueue_init_indirect_list(struct virtqueue * vq,struct vring_desc * indirect)282cb6e1f75SImre Vadász virtqueue_init_indirect_list(struct virtqueue *vq,
283cb6e1f75SImre Vadász struct vring_desc *indirect)
284cb6e1f75SImre Vadász {
285cb6e1f75SImre Vadász int i;
286cb6e1f75SImre Vadász
287cb6e1f75SImre Vadász bzero(indirect, vq->vq_indirect_mem_size);
288cb6e1f75SImre Vadász
289cb6e1f75SImre Vadász for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
290cb6e1f75SImre Vadász indirect[i].next = i + 1;
291cb6e1f75SImre Vadász indirect[i].next = VQ_RING_DESC_CHAIN_END;
292cb6e1f75SImre Vadász }
293cb6e1f75SImre Vadász
29411447b59SVenkatesh Srinivas int
virtqueue_reinit(struct virtqueue * vq,uint16_t size)29511447b59SVenkatesh Srinivas virtqueue_reinit(struct virtqueue *vq, uint16_t size)
29611447b59SVenkatesh Srinivas {
29711447b59SVenkatesh Srinivas struct vq_desc_extra *dxp;
29811447b59SVenkatesh Srinivas int i;
29911447b59SVenkatesh Srinivas
30011447b59SVenkatesh Srinivas if (vq->vq_nentries != size) {
30111447b59SVenkatesh Srinivas device_printf(vq->vq_dev,
30211447b59SVenkatesh Srinivas "%s: '%s' changed size; old=%hu, new=%hu\n",
30311447b59SVenkatesh Srinivas __func__, vq->vq_name, vq->vq_nentries, size);
30411447b59SVenkatesh Srinivas return (EINVAL);
30511447b59SVenkatesh Srinivas }
30611447b59SVenkatesh Srinivas
30711447b59SVenkatesh Srinivas /* Warn if the virtqueue was not properly cleaned up. */
30811447b59SVenkatesh Srinivas if (vq->vq_free_cnt != vq->vq_nentries) {
30911447b59SVenkatesh Srinivas device_printf(vq->vq_dev,
31011447b59SVenkatesh Srinivas "%s: warning, '%s' virtqueue not empty, "
31111447b59SVenkatesh Srinivas "leaking %d entries\n", __func__, vq->vq_name,
31211447b59SVenkatesh Srinivas vq->vq_nentries - vq->vq_free_cnt);
31311447b59SVenkatesh Srinivas }
31411447b59SVenkatesh Srinivas
31511447b59SVenkatesh Srinivas vq->vq_desc_head_idx = 0;
31611447b59SVenkatesh Srinivas vq->vq_used_cons_idx = 0;
31711447b59SVenkatesh Srinivas vq->vq_queued_cnt = 0;
31811447b59SVenkatesh Srinivas vq->vq_free_cnt = vq->vq_nentries;
31911447b59SVenkatesh Srinivas
32011447b59SVenkatesh Srinivas /* To be safe, reset all our allocated memory. */
32111447b59SVenkatesh Srinivas bzero(vq->vq_ring_mem, vq->vq_ring_size);
32211447b59SVenkatesh Srinivas for (i = 0; i < vq->vq_nentries; i++) {
32311447b59SVenkatesh Srinivas dxp = &vq->vq_descx[i];
32411447b59SVenkatesh Srinivas dxp->cookie = NULL;
32511447b59SVenkatesh Srinivas dxp->ndescs = 0;
326cb6e1f75SImre Vadász if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
327cb6e1f75SImre Vadász virtqueue_init_indirect_list(vq, dxp->indirect);
32811447b59SVenkatesh Srinivas }
32911447b59SVenkatesh Srinivas
33011447b59SVenkatesh Srinivas vq_ring_init(vq);
33111447b59SVenkatesh Srinivas virtqueue_disable_intr(vq);
33211447b59SVenkatesh Srinivas
33311447b59SVenkatesh Srinivas return (0);
33411447b59SVenkatesh Srinivas }
33511447b59SVenkatesh Srinivas
33611447b59SVenkatesh Srinivas void
virtqueue_free(struct virtqueue * vq)33711447b59SVenkatesh Srinivas virtqueue_free(struct virtqueue *vq)
33811447b59SVenkatesh Srinivas {
33911447b59SVenkatesh Srinivas
34011447b59SVenkatesh Srinivas if (vq->vq_free_cnt != vq->vq_nentries) {
34111447b59SVenkatesh Srinivas device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, "
34211447b59SVenkatesh Srinivas "leaking %d entries\n", vq->vq_name,
34311447b59SVenkatesh Srinivas vq->vq_nentries - vq->vq_free_cnt);
34411447b59SVenkatesh Srinivas }
34511447b59SVenkatesh Srinivas
346cb6e1f75SImre Vadász if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
347cb6e1f75SImre Vadász virtqueue_free_indirect(vq);
348cb6e1f75SImre Vadász
34911447b59SVenkatesh Srinivas if (vq->vq_ring_mem != NULL) {
35011447b59SVenkatesh Srinivas contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF);
35111447b59SVenkatesh Srinivas vq->vq_ring_size = 0;
35211447b59SVenkatesh Srinivas vq->vq_ring_mem = NULL;
35311447b59SVenkatesh Srinivas }
35411447b59SVenkatesh Srinivas
35511447b59SVenkatesh Srinivas kfree(vq, M_DEVBUF);
35611447b59SVenkatesh Srinivas }
35711447b59SVenkatesh Srinivas
35811447b59SVenkatesh Srinivas vm_paddr_t
virtqueue_paddr(struct virtqueue * vq)35911447b59SVenkatesh Srinivas virtqueue_paddr(struct virtqueue *vq)
36011447b59SVenkatesh Srinivas {
36111447b59SVenkatesh Srinivas return (vtophys(vq->vq_ring_mem));
36211447b59SVenkatesh Srinivas }
36311447b59SVenkatesh Srinivas
36411447b59SVenkatesh Srinivas int
virtqueue_size(struct virtqueue * vq)36511447b59SVenkatesh Srinivas virtqueue_size(struct virtqueue *vq)
36611447b59SVenkatesh Srinivas {
36711447b59SVenkatesh Srinivas return (vq->vq_nentries);
36811447b59SVenkatesh Srinivas }
36911447b59SVenkatesh Srinivas
37011447b59SVenkatesh Srinivas int
virtqueue_empty(struct virtqueue * vq)37111447b59SVenkatesh Srinivas virtqueue_empty(struct virtqueue *vq)
37211447b59SVenkatesh Srinivas {
37311447b59SVenkatesh Srinivas
37411447b59SVenkatesh Srinivas return (vq->vq_nentries == vq->vq_free_cnt);
37511447b59SVenkatesh Srinivas }
37611447b59SVenkatesh Srinivas
37711447b59SVenkatesh Srinivas int
virtqueue_full(struct virtqueue * vq)37811447b59SVenkatesh Srinivas virtqueue_full(struct virtqueue *vq)
37911447b59SVenkatesh Srinivas {
38011447b59SVenkatesh Srinivas
38111447b59SVenkatesh Srinivas return (vq->vq_free_cnt == 0);
38211447b59SVenkatesh Srinivas }
38311447b59SVenkatesh Srinivas
38411447b59SVenkatesh Srinivas void
virtqueue_notify(struct virtqueue * vq,lwkt_serialize_t interlock)3852f1382caSVenkatesh Srinivas virtqueue_notify(struct virtqueue *vq, lwkt_serialize_t interlock)
38611447b59SVenkatesh Srinivas {
38711447b59SVenkatesh Srinivas /* Ensure updated avail->idx is visible to host. */
38811447b59SVenkatesh Srinivas cpu_mfence();
38911447b59SVenkatesh Srinivas
39011447b59SVenkatesh Srinivas if (vq_ring_must_notify_host(vq)) {
39158db6936SImre Vadasz if (interlock != NULL)
3922f1382caSVenkatesh Srinivas lwkt_serialize_exit(interlock);
39311447b59SVenkatesh Srinivas vq_ring_notify_host(vq);
39458db6936SImre Vadasz if (interlock != NULL)
3952f1382caSVenkatesh Srinivas lwkt_serialize_enter(interlock);
39611447b59SVenkatesh Srinivas }
39711447b59SVenkatesh Srinivas vq->vq_queued_cnt = 0;
39811447b59SVenkatesh Srinivas }
39911447b59SVenkatesh Srinivas
40011447b59SVenkatesh Srinivas int
virtqueue_nused(struct virtqueue * vq)40111447b59SVenkatesh Srinivas virtqueue_nused(struct virtqueue *vq)
40211447b59SVenkatesh Srinivas {
40311447b59SVenkatesh Srinivas uint16_t used_idx, nused;
40411447b59SVenkatesh Srinivas
40511447b59SVenkatesh Srinivas used_idx = vq->vq_ring.used->idx;
40611447b59SVenkatesh Srinivas nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
40711447b59SVenkatesh Srinivas VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
40811447b59SVenkatesh Srinivas
40911447b59SVenkatesh Srinivas return (nused);
41011447b59SVenkatesh Srinivas }
41111447b59SVenkatesh Srinivas
41211447b59SVenkatesh Srinivas int
virtqueue_pending(struct virtqueue * vq)413*9d96478cSImre Vadász virtqueue_pending(struct virtqueue *vq)
41411447b59SVenkatesh Srinivas {
415*9d96478cSImre Vadász return (vq->vq_used_cons_idx != vq->vq_ring.used->idx);
41611447b59SVenkatesh Srinivas }
41711447b59SVenkatesh Srinivas
41811447b59SVenkatesh Srinivas /*
41911447b59SVenkatesh Srinivas * Enable interrupts on a given virtqueue. Returns 1 if there are
42011447b59SVenkatesh Srinivas * additional entries to process on the virtqueue after we return.
42111447b59SVenkatesh Srinivas */
42211447b59SVenkatesh Srinivas int
virtqueue_enable_intr(struct virtqueue * vq)42311447b59SVenkatesh Srinivas virtqueue_enable_intr(struct virtqueue *vq)
42411447b59SVenkatesh Srinivas {
42511447b59SVenkatesh Srinivas /*
42611447b59SVenkatesh Srinivas * Enable interrupts, making sure we get the latest
42711447b59SVenkatesh Srinivas * index of what's already been consumed.
42811447b59SVenkatesh Srinivas */
42911447b59SVenkatesh Srinivas vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
43011447b59SVenkatesh Srinivas if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
43111447b59SVenkatesh Srinivas vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx;
43211447b59SVenkatesh Srinivas } else {
43311447b59SVenkatesh Srinivas vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
43411447b59SVenkatesh Srinivas }
43511447b59SVenkatesh Srinivas
43611447b59SVenkatesh Srinivas cpu_mfence();
43711447b59SVenkatesh Srinivas
43811447b59SVenkatesh Srinivas /*
43911447b59SVenkatesh Srinivas * Additional items may have been consumed in the time between
44011447b59SVenkatesh Srinivas * since we last checked and enabled interrupts above. Let our
44111447b59SVenkatesh Srinivas * caller know so it processes the new entries.
44211447b59SVenkatesh Srinivas */
44311447b59SVenkatesh Srinivas if (vq->vq_used_cons_idx != vq->vq_ring.used->idx)
44411447b59SVenkatesh Srinivas return (1);
44511447b59SVenkatesh Srinivas
44611447b59SVenkatesh Srinivas return (0);
44711447b59SVenkatesh Srinivas }
44811447b59SVenkatesh Srinivas
44911447b59SVenkatesh Srinivas int
virtqueue_postpone_intr(struct virtqueue * vq)45011447b59SVenkatesh Srinivas virtqueue_postpone_intr(struct virtqueue *vq)
45111447b59SVenkatesh Srinivas {
45211447b59SVenkatesh Srinivas uint16_t ndesc;
45311447b59SVenkatesh Srinivas
45411447b59SVenkatesh Srinivas /*
45511447b59SVenkatesh Srinivas * Postpone until at least half of the available descriptors
45611447b59SVenkatesh Srinivas * have been consumed.
45711447b59SVenkatesh Srinivas *
45811447b59SVenkatesh Srinivas * XXX Adaptive factor? (Linux uses 3/4)
45911447b59SVenkatesh Srinivas */
46011447b59SVenkatesh Srinivas ndesc = (uint16_t)(vq->vq_ring.avail->idx - vq->vq_used_cons_idx) / 2;
46111447b59SVenkatesh Srinivas
46211447b59SVenkatesh Srinivas if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
46311447b59SVenkatesh Srinivas vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc;
46411447b59SVenkatesh Srinivas else
46511447b59SVenkatesh Srinivas vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
46611447b59SVenkatesh Srinivas
46711447b59SVenkatesh Srinivas cpu_mfence();
46811447b59SVenkatesh Srinivas
46911447b59SVenkatesh Srinivas /*
47011447b59SVenkatesh Srinivas * Enough items may have already been consumed to meet our
47111447b59SVenkatesh Srinivas * threshold since we last checked. Let our caller know so
47211447b59SVenkatesh Srinivas * it processes the new entries.
47311447b59SVenkatesh Srinivas */
47411447b59SVenkatesh Srinivas if (virtqueue_nused(vq) > ndesc)
47511447b59SVenkatesh Srinivas return (1);
47611447b59SVenkatesh Srinivas
47711447b59SVenkatesh Srinivas return (0);
47811447b59SVenkatesh Srinivas }
47911447b59SVenkatesh Srinivas
48011447b59SVenkatesh Srinivas void
virtqueue_disable_intr(struct virtqueue * vq)48111447b59SVenkatesh Srinivas virtqueue_disable_intr(struct virtqueue *vq)
48211447b59SVenkatesh Srinivas {
48311447b59SVenkatesh Srinivas /*
48411447b59SVenkatesh Srinivas * Note this is only considered a hint to the host.
48511447b59SVenkatesh Srinivas */
48611447b59SVenkatesh Srinivas if ((vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) == 0)
48711447b59SVenkatesh Srinivas vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
48811447b59SVenkatesh Srinivas }
48911447b59SVenkatesh Srinivas
49011447b59SVenkatesh Srinivas int
virtqueue_enqueue(struct virtqueue * vq,void * cookie,struct sglist * sg,int readable,int writable)49111447b59SVenkatesh Srinivas virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
49211447b59SVenkatesh Srinivas int readable, int writable)
49311447b59SVenkatesh Srinivas {
49411447b59SVenkatesh Srinivas struct vq_desc_extra *dxp;
49511447b59SVenkatesh Srinivas int needed;
49611447b59SVenkatesh Srinivas uint16_t head_idx, idx;
49711447b59SVenkatesh Srinivas
49811447b59SVenkatesh Srinivas needed = readable + writable;
49911447b59SVenkatesh Srinivas
50011447b59SVenkatesh Srinivas VQASSERT(vq, cookie != NULL, "enqueuing with no cookie");
50111447b59SVenkatesh Srinivas VQASSERT(vq, needed == sg->sg_nseg,
50211447b59SVenkatesh Srinivas "segment count mismatch, %d, %d", needed, sg->sg_nseg);
503cb6e1f75SImre Vadász VQASSERT(vq,
504cb6e1f75SImre Vadász needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size,
505cb6e1f75SImre Vadász "too many segments to enqueue: %d, %d/%d", needed,
506cb6e1f75SImre Vadász vq->vq_nentries, vq->vq_max_indirect_size);
50711447b59SVenkatesh Srinivas
50811447b59SVenkatesh Srinivas if (needed < 1)
50911447b59SVenkatesh Srinivas return (EINVAL);
51011447b59SVenkatesh Srinivas if (vq->vq_free_cnt == 0)
51111447b59SVenkatesh Srinivas return (ENOSPC);
512cb6e1f75SImre Vadász
513cb6e1f75SImre Vadász if (vq_ring_use_indirect(vq, needed)) {
514cb6e1f75SImre Vadász vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable);
515cb6e1f75SImre Vadász return (0);
516cb6e1f75SImre Vadász } else if (vq->vq_free_cnt < needed)
51711447b59SVenkatesh Srinivas return (EMSGSIZE);
51811447b59SVenkatesh Srinivas
51911447b59SVenkatesh Srinivas head_idx = vq->vq_desc_head_idx;
52011447b59SVenkatesh Srinivas VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
52111447b59SVenkatesh Srinivas dxp = &vq->vq_descx[head_idx];
52211447b59SVenkatesh Srinivas
52311447b59SVenkatesh Srinivas VQASSERT(vq, dxp->cookie == NULL,
52411447b59SVenkatesh Srinivas "cookie already exists for index %d", head_idx);
52511447b59SVenkatesh Srinivas dxp->cookie = cookie;
52611447b59SVenkatesh Srinivas dxp->ndescs = needed;
52711447b59SVenkatesh Srinivas
52811447b59SVenkatesh Srinivas idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
52911447b59SVenkatesh Srinivas sg, readable, writable);
53011447b59SVenkatesh Srinivas
53111447b59SVenkatesh Srinivas vq->vq_desc_head_idx = idx;
53211447b59SVenkatesh Srinivas vq->vq_free_cnt -= needed;
53311447b59SVenkatesh Srinivas if (vq->vq_free_cnt == 0)
53411447b59SVenkatesh Srinivas VQ_RING_ASSERT_CHAIN_TERM(vq);
53511447b59SVenkatesh Srinivas else
53611447b59SVenkatesh Srinivas VQ_RING_ASSERT_VALID_IDX(vq, idx);
53711447b59SVenkatesh Srinivas
53811447b59SVenkatesh Srinivas vq_ring_update_avail(vq, head_idx);
53911447b59SVenkatesh Srinivas
54011447b59SVenkatesh Srinivas return (0);
54111447b59SVenkatesh Srinivas }
54211447b59SVenkatesh Srinivas
54311447b59SVenkatesh Srinivas void *
virtqueue_dequeue(struct virtqueue * vq,uint32_t * len)54411447b59SVenkatesh Srinivas virtqueue_dequeue(struct virtqueue *vq, uint32_t *len)
54511447b59SVenkatesh Srinivas {
54611447b59SVenkatesh Srinivas struct vring_used_elem *uep;
54711447b59SVenkatesh Srinivas void *cookie;
54811447b59SVenkatesh Srinivas uint16_t used_idx, desc_idx;
54911447b59SVenkatesh Srinivas
55011447b59SVenkatesh Srinivas if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
55111447b59SVenkatesh Srinivas return (NULL);
55211447b59SVenkatesh Srinivas
55311447b59SVenkatesh Srinivas used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
55411447b59SVenkatesh Srinivas uep = &vq->vq_ring.used->ring[used_idx];
55511447b59SVenkatesh Srinivas
556dfa40836SImre Vadász cpu_lfence();
55711447b59SVenkatesh Srinivas desc_idx = (uint16_t) uep->id;
55811447b59SVenkatesh Srinivas if (len != NULL)
55911447b59SVenkatesh Srinivas *len = uep->len;
56011447b59SVenkatesh Srinivas
56111447b59SVenkatesh Srinivas vq_ring_free_chain(vq, desc_idx);
56211447b59SVenkatesh Srinivas
56311447b59SVenkatesh Srinivas cookie = vq->vq_descx[desc_idx].cookie;
56411447b59SVenkatesh Srinivas VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx);
56511447b59SVenkatesh Srinivas vq->vq_descx[desc_idx].cookie = NULL;
56611447b59SVenkatesh Srinivas
56711447b59SVenkatesh Srinivas return (cookie);
56811447b59SVenkatesh Srinivas }
56911447b59SVenkatesh Srinivas
57011447b59SVenkatesh Srinivas void *
virtqueue_poll(struct virtqueue * vq,uint32_t * len)57111447b59SVenkatesh Srinivas virtqueue_poll(struct virtqueue *vq, uint32_t *len)
57211447b59SVenkatesh Srinivas {
57311447b59SVenkatesh Srinivas void *cookie;
57411447b59SVenkatesh Srinivas
57511447b59SVenkatesh Srinivas /* We only poll the virtqueue when dumping to virtio-blk */
57611447b59SVenkatesh Srinivas while ((cookie = virtqueue_dequeue(vq, len)) == NULL)
57711447b59SVenkatesh Srinivas ;
57811447b59SVenkatesh Srinivas
57911447b59SVenkatesh Srinivas return (cookie);
58011447b59SVenkatesh Srinivas }
58111447b59SVenkatesh Srinivas
58211447b59SVenkatesh Srinivas void *
virtqueue_drain(struct virtqueue * vq,int * last)58311447b59SVenkatesh Srinivas virtqueue_drain(struct virtqueue *vq, int *last)
58411447b59SVenkatesh Srinivas {
58511447b59SVenkatesh Srinivas void *cookie;
58611447b59SVenkatesh Srinivas int idx;
58711447b59SVenkatesh Srinivas
58811447b59SVenkatesh Srinivas cookie = NULL;
58911447b59SVenkatesh Srinivas idx = *last;
59011447b59SVenkatesh Srinivas
59111447b59SVenkatesh Srinivas while (idx < vq->vq_nentries && cookie == NULL) {
59211447b59SVenkatesh Srinivas if ((cookie = vq->vq_descx[idx].cookie) != NULL) {
59311447b59SVenkatesh Srinivas vq->vq_descx[idx].cookie = NULL;
59411447b59SVenkatesh Srinivas /* Free chain to keep free count consistent. */
59511447b59SVenkatesh Srinivas vq_ring_free_chain(vq, idx);
59611447b59SVenkatesh Srinivas }
59711447b59SVenkatesh Srinivas idx++;
59811447b59SVenkatesh Srinivas }
59911447b59SVenkatesh Srinivas
60011447b59SVenkatesh Srinivas *last = idx;
60111447b59SVenkatesh Srinivas
60211447b59SVenkatesh Srinivas return (cookie);
60311447b59SVenkatesh Srinivas }
60411447b59SVenkatesh Srinivas
60511447b59SVenkatesh Srinivas void
virtqueue_dump(struct virtqueue * vq)60611447b59SVenkatesh Srinivas virtqueue_dump(struct virtqueue *vq)
60711447b59SVenkatesh Srinivas {
60811447b59SVenkatesh Srinivas
60911447b59SVenkatesh Srinivas if (vq == NULL)
61011447b59SVenkatesh Srinivas return;
61111447b59SVenkatesh Srinivas
61211447b59SVenkatesh Srinivas kprintf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
61311447b59SVenkatesh Srinivas "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
61411447b59SVenkatesh Srinivas "used.idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
61511447b59SVenkatesh Srinivas vq->vq_name, vq->vq_nentries, vq->vq_free_cnt,
61611447b59SVenkatesh Srinivas virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx,
61711447b59SVenkatesh Srinivas vq->vq_ring.avail->idx, vq->vq_used_cons_idx,
61811447b59SVenkatesh Srinivas vq->vq_ring.used->idx, vq->vq_ring.avail->flags,
61911447b59SVenkatesh Srinivas vq->vq_ring.used->flags);
62011447b59SVenkatesh Srinivas }
62111447b59SVenkatesh Srinivas
62211447b59SVenkatesh Srinivas static void
vq_ring_init(struct virtqueue * vq)62311447b59SVenkatesh Srinivas vq_ring_init(struct virtqueue *vq)
62411447b59SVenkatesh Srinivas {
62511447b59SVenkatesh Srinivas struct vring *vr;
62611447b59SVenkatesh Srinivas char *ring_mem;
62711447b59SVenkatesh Srinivas int i, size;
62811447b59SVenkatesh Srinivas
62911447b59SVenkatesh Srinivas ring_mem = vq->vq_ring_mem;
63011447b59SVenkatesh Srinivas size = vq->vq_nentries;
63111447b59SVenkatesh Srinivas vr = &vq->vq_ring;
63211447b59SVenkatesh Srinivas
63311447b59SVenkatesh Srinivas vring_init(vr, size, ring_mem, vq->vq_alignment);
63411447b59SVenkatesh Srinivas
63511447b59SVenkatesh Srinivas for (i = 0; i < size - 1; i++)
63611447b59SVenkatesh Srinivas vr->desc[i].next = i + 1;
63711447b59SVenkatesh Srinivas vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
63811447b59SVenkatesh Srinivas }
63911447b59SVenkatesh Srinivas
64011447b59SVenkatesh Srinivas static void
vq_ring_update_avail(struct virtqueue * vq,uint16_t desc_idx)64111447b59SVenkatesh Srinivas vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
64211447b59SVenkatesh Srinivas {
64311447b59SVenkatesh Srinivas uint16_t avail_idx;
64411447b59SVenkatesh Srinivas
64511447b59SVenkatesh Srinivas /*
64611447b59SVenkatesh Srinivas * Place the head of the descriptor chain into the next slot and make
64711447b59SVenkatesh Srinivas * it usable to the host. The chain is made available now rather than
64811447b59SVenkatesh Srinivas * deferring to virtqueue_notify() in the hopes that if the host is
64911447b59SVenkatesh Srinivas * currently running on another CPU, we can keep it processing the new
65011447b59SVenkatesh Srinivas * descriptor.
65111447b59SVenkatesh Srinivas */
65211447b59SVenkatesh Srinivas avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1);
65311447b59SVenkatesh Srinivas vq->vq_ring.avail->ring[avail_idx] = desc_idx;
65411447b59SVenkatesh Srinivas
655dfa40836SImre Vadász cpu_sfence();
65611447b59SVenkatesh Srinivas vq->vq_ring.avail->idx++;
65711447b59SVenkatesh Srinivas
65811447b59SVenkatesh Srinivas /* Keep pending count until virtqueue_notify() for debugging. */
65911447b59SVenkatesh Srinivas vq->vq_queued_cnt++;
66011447b59SVenkatesh Srinivas }
66111447b59SVenkatesh Srinivas
66211447b59SVenkatesh Srinivas static uint16_t
vq_ring_enqueue_segments(struct virtqueue * vq,struct vring_desc * desc,uint16_t head_idx,struct sglist * sg,int readable,int writable)66311447b59SVenkatesh Srinivas vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
66411447b59SVenkatesh Srinivas uint16_t head_idx, struct sglist *sg, int readable, int writable)
66511447b59SVenkatesh Srinivas {
66611447b59SVenkatesh Srinivas struct sglist_seg *seg;
66711447b59SVenkatesh Srinivas struct vring_desc *dp;
66811447b59SVenkatesh Srinivas int i, needed;
66911447b59SVenkatesh Srinivas uint16_t idx;
67011447b59SVenkatesh Srinivas
67111447b59SVenkatesh Srinivas needed = readable + writable;
67211447b59SVenkatesh Srinivas
67311447b59SVenkatesh Srinivas for (i = 0, idx = head_idx, seg = sg->sg_segs;
67411447b59SVenkatesh Srinivas i < needed;
67511447b59SVenkatesh Srinivas i++, idx = dp->next, seg++) {
67611447b59SVenkatesh Srinivas VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
67711447b59SVenkatesh Srinivas "premature end of free desc chain");
67811447b59SVenkatesh Srinivas
67911447b59SVenkatesh Srinivas dp = &desc[idx];
68011447b59SVenkatesh Srinivas dp->addr = seg->ss_paddr;
68111447b59SVenkatesh Srinivas dp->len = seg->ss_len;
68211447b59SVenkatesh Srinivas dp->flags = 0;
68311447b59SVenkatesh Srinivas
68411447b59SVenkatesh Srinivas if (i < needed - 1)
68511447b59SVenkatesh Srinivas dp->flags |= VRING_DESC_F_NEXT;
68611447b59SVenkatesh Srinivas if (i >= readable)
68711447b59SVenkatesh Srinivas dp->flags |= VRING_DESC_F_WRITE;
68811447b59SVenkatesh Srinivas }
68911447b59SVenkatesh Srinivas
69011447b59SVenkatesh Srinivas return (idx);
69111447b59SVenkatesh Srinivas }
69211447b59SVenkatesh Srinivas
69311447b59SVenkatesh Srinivas static int
vq_ring_use_indirect(struct virtqueue * vq,int needed)694cb6e1f75SImre Vadász vq_ring_use_indirect(struct virtqueue *vq, int needed)
695cb6e1f75SImre Vadász {
696cb6e1f75SImre Vadász
697cb6e1f75SImre Vadász if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0)
698cb6e1f75SImre Vadász return (0);
699cb6e1f75SImre Vadász
700cb6e1f75SImre Vadász if (vq->vq_max_indirect_size < needed)
701cb6e1f75SImre Vadász return (0);
702cb6e1f75SImre Vadász
703cb6e1f75SImre Vadász if (needed < 2)
704cb6e1f75SImre Vadász return (0);
705cb6e1f75SImre Vadász
706cb6e1f75SImre Vadász return (1);
707cb6e1f75SImre Vadász }
708cb6e1f75SImre Vadász
709cb6e1f75SImre Vadász static void
vq_ring_enqueue_indirect(struct virtqueue * vq,void * cookie,struct sglist * sg,int readable,int writable)710cb6e1f75SImre Vadász vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
711cb6e1f75SImre Vadász struct sglist *sg, int readable, int writable)
712cb6e1f75SImre Vadász {
713cb6e1f75SImre Vadász struct vring_desc *dp;
714cb6e1f75SImre Vadász struct vq_desc_extra *dxp;
715cb6e1f75SImre Vadász int needed;
716cb6e1f75SImre Vadász uint16_t head_idx;
717cb6e1f75SImre Vadász
718cb6e1f75SImre Vadász needed = readable + writable;
719cb6e1f75SImre Vadász VQASSERT(vq, needed <= vq->vq_max_indirect_size,
720cb6e1f75SImre Vadász "enqueuing too many indirect descriptors");
721cb6e1f75SImre Vadász
722cb6e1f75SImre Vadász head_idx = vq->vq_desc_head_idx;
723cb6e1f75SImre Vadász VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
724cb6e1f75SImre Vadász dp = &vq->vq_ring.desc[head_idx];
725cb6e1f75SImre Vadász dxp = &vq->vq_descx[head_idx];
726cb6e1f75SImre Vadász
727cb6e1f75SImre Vadász VQASSERT(vq, dxp->cookie == NULL,
728cb6e1f75SImre Vadász "cookie already exists for index %d", head_idx);
729cb6e1f75SImre Vadász dxp->cookie = cookie;
730cb6e1f75SImre Vadász dxp->ndescs = 1;
731cb6e1f75SImre Vadász
732cb6e1f75SImre Vadász dp->addr = dxp->indirect_paddr;
733cb6e1f75SImre Vadász dp->len = needed * sizeof(struct vring_desc);
734cb6e1f75SImre Vadász dp->flags = VRING_DESC_F_INDIRECT;
735cb6e1f75SImre Vadász
736cb6e1f75SImre Vadász vq_ring_enqueue_segments(vq, dxp->indirect, 0,
737cb6e1f75SImre Vadász sg, readable, writable);
738cb6e1f75SImre Vadász
739cb6e1f75SImre Vadász vq->vq_desc_head_idx = dp->next;
740cb6e1f75SImre Vadász vq->vq_free_cnt--;
741cb6e1f75SImre Vadász if (vq->vq_free_cnt == 0)
742cb6e1f75SImre Vadász VQ_RING_ASSERT_CHAIN_TERM(vq);
743cb6e1f75SImre Vadász else
744cb6e1f75SImre Vadász VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
745cb6e1f75SImre Vadász
746cb6e1f75SImre Vadász vq_ring_update_avail(vq, head_idx);
747cb6e1f75SImre Vadász }
748cb6e1f75SImre Vadász
749cb6e1f75SImre Vadász static int
vq_ring_must_notify_host(struct virtqueue * vq)75011447b59SVenkatesh Srinivas vq_ring_must_notify_host(struct virtqueue *vq)
75111447b59SVenkatesh Srinivas {
75211447b59SVenkatesh Srinivas uint16_t new_idx, prev_idx, event_idx;
75311447b59SVenkatesh Srinivas
75411447b59SVenkatesh Srinivas if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
75511447b59SVenkatesh Srinivas new_idx = vq->vq_ring.avail->idx;
75611447b59SVenkatesh Srinivas prev_idx = new_idx - vq->vq_queued_cnt;
75711447b59SVenkatesh Srinivas event_idx = vring_avail_event(&vq->vq_ring);
75811447b59SVenkatesh Srinivas
75911447b59SVenkatesh Srinivas return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
76011447b59SVenkatesh Srinivas }
76111447b59SVenkatesh Srinivas
76211447b59SVenkatesh Srinivas return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0);
76311447b59SVenkatesh Srinivas }
76411447b59SVenkatesh Srinivas
76511447b59SVenkatesh Srinivas static void
vq_ring_notify_host(struct virtqueue * vq)76611447b59SVenkatesh Srinivas vq_ring_notify_host(struct virtqueue *vq)
76711447b59SVenkatesh Srinivas {
76811447b59SVenkatesh Srinivas VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index);
76911447b59SVenkatesh Srinivas }
77011447b59SVenkatesh Srinivas
77111447b59SVenkatesh Srinivas static void
vq_ring_free_chain(struct virtqueue * vq,uint16_t desc_idx)77211447b59SVenkatesh Srinivas vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
77311447b59SVenkatesh Srinivas {
77411447b59SVenkatesh Srinivas struct vring_desc *dp;
77511447b59SVenkatesh Srinivas struct vq_desc_extra *dxp;
77611447b59SVenkatesh Srinivas
77711447b59SVenkatesh Srinivas VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
77811447b59SVenkatesh Srinivas dp = &vq->vq_ring.desc[desc_idx];
77911447b59SVenkatesh Srinivas dxp = &vq->vq_descx[desc_idx];
78011447b59SVenkatesh Srinivas
78111447b59SVenkatesh Srinivas if (vq->vq_free_cnt == 0)
78211447b59SVenkatesh Srinivas VQ_RING_ASSERT_CHAIN_TERM(vq);
78311447b59SVenkatesh Srinivas
78411447b59SVenkatesh Srinivas vq->vq_free_cnt += dxp->ndescs;
78511447b59SVenkatesh Srinivas dxp->ndescs--;
78611447b59SVenkatesh Srinivas
787cb6e1f75SImre Vadász if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
78811447b59SVenkatesh Srinivas while (dp->flags & VRING_DESC_F_NEXT) {
78911447b59SVenkatesh Srinivas VQ_RING_ASSERT_VALID_IDX(vq, dp->next);
79011447b59SVenkatesh Srinivas dp = &vq->vq_ring.desc[dp->next];
79111447b59SVenkatesh Srinivas dxp->ndescs--;
79211447b59SVenkatesh Srinivas }
793cb6e1f75SImre Vadász }
79411447b59SVenkatesh Srinivas VQASSERT(vq, dxp->ndescs == 0, "failed to free entire desc chain");
79511447b59SVenkatesh Srinivas
79611447b59SVenkatesh Srinivas /*
79711447b59SVenkatesh Srinivas * We must append the existing free chain, if any, to the end of
79811447b59SVenkatesh Srinivas * newly freed chain. If the virtqueue was completely used, then
79911447b59SVenkatesh Srinivas * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
80011447b59SVenkatesh Srinivas */
80111447b59SVenkatesh Srinivas dp->next = vq->vq_desc_head_idx;
80211447b59SVenkatesh Srinivas vq->vq_desc_head_idx = desc_idx;
80311447b59SVenkatesh Srinivas }
804