110857ec0SEugenio Pérez /*
210857ec0SEugenio Pérez * vhost shadow virtqueue
310857ec0SEugenio Pérez *
410857ec0SEugenio Pérez * SPDX-FileCopyrightText: Red Hat, Inc. 2021
510857ec0SEugenio Pérez * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
610857ec0SEugenio Pérez *
710857ec0SEugenio Pérez * SPDX-License-Identifier: GPL-2.0-or-later
810857ec0SEugenio Pérez */
910857ec0SEugenio Pérez
1010857ec0SEugenio Pérez #include "qemu/osdep.h"
1110857ec0SEugenio Pérez #include "hw/virtio/vhost-shadow-virtqueue.h"
1210857ec0SEugenio Pérez
1310857ec0SEugenio Pérez #include "qemu/error-report.h"
144725a418SEugenio Pérez #include "qapi/error.h"
15dff4426fSEugenio Pérez #include "qemu/main-loop.h"
16100890f7SEugenio Pérez #include "qemu/log.h"
17100890f7SEugenio Pérez #include "qemu/memalign.h"
18dff4426fSEugenio Pérez #include "linux-headers/linux/vhost.h"
19dff4426fSEugenio Pérez
20dff4426fSEugenio Pérez /**
214725a418SEugenio Pérez * Validate the transport device features that both guests can use with the SVQ
224725a418SEugenio Pérez * and SVQs can use with the device.
234725a418SEugenio Pérez *
244725a418SEugenio Pérez * @dev_features: The features
254725a418SEugenio Pérez * @errp: Error pointer
264725a418SEugenio Pérez */
vhost_svq_valid_features(uint64_t features,Error ** errp)274725a418SEugenio Pérez bool vhost_svq_valid_features(uint64_t features, Error **errp)
284725a418SEugenio Pérez {
294725a418SEugenio Pérez bool ok = true;
304725a418SEugenio Pérez uint64_t svq_features = features;
314725a418SEugenio Pérez
324725a418SEugenio Pérez for (uint64_t b = VIRTIO_TRANSPORT_F_START; b <= VIRTIO_TRANSPORT_F_END;
334725a418SEugenio Pérez ++b) {
344725a418SEugenio Pérez switch (b) {
354725a418SEugenio Pérez case VIRTIO_F_ANY_LAYOUT:
36396d5126SEugenio Pérez case VIRTIO_RING_F_EVENT_IDX:
374725a418SEugenio Pérez continue;
384725a418SEugenio Pérez
394725a418SEugenio Pérez case VIRTIO_F_ACCESS_PLATFORM:
404725a418SEugenio Pérez /* SVQ trust in the host's IOMMU to translate addresses */
414725a418SEugenio Pérez case VIRTIO_F_VERSION_1:
424725a418SEugenio Pérez /* SVQ trust that the guest vring is little endian */
434725a418SEugenio Pérez if (!(svq_features & BIT_ULL(b))) {
444725a418SEugenio Pérez svq_features |= BIT_ULL(b);
454725a418SEugenio Pérez ok = false;
464725a418SEugenio Pérez }
474725a418SEugenio Pérez continue;
484725a418SEugenio Pérez
494725a418SEugenio Pérez default:
504725a418SEugenio Pérez if (svq_features & BIT_ULL(b)) {
514725a418SEugenio Pérez svq_features &= ~BIT_ULL(b);
524725a418SEugenio Pérez ok = false;
534725a418SEugenio Pérez }
544725a418SEugenio Pérez }
554725a418SEugenio Pérez }
564725a418SEugenio Pérez
574725a418SEugenio Pérez if (!ok) {
584725a418SEugenio Pérez error_setg(errp, "SVQ Invalid device feature flags, offer: 0x%"PRIx64
594725a418SEugenio Pérez ", ok: 0x%"PRIx64, features, svq_features);
604725a418SEugenio Pérez }
614725a418SEugenio Pérez return ok;
624725a418SEugenio Pérez }
634725a418SEugenio Pérez
644725a418SEugenio Pérez /**
65100890f7SEugenio Pérez * Number of descriptors that the SVQ can make available from the guest.
66dff4426fSEugenio Pérez *
67100890f7SEugenio Pérez * @svq: The svq
68dff4426fSEugenio Pérez */
vhost_svq_available_slots(const VhostShadowVirtqueue * svq)6999d6a324SHawkins Jiawei uint16_t vhost_svq_available_slots(const VhostShadowVirtqueue *svq)
70dff4426fSEugenio Pérez {
715d410557SHawkins Jiawei return svq->num_free;
72100890f7SEugenio Pérez }
73100890f7SEugenio Pérez
7434e3c94eSEugenio Pérez /**
7534e3c94eSEugenio Pérez * Translate addresses between the qemu's virtual address and the SVQ IOVA
7634e3c94eSEugenio Pérez *
7734e3c94eSEugenio Pérez * @svq: Shadow VirtQueue
7834e3c94eSEugenio Pérez * @vaddr: Translated IOVA addresses
7934e3c94eSEugenio Pérez * @iovec: Source qemu's VA addresses
8034e3c94eSEugenio Pérez * @num: Length of iovec and minimum length of vaddr
8134e3c94eSEugenio Pérez */
vhost_svq_translate_addr(const VhostShadowVirtqueue * svq,hwaddr * addrs,const struct iovec * iovec,size_t num)8234e3c94eSEugenio Pérez static bool vhost_svq_translate_addr(const VhostShadowVirtqueue *svq,
8334e3c94eSEugenio Pérez hwaddr *addrs, const struct iovec *iovec,
8434e3c94eSEugenio Pérez size_t num)
8534e3c94eSEugenio Pérez {
8634e3c94eSEugenio Pérez if (num == 0) {
8734e3c94eSEugenio Pérez return true;
8834e3c94eSEugenio Pérez }
8934e3c94eSEugenio Pérez
9034e3c94eSEugenio Pérez for (size_t i = 0; i < num; ++i) {
9134e3c94eSEugenio Pérez DMAMap needle = {
9234e3c94eSEugenio Pérez .translated_addr = (hwaddr)(uintptr_t)iovec[i].iov_base,
9334e3c94eSEugenio Pérez .size = iovec[i].iov_len,
9434e3c94eSEugenio Pérez };
9534e3c94eSEugenio Pérez Int128 needle_last, map_last;
9634e3c94eSEugenio Pérez size_t off;
9734e3c94eSEugenio Pérez
9834e3c94eSEugenio Pérez const DMAMap *map = vhost_iova_tree_find_iova(svq->iova_tree, &needle);
9934e3c94eSEugenio Pérez /*
10034e3c94eSEugenio Pérez * Map cannot be NULL since iova map contains all guest space and
10134e3c94eSEugenio Pérez * qemu already has a physical address mapped
10234e3c94eSEugenio Pérez */
10334e3c94eSEugenio Pérez if (unlikely(!map)) {
10434e3c94eSEugenio Pérez qemu_log_mask(LOG_GUEST_ERROR,
10534e3c94eSEugenio Pérez "Invalid address 0x%"HWADDR_PRIx" given by guest",
10634e3c94eSEugenio Pérez needle.translated_addr);
10734e3c94eSEugenio Pérez return false;
10834e3c94eSEugenio Pérez }
10934e3c94eSEugenio Pérez
11034e3c94eSEugenio Pérez off = needle.translated_addr - map->translated_addr;
11134e3c94eSEugenio Pérez addrs[i] = map->iova + off;
11234e3c94eSEugenio Pérez
11334e3c94eSEugenio Pérez needle_last = int128_add(int128_make64(needle.translated_addr),
114b77a5f22SHawkins Jiawei int128_makes64(iovec[i].iov_len - 1));
11534e3c94eSEugenio Pérez map_last = int128_make64(map->translated_addr + map->size);
11634e3c94eSEugenio Pérez if (unlikely(int128_gt(needle_last, map_last))) {
11734e3c94eSEugenio Pérez qemu_log_mask(LOG_GUEST_ERROR,
11834e3c94eSEugenio Pérez "Guest buffer expands over iova range");
11934e3c94eSEugenio Pérez return false;
12034e3c94eSEugenio Pérez }
12134e3c94eSEugenio Pérez }
12234e3c94eSEugenio Pérez
12334e3c94eSEugenio Pérez return true;
12434e3c94eSEugenio Pérez }
12534e3c94eSEugenio Pérez
126009c2549SEugenio Pérez /**
127009c2549SEugenio Pérez * Write descriptors to SVQ vring
128009c2549SEugenio Pérez *
129009c2549SEugenio Pérez * @svq: The shadow virtqueue
130009c2549SEugenio Pérez * @sg: Cache for hwaddr
131009c2549SEugenio Pérez * @iovec: The iovec from the guest
132009c2549SEugenio Pérez * @num: iovec length
133009c2549SEugenio Pérez * @more_descs: True if more descriptors come in the chain
134009c2549SEugenio Pérez * @write: True if they are writeable descriptors
135009c2549SEugenio Pérez *
136009c2549SEugenio Pérez * Return true if success, false otherwise and print error.
137009c2549SEugenio Pérez */
vhost_svq_vring_write_descs(VhostShadowVirtqueue * svq,hwaddr * sg,const struct iovec * iovec,size_t num,bool more_descs,bool write)138009c2549SEugenio Pérez static bool vhost_svq_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg,
139100890f7SEugenio Pérez const struct iovec *iovec, size_t num,
140100890f7SEugenio Pérez bool more_descs, bool write)
141100890f7SEugenio Pérez {
142100890f7SEugenio Pérez uint16_t i = svq->free_head, last = svq->free_head;
143100890f7SEugenio Pérez unsigned n;
144100890f7SEugenio Pérez uint16_t flags = write ? cpu_to_le16(VRING_DESC_F_WRITE) : 0;
145100890f7SEugenio Pérez vring_desc_t *descs = svq->vring.desc;
146009c2549SEugenio Pérez bool ok;
147100890f7SEugenio Pérez
148100890f7SEugenio Pérez if (num == 0) {
149009c2549SEugenio Pérez return true;
150009c2549SEugenio Pérez }
151009c2549SEugenio Pérez
152009c2549SEugenio Pérez ok = vhost_svq_translate_addr(svq, sg, iovec, num);
153009c2549SEugenio Pérez if (unlikely(!ok)) {
154009c2549SEugenio Pérez return false;
155100890f7SEugenio Pérez }
156100890f7SEugenio Pérez
157100890f7SEugenio Pérez for (n = 0; n < num; n++) {
158100890f7SEugenio Pérez if (more_descs || (n + 1 < num)) {
159100890f7SEugenio Pérez descs[i].flags = flags | cpu_to_le16(VRING_DESC_F_NEXT);
160495fe3a7SEugenio Pérez descs[i].next = cpu_to_le16(svq->desc_next[i]);
161100890f7SEugenio Pérez } else {
162100890f7SEugenio Pérez descs[i].flags = flags;
163100890f7SEugenio Pérez }
16434e3c94eSEugenio Pérez descs[i].addr = cpu_to_le64(sg[n]);
165100890f7SEugenio Pérez descs[i].len = cpu_to_le32(iovec[n].iov_len);
166100890f7SEugenio Pérez
167100890f7SEugenio Pérez last = i;
168495fe3a7SEugenio Pérez i = cpu_to_le16(svq->desc_next[i]);
169100890f7SEugenio Pérez }
170100890f7SEugenio Pérez
171495fe3a7SEugenio Pérez svq->free_head = le16_to_cpu(svq->desc_next[last]);
172009c2549SEugenio Pérez return true;
173100890f7SEugenio Pérez }
174100890f7SEugenio Pérez
vhost_svq_add_split(VhostShadowVirtqueue * svq,const struct iovec * out_sg,size_t out_num,const struct iovec * in_sg,size_t in_num,unsigned * head)175100890f7SEugenio Pérez static bool vhost_svq_add_split(VhostShadowVirtqueue *svq,
1761f46ae65SEugenio Pérez const struct iovec *out_sg, size_t out_num,
1771f46ae65SEugenio Pérez const struct iovec *in_sg, size_t in_num,
1781f46ae65SEugenio Pérez unsigned *head)
179100890f7SEugenio Pérez {
180100890f7SEugenio Pérez unsigned avail_idx;
181100890f7SEugenio Pérez vring_avail_t *avail = svq->vring.avail;
18234e3c94eSEugenio Pérez bool ok;
1831f46ae65SEugenio Pérez g_autofree hwaddr *sgs = g_new(hwaddr, MAX(out_num, in_num));
184100890f7SEugenio Pérez
185100890f7SEugenio Pérez *head = svq->free_head;
186100890f7SEugenio Pérez
187100890f7SEugenio Pérez /* We need some descriptors here */
1881f46ae65SEugenio Pérez if (unlikely(!out_num && !in_num)) {
189100890f7SEugenio Pérez qemu_log_mask(LOG_GUEST_ERROR,
190100890f7SEugenio Pérez "Guest provided element with no descriptors");
191100890f7SEugenio Pérez return false;
192100890f7SEugenio Pérez }
193100890f7SEugenio Pérez
1941f46ae65SEugenio Pérez ok = vhost_svq_vring_write_descs(svq, sgs, out_sg, out_num, in_num > 0,
1951f46ae65SEugenio Pérez false);
19634e3c94eSEugenio Pérez if (unlikely(!ok)) {
19734e3c94eSEugenio Pérez return false;
19834e3c94eSEugenio Pérez }
19934e3c94eSEugenio Pérez
2001f46ae65SEugenio Pérez ok = vhost_svq_vring_write_descs(svq, sgs, in_sg, in_num, false, true);
201009c2549SEugenio Pérez if (unlikely(!ok)) {
202009c2549SEugenio Pérez return false;
203009c2549SEugenio Pérez }
204100890f7SEugenio Pérez
205100890f7SEugenio Pérez /*
206100890f7SEugenio Pérez * Put the entry in the available array (but don't update avail->idx until
207100890f7SEugenio Pérez * they do sync).
208100890f7SEugenio Pérez */
209100890f7SEugenio Pérez avail_idx = svq->shadow_avail_idx & (svq->vring.num - 1);
210100890f7SEugenio Pérez avail->ring[avail_idx] = cpu_to_le16(*head);
211100890f7SEugenio Pérez svq->shadow_avail_idx++;
212100890f7SEugenio Pérez
213100890f7SEugenio Pérez /* Update the avail index after write the descriptor */
214100890f7SEugenio Pérez smp_wmb();
215100890f7SEugenio Pérez avail->idx = cpu_to_le16(svq->shadow_avail_idx);
216100890f7SEugenio Pérez
217100890f7SEugenio Pérez return true;
218100890f7SEugenio Pérez }
219100890f7SEugenio Pérez
vhost_svq_kick(VhostShadowVirtqueue * svq)220d93a2405SEugenio Pérez static void vhost_svq_kick(VhostShadowVirtqueue *svq)
221d93a2405SEugenio Pérez {
22222a6840fSEugenio Pérez bool needs_kick;
22322a6840fSEugenio Pérez
224d93a2405SEugenio Pérez /*
225d93a2405SEugenio Pérez * We need to expose the available array entries before checking the used
226d93a2405SEugenio Pérez * flags
227d93a2405SEugenio Pérez */
228d93a2405SEugenio Pérez smp_mb();
22922a6840fSEugenio Pérez
23022a6840fSEugenio Pérez if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
23122a6840fSEugenio Pérez uint16_t avail_event = *(uint16_t *)(&svq->vring.used->ring[svq->vring.num]);
23222a6840fSEugenio Pérez needs_kick = vring_need_event(avail_event, svq->shadow_avail_idx, svq->shadow_avail_idx - 1);
23322a6840fSEugenio Pérez } else {
23422a6840fSEugenio Pérez needs_kick = !(svq->vring.used->flags & VRING_USED_F_NO_NOTIFY);
23522a6840fSEugenio Pérez }
23622a6840fSEugenio Pérez
23722a6840fSEugenio Pérez if (!needs_kick) {
238d93a2405SEugenio Pérez return;
239d93a2405SEugenio Pérez }
240d93a2405SEugenio Pérez
241d93a2405SEugenio Pérez event_notifier_set(&svq->hdev_kick);
242d93a2405SEugenio Pérez }
243d93a2405SEugenio Pérez
2445181db13SEugenio Pérez /**
2455181db13SEugenio Pérez * Add an element to a SVQ.
2465181db13SEugenio Pérez *
247f20b70ebSEugenio Pérez * Return -EINVAL if element is invalid, -ENOSPC if dev queue is full
2485181db13SEugenio Pérez */
vhost_svq_add(VhostShadowVirtqueue * svq,const struct iovec * out_sg,size_t out_num,const struct iovec * in_sg,size_t in_num,VirtQueueElement * elem)249d0291f3fSEugenio Pérez int vhost_svq_add(VhostShadowVirtqueue *svq, const struct iovec *out_sg,
250d0291f3fSEugenio Pérez size_t out_num, const struct iovec *in_sg, size_t in_num,
251d0291f3fSEugenio Pérez VirtQueueElement *elem)
252100890f7SEugenio Pérez {
253100890f7SEugenio Pérez unsigned qemu_head;
2541f46ae65SEugenio Pérez unsigned ndescs = in_num + out_num;
255f20b70ebSEugenio Pérez bool ok;
256f20b70ebSEugenio Pérez
257f20b70ebSEugenio Pérez if (unlikely(ndescs > vhost_svq_available_slots(svq))) {
258f20b70ebSEugenio Pérez return -ENOSPC;
259f20b70ebSEugenio Pérez }
260f20b70ebSEugenio Pérez
2611f46ae65SEugenio Pérez ok = vhost_svq_add_split(svq, out_sg, out_num, in_sg, in_num, &qemu_head);
262100890f7SEugenio Pérez if (unlikely(!ok)) {
263f20b70ebSEugenio Pérez return -EINVAL;
264100890f7SEugenio Pérez }
265100890f7SEugenio Pérez
2665d410557SHawkins Jiawei svq->num_free -= ndescs;
2679e87868fSEugenio Pérez svq->desc_state[qemu_head].elem = elem;
268ac4cfdc6SEugenio Pérez svq->desc_state[qemu_head].ndescs = ndescs;
26998b5adefSEugenio Pérez vhost_svq_kick(svq);
270f20b70ebSEugenio Pérez return 0;
271100890f7SEugenio Pérez }
272100890f7SEugenio Pérez
2731f46ae65SEugenio Pérez /* Convenience wrapper to add a guest's element to SVQ */
vhost_svq_add_element(VhostShadowVirtqueue * svq,VirtQueueElement * elem)2741f46ae65SEugenio Pérez static int vhost_svq_add_element(VhostShadowVirtqueue *svq,
2751f46ae65SEugenio Pérez VirtQueueElement *elem)
2761f46ae65SEugenio Pérez {
2771f46ae65SEugenio Pérez return vhost_svq_add(svq, elem->out_sg, elem->out_num, elem->in_sg,
2781f46ae65SEugenio Pérez elem->in_num, elem);
2791f46ae65SEugenio Pérez }
2801f46ae65SEugenio Pérez
281dff4426fSEugenio Pérez /**
282100890f7SEugenio Pérez * Forward available buffers.
283100890f7SEugenio Pérez *
284100890f7SEugenio Pérez * @svq: Shadow VirtQueue
285100890f7SEugenio Pérez *
286100890f7SEugenio Pérez * Note that this function does not guarantee that all guest's available
287100890f7SEugenio Pérez * buffers are available to the device in SVQ avail ring. The guest may have
288100890f7SEugenio Pérez * exposed a GPA / GIOVA contiguous buffer, but it may not be contiguous in
289100890f7SEugenio Pérez * qemu vaddr.
290100890f7SEugenio Pérez *
291100890f7SEugenio Pérez * If that happens, guest's kick notifications will be disabled until the
292100890f7SEugenio Pérez * device uses some buffers.
293100890f7SEugenio Pérez */
vhost_handle_guest_kick(VhostShadowVirtqueue * svq)294100890f7SEugenio Pérez static void vhost_handle_guest_kick(VhostShadowVirtqueue *svq)
295100890f7SEugenio Pérez {
296100890f7SEugenio Pérez /* Clear event notifier */
297100890f7SEugenio Pérez event_notifier_test_and_clear(&svq->svq_kick);
298100890f7SEugenio Pérez
299100890f7SEugenio Pérez /* Forward to the device as many available buffers as possible */
300100890f7SEugenio Pérez do {
301100890f7SEugenio Pérez virtio_queue_set_notification(svq->vq, false);
302100890f7SEugenio Pérez
303100890f7SEugenio Pérez while (true) {
304ad9f958dSBernhard Beschow g_autofree VirtQueueElement *elem = NULL;
305f20b70ebSEugenio Pérez int r;
306100890f7SEugenio Pérez
307100890f7SEugenio Pérez if (svq->next_guest_avail_elem) {
308100890f7SEugenio Pérez elem = g_steal_pointer(&svq->next_guest_avail_elem);
309100890f7SEugenio Pérez } else {
310100890f7SEugenio Pérez elem = virtqueue_pop(svq->vq, sizeof(*elem));
311100890f7SEugenio Pérez }
312100890f7SEugenio Pérez
313100890f7SEugenio Pérez if (!elem) {
314100890f7SEugenio Pérez break;
315100890f7SEugenio Pérez }
316100890f7SEugenio Pérez
317e966c0b7SEugenio Pérez if (svq->ops) {
318e966c0b7SEugenio Pérez r = svq->ops->avail_handler(svq, elem, svq->ops_opaque);
319e966c0b7SEugenio Pérez } else {
3201f46ae65SEugenio Pérez r = vhost_svq_add_element(svq, elem);
321e966c0b7SEugenio Pérez }
322f20b70ebSEugenio Pérez if (unlikely(r != 0)) {
323f20b70ebSEugenio Pérez if (r == -ENOSPC) {
324100890f7SEugenio Pérez /*
325f20b70ebSEugenio Pérez * This condition is possible since a contiguous buffer in
326f20b70ebSEugenio Pérez * GPA does not imply a contiguous buffer in qemu's VA
327f20b70ebSEugenio Pérez * scatter-gather segments. If that happens, the buffer
328f20b70ebSEugenio Pérez * exposed to the device needs to be a chain of descriptors
329f20b70ebSEugenio Pérez * at this moment.
330100890f7SEugenio Pérez *
331100890f7SEugenio Pérez * SVQ cannot hold more available buffers if we are here:
332f20b70ebSEugenio Pérez * queue the current guest descriptor and ignore kicks
333100890f7SEugenio Pérez * until some elements are used.
334100890f7SEugenio Pérez */
3359c2ab2f1SEugenio Pérez svq->next_guest_avail_elem = g_steal_pointer(&elem);
336100890f7SEugenio Pérez }
337100890f7SEugenio Pérez
338f20b70ebSEugenio Pérez /* VQ is full or broken, just return and ignore kicks */
339100890f7SEugenio Pérez return;
340100890f7SEugenio Pérez }
3419c2ab2f1SEugenio Pérez /* elem belongs to SVQ or external caller now */
3429c2ab2f1SEugenio Pérez elem = NULL;
343100890f7SEugenio Pérez }
344100890f7SEugenio Pérez
345100890f7SEugenio Pérez virtio_queue_set_notification(svq->vq, true);
346100890f7SEugenio Pérez } while (!virtio_queue_empty(svq->vq));
347100890f7SEugenio Pérez }
348100890f7SEugenio Pérez
349100890f7SEugenio Pérez /**
350100890f7SEugenio Pérez * Handle guest's kick.
351100890f7SEugenio Pérez *
352100890f7SEugenio Pérez * @n: guest kick event notifier, the one that guest set to notify svq.
353100890f7SEugenio Pérez */
vhost_handle_guest_kick_notifier(EventNotifier * n)354100890f7SEugenio Pérez static void vhost_handle_guest_kick_notifier(EventNotifier *n)
355100890f7SEugenio Pérez {
356100890f7SEugenio Pérez VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, svq_kick);
357100890f7SEugenio Pérez event_notifier_test_and_clear(n);
358100890f7SEugenio Pérez vhost_handle_guest_kick(svq);
359100890f7SEugenio Pérez }
360100890f7SEugenio Pérez
vhost_svq_more_used(VhostShadowVirtqueue * svq)361100890f7SEugenio Pérez static bool vhost_svq_more_used(VhostShadowVirtqueue *svq)
362100890f7SEugenio Pérez {
363c381abc3SEugenio Pérez uint16_t *used_idx = &svq->vring.used->idx;
364100890f7SEugenio Pérez if (svq->last_used_idx != svq->shadow_used_idx) {
365100890f7SEugenio Pérez return true;
366100890f7SEugenio Pérez }
367100890f7SEugenio Pérez
368c381abc3SEugenio Pérez svq->shadow_used_idx = cpu_to_le16(*(volatile uint16_t *)used_idx);
369100890f7SEugenio Pérez
370100890f7SEugenio Pérez return svq->last_used_idx != svq->shadow_used_idx;
371100890f7SEugenio Pérez }
372100890f7SEugenio Pérez
373100890f7SEugenio Pérez /**
374100890f7SEugenio Pérez * Enable vhost device calls after disable them.
375100890f7SEugenio Pérez *
376100890f7SEugenio Pérez * @svq: The svq
377100890f7SEugenio Pérez *
378100890f7SEugenio Pérez * It returns false if there are pending used buffers from the vhost device,
379100890f7SEugenio Pérez * avoiding the possible races between SVQ checking for more work and enabling
380100890f7SEugenio Pérez * callbacks. True if SVQ used vring has no more pending buffers.
381100890f7SEugenio Pérez */
vhost_svq_enable_notification(VhostShadowVirtqueue * svq)382100890f7SEugenio Pérez static bool vhost_svq_enable_notification(VhostShadowVirtqueue *svq)
383100890f7SEugenio Pérez {
38401f8beacSEugenio Pérez if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
38501f8beacSEugenio Pérez uint16_t *used_event = (uint16_t *)&svq->vring.avail->ring[svq->vring.num];
38601f8beacSEugenio Pérez *used_event = svq->shadow_used_idx;
38701f8beacSEugenio Pérez } else {
388100890f7SEugenio Pérez svq->vring.avail->flags &= ~cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
38901f8beacSEugenio Pérez }
39001f8beacSEugenio Pérez
39101f8beacSEugenio Pérez /* Make sure the event is enabled before the read of used_idx */
392100890f7SEugenio Pérez smp_mb();
393100890f7SEugenio Pérez return !vhost_svq_more_used(svq);
394100890f7SEugenio Pérez }
395100890f7SEugenio Pérez
vhost_svq_disable_notification(VhostShadowVirtqueue * svq)396100890f7SEugenio Pérez static void vhost_svq_disable_notification(VhostShadowVirtqueue *svq)
397100890f7SEugenio Pérez {
39801f8beacSEugenio Pérez /*
39901f8beacSEugenio Pérez * No need to disable notification in the event idx case, since used event
40001f8beacSEugenio Pérez * index is already an index too far away.
40101f8beacSEugenio Pérez */
40201f8beacSEugenio Pérez if (!virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
403100890f7SEugenio Pérez svq->vring.avail->flags |= cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
404100890f7SEugenio Pérez }
40501f8beacSEugenio Pérez }
406100890f7SEugenio Pérez
vhost_svq_last_desc_of_chain(const VhostShadowVirtqueue * svq,uint16_t num,uint16_t i)40781abfa57SEugenio Pérez static uint16_t vhost_svq_last_desc_of_chain(const VhostShadowVirtqueue *svq,
40881abfa57SEugenio Pérez uint16_t num, uint16_t i)
40981abfa57SEugenio Pérez {
41081abfa57SEugenio Pérez for (uint16_t j = 0; j < (num - 1); ++j) {
41181abfa57SEugenio Pérez i = le16_to_cpu(svq->desc_next[i]);
41281abfa57SEugenio Pérez }
41381abfa57SEugenio Pérez
41481abfa57SEugenio Pérez return i;
41581abfa57SEugenio Pérez }
41681abfa57SEugenio Pérez
vhost_svq_get_buf(VhostShadowVirtqueue * svq,uint32_t * len)417100890f7SEugenio Pérez static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq,
418100890f7SEugenio Pérez uint32_t *len)
419100890f7SEugenio Pérez {
420100890f7SEugenio Pérez const vring_used_t *used = svq->vring.used;
421100890f7SEugenio Pérez vring_used_elem_t used_elem;
42281abfa57SEugenio Pérez uint16_t last_used, last_used_chain, num;
423100890f7SEugenio Pérez
424100890f7SEugenio Pérez if (!vhost_svq_more_used(svq)) {
425100890f7SEugenio Pérez return NULL;
426100890f7SEugenio Pérez }
427100890f7SEugenio Pérez
428100890f7SEugenio Pérez /* Only get used array entries after they have been exposed by dev */
429100890f7SEugenio Pérez smp_rmb();
430100890f7SEugenio Pérez last_used = svq->last_used_idx & (svq->vring.num - 1);
431100890f7SEugenio Pérez used_elem.id = le32_to_cpu(used->ring[last_used].id);
432100890f7SEugenio Pérez used_elem.len = le32_to_cpu(used->ring[last_used].len);
433100890f7SEugenio Pérez
434100890f7SEugenio Pérez svq->last_used_idx++;
435100890f7SEugenio Pérez if (unlikely(used_elem.id >= svq->vring.num)) {
436100890f7SEugenio Pérez qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used",
437100890f7SEugenio Pérez svq->vdev->name, used_elem.id);
438100890f7SEugenio Pérez return NULL;
439100890f7SEugenio Pérez }
440100890f7SEugenio Pérez
44186f5f254SEugenio Pérez if (unlikely(!svq->desc_state[used_elem.id].ndescs)) {
442100890f7SEugenio Pérez qemu_log_mask(LOG_GUEST_ERROR,
443100890f7SEugenio Pérez "Device %s says index %u is used, but it was not available",
444100890f7SEugenio Pérez svq->vdev->name, used_elem.id);
445100890f7SEugenio Pérez return NULL;
446100890f7SEugenio Pérez }
447100890f7SEugenio Pérez
448ac4cfdc6SEugenio Pérez num = svq->desc_state[used_elem.id].ndescs;
44986f5f254SEugenio Pérez svq->desc_state[used_elem.id].ndescs = 0;
45081abfa57SEugenio Pérez last_used_chain = vhost_svq_last_desc_of_chain(svq, num, used_elem.id);
45181abfa57SEugenio Pérez svq->desc_next[last_used_chain] = svq->free_head;
452100890f7SEugenio Pérez svq->free_head = used_elem.id;
4535d410557SHawkins Jiawei svq->num_free += num;
454100890f7SEugenio Pérez
455100890f7SEugenio Pérez *len = used_elem.len;
4569e87868fSEugenio Pérez return g_steal_pointer(&svq->desc_state[used_elem.id].elem);
457100890f7SEugenio Pérez }
458100890f7SEugenio Pérez
459432efd14SEugenio Pérez /**
460432efd14SEugenio Pérez * Push an element to SVQ, returning it to the guest.
461432efd14SEugenio Pérez */
vhost_svq_push_elem(VhostShadowVirtqueue * svq,const VirtQueueElement * elem,uint32_t len)462432efd14SEugenio Pérez void vhost_svq_push_elem(VhostShadowVirtqueue *svq,
463432efd14SEugenio Pérez const VirtQueueElement *elem, uint32_t len)
464432efd14SEugenio Pérez {
465432efd14SEugenio Pérez virtqueue_push(svq->vq, elem, len);
466432efd14SEugenio Pérez if (svq->next_guest_avail_elem) {
467432efd14SEugenio Pérez /*
468432efd14SEugenio Pérez * Avail ring was full when vhost_svq_flush was called, so it's a
469432efd14SEugenio Pérez * good moment to make more descriptors available if possible.
470432efd14SEugenio Pérez */
471432efd14SEugenio Pérez vhost_handle_guest_kick(svq);
472432efd14SEugenio Pérez }
473432efd14SEugenio Pérez }
474432efd14SEugenio Pérez
vhost_svq_flush(VhostShadowVirtqueue * svq,bool check_for_avail_queue)475100890f7SEugenio Pérez static void vhost_svq_flush(VhostShadowVirtqueue *svq,
476100890f7SEugenio Pérez bool check_for_avail_queue)
477100890f7SEugenio Pérez {
478100890f7SEugenio Pérez VirtQueue *vq = svq->vq;
479100890f7SEugenio Pérez
480100890f7SEugenio Pérez /* Forward as many used buffers as possible. */
481100890f7SEugenio Pérez do {
482100890f7SEugenio Pérez unsigned i = 0;
483100890f7SEugenio Pérez
484100890f7SEugenio Pérez vhost_svq_disable_notification(svq);
485100890f7SEugenio Pérez while (true) {
486100890f7SEugenio Pérez uint32_t len;
487100890f7SEugenio Pérez g_autofree VirtQueueElement *elem = vhost_svq_get_buf(svq, &len);
488100890f7SEugenio Pérez if (!elem) {
489100890f7SEugenio Pérez break;
490100890f7SEugenio Pérez }
491100890f7SEugenio Pérez
492100890f7SEugenio Pérez if (unlikely(i >= svq->vring.num)) {
493100890f7SEugenio Pérez qemu_log_mask(LOG_GUEST_ERROR,
494100890f7SEugenio Pérez "More than %u used buffers obtained in a %u size SVQ",
495100890f7SEugenio Pérez i, svq->vring.num);
496100890f7SEugenio Pérez virtqueue_fill(vq, elem, len, i);
497100890f7SEugenio Pérez virtqueue_flush(vq, i);
498100890f7SEugenio Pérez return;
499100890f7SEugenio Pérez }
500100890f7SEugenio Pérez virtqueue_fill(vq, elem, len, i++);
501100890f7SEugenio Pérez }
502100890f7SEugenio Pérez
503100890f7SEugenio Pérez virtqueue_flush(vq, i);
504100890f7SEugenio Pérez event_notifier_set(&svq->svq_call);
505100890f7SEugenio Pérez
506100890f7SEugenio Pérez if (check_for_avail_queue && svq->next_guest_avail_elem) {
507100890f7SEugenio Pérez /*
508100890f7SEugenio Pérez * Avail ring was full when vhost_svq_flush was called, so it's a
509100890f7SEugenio Pérez * good moment to make more descriptors available if possible.
510100890f7SEugenio Pérez */
511100890f7SEugenio Pérez vhost_handle_guest_kick(svq);
512100890f7SEugenio Pérez }
513100890f7SEugenio Pérez } while (!vhost_svq_enable_notification(svq));
514100890f7SEugenio Pérez }
515100890f7SEugenio Pérez
516100890f7SEugenio Pérez /**
517b0de17a2SHawkins Jiawei * Poll the SVQ to wait for the device to use the specified number
518b0de17a2SHawkins Jiawei * of elements and return the total length written by the device.
5193f44d13dSEugenio Pérez *
5203f44d13dSEugenio Pérez * This function race with main event loop SVQ polling, so extra
5213f44d13dSEugenio Pérez * synchronization is needed.
5223f44d13dSEugenio Pérez *
523b0de17a2SHawkins Jiawei * @svq: The svq
524b0de17a2SHawkins Jiawei * @num: The number of elements that need to be used
5253f44d13dSEugenio Pérez */
vhost_svq_poll(VhostShadowVirtqueue * svq,size_t num)526b0de17a2SHawkins Jiawei size_t vhost_svq_poll(VhostShadowVirtqueue *svq, size_t num)
5273f44d13dSEugenio Pérez {
528b0de17a2SHawkins Jiawei size_t len = 0;
529b0de17a2SHawkins Jiawei uint32_t r;
530b0de17a2SHawkins Jiawei
531b0de17a2SHawkins Jiawei while (num--) {
5323f44d13dSEugenio Pérez int64_t start_us = g_get_monotonic_time();
533d368c0b0SEugenio Pérez
534d368c0b0SEugenio Pérez do {
535d368c0b0SEugenio Pérez if (vhost_svq_more_used(svq)) {
536d368c0b0SEugenio Pérez break;
5373f44d13dSEugenio Pérez }
5383f44d13dSEugenio Pérez
5393f44d13dSEugenio Pérez if (unlikely(g_get_monotonic_time() - start_us > 10e6)) {
540b0de17a2SHawkins Jiawei return len;
5413f44d13dSEugenio Pérez }
5423f44d13dSEugenio Pérez } while (true);
543d368c0b0SEugenio Pérez
544b0de17a2SHawkins Jiawei vhost_svq_get_buf(svq, &r);
545b0de17a2SHawkins Jiawei len += r;
546b0de17a2SHawkins Jiawei }
547b0de17a2SHawkins Jiawei
548d368c0b0SEugenio Pérez return len;
5493f44d13dSEugenio Pérez }
5503f44d13dSEugenio Pérez
5513f44d13dSEugenio Pérez /**
552100890f7SEugenio Pérez * Forward used buffers.
553a8ac8858SEugenio Pérez *
554a8ac8858SEugenio Pérez * @n: hdev call event notifier, the one that device set to notify svq.
555100890f7SEugenio Pérez *
556100890f7SEugenio Pérez * Note that we are not making any buffers available in the loop, there is no
557100890f7SEugenio Pérez * way that it runs more than virtqueue size times.
558a8ac8858SEugenio Pérez */
vhost_svq_handle_call(EventNotifier * n)559a8ac8858SEugenio Pérez static void vhost_svq_handle_call(EventNotifier *n)
560a8ac8858SEugenio Pérez {
561a8ac8858SEugenio Pérez VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue,
562a8ac8858SEugenio Pérez hdev_call);
563a8ac8858SEugenio Pérez event_notifier_test_and_clear(n);
564100890f7SEugenio Pérez vhost_svq_flush(svq, true);
565a8ac8858SEugenio Pérez }
566a8ac8858SEugenio Pérez
567a8ac8858SEugenio Pérez /**
568a8ac8858SEugenio Pérez * Set the call notifier for the SVQ to call the guest
569a8ac8858SEugenio Pérez *
570a8ac8858SEugenio Pérez * @svq: Shadow virtqueue
571a8ac8858SEugenio Pérez * @call_fd: call notifier
572a8ac8858SEugenio Pérez *
573a8ac8858SEugenio Pérez * Called on BQL context.
574a8ac8858SEugenio Pérez */
vhost_svq_set_svq_call_fd(VhostShadowVirtqueue * svq,int call_fd)575a8ac8858SEugenio Pérez void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd)
576a8ac8858SEugenio Pérez {
577a8ac8858SEugenio Pérez if (call_fd == VHOST_FILE_UNBIND) {
578a8ac8858SEugenio Pérez /*
579a8ac8858SEugenio Pérez * Fail event_notifier_set if called handling device call.
580a8ac8858SEugenio Pérez *
581a8ac8858SEugenio Pérez * SVQ still needs device notifications, since it needs to keep
582a8ac8858SEugenio Pérez * forwarding used buffers even with the unbind.
583a8ac8858SEugenio Pérez */
584a8ac8858SEugenio Pérez memset(&svq->svq_call, 0, sizeof(svq->svq_call));
585a8ac8858SEugenio Pérez } else {
586a8ac8858SEugenio Pérez event_notifier_init_fd(&svq->svq_call, call_fd);
587a8ac8858SEugenio Pérez }
588a8ac8858SEugenio Pérez }
589a8ac8858SEugenio Pérez
590a8ac8858SEugenio Pérez /**
591dafb34c9SEugenio Pérez * Get the shadow vq vring address.
592dafb34c9SEugenio Pérez * @svq: Shadow virtqueue
593dafb34c9SEugenio Pérez * @addr: Destination to store address
594dafb34c9SEugenio Pérez */
vhost_svq_get_vring_addr(const VhostShadowVirtqueue * svq,struct vhost_vring_addr * addr)595dafb34c9SEugenio Pérez void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
596dafb34c9SEugenio Pérez struct vhost_vring_addr *addr)
597dafb34c9SEugenio Pérez {
59834e3c94eSEugenio Pérez addr->desc_user_addr = (uint64_t)(uintptr_t)svq->vring.desc;
59934e3c94eSEugenio Pérez addr->avail_user_addr = (uint64_t)(uintptr_t)svq->vring.avail;
60034e3c94eSEugenio Pérez addr->used_user_addr = (uint64_t)(uintptr_t)svq->vring.used;
601dafb34c9SEugenio Pérez }
602dafb34c9SEugenio Pérez
vhost_svq_driver_area_size(const VhostShadowVirtqueue * svq)603dafb34c9SEugenio Pérez size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq)
604dafb34c9SEugenio Pérez {
605dafb34c9SEugenio Pérez size_t desc_size = sizeof(vring_desc_t) * svq->vring.num;
606f0c48e05SEugenio Pérez size_t avail_size = offsetof(vring_avail_t, ring[svq->vring.num]) +
607f0c48e05SEugenio Pérez sizeof(uint16_t);
608dafb34c9SEugenio Pérez
6098e3b0cbbSMarc-André Lureau return ROUND_UP(desc_size + avail_size, qemu_real_host_page_size());
610dafb34c9SEugenio Pérez }
611dafb34c9SEugenio Pérez
vhost_svq_device_area_size(const VhostShadowVirtqueue * svq)612dafb34c9SEugenio Pérez size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq)
613dafb34c9SEugenio Pérez {
614f0c48e05SEugenio Pérez size_t used_size = offsetof(vring_used_t, ring[svq->vring.num]) +
615f0c48e05SEugenio Pérez sizeof(uint16_t);
6168e3b0cbbSMarc-André Lureau return ROUND_UP(used_size, qemu_real_host_page_size());
617dafb34c9SEugenio Pérez }
618dafb34c9SEugenio Pérez
619dafb34c9SEugenio Pérez /**
620dff4426fSEugenio Pérez * Set a new file descriptor for the guest to kick the SVQ and notify for avail
621dff4426fSEugenio Pérez *
622dff4426fSEugenio Pérez * @svq: The svq
623dff4426fSEugenio Pérez * @svq_kick_fd: The svq kick fd
624dff4426fSEugenio Pérez *
625dff4426fSEugenio Pérez * Note that the SVQ will never close the old file descriptor.
626dff4426fSEugenio Pérez */
vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue * svq,int svq_kick_fd)627dff4426fSEugenio Pérez void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
628dff4426fSEugenio Pérez {
629dff4426fSEugenio Pérez EventNotifier *svq_kick = &svq->svq_kick;
630dff4426fSEugenio Pérez bool poll_stop = VHOST_FILE_UNBIND != event_notifier_get_fd(svq_kick);
631dff4426fSEugenio Pérez bool poll_start = svq_kick_fd != VHOST_FILE_UNBIND;
632dff4426fSEugenio Pérez
633dff4426fSEugenio Pérez if (poll_stop) {
634dff4426fSEugenio Pérez event_notifier_set_handler(svq_kick, NULL);
635dff4426fSEugenio Pérez }
636dff4426fSEugenio Pérez
6378b64e486SEugenio Pérez event_notifier_init_fd(svq_kick, svq_kick_fd);
638dff4426fSEugenio Pérez /*
639dff4426fSEugenio Pérez * event_notifier_set_handler already checks for guest's notifications if
640dff4426fSEugenio Pérez * they arrive at the new file descriptor in the switch, so there is no
641dff4426fSEugenio Pérez * need to explicitly check for them.
642dff4426fSEugenio Pérez */
643dff4426fSEugenio Pérez if (poll_start) {
644dff4426fSEugenio Pérez event_notifier_set(svq_kick);
645100890f7SEugenio Pérez event_notifier_set_handler(svq_kick, vhost_handle_guest_kick_notifier);
646100890f7SEugenio Pérez }
647100890f7SEugenio Pérez }
648100890f7SEugenio Pérez
649100890f7SEugenio Pérez /**
650100890f7SEugenio Pérez * Start the shadow virtqueue operation.
651100890f7SEugenio Pérez *
652100890f7SEugenio Pérez * @svq: Shadow Virtqueue
653100890f7SEugenio Pérez * @vdev: VirtIO device
654100890f7SEugenio Pérez * @vq: Virtqueue to shadow
6555fde952bSEugenio Pérez * @iova_tree: Tree to perform descriptors translations
656100890f7SEugenio Pérez */
vhost_svq_start(VhostShadowVirtqueue * svq,VirtIODevice * vdev,VirtQueue * vq,VhostIOVATree * iova_tree)657100890f7SEugenio Pérez void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
6585fde952bSEugenio Pérez VirtQueue *vq, VhostIOVATree *iova_tree)
659100890f7SEugenio Pérez {
660babf8b87SEugenio Pérez size_t desc_size;
661100890f7SEugenio Pérez
66220e7412bSEugenio Pérez event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call);
663100890f7SEugenio Pérez svq->next_guest_avail_elem = NULL;
664100890f7SEugenio Pérez svq->shadow_avail_idx = 0;
665100890f7SEugenio Pérez svq->shadow_used_idx = 0;
666100890f7SEugenio Pérez svq->last_used_idx = 0;
667100890f7SEugenio Pérez svq->vdev = vdev;
668100890f7SEugenio Pérez svq->vq = vq;
6695fde952bSEugenio Pérez svq->iova_tree = iova_tree;
670100890f7SEugenio Pérez
671100890f7SEugenio Pérez svq->vring.num = virtio_queue_get_num(vdev, virtio_get_queue_index(vq));
6725d410557SHawkins Jiawei svq->num_free = svq->vring.num;
673babf8b87SEugenio Pérez svq->vring.desc = mmap(NULL, vhost_svq_driver_area_size(svq),
674babf8b87SEugenio Pérez PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
675babf8b87SEugenio Pérez -1, 0);
676100890f7SEugenio Pérez desc_size = sizeof(vring_desc_t) * svq->vring.num;
677100890f7SEugenio Pérez svq->vring.avail = (void *)((char *)svq->vring.desc + desc_size);
678babf8b87SEugenio Pérez svq->vring.used = mmap(NULL, vhost_svq_device_area_size(svq),
679babf8b87SEugenio Pérez PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
680babf8b87SEugenio Pérez -1, 0);
6819e87868fSEugenio Pérez svq->desc_state = g_new0(SVQDescState, svq->vring.num);
682495fe3a7SEugenio Pérez svq->desc_next = g_new0(uint16_t, svq->vring.num);
683100890f7SEugenio Pérez for (unsigned i = 0; i < svq->vring.num - 1; i++) {
684495fe3a7SEugenio Pérez svq->desc_next[i] = cpu_to_le16(i + 1);
685dff4426fSEugenio Pérez }
686dff4426fSEugenio Pérez }
687dff4426fSEugenio Pérez
688dff4426fSEugenio Pérez /**
689dff4426fSEugenio Pérez * Stop the shadow virtqueue operation.
690dff4426fSEugenio Pérez * @svq: Shadow Virtqueue
691dff4426fSEugenio Pérez */
vhost_svq_stop(VhostShadowVirtqueue * svq)692dff4426fSEugenio Pérez void vhost_svq_stop(VhostShadowVirtqueue *svq)
693dff4426fSEugenio Pérez {
6948b64e486SEugenio Pérez vhost_svq_set_svq_kick_fd(svq, VHOST_FILE_UNBIND);
695100890f7SEugenio Pérez g_autofree VirtQueueElement *next_avail_elem = NULL;
696100890f7SEugenio Pérez
697100890f7SEugenio Pérez if (!svq->vq) {
698100890f7SEugenio Pérez return;
699100890f7SEugenio Pérez }
700100890f7SEugenio Pérez
701100890f7SEugenio Pérez /* Send all pending used descriptors to guest */
702100890f7SEugenio Pérez vhost_svq_flush(svq, false);
703100890f7SEugenio Pérez
704100890f7SEugenio Pérez for (unsigned i = 0; i < svq->vring.num; ++i) {
705100890f7SEugenio Pérez g_autofree VirtQueueElement *elem = NULL;
7069e87868fSEugenio Pérez elem = g_steal_pointer(&svq->desc_state[i].elem);
707100890f7SEugenio Pérez if (elem) {
7084241e8bdSEugenio Pérez /*
7094241e8bdSEugenio Pérez * TODO: This is ok for networking, but other kinds of devices
7104241e8bdSEugenio Pérez * might have problems with just unpop these.
7114241e8bdSEugenio Pérez */
7124241e8bdSEugenio Pérez virtqueue_unpop(svq->vq, elem, 0);
713100890f7SEugenio Pérez }
714100890f7SEugenio Pérez }
715100890f7SEugenio Pérez
716100890f7SEugenio Pérez next_avail_elem = g_steal_pointer(&svq->next_guest_avail_elem);
717100890f7SEugenio Pérez if (next_avail_elem) {
7184241e8bdSEugenio Pérez virtqueue_unpop(svq->vq, next_avail_elem, 0);
719100890f7SEugenio Pérez }
720100890f7SEugenio Pérez svq->vq = NULL;
721495fe3a7SEugenio Pérez g_free(svq->desc_next);
7229e87868fSEugenio Pérez g_free(svq->desc_state);
723babf8b87SEugenio Pérez munmap(svq->vring.desc, vhost_svq_driver_area_size(svq));
724babf8b87SEugenio Pérez munmap(svq->vring.used, vhost_svq_device_area_size(svq));
72520e7412bSEugenio Pérez event_notifier_set_handler(&svq->hdev_call, NULL);
726dff4426fSEugenio Pérez }
72710857ec0SEugenio Pérez
72810857ec0SEugenio Pérez /**
72910857ec0SEugenio Pérez * Creates vhost shadow virtqueue, and instructs the vhost device to use the
73010857ec0SEugenio Pérez * shadow methods and file descriptors.
73110857ec0SEugenio Pérez *
732e966c0b7SEugenio Pérez * @ops: SVQ owner callbacks
733e966c0b7SEugenio Pérez * @ops_opaque: ops opaque pointer
73410857ec0SEugenio Pérez */
vhost_svq_new(const VhostShadowVirtqueueOps * ops,void * ops_opaque)7355fde952bSEugenio Pérez VhostShadowVirtqueue *vhost_svq_new(const VhostShadowVirtqueueOps *ops,
736e966c0b7SEugenio Pérez void *ops_opaque)
73710857ec0SEugenio Pérez {
7383cfb4d06SEugenio Pérez VhostShadowVirtqueue *svq = g_new0(VhostShadowVirtqueue, 1);
73910857ec0SEugenio Pérez
740dff4426fSEugenio Pérez event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND);
741e966c0b7SEugenio Pérez svq->ops = ops;
742e966c0b7SEugenio Pérez svq->ops_opaque = ops_opaque;
7433cfb4d06SEugenio Pérez return svq;
74410857ec0SEugenio Pérez }
74510857ec0SEugenio Pérez
74610857ec0SEugenio Pérez /**
74710857ec0SEugenio Pérez * Free the resources of the shadow virtqueue.
74810857ec0SEugenio Pérez *
74910857ec0SEugenio Pérez * @pvq: gpointer to SVQ so it can be used by autofree functions.
75010857ec0SEugenio Pérez */
vhost_svq_free(gpointer pvq)75110857ec0SEugenio Pérez void vhost_svq_free(gpointer pvq)
75210857ec0SEugenio Pérez {
75310857ec0SEugenio Pérez VhostShadowVirtqueue *vq = pvq;
754dff4426fSEugenio Pérez vhost_svq_stop(vq);
75510857ec0SEugenio Pérez g_free(vq);
75610857ec0SEugenio Pérez }
757