110857ec0SEugenio Pérez /* 210857ec0SEugenio Pérez * vhost shadow virtqueue 310857ec0SEugenio Pérez * 410857ec0SEugenio Pérez * SPDX-FileCopyrightText: Red Hat, Inc. 2021 510857ec0SEugenio Pérez * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com> 610857ec0SEugenio Pérez * 710857ec0SEugenio Pérez * SPDX-License-Identifier: GPL-2.0-or-later 810857ec0SEugenio Pérez */ 910857ec0SEugenio Pérez 1010857ec0SEugenio Pérez #include "qemu/osdep.h" 1110857ec0SEugenio Pérez #include "hw/virtio/vhost-shadow-virtqueue.h" 1210857ec0SEugenio Pérez 1310857ec0SEugenio Pérez #include "qemu/error-report.h" 144725a418SEugenio Pérez #include "qapi/error.h" 15dff4426fSEugenio Pérez #include "qemu/main-loop.h" 16100890f7SEugenio Pérez #include "qemu/log.h" 17100890f7SEugenio Pérez #include "qemu/memalign.h" 18dff4426fSEugenio Pérez #include "linux-headers/linux/vhost.h" 19dff4426fSEugenio Pérez 20dff4426fSEugenio Pérez /** 214725a418SEugenio Pérez * Validate the transport device features that both guests can use with the SVQ 224725a418SEugenio Pérez * and SVQs can use with the device. 234725a418SEugenio Pérez * 244725a418SEugenio Pérez * @dev_features: The features 254725a418SEugenio Pérez * @errp: Error pointer 264725a418SEugenio Pérez */ 274725a418SEugenio Pérez bool vhost_svq_valid_features(uint64_t features, Error **errp) 284725a418SEugenio Pérez { 294725a418SEugenio Pérez bool ok = true; 304725a418SEugenio Pérez uint64_t svq_features = features; 314725a418SEugenio Pérez 324725a418SEugenio Pérez for (uint64_t b = VIRTIO_TRANSPORT_F_START; b <= VIRTIO_TRANSPORT_F_END; 334725a418SEugenio Pérez ++b) { 344725a418SEugenio Pérez switch (b) { 354725a418SEugenio Pérez case VIRTIO_F_ANY_LAYOUT: 36396d5126SEugenio Pérez case VIRTIO_RING_F_EVENT_IDX: 374725a418SEugenio Pérez continue; 384725a418SEugenio Pérez 394725a418SEugenio Pérez case VIRTIO_F_ACCESS_PLATFORM: 404725a418SEugenio Pérez /* SVQ trust in the host's IOMMU to translate addresses */ 414725a418SEugenio Pérez case VIRTIO_F_VERSION_1: 424725a418SEugenio Pérez /* SVQ trust that the guest vring is little endian */ 434725a418SEugenio Pérez if (!(svq_features & BIT_ULL(b))) { 444725a418SEugenio Pérez svq_features |= BIT_ULL(b); 454725a418SEugenio Pérez ok = false; 464725a418SEugenio Pérez } 474725a418SEugenio Pérez continue; 484725a418SEugenio Pérez 494725a418SEugenio Pérez default: 504725a418SEugenio Pérez if (svq_features & BIT_ULL(b)) { 514725a418SEugenio Pérez svq_features &= ~BIT_ULL(b); 524725a418SEugenio Pérez ok = false; 534725a418SEugenio Pérez } 544725a418SEugenio Pérez } 554725a418SEugenio Pérez } 564725a418SEugenio Pérez 574725a418SEugenio Pérez if (!ok) { 584725a418SEugenio Pérez error_setg(errp, "SVQ Invalid device feature flags, offer: 0x%"PRIx64 594725a418SEugenio Pérez ", ok: 0x%"PRIx64, features, svq_features); 604725a418SEugenio Pérez } 614725a418SEugenio Pérez return ok; 624725a418SEugenio Pérez } 634725a418SEugenio Pérez 644725a418SEugenio Pérez /** 65100890f7SEugenio Pérez * Number of descriptors that the SVQ can make available from the guest. 66dff4426fSEugenio Pérez * 67100890f7SEugenio Pérez * @svq: The svq 68dff4426fSEugenio Pérez */ 69100890f7SEugenio Pérez static uint16_t vhost_svq_available_slots(const VhostShadowVirtqueue *svq) 70dff4426fSEugenio Pérez { 715d410557SHawkins Jiawei return svq->num_free; 72100890f7SEugenio Pérez } 73100890f7SEugenio Pérez 7434e3c94eSEugenio Pérez /** 7534e3c94eSEugenio Pérez * Translate addresses between the qemu's virtual address and the SVQ IOVA 7634e3c94eSEugenio Pérez * 7734e3c94eSEugenio Pérez * @svq: Shadow VirtQueue 7834e3c94eSEugenio Pérez * @vaddr: Translated IOVA addresses 7934e3c94eSEugenio Pérez * @iovec: Source qemu's VA addresses 8034e3c94eSEugenio Pérez * @num: Length of iovec and minimum length of vaddr 8134e3c94eSEugenio Pérez */ 8234e3c94eSEugenio Pérez static bool vhost_svq_translate_addr(const VhostShadowVirtqueue *svq, 8334e3c94eSEugenio Pérez hwaddr *addrs, const struct iovec *iovec, 8434e3c94eSEugenio Pérez size_t num) 8534e3c94eSEugenio Pérez { 8634e3c94eSEugenio Pérez if (num == 0) { 8734e3c94eSEugenio Pérez return true; 8834e3c94eSEugenio Pérez } 8934e3c94eSEugenio Pérez 9034e3c94eSEugenio Pérez for (size_t i = 0; i < num; ++i) { 9134e3c94eSEugenio Pérez DMAMap needle = { 9234e3c94eSEugenio Pérez .translated_addr = (hwaddr)(uintptr_t)iovec[i].iov_base, 9334e3c94eSEugenio Pérez .size = iovec[i].iov_len, 9434e3c94eSEugenio Pérez }; 9534e3c94eSEugenio Pérez Int128 needle_last, map_last; 9634e3c94eSEugenio Pérez size_t off; 9734e3c94eSEugenio Pérez 9834e3c94eSEugenio Pérez const DMAMap *map = vhost_iova_tree_find_iova(svq->iova_tree, &needle); 9934e3c94eSEugenio Pérez /* 10034e3c94eSEugenio Pérez * Map cannot be NULL since iova map contains all guest space and 10134e3c94eSEugenio Pérez * qemu already has a physical address mapped 10234e3c94eSEugenio Pérez */ 10334e3c94eSEugenio Pérez if (unlikely(!map)) { 10434e3c94eSEugenio Pérez qemu_log_mask(LOG_GUEST_ERROR, 10534e3c94eSEugenio Pérez "Invalid address 0x%"HWADDR_PRIx" given by guest", 10634e3c94eSEugenio Pérez needle.translated_addr); 10734e3c94eSEugenio Pérez return false; 10834e3c94eSEugenio Pérez } 10934e3c94eSEugenio Pérez 11034e3c94eSEugenio Pérez off = needle.translated_addr - map->translated_addr; 11134e3c94eSEugenio Pérez addrs[i] = map->iova + off; 11234e3c94eSEugenio Pérez 11334e3c94eSEugenio Pérez needle_last = int128_add(int128_make64(needle.translated_addr), 11434e3c94eSEugenio Pérez int128_make64(iovec[i].iov_len)); 11534e3c94eSEugenio Pérez map_last = int128_make64(map->translated_addr + map->size); 11634e3c94eSEugenio Pérez if (unlikely(int128_gt(needle_last, map_last))) { 11734e3c94eSEugenio Pérez qemu_log_mask(LOG_GUEST_ERROR, 11834e3c94eSEugenio Pérez "Guest buffer expands over iova range"); 11934e3c94eSEugenio Pérez return false; 12034e3c94eSEugenio Pérez } 12134e3c94eSEugenio Pérez } 12234e3c94eSEugenio Pérez 12334e3c94eSEugenio Pérez return true; 12434e3c94eSEugenio Pérez } 12534e3c94eSEugenio Pérez 126009c2549SEugenio Pérez /** 127009c2549SEugenio Pérez * Write descriptors to SVQ vring 128009c2549SEugenio Pérez * 129009c2549SEugenio Pérez * @svq: The shadow virtqueue 130009c2549SEugenio Pérez * @sg: Cache for hwaddr 131009c2549SEugenio Pérez * @iovec: The iovec from the guest 132009c2549SEugenio Pérez * @num: iovec length 133009c2549SEugenio Pérez * @more_descs: True if more descriptors come in the chain 134009c2549SEugenio Pérez * @write: True if they are writeable descriptors 135009c2549SEugenio Pérez * 136009c2549SEugenio Pérez * Return true if success, false otherwise and print error. 137009c2549SEugenio Pérez */ 138009c2549SEugenio Pérez static bool vhost_svq_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg, 139100890f7SEugenio Pérez const struct iovec *iovec, size_t num, 140100890f7SEugenio Pérez bool more_descs, bool write) 141100890f7SEugenio Pérez { 142100890f7SEugenio Pérez uint16_t i = svq->free_head, last = svq->free_head; 143100890f7SEugenio Pérez unsigned n; 144100890f7SEugenio Pérez uint16_t flags = write ? cpu_to_le16(VRING_DESC_F_WRITE) : 0; 145100890f7SEugenio Pérez vring_desc_t *descs = svq->vring.desc; 146009c2549SEugenio Pérez bool ok; 147100890f7SEugenio Pérez 148100890f7SEugenio Pérez if (num == 0) { 149009c2549SEugenio Pérez return true; 150009c2549SEugenio Pérez } 151009c2549SEugenio Pérez 152009c2549SEugenio Pérez ok = vhost_svq_translate_addr(svq, sg, iovec, num); 153009c2549SEugenio Pérez if (unlikely(!ok)) { 154009c2549SEugenio Pérez return false; 155100890f7SEugenio Pérez } 156100890f7SEugenio Pérez 157100890f7SEugenio Pérez for (n = 0; n < num; n++) { 158100890f7SEugenio Pérez if (more_descs || (n + 1 < num)) { 159100890f7SEugenio Pérez descs[i].flags = flags | cpu_to_le16(VRING_DESC_F_NEXT); 160495fe3a7SEugenio Pérez descs[i].next = cpu_to_le16(svq->desc_next[i]); 161100890f7SEugenio Pérez } else { 162100890f7SEugenio Pérez descs[i].flags = flags; 163100890f7SEugenio Pérez } 16434e3c94eSEugenio Pérez descs[i].addr = cpu_to_le64(sg[n]); 165100890f7SEugenio Pérez descs[i].len = cpu_to_le32(iovec[n].iov_len); 166100890f7SEugenio Pérez 167100890f7SEugenio Pérez last = i; 168495fe3a7SEugenio Pérez i = cpu_to_le16(svq->desc_next[i]); 169100890f7SEugenio Pérez } 170100890f7SEugenio Pérez 171495fe3a7SEugenio Pérez svq->free_head = le16_to_cpu(svq->desc_next[last]); 172009c2549SEugenio Pérez return true; 173100890f7SEugenio Pérez } 174100890f7SEugenio Pérez 175100890f7SEugenio Pérez static bool vhost_svq_add_split(VhostShadowVirtqueue *svq, 1761f46ae65SEugenio Pérez const struct iovec *out_sg, size_t out_num, 1771f46ae65SEugenio Pérez const struct iovec *in_sg, size_t in_num, 1781f46ae65SEugenio Pérez unsigned *head) 179100890f7SEugenio Pérez { 180100890f7SEugenio Pérez unsigned avail_idx; 181100890f7SEugenio Pérez vring_avail_t *avail = svq->vring.avail; 18234e3c94eSEugenio Pérez bool ok; 1831f46ae65SEugenio Pérez g_autofree hwaddr *sgs = g_new(hwaddr, MAX(out_num, in_num)); 184100890f7SEugenio Pérez 185100890f7SEugenio Pérez *head = svq->free_head; 186100890f7SEugenio Pérez 187100890f7SEugenio Pérez /* We need some descriptors here */ 1881f46ae65SEugenio Pérez if (unlikely(!out_num && !in_num)) { 189100890f7SEugenio Pérez qemu_log_mask(LOG_GUEST_ERROR, 190100890f7SEugenio Pérez "Guest provided element with no descriptors"); 191100890f7SEugenio Pérez return false; 192100890f7SEugenio Pérez } 193100890f7SEugenio Pérez 1941f46ae65SEugenio Pérez ok = vhost_svq_vring_write_descs(svq, sgs, out_sg, out_num, in_num > 0, 1951f46ae65SEugenio Pérez false); 19634e3c94eSEugenio Pérez if (unlikely(!ok)) { 19734e3c94eSEugenio Pérez return false; 19834e3c94eSEugenio Pérez } 19934e3c94eSEugenio Pérez 2001f46ae65SEugenio Pérez ok = vhost_svq_vring_write_descs(svq, sgs, in_sg, in_num, false, true); 201009c2549SEugenio Pérez if (unlikely(!ok)) { 202009c2549SEugenio Pérez return false; 203009c2549SEugenio Pérez } 204100890f7SEugenio Pérez 205100890f7SEugenio Pérez /* 206100890f7SEugenio Pérez * Put the entry in the available array (but don't update avail->idx until 207100890f7SEugenio Pérez * they do sync). 208100890f7SEugenio Pérez */ 209100890f7SEugenio Pérez avail_idx = svq->shadow_avail_idx & (svq->vring.num - 1); 210100890f7SEugenio Pérez avail->ring[avail_idx] = cpu_to_le16(*head); 211100890f7SEugenio Pérez svq->shadow_avail_idx++; 212100890f7SEugenio Pérez 213100890f7SEugenio Pérez /* Update the avail index after write the descriptor */ 214100890f7SEugenio Pérez smp_wmb(); 215100890f7SEugenio Pérez avail->idx = cpu_to_le16(svq->shadow_avail_idx); 216100890f7SEugenio Pérez 217100890f7SEugenio Pérez return true; 218100890f7SEugenio Pérez } 219100890f7SEugenio Pérez 220d93a2405SEugenio Pérez static void vhost_svq_kick(VhostShadowVirtqueue *svq) 221d93a2405SEugenio Pérez { 22222a6840fSEugenio Pérez bool needs_kick; 22322a6840fSEugenio Pérez 224d93a2405SEugenio Pérez /* 225d93a2405SEugenio Pérez * We need to expose the available array entries before checking the used 226d93a2405SEugenio Pérez * flags 227d93a2405SEugenio Pérez */ 228d93a2405SEugenio Pérez smp_mb(); 22922a6840fSEugenio Pérez 23022a6840fSEugenio Pérez if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 23122a6840fSEugenio Pérez uint16_t avail_event = *(uint16_t *)(&svq->vring.used->ring[svq->vring.num]); 23222a6840fSEugenio Pérez needs_kick = vring_need_event(avail_event, svq->shadow_avail_idx, svq->shadow_avail_idx - 1); 23322a6840fSEugenio Pérez } else { 23422a6840fSEugenio Pérez needs_kick = !(svq->vring.used->flags & VRING_USED_F_NO_NOTIFY); 23522a6840fSEugenio Pérez } 23622a6840fSEugenio Pérez 23722a6840fSEugenio Pérez if (!needs_kick) { 238d93a2405SEugenio Pérez return; 239d93a2405SEugenio Pérez } 240d93a2405SEugenio Pérez 241d93a2405SEugenio Pérez event_notifier_set(&svq->hdev_kick); 242d93a2405SEugenio Pérez } 243d93a2405SEugenio Pérez 2445181db13SEugenio Pérez /** 2455181db13SEugenio Pérez * Add an element to a SVQ. 2465181db13SEugenio Pérez * 247f20b70ebSEugenio Pérez * Return -EINVAL if element is invalid, -ENOSPC if dev queue is full 2485181db13SEugenio Pérez */ 249d0291f3fSEugenio Pérez int vhost_svq_add(VhostShadowVirtqueue *svq, const struct iovec *out_sg, 250d0291f3fSEugenio Pérez size_t out_num, const struct iovec *in_sg, size_t in_num, 251d0291f3fSEugenio Pérez VirtQueueElement *elem) 252100890f7SEugenio Pérez { 253100890f7SEugenio Pérez unsigned qemu_head; 2541f46ae65SEugenio Pérez unsigned ndescs = in_num + out_num; 255f20b70ebSEugenio Pérez bool ok; 256f20b70ebSEugenio Pérez 257f20b70ebSEugenio Pérez if (unlikely(ndescs > vhost_svq_available_slots(svq))) { 258f20b70ebSEugenio Pérez return -ENOSPC; 259f20b70ebSEugenio Pérez } 260f20b70ebSEugenio Pérez 2611f46ae65SEugenio Pérez ok = vhost_svq_add_split(svq, out_sg, out_num, in_sg, in_num, &qemu_head); 262100890f7SEugenio Pérez if (unlikely(!ok)) { 263f20b70ebSEugenio Pérez return -EINVAL; 264100890f7SEugenio Pérez } 265100890f7SEugenio Pérez 2665d410557SHawkins Jiawei svq->num_free -= ndescs; 2679e87868fSEugenio Pérez svq->desc_state[qemu_head].elem = elem; 268ac4cfdc6SEugenio Pérez svq->desc_state[qemu_head].ndescs = ndescs; 26998b5adefSEugenio Pérez vhost_svq_kick(svq); 270f20b70ebSEugenio Pérez return 0; 271100890f7SEugenio Pérez } 272100890f7SEugenio Pérez 2731f46ae65SEugenio Pérez /* Convenience wrapper to add a guest's element to SVQ */ 2741f46ae65SEugenio Pérez static int vhost_svq_add_element(VhostShadowVirtqueue *svq, 2751f46ae65SEugenio Pérez VirtQueueElement *elem) 2761f46ae65SEugenio Pérez { 2771f46ae65SEugenio Pérez return vhost_svq_add(svq, elem->out_sg, elem->out_num, elem->in_sg, 2781f46ae65SEugenio Pérez elem->in_num, elem); 2791f46ae65SEugenio Pérez } 2801f46ae65SEugenio Pérez 281dff4426fSEugenio Pérez /** 282100890f7SEugenio Pérez * Forward available buffers. 283100890f7SEugenio Pérez * 284100890f7SEugenio Pérez * @svq: Shadow VirtQueue 285100890f7SEugenio Pérez * 286100890f7SEugenio Pérez * Note that this function does not guarantee that all guest's available 287100890f7SEugenio Pérez * buffers are available to the device in SVQ avail ring. The guest may have 288100890f7SEugenio Pérez * exposed a GPA / GIOVA contiguous buffer, but it may not be contiguous in 289100890f7SEugenio Pérez * qemu vaddr. 290100890f7SEugenio Pérez * 291100890f7SEugenio Pérez * If that happens, guest's kick notifications will be disabled until the 292100890f7SEugenio Pérez * device uses some buffers. 293100890f7SEugenio Pérez */ 294100890f7SEugenio Pérez static void vhost_handle_guest_kick(VhostShadowVirtqueue *svq) 295100890f7SEugenio Pérez { 296100890f7SEugenio Pérez /* Clear event notifier */ 297100890f7SEugenio Pérez event_notifier_test_and_clear(&svq->svq_kick); 298100890f7SEugenio Pérez 299100890f7SEugenio Pérez /* Forward to the device as many available buffers as possible */ 300100890f7SEugenio Pérez do { 301100890f7SEugenio Pérez virtio_queue_set_notification(svq->vq, false); 302100890f7SEugenio Pérez 303100890f7SEugenio Pérez while (true) { 304ad9f958dSBernhard Beschow g_autofree VirtQueueElement *elem = NULL; 305f20b70ebSEugenio Pérez int r; 306100890f7SEugenio Pérez 307100890f7SEugenio Pérez if (svq->next_guest_avail_elem) { 308100890f7SEugenio Pérez elem = g_steal_pointer(&svq->next_guest_avail_elem); 309100890f7SEugenio Pérez } else { 310100890f7SEugenio Pérez elem = virtqueue_pop(svq->vq, sizeof(*elem)); 311100890f7SEugenio Pérez } 312100890f7SEugenio Pérez 313100890f7SEugenio Pérez if (!elem) { 314100890f7SEugenio Pérez break; 315100890f7SEugenio Pérez } 316100890f7SEugenio Pérez 317e966c0b7SEugenio Pérez if (svq->ops) { 318e966c0b7SEugenio Pérez r = svq->ops->avail_handler(svq, elem, svq->ops_opaque); 319e966c0b7SEugenio Pérez } else { 3201f46ae65SEugenio Pérez r = vhost_svq_add_element(svq, elem); 321e966c0b7SEugenio Pérez } 322f20b70ebSEugenio Pérez if (unlikely(r != 0)) { 323f20b70ebSEugenio Pérez if (r == -ENOSPC) { 324100890f7SEugenio Pérez /* 325f20b70ebSEugenio Pérez * This condition is possible since a contiguous buffer in 326f20b70ebSEugenio Pérez * GPA does not imply a contiguous buffer in qemu's VA 327f20b70ebSEugenio Pérez * scatter-gather segments. If that happens, the buffer 328f20b70ebSEugenio Pérez * exposed to the device needs to be a chain of descriptors 329f20b70ebSEugenio Pérez * at this moment. 330100890f7SEugenio Pérez * 331100890f7SEugenio Pérez * SVQ cannot hold more available buffers if we are here: 332f20b70ebSEugenio Pérez * queue the current guest descriptor and ignore kicks 333100890f7SEugenio Pérez * until some elements are used. 334100890f7SEugenio Pérez */ 3359c2ab2f1SEugenio Pérez svq->next_guest_avail_elem = g_steal_pointer(&elem); 336100890f7SEugenio Pérez } 337100890f7SEugenio Pérez 338f20b70ebSEugenio Pérez /* VQ is full or broken, just return and ignore kicks */ 339100890f7SEugenio Pérez return; 340100890f7SEugenio Pérez } 3419c2ab2f1SEugenio Pérez /* elem belongs to SVQ or external caller now */ 3429c2ab2f1SEugenio Pérez elem = NULL; 343100890f7SEugenio Pérez } 344100890f7SEugenio Pérez 345100890f7SEugenio Pérez virtio_queue_set_notification(svq->vq, true); 346100890f7SEugenio Pérez } while (!virtio_queue_empty(svq->vq)); 347100890f7SEugenio Pérez } 348100890f7SEugenio Pérez 349100890f7SEugenio Pérez /** 350100890f7SEugenio Pérez * Handle guest's kick. 351100890f7SEugenio Pérez * 352100890f7SEugenio Pérez * @n: guest kick event notifier, the one that guest set to notify svq. 353100890f7SEugenio Pérez */ 354100890f7SEugenio Pérez static void vhost_handle_guest_kick_notifier(EventNotifier *n) 355100890f7SEugenio Pérez { 356100890f7SEugenio Pérez VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, svq_kick); 357100890f7SEugenio Pérez event_notifier_test_and_clear(n); 358100890f7SEugenio Pérez vhost_handle_guest_kick(svq); 359100890f7SEugenio Pérez } 360100890f7SEugenio Pérez 361100890f7SEugenio Pérez static bool vhost_svq_more_used(VhostShadowVirtqueue *svq) 362100890f7SEugenio Pérez { 363c381abc3SEugenio Pérez uint16_t *used_idx = &svq->vring.used->idx; 364100890f7SEugenio Pérez if (svq->last_used_idx != svq->shadow_used_idx) { 365100890f7SEugenio Pérez return true; 366100890f7SEugenio Pérez } 367100890f7SEugenio Pérez 368c381abc3SEugenio Pérez svq->shadow_used_idx = cpu_to_le16(*(volatile uint16_t *)used_idx); 369100890f7SEugenio Pérez 370100890f7SEugenio Pérez return svq->last_used_idx != svq->shadow_used_idx; 371100890f7SEugenio Pérez } 372100890f7SEugenio Pérez 373100890f7SEugenio Pérez /** 374100890f7SEugenio Pérez * Enable vhost device calls after disable them. 375100890f7SEugenio Pérez * 376100890f7SEugenio Pérez * @svq: The svq 377100890f7SEugenio Pérez * 378100890f7SEugenio Pérez * It returns false if there are pending used buffers from the vhost device, 379100890f7SEugenio Pérez * avoiding the possible races between SVQ checking for more work and enabling 380100890f7SEugenio Pérez * callbacks. True if SVQ used vring has no more pending buffers. 381100890f7SEugenio Pérez */ 382100890f7SEugenio Pérez static bool vhost_svq_enable_notification(VhostShadowVirtqueue *svq) 383100890f7SEugenio Pérez { 38401f8beacSEugenio Pérez if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 38501f8beacSEugenio Pérez uint16_t *used_event = (uint16_t *)&svq->vring.avail->ring[svq->vring.num]; 38601f8beacSEugenio Pérez *used_event = svq->shadow_used_idx; 38701f8beacSEugenio Pérez } else { 388100890f7SEugenio Pérez svq->vring.avail->flags &= ~cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT); 38901f8beacSEugenio Pérez } 39001f8beacSEugenio Pérez 39101f8beacSEugenio Pérez /* Make sure the event is enabled before the read of used_idx */ 392100890f7SEugenio Pérez smp_mb(); 393100890f7SEugenio Pérez return !vhost_svq_more_used(svq); 394100890f7SEugenio Pérez } 395100890f7SEugenio Pérez 396100890f7SEugenio Pérez static void vhost_svq_disable_notification(VhostShadowVirtqueue *svq) 397100890f7SEugenio Pérez { 39801f8beacSEugenio Pérez /* 39901f8beacSEugenio Pérez * No need to disable notification in the event idx case, since used event 40001f8beacSEugenio Pérez * index is already an index too far away. 40101f8beacSEugenio Pérez */ 40201f8beacSEugenio Pérez if (!virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) { 403100890f7SEugenio Pérez svq->vring.avail->flags |= cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT); 404100890f7SEugenio Pérez } 40501f8beacSEugenio Pérez } 406100890f7SEugenio Pérez 40781abfa57SEugenio Pérez static uint16_t vhost_svq_last_desc_of_chain(const VhostShadowVirtqueue *svq, 40881abfa57SEugenio Pérez uint16_t num, uint16_t i) 40981abfa57SEugenio Pérez { 41081abfa57SEugenio Pérez for (uint16_t j = 0; j < (num - 1); ++j) { 41181abfa57SEugenio Pérez i = le16_to_cpu(svq->desc_next[i]); 41281abfa57SEugenio Pérez } 41381abfa57SEugenio Pérez 41481abfa57SEugenio Pérez return i; 41581abfa57SEugenio Pérez } 41681abfa57SEugenio Pérez 417100890f7SEugenio Pérez static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq, 418100890f7SEugenio Pérez uint32_t *len) 419100890f7SEugenio Pérez { 420100890f7SEugenio Pérez const vring_used_t *used = svq->vring.used; 421100890f7SEugenio Pérez vring_used_elem_t used_elem; 42281abfa57SEugenio Pérez uint16_t last_used, last_used_chain, num; 423100890f7SEugenio Pérez 424100890f7SEugenio Pérez if (!vhost_svq_more_used(svq)) { 425100890f7SEugenio Pérez return NULL; 426100890f7SEugenio Pérez } 427100890f7SEugenio Pérez 428100890f7SEugenio Pérez /* Only get used array entries after they have been exposed by dev */ 429100890f7SEugenio Pérez smp_rmb(); 430100890f7SEugenio Pérez last_used = svq->last_used_idx & (svq->vring.num - 1); 431100890f7SEugenio Pérez used_elem.id = le32_to_cpu(used->ring[last_used].id); 432100890f7SEugenio Pérez used_elem.len = le32_to_cpu(used->ring[last_used].len); 433100890f7SEugenio Pérez 434100890f7SEugenio Pérez svq->last_used_idx++; 435100890f7SEugenio Pérez if (unlikely(used_elem.id >= svq->vring.num)) { 436100890f7SEugenio Pérez qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used", 437100890f7SEugenio Pérez svq->vdev->name, used_elem.id); 438100890f7SEugenio Pérez return NULL; 439100890f7SEugenio Pérez } 440100890f7SEugenio Pérez 44186f5f254SEugenio Pérez if (unlikely(!svq->desc_state[used_elem.id].ndescs)) { 442100890f7SEugenio Pérez qemu_log_mask(LOG_GUEST_ERROR, 443100890f7SEugenio Pérez "Device %s says index %u is used, but it was not available", 444100890f7SEugenio Pérez svq->vdev->name, used_elem.id); 445100890f7SEugenio Pérez return NULL; 446100890f7SEugenio Pérez } 447100890f7SEugenio Pérez 448ac4cfdc6SEugenio Pérez num = svq->desc_state[used_elem.id].ndescs; 44986f5f254SEugenio Pérez svq->desc_state[used_elem.id].ndescs = 0; 45081abfa57SEugenio Pérez last_used_chain = vhost_svq_last_desc_of_chain(svq, num, used_elem.id); 45181abfa57SEugenio Pérez svq->desc_next[last_used_chain] = svq->free_head; 452100890f7SEugenio Pérez svq->free_head = used_elem.id; 4535d410557SHawkins Jiawei svq->num_free += num; 454100890f7SEugenio Pérez 455100890f7SEugenio Pérez *len = used_elem.len; 4569e87868fSEugenio Pérez return g_steal_pointer(&svq->desc_state[used_elem.id].elem); 457100890f7SEugenio Pérez } 458100890f7SEugenio Pérez 459432efd14SEugenio Pérez /** 460432efd14SEugenio Pérez * Push an element to SVQ, returning it to the guest. 461432efd14SEugenio Pérez */ 462432efd14SEugenio Pérez void vhost_svq_push_elem(VhostShadowVirtqueue *svq, 463432efd14SEugenio Pérez const VirtQueueElement *elem, uint32_t len) 464432efd14SEugenio Pérez { 465432efd14SEugenio Pérez virtqueue_push(svq->vq, elem, len); 466432efd14SEugenio Pérez if (svq->next_guest_avail_elem) { 467432efd14SEugenio Pérez /* 468432efd14SEugenio Pérez * Avail ring was full when vhost_svq_flush was called, so it's a 469432efd14SEugenio Pérez * good moment to make more descriptors available if possible. 470432efd14SEugenio Pérez */ 471432efd14SEugenio Pérez vhost_handle_guest_kick(svq); 472432efd14SEugenio Pérez } 473432efd14SEugenio Pérez } 474432efd14SEugenio Pérez 475100890f7SEugenio Pérez static void vhost_svq_flush(VhostShadowVirtqueue *svq, 476100890f7SEugenio Pérez bool check_for_avail_queue) 477100890f7SEugenio Pérez { 478100890f7SEugenio Pérez VirtQueue *vq = svq->vq; 479100890f7SEugenio Pérez 480100890f7SEugenio Pérez /* Forward as many used buffers as possible. */ 481100890f7SEugenio Pérez do { 482100890f7SEugenio Pérez unsigned i = 0; 483100890f7SEugenio Pérez 484100890f7SEugenio Pérez vhost_svq_disable_notification(svq); 485100890f7SEugenio Pérez while (true) { 486100890f7SEugenio Pérez uint32_t len; 487100890f7SEugenio Pérez g_autofree VirtQueueElement *elem = vhost_svq_get_buf(svq, &len); 488100890f7SEugenio Pérez if (!elem) { 489100890f7SEugenio Pérez break; 490100890f7SEugenio Pérez } 491100890f7SEugenio Pérez 492100890f7SEugenio Pérez if (unlikely(i >= svq->vring.num)) { 493100890f7SEugenio Pérez qemu_log_mask(LOG_GUEST_ERROR, 494100890f7SEugenio Pérez "More than %u used buffers obtained in a %u size SVQ", 495100890f7SEugenio Pérez i, svq->vring.num); 496100890f7SEugenio Pérez virtqueue_fill(vq, elem, len, i); 497100890f7SEugenio Pérez virtqueue_flush(vq, i); 498100890f7SEugenio Pérez return; 499100890f7SEugenio Pérez } 500100890f7SEugenio Pérez virtqueue_fill(vq, elem, len, i++); 501100890f7SEugenio Pérez } 502100890f7SEugenio Pérez 503100890f7SEugenio Pérez virtqueue_flush(vq, i); 504100890f7SEugenio Pérez event_notifier_set(&svq->svq_call); 505100890f7SEugenio Pérez 506100890f7SEugenio Pérez if (check_for_avail_queue && svq->next_guest_avail_elem) { 507100890f7SEugenio Pérez /* 508100890f7SEugenio Pérez * Avail ring was full when vhost_svq_flush was called, so it's a 509100890f7SEugenio Pérez * good moment to make more descriptors available if possible. 510100890f7SEugenio Pérez */ 511100890f7SEugenio Pérez vhost_handle_guest_kick(svq); 512100890f7SEugenio Pérez } 513100890f7SEugenio Pérez } while (!vhost_svq_enable_notification(svq)); 514100890f7SEugenio Pérez } 515100890f7SEugenio Pérez 516100890f7SEugenio Pérez /** 5173f44d13dSEugenio Pérez * Poll the SVQ for one device used buffer. 5183f44d13dSEugenio Pérez * 5193f44d13dSEugenio Pérez * This function race with main event loop SVQ polling, so extra 5203f44d13dSEugenio Pérez * synchronization is needed. 5213f44d13dSEugenio Pérez * 5223f44d13dSEugenio Pérez * Return the length written by the device. 5233f44d13dSEugenio Pérez */ 5243f44d13dSEugenio Pérez size_t vhost_svq_poll(VhostShadowVirtqueue *svq) 5253f44d13dSEugenio Pérez { 5263f44d13dSEugenio Pérez int64_t start_us = g_get_monotonic_time(); 527e4dd39c6SCarlos López uint32_t len = 0; 528d368c0b0SEugenio Pérez 529d368c0b0SEugenio Pérez do { 530d368c0b0SEugenio Pérez if (vhost_svq_more_used(svq)) { 531d368c0b0SEugenio Pérez break; 5323f44d13dSEugenio Pérez } 5333f44d13dSEugenio Pérez 5343f44d13dSEugenio Pérez if (unlikely(g_get_monotonic_time() - start_us > 10e6)) { 5353f44d13dSEugenio Pérez return 0; 5363f44d13dSEugenio Pérez } 5373f44d13dSEugenio Pérez } while (true); 538d368c0b0SEugenio Pérez 539d368c0b0SEugenio Pérez vhost_svq_get_buf(svq, &len); 540d368c0b0SEugenio Pérez return len; 5413f44d13dSEugenio Pérez } 5423f44d13dSEugenio Pérez 5433f44d13dSEugenio Pérez /** 544100890f7SEugenio Pérez * Forward used buffers. 545a8ac8858SEugenio Pérez * 546a8ac8858SEugenio Pérez * @n: hdev call event notifier, the one that device set to notify svq. 547100890f7SEugenio Pérez * 548100890f7SEugenio Pérez * Note that we are not making any buffers available in the loop, there is no 549100890f7SEugenio Pérez * way that it runs more than virtqueue size times. 550a8ac8858SEugenio Pérez */ 551a8ac8858SEugenio Pérez static void vhost_svq_handle_call(EventNotifier *n) 552a8ac8858SEugenio Pérez { 553a8ac8858SEugenio Pérez VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, 554a8ac8858SEugenio Pérez hdev_call); 555a8ac8858SEugenio Pérez event_notifier_test_and_clear(n); 556100890f7SEugenio Pérez vhost_svq_flush(svq, true); 557a8ac8858SEugenio Pérez } 558a8ac8858SEugenio Pérez 559a8ac8858SEugenio Pérez /** 560a8ac8858SEugenio Pérez * Set the call notifier for the SVQ to call the guest 561a8ac8858SEugenio Pérez * 562a8ac8858SEugenio Pérez * @svq: Shadow virtqueue 563a8ac8858SEugenio Pérez * @call_fd: call notifier 564a8ac8858SEugenio Pérez * 565a8ac8858SEugenio Pérez * Called on BQL context. 566a8ac8858SEugenio Pérez */ 567a8ac8858SEugenio Pérez void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd) 568a8ac8858SEugenio Pérez { 569a8ac8858SEugenio Pérez if (call_fd == VHOST_FILE_UNBIND) { 570a8ac8858SEugenio Pérez /* 571a8ac8858SEugenio Pérez * Fail event_notifier_set if called handling device call. 572a8ac8858SEugenio Pérez * 573a8ac8858SEugenio Pérez * SVQ still needs device notifications, since it needs to keep 574a8ac8858SEugenio Pérez * forwarding used buffers even with the unbind. 575a8ac8858SEugenio Pérez */ 576a8ac8858SEugenio Pérez memset(&svq->svq_call, 0, sizeof(svq->svq_call)); 577a8ac8858SEugenio Pérez } else { 578a8ac8858SEugenio Pérez event_notifier_init_fd(&svq->svq_call, call_fd); 579a8ac8858SEugenio Pérez } 580a8ac8858SEugenio Pérez } 581a8ac8858SEugenio Pérez 582a8ac8858SEugenio Pérez /** 583dafb34c9SEugenio Pérez * Get the shadow vq vring address. 584dafb34c9SEugenio Pérez * @svq: Shadow virtqueue 585dafb34c9SEugenio Pérez * @addr: Destination to store address 586dafb34c9SEugenio Pérez */ 587dafb34c9SEugenio Pérez void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq, 588dafb34c9SEugenio Pérez struct vhost_vring_addr *addr) 589dafb34c9SEugenio Pérez { 59034e3c94eSEugenio Pérez addr->desc_user_addr = (uint64_t)(uintptr_t)svq->vring.desc; 59134e3c94eSEugenio Pérez addr->avail_user_addr = (uint64_t)(uintptr_t)svq->vring.avail; 59234e3c94eSEugenio Pérez addr->used_user_addr = (uint64_t)(uintptr_t)svq->vring.used; 593dafb34c9SEugenio Pérez } 594dafb34c9SEugenio Pérez 595dafb34c9SEugenio Pérez size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq) 596dafb34c9SEugenio Pérez { 597dafb34c9SEugenio Pérez size_t desc_size = sizeof(vring_desc_t) * svq->vring.num; 598f0c48e05SEugenio Pérez size_t avail_size = offsetof(vring_avail_t, ring[svq->vring.num]) + 599f0c48e05SEugenio Pérez sizeof(uint16_t); 600dafb34c9SEugenio Pérez 6018e3b0cbbSMarc-André Lureau return ROUND_UP(desc_size + avail_size, qemu_real_host_page_size()); 602dafb34c9SEugenio Pérez } 603dafb34c9SEugenio Pérez 604dafb34c9SEugenio Pérez size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq) 605dafb34c9SEugenio Pérez { 606f0c48e05SEugenio Pérez size_t used_size = offsetof(vring_used_t, ring[svq->vring.num]) + 607f0c48e05SEugenio Pérez sizeof(uint16_t); 6088e3b0cbbSMarc-André Lureau return ROUND_UP(used_size, qemu_real_host_page_size()); 609dafb34c9SEugenio Pérez } 610dafb34c9SEugenio Pérez 611dafb34c9SEugenio Pérez /** 612dff4426fSEugenio Pérez * Set a new file descriptor for the guest to kick the SVQ and notify for avail 613dff4426fSEugenio Pérez * 614dff4426fSEugenio Pérez * @svq: The svq 615dff4426fSEugenio Pérez * @svq_kick_fd: The svq kick fd 616dff4426fSEugenio Pérez * 617dff4426fSEugenio Pérez * Note that the SVQ will never close the old file descriptor. 618dff4426fSEugenio Pérez */ 619dff4426fSEugenio Pérez void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd) 620dff4426fSEugenio Pérez { 621dff4426fSEugenio Pérez EventNotifier *svq_kick = &svq->svq_kick; 622dff4426fSEugenio Pérez bool poll_stop = VHOST_FILE_UNBIND != event_notifier_get_fd(svq_kick); 623dff4426fSEugenio Pérez bool poll_start = svq_kick_fd != VHOST_FILE_UNBIND; 624dff4426fSEugenio Pérez 625dff4426fSEugenio Pérez if (poll_stop) { 626dff4426fSEugenio Pérez event_notifier_set_handler(svq_kick, NULL); 627dff4426fSEugenio Pérez } 628dff4426fSEugenio Pérez 6298b64e486SEugenio Pérez event_notifier_init_fd(svq_kick, svq_kick_fd); 630dff4426fSEugenio Pérez /* 631dff4426fSEugenio Pérez * event_notifier_set_handler already checks for guest's notifications if 632dff4426fSEugenio Pérez * they arrive at the new file descriptor in the switch, so there is no 633dff4426fSEugenio Pérez * need to explicitly check for them. 634dff4426fSEugenio Pérez */ 635dff4426fSEugenio Pérez if (poll_start) { 636dff4426fSEugenio Pérez event_notifier_set(svq_kick); 637100890f7SEugenio Pérez event_notifier_set_handler(svq_kick, vhost_handle_guest_kick_notifier); 638100890f7SEugenio Pérez } 639100890f7SEugenio Pérez } 640100890f7SEugenio Pérez 641100890f7SEugenio Pérez /** 642100890f7SEugenio Pérez * Start the shadow virtqueue operation. 643100890f7SEugenio Pérez * 644100890f7SEugenio Pérez * @svq: Shadow Virtqueue 645100890f7SEugenio Pérez * @vdev: VirtIO device 646100890f7SEugenio Pérez * @vq: Virtqueue to shadow 6475fde952bSEugenio Pérez * @iova_tree: Tree to perform descriptors translations 648100890f7SEugenio Pérez */ 649100890f7SEugenio Pérez void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev, 6505fde952bSEugenio Pérez VirtQueue *vq, VhostIOVATree *iova_tree) 651100890f7SEugenio Pérez { 652*babf8b87SEugenio Pérez size_t desc_size; 653100890f7SEugenio Pérez 65420e7412bSEugenio Pérez event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call); 655100890f7SEugenio Pérez svq->next_guest_avail_elem = NULL; 656100890f7SEugenio Pérez svq->shadow_avail_idx = 0; 657100890f7SEugenio Pérez svq->shadow_used_idx = 0; 658100890f7SEugenio Pérez svq->last_used_idx = 0; 659100890f7SEugenio Pérez svq->vdev = vdev; 660100890f7SEugenio Pérez svq->vq = vq; 6615fde952bSEugenio Pérez svq->iova_tree = iova_tree; 662100890f7SEugenio Pérez 663100890f7SEugenio Pérez svq->vring.num = virtio_queue_get_num(vdev, virtio_get_queue_index(vq)); 6645d410557SHawkins Jiawei svq->num_free = svq->vring.num; 665*babf8b87SEugenio Pérez svq->vring.desc = mmap(NULL, vhost_svq_driver_area_size(svq), 666*babf8b87SEugenio Pérez PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, 667*babf8b87SEugenio Pérez -1, 0); 668100890f7SEugenio Pérez desc_size = sizeof(vring_desc_t) * svq->vring.num; 669100890f7SEugenio Pérez svq->vring.avail = (void *)((char *)svq->vring.desc + desc_size); 670*babf8b87SEugenio Pérez svq->vring.used = mmap(NULL, vhost_svq_device_area_size(svq), 671*babf8b87SEugenio Pérez PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, 672*babf8b87SEugenio Pérez -1, 0); 6739e87868fSEugenio Pérez svq->desc_state = g_new0(SVQDescState, svq->vring.num); 674495fe3a7SEugenio Pérez svq->desc_next = g_new0(uint16_t, svq->vring.num); 675100890f7SEugenio Pérez for (unsigned i = 0; i < svq->vring.num - 1; i++) { 676495fe3a7SEugenio Pérez svq->desc_next[i] = cpu_to_le16(i + 1); 677dff4426fSEugenio Pérez } 678dff4426fSEugenio Pérez } 679dff4426fSEugenio Pérez 680dff4426fSEugenio Pérez /** 681dff4426fSEugenio Pérez * Stop the shadow virtqueue operation. 682dff4426fSEugenio Pérez * @svq: Shadow Virtqueue 683dff4426fSEugenio Pérez */ 684dff4426fSEugenio Pérez void vhost_svq_stop(VhostShadowVirtqueue *svq) 685dff4426fSEugenio Pérez { 6868b64e486SEugenio Pérez vhost_svq_set_svq_kick_fd(svq, VHOST_FILE_UNBIND); 687100890f7SEugenio Pérez g_autofree VirtQueueElement *next_avail_elem = NULL; 688100890f7SEugenio Pérez 689100890f7SEugenio Pérez if (!svq->vq) { 690100890f7SEugenio Pérez return; 691100890f7SEugenio Pérez } 692100890f7SEugenio Pérez 693100890f7SEugenio Pérez /* Send all pending used descriptors to guest */ 694100890f7SEugenio Pérez vhost_svq_flush(svq, false); 695100890f7SEugenio Pérez 696100890f7SEugenio Pérez for (unsigned i = 0; i < svq->vring.num; ++i) { 697100890f7SEugenio Pérez g_autofree VirtQueueElement *elem = NULL; 6989e87868fSEugenio Pérez elem = g_steal_pointer(&svq->desc_state[i].elem); 699100890f7SEugenio Pérez if (elem) { 7004241e8bdSEugenio Pérez /* 7014241e8bdSEugenio Pérez * TODO: This is ok for networking, but other kinds of devices 7024241e8bdSEugenio Pérez * might have problems with just unpop these. 7034241e8bdSEugenio Pérez */ 7044241e8bdSEugenio Pérez virtqueue_unpop(svq->vq, elem, 0); 705100890f7SEugenio Pérez } 706100890f7SEugenio Pérez } 707100890f7SEugenio Pérez 708100890f7SEugenio Pérez next_avail_elem = g_steal_pointer(&svq->next_guest_avail_elem); 709100890f7SEugenio Pérez if (next_avail_elem) { 7104241e8bdSEugenio Pérez virtqueue_unpop(svq->vq, next_avail_elem, 0); 711100890f7SEugenio Pérez } 712100890f7SEugenio Pérez svq->vq = NULL; 713495fe3a7SEugenio Pérez g_free(svq->desc_next); 7149e87868fSEugenio Pérez g_free(svq->desc_state); 715*babf8b87SEugenio Pérez munmap(svq->vring.desc, vhost_svq_driver_area_size(svq)); 716*babf8b87SEugenio Pérez munmap(svq->vring.used, vhost_svq_device_area_size(svq)); 71720e7412bSEugenio Pérez event_notifier_set_handler(&svq->hdev_call, NULL); 718dff4426fSEugenio Pérez } 71910857ec0SEugenio Pérez 72010857ec0SEugenio Pérez /** 72110857ec0SEugenio Pérez * Creates vhost shadow virtqueue, and instructs the vhost device to use the 72210857ec0SEugenio Pérez * shadow methods and file descriptors. 72310857ec0SEugenio Pérez * 724e966c0b7SEugenio Pérez * @ops: SVQ owner callbacks 725e966c0b7SEugenio Pérez * @ops_opaque: ops opaque pointer 72610857ec0SEugenio Pérez */ 7275fde952bSEugenio Pérez VhostShadowVirtqueue *vhost_svq_new(const VhostShadowVirtqueueOps *ops, 728e966c0b7SEugenio Pérez void *ops_opaque) 72910857ec0SEugenio Pérez { 7303cfb4d06SEugenio Pérez VhostShadowVirtqueue *svq = g_new0(VhostShadowVirtqueue, 1); 73110857ec0SEugenio Pérez 732dff4426fSEugenio Pérez event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND); 733e966c0b7SEugenio Pérez svq->ops = ops; 734e966c0b7SEugenio Pérez svq->ops_opaque = ops_opaque; 7353cfb4d06SEugenio Pérez return svq; 73610857ec0SEugenio Pérez } 73710857ec0SEugenio Pérez 73810857ec0SEugenio Pérez /** 73910857ec0SEugenio Pérez * Free the resources of the shadow virtqueue. 74010857ec0SEugenio Pérez * 74110857ec0SEugenio Pérez * @pvq: gpointer to SVQ so it can be used by autofree functions. 74210857ec0SEugenio Pérez */ 74310857ec0SEugenio Pérez void vhost_svq_free(gpointer pvq) 74410857ec0SEugenio Pérez { 74510857ec0SEugenio Pérez VhostShadowVirtqueue *vq = pvq; 746dff4426fSEugenio Pérez vhost_svq_stop(vq); 74710857ec0SEugenio Pérez g_free(vq); 74810857ec0SEugenio Pérez } 749