xref: /qemu/net/vhost-vdpa.c (revision c23a9563) 
1 /*
2  * vhost-vdpa.c
3  *
4  * Copyright(c) 2017-2018 Intel Corporation.
5  * Copyright(c) 2020 Red Hat, Inc.
6  *
7  * This work is licensed under the terms of the GNU GPL, version 2 or later.
8  * See the COPYING file in the top-level directory.
9  *
10  */
11 
12 #include "qemu/osdep.h"
13 #include "clients.h"
14 #include "hw/virtio/virtio-net.h"
15 #include "net/vhost_net.h"
16 #include "net/vhost-vdpa.h"
17 #include "hw/virtio/vhost-vdpa.h"
18 #include "qemu/config-file.h"
19 #include "qemu/error-report.h"
20 #include "qemu/log.h"
21 #include "qemu/memalign.h"
22 #include "qemu/option.h"
23 #include "qapi/error.h"
24 #include <linux/vhost.h>
25 #include <sys/ioctl.h>
26 #include <err.h>
27 #include "standard-headers/linux/virtio_net.h"
28 #include "monitor/monitor.h"
29 #include "hw/virtio/vhost.h"
30 
31 /* Todo:need to add the multiqueue support here */
32 typedef struct VhostVDPAState {
33     NetClientState nc;
34     struct vhost_vdpa vhost_vdpa;
35     VHostNetState *vhost_net;
36 
37     /* Control commands shadow buffers */
38     void *cvq_cmd_out_buffer;
39     virtio_net_ctrl_ack *status;
40 
41     bool started;
42 } VhostVDPAState;
43 
44 const int vdpa_feature_bits[] = {
45     VIRTIO_F_NOTIFY_ON_EMPTY,
46     VIRTIO_RING_F_INDIRECT_DESC,
47     VIRTIO_RING_F_EVENT_IDX,
48     VIRTIO_F_ANY_LAYOUT,
49     VIRTIO_F_VERSION_1,
50     VIRTIO_NET_F_CSUM,
51     VIRTIO_NET_F_GUEST_CSUM,
52     VIRTIO_NET_F_GSO,
53     VIRTIO_NET_F_GUEST_TSO4,
54     VIRTIO_NET_F_GUEST_TSO6,
55     VIRTIO_NET_F_GUEST_ECN,
56     VIRTIO_NET_F_GUEST_UFO,
57     VIRTIO_NET_F_HOST_TSO4,
58     VIRTIO_NET_F_HOST_TSO6,
59     VIRTIO_NET_F_HOST_ECN,
60     VIRTIO_NET_F_HOST_UFO,
61     VIRTIO_NET_F_MRG_RXBUF,
62     VIRTIO_NET_F_MTU,
63     VIRTIO_NET_F_CTRL_RX,
64     VIRTIO_NET_F_CTRL_RX_EXTRA,
65     VIRTIO_NET_F_CTRL_VLAN,
66     VIRTIO_NET_F_CTRL_MAC_ADDR,
67     VIRTIO_NET_F_RSS,
68     VIRTIO_NET_F_MQ,
69     VIRTIO_NET_F_CTRL_VQ,
70     VIRTIO_F_IOMMU_PLATFORM,
71     VIRTIO_F_RING_PACKED,
72     VIRTIO_F_RING_RESET,
73     VIRTIO_NET_F_RSS,
74     VIRTIO_NET_F_HASH_REPORT,
75     VIRTIO_NET_F_GUEST_ANNOUNCE,
76     VIRTIO_NET_F_STATUS,
77     VHOST_INVALID_FEATURE_BIT
78 };
79 
80 /** Supported device specific feature bits with SVQ */
81 static const uint64_t vdpa_svq_device_features =
82     BIT_ULL(VIRTIO_NET_F_CSUM) |
83     BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) |
84     BIT_ULL(VIRTIO_NET_F_MTU) |
85     BIT_ULL(VIRTIO_NET_F_MAC) |
86     BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) |
87     BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) |
88     BIT_ULL(VIRTIO_NET_F_GUEST_ECN) |
89     BIT_ULL(VIRTIO_NET_F_GUEST_UFO) |
90     BIT_ULL(VIRTIO_NET_F_HOST_TSO4) |
91     BIT_ULL(VIRTIO_NET_F_HOST_TSO6) |
92     BIT_ULL(VIRTIO_NET_F_HOST_ECN) |
93     BIT_ULL(VIRTIO_NET_F_HOST_UFO) |
94     BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) |
95     BIT_ULL(VIRTIO_NET_F_STATUS) |
96     BIT_ULL(VIRTIO_NET_F_CTRL_VQ) |
97     BIT_ULL(VIRTIO_NET_F_MQ) |
98     BIT_ULL(VIRTIO_F_ANY_LAYOUT) |
99     BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) |
100     BIT_ULL(VIRTIO_NET_F_RSC_EXT) |
101     BIT_ULL(VIRTIO_NET_F_STANDBY);
102 
103 VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
104 {
105     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
106     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
107     return s->vhost_net;
108 }
109 
110 static int vhost_vdpa_net_check_device_id(struct vhost_net *net)
111 {
112     uint32_t device_id;
113     int ret;
114     struct vhost_dev *hdev;
115 
116     hdev = (struct vhost_dev *)&net->dev;
117     ret = hdev->vhost_ops->vhost_get_device_id(hdev, &device_id);
118     if (device_id != VIRTIO_ID_NET) {
119         return -ENOTSUP;
120     }
121     return ret;
122 }
123 
124 static int vhost_vdpa_add(NetClientState *ncs, void *be,
125                           int queue_pair_index, int nvqs)
126 {
127     VhostNetOptions options;
128     struct vhost_net *net = NULL;
129     VhostVDPAState *s;
130     int ret;
131 
132     options.backend_type = VHOST_BACKEND_TYPE_VDPA;
133     assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
134     s = DO_UPCAST(VhostVDPAState, nc, ncs);
135     options.net_backend = ncs;
136     options.opaque      = be;
137     options.busyloop_timeout = 0;
138     options.nvqs = nvqs;
139 
140     net = vhost_net_init(&options);
141     if (!net) {
142         error_report("failed to init vhost_net for queue");
143         goto err_init;
144     }
145     s->vhost_net = net;
146     ret = vhost_vdpa_net_check_device_id(net);
147     if (ret) {
148         goto err_check;
149     }
150     return 0;
151 err_check:
152     vhost_net_cleanup(net);
153     g_free(net);
154 err_init:
155     return -1;
156 }
157 
158 static void vhost_vdpa_cleanup(NetClientState *nc)
159 {
160     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
161     struct vhost_dev *dev = &s->vhost_net->dev;
162 
163     qemu_vfree(s->cvq_cmd_out_buffer);
164     qemu_vfree(s->status);
165     if (dev->vq_index + dev->nvqs == dev->vq_index_end) {
166         g_clear_pointer(&s->vhost_vdpa.iova_tree, vhost_iova_tree_delete);
167     }
168     if (s->vhost_net) {
169         vhost_net_cleanup(s->vhost_net);
170         g_free(s->vhost_net);
171         s->vhost_net = NULL;
172     }
173      if (s->vhost_vdpa.device_fd >= 0) {
174         qemu_close(s->vhost_vdpa.device_fd);
175         s->vhost_vdpa.device_fd = -1;
176     }
177 }
178 
179 static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
180 {
181     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
182 
183     return true;
184 }
185 
186 static bool vhost_vdpa_has_ufo(NetClientState *nc)
187 {
188     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
189     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
190     uint64_t features = 0;
191     features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
192     features = vhost_net_get_features(s->vhost_net, features);
193     return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
194 
195 }
196 
197 static bool vhost_vdpa_check_peer_type(NetClientState *nc, ObjectClass *oc,
198                                        Error **errp)
199 {
200     const char *driver = object_class_get_name(oc);
201 
202     if (!g_str_has_prefix(driver, "virtio-net-")) {
203         error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
204         return false;
205     }
206 
207     return true;
208 }
209 
210 /** Dummy receive in case qemu falls back to userland tap networking */
211 static ssize_t vhost_vdpa_receive(NetClientState *nc, const uint8_t *buf,
212                                   size_t size)
213 {
214     return size;
215 }
216 
217 static NetClientInfo net_vhost_vdpa_info = {
218         .type = NET_CLIENT_DRIVER_VHOST_VDPA,
219         .size = sizeof(VhostVDPAState),
220         .receive = vhost_vdpa_receive,
221         .cleanup = vhost_vdpa_cleanup,
222         .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
223         .has_ufo = vhost_vdpa_has_ufo,
224         .check_peer_type = vhost_vdpa_check_peer_type,
225 };
226 
227 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa *v, void *addr)
228 {
229     VhostIOVATree *tree = v->iova_tree;
230     DMAMap needle = {
231         /*
232          * No need to specify size or to look for more translations since
233          * this contiguous chunk was allocated by us.
234          */
235         .translated_addr = (hwaddr)(uintptr_t)addr,
236     };
237     const DMAMap *map = vhost_iova_tree_find_iova(tree, &needle);
238     int r;
239 
240     if (unlikely(!map)) {
241         error_report("Cannot locate expected map");
242         return;
243     }
244 
245     r = vhost_vdpa_dma_unmap(v, map->iova, map->size + 1);
246     if (unlikely(r != 0)) {
247         error_report("Device cannot unmap: %s(%d)", g_strerror(r), r);
248     }
249 
250     vhost_iova_tree_remove(tree, *map);
251 }
252 
253 static size_t vhost_vdpa_net_cvq_cmd_len(void)
254 {
255     /*
256      * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
257      * In buffer is always 1 byte, so it should fit here
258      */
259     return sizeof(struct virtio_net_ctrl_hdr) +
260            2 * sizeof(struct virtio_net_ctrl_mac) +
261            MAC_TABLE_ENTRIES * ETH_ALEN;
262 }
263 
264 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
265 {
266     return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
267 }
268 
269 /** Map CVQ buffer. */
270 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa *v, void *buf, size_t size,
271                                   bool write)
272 {
273     DMAMap map = {};
274     int r;
275 
276     map.translated_addr = (hwaddr)(uintptr_t)buf;
277     map.size = size - 1;
278     map.perm = write ? IOMMU_RW : IOMMU_RO,
279     r = vhost_iova_tree_map_alloc(v->iova_tree, &map);
280     if (unlikely(r != IOVA_OK)) {
281         error_report("Cannot map injected element");
282         return r;
283     }
284 
285     r = vhost_vdpa_dma_map(v, map.iova, vhost_vdpa_net_cvq_cmd_page_len(), buf,
286                            !write);
287     if (unlikely(r < 0)) {
288         goto dma_map_err;
289     }
290 
291     return 0;
292 
293 dma_map_err:
294     vhost_iova_tree_remove(v->iova_tree, map);
295     return r;
296 }
297 
298 static int vhost_vdpa_net_cvq_start(NetClientState *nc)
299 {
300     VhostVDPAState *s;
301     int r;
302 
303     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
304 
305     s = DO_UPCAST(VhostVDPAState, nc, nc);
306     if (!s->vhost_vdpa.shadow_vqs_enabled) {
307         return 0;
308     }
309 
310     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer,
311                                vhost_vdpa_net_cvq_cmd_page_len(), false);
312     if (unlikely(r < 0)) {
313         return r;
314     }
315 
316     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->status,
317                                vhost_vdpa_net_cvq_cmd_page_len(), true);
318     if (unlikely(r < 0)) {
319         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
320     }
321 
322     return r;
323 }
324 
325 static void vhost_vdpa_net_cvq_stop(NetClientState *nc)
326 {
327     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
328 
329     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
330 
331     if (s->vhost_vdpa.shadow_vqs_enabled) {
332         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
333         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->status);
334     }
335 }
336 
337 static ssize_t vhost_vdpa_net_cvq_add(VhostVDPAState *s, size_t out_len,
338                                       size_t in_len)
339 {
340     /* Buffers for the device */
341     const struct iovec out = {
342         .iov_base = s->cvq_cmd_out_buffer,
343         .iov_len = out_len,
344     };
345     const struct iovec in = {
346         .iov_base = s->status,
347         .iov_len = sizeof(virtio_net_ctrl_ack),
348     };
349     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
350     int r;
351 
352     r = vhost_svq_add(svq, &out, 1, &in, 1, NULL);
353     if (unlikely(r != 0)) {
354         if (unlikely(r == -ENOSPC)) {
355             qemu_log_mask(LOG_GUEST_ERROR, "%s: No space on device queue\n",
356                           __func__);
357         }
358         return r;
359     }
360 
361     /*
362      * We can poll here since we've had BQL from the time we sent the
363      * descriptor. Also, we need to take the answer before SVQ pulls by itself,
364      * when BQL is released
365      */
366     return vhost_svq_poll(svq);
367 }
368 
369 static ssize_t vhost_vdpa_net_load_cmd(VhostVDPAState *s, uint8_t class,
370                                        uint8_t cmd, const void *data,
371                                        size_t data_size)
372 {
373     const struct virtio_net_ctrl_hdr ctrl = {
374         .class = class,
375         .cmd = cmd,
376     };
377 
378     assert(data_size < vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl));
379 
380     memcpy(s->cvq_cmd_out_buffer, &ctrl, sizeof(ctrl));
381     memcpy(s->cvq_cmd_out_buffer + sizeof(ctrl), data, data_size);
382 
383     return vhost_vdpa_net_cvq_add(s, sizeof(ctrl) + data_size,
384                                   sizeof(virtio_net_ctrl_ack));
385 }
386 
387 static int vhost_vdpa_net_load_mac(VhostVDPAState *s, const VirtIONet *n)
388 {
389     uint64_t features = n->parent_obj.guest_features;
390     if (features & BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR)) {
391         ssize_t dev_written = vhost_vdpa_net_load_cmd(s, VIRTIO_NET_CTRL_MAC,
392                                                   VIRTIO_NET_CTRL_MAC_ADDR_SET,
393                                                   n->mac, sizeof(n->mac));
394         if (unlikely(dev_written < 0)) {
395             return dev_written;
396         }
397 
398         return *s->status != VIRTIO_NET_OK;
399     }
400 
401     return 0;
402 }
403 
404 static int vhost_vdpa_net_load_mq(VhostVDPAState *s,
405                                   const VirtIONet *n)
406 {
407     struct virtio_net_ctrl_mq mq;
408     uint64_t features = n->parent_obj.guest_features;
409     ssize_t dev_written;
410 
411     if (!(features & BIT_ULL(VIRTIO_NET_F_MQ))) {
412         return 0;
413     }
414 
415     mq.virtqueue_pairs = cpu_to_le16(n->curr_queue_pairs);
416     dev_written = vhost_vdpa_net_load_cmd(s, VIRTIO_NET_CTRL_MQ,
417                                           VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &mq,
418                                           sizeof(mq));
419     if (unlikely(dev_written < 0)) {
420         return dev_written;
421     }
422 
423     return *s->status != VIRTIO_NET_OK;
424 }
425 
426 static int vhost_vdpa_net_load(NetClientState *nc)
427 {
428     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
429     struct vhost_vdpa *v = &s->vhost_vdpa;
430     const VirtIONet *n;
431     int r;
432 
433     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
434 
435     if (!v->shadow_vqs_enabled) {
436         return 0;
437     }
438 
439     n = VIRTIO_NET(v->dev->vdev);
440     r = vhost_vdpa_net_load_mac(s, n);
441     if (unlikely(r < 0)) {
442         return r;
443     }
444     r = vhost_vdpa_net_load_mq(s, n);
445     if (unlikely(r)) {
446         return r;
447     }
448 
449     return 0;
450 }
451 
452 static NetClientInfo net_vhost_vdpa_cvq_info = {
453     .type = NET_CLIENT_DRIVER_VHOST_VDPA,
454     .size = sizeof(VhostVDPAState),
455     .receive = vhost_vdpa_receive,
456     .start = vhost_vdpa_net_cvq_start,
457     .load = vhost_vdpa_net_load,
458     .stop = vhost_vdpa_net_cvq_stop,
459     .cleanup = vhost_vdpa_cleanup,
460     .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
461     .has_ufo = vhost_vdpa_has_ufo,
462     .check_peer_type = vhost_vdpa_check_peer_type,
463 };
464 
465 /**
466  * Validate and copy control virtqueue commands.
467  *
468  * Following QEMU guidelines, we offer a copy of the buffers to the device to
469  * prevent TOCTOU bugs.
470  */
471 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue *svq,
472                                             VirtQueueElement *elem,
473                                             void *opaque)
474 {
475     VhostVDPAState *s = opaque;
476     size_t in_len;
477     virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
478     /* Out buffer sent to both the vdpa device and the device model */
479     struct iovec out = {
480         .iov_base = s->cvq_cmd_out_buffer,
481     };
482     /* in buffer used for device model */
483     const struct iovec in = {
484         .iov_base = &status,
485         .iov_len = sizeof(status),
486     };
487     ssize_t dev_written = -EINVAL;
488 
489     out.iov_len = iov_to_buf(elem->out_sg, elem->out_num, 0,
490                              s->cvq_cmd_out_buffer,
491                              vhost_vdpa_net_cvq_cmd_len());
492     dev_written = vhost_vdpa_net_cvq_add(s, out.iov_len, sizeof(status));
493     if (unlikely(dev_written < 0)) {
494         goto out;
495     }
496 
497     if (unlikely(dev_written < sizeof(status))) {
498         error_report("Insufficient written data (%zu)", dev_written);
499         goto out;
500     }
501 
502     if (*s->status != VIRTIO_NET_OK) {
503         return VIRTIO_NET_ERR;
504     }
505 
506     status = VIRTIO_NET_ERR;
507     virtio_net_handle_ctrl_iov(svq->vdev, &in, 1, &out, 1);
508     if (status != VIRTIO_NET_OK) {
509         error_report("Bad CVQ processing in model");
510     }
511 
512 out:
513     in_len = iov_from_buf(elem->in_sg, elem->in_num, 0, &status,
514                           sizeof(status));
515     if (unlikely(in_len < sizeof(status))) {
516         error_report("Bad device CVQ written length");
517     }
518     vhost_svq_push_elem(svq, elem, MIN(in_len, sizeof(status)));
519     g_free(elem);
520     return dev_written < 0 ? dev_written : 0;
521 }
522 
523 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops = {
524     .avail_handler = vhost_vdpa_net_handle_ctrl_avail,
525 };
526 
527 static NetClientState *net_vhost_vdpa_init(NetClientState *peer,
528                                            const char *device,
529                                            const char *name,
530                                            int vdpa_device_fd,
531                                            int queue_pair_index,
532                                            int nvqs,
533                                            bool is_datapath,
534                                            bool svq,
535                                            VhostIOVATree *iova_tree)
536 {
537     NetClientState *nc = NULL;
538     VhostVDPAState *s;
539     int ret = 0;
540     assert(name);
541     if (is_datapath) {
542         nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device,
543                                  name);
544     } else {
545         nc = qemu_new_net_control_client(&net_vhost_vdpa_cvq_info, peer,
546                                          device, name);
547     }
548     qemu_set_info_str(nc, TYPE_VHOST_VDPA);
549     s = DO_UPCAST(VhostVDPAState, nc, nc);
550 
551     s->vhost_vdpa.device_fd = vdpa_device_fd;
552     s->vhost_vdpa.index = queue_pair_index;
553     s->vhost_vdpa.shadow_vqs_enabled = svq;
554     s->vhost_vdpa.iova_tree = iova_tree;
555     if (!is_datapath) {
556         s->cvq_cmd_out_buffer = qemu_memalign(qemu_real_host_page_size(),
557                                             vhost_vdpa_net_cvq_cmd_page_len());
558         memset(s->cvq_cmd_out_buffer, 0, vhost_vdpa_net_cvq_cmd_page_len());
559         s->status = qemu_memalign(qemu_real_host_page_size(),
560                                   vhost_vdpa_net_cvq_cmd_page_len());
561         memset(s->status, 0, vhost_vdpa_net_cvq_cmd_page_len());
562 
563         s->vhost_vdpa.shadow_vq_ops = &vhost_vdpa_net_svq_ops;
564         s->vhost_vdpa.shadow_vq_ops_opaque = s;
565     }
566     ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa, queue_pair_index, nvqs);
567     if (ret) {
568         qemu_del_net_client(nc);
569         return NULL;
570     }
571     return nc;
572 }
573 
574 static int vhost_vdpa_get_iova_range(int fd,
575                                      struct vhost_vdpa_iova_range *iova_range)
576 {
577     int ret = ioctl(fd, VHOST_VDPA_GET_IOVA_RANGE, iova_range);
578 
579     return ret < 0 ? -errno : 0;
580 }
581 
582 static int vhost_vdpa_get_features(int fd, uint64_t *features, Error **errp)
583 {
584     int ret = ioctl(fd, VHOST_GET_FEATURES, features);
585     if (unlikely(ret < 0)) {
586         error_setg_errno(errp, errno,
587                          "Fail to query features from vhost-vDPA device");
588     }
589     return ret;
590 }
591 
592 static int vhost_vdpa_get_max_queue_pairs(int fd, uint64_t features,
593                                           int *has_cvq, Error **errp)
594 {
595     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
596     g_autofree struct vhost_vdpa_config *config = NULL;
597     __virtio16 *max_queue_pairs;
598     int ret;
599 
600     if (features & (1 << VIRTIO_NET_F_CTRL_VQ)) {
601         *has_cvq = 1;
602     } else {
603         *has_cvq = 0;
604     }
605 
606     if (features & (1 << VIRTIO_NET_F_MQ)) {
607         config = g_malloc0(config_size + sizeof(*max_queue_pairs));
608         config->off = offsetof(struct virtio_net_config, max_virtqueue_pairs);
609         config->len = sizeof(*max_queue_pairs);
610 
611         ret = ioctl(fd, VHOST_VDPA_GET_CONFIG, config);
612         if (ret) {
613             error_setg(errp, "Fail to get config from vhost-vDPA device");
614             return -ret;
615         }
616 
617         max_queue_pairs = (__virtio16 *)&config->buf;
618 
619         return lduw_le_p(max_queue_pairs);
620     }
621 
622     return 1;
623 }
624 
625 int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
626                         NetClientState *peer, Error **errp)
627 {
628     const NetdevVhostVDPAOptions *opts;
629     uint64_t features;
630     int vdpa_device_fd;
631     g_autofree NetClientState **ncs = NULL;
632     g_autoptr(VhostIOVATree) iova_tree = NULL;
633     NetClientState *nc;
634     int queue_pairs, r, i = 0, has_cvq = 0;
635 
636     assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
637     opts = &netdev->u.vhost_vdpa;
638     if (!opts->has_vhostdev && !opts->has_vhostfd) {
639         error_setg(errp,
640                    "vhost-vdpa: neither vhostdev= nor vhostfd= was specified");
641         return -1;
642     }
643 
644     if (opts->has_vhostdev && opts->has_vhostfd) {
645         error_setg(errp,
646                    "vhost-vdpa: vhostdev= and vhostfd= are mutually exclusive");
647         return -1;
648     }
649 
650     if (opts->has_vhostdev) {
651         vdpa_device_fd = qemu_open(opts->vhostdev, O_RDWR, errp);
652         if (vdpa_device_fd == -1) {
653             return -errno;
654         }
655     } else {
656         /* has_vhostfd */
657         vdpa_device_fd = monitor_fd_param(monitor_cur(), opts->vhostfd, errp);
658         if (vdpa_device_fd == -1) {
659             error_prepend(errp, "vhost-vdpa: unable to parse vhostfd: ");
660             return -1;
661         }
662     }
663 
664     r = vhost_vdpa_get_features(vdpa_device_fd, &features, errp);
665     if (unlikely(r < 0)) {
666         goto err;
667     }
668 
669     queue_pairs = vhost_vdpa_get_max_queue_pairs(vdpa_device_fd, features,
670                                                  &has_cvq, errp);
671     if (queue_pairs < 0) {
672         qemu_close(vdpa_device_fd);
673         return queue_pairs;
674     }
675 
676     if (opts->x_svq) {
677         struct vhost_vdpa_iova_range iova_range;
678 
679         uint64_t invalid_dev_features =
680             features & ~vdpa_svq_device_features &
681             /* Transport are all accepted at this point */
682             ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START,
683                              VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START);
684 
685         if (invalid_dev_features) {
686             error_setg(errp, "vdpa svq does not work with features 0x%" PRIx64,
687                        invalid_dev_features);
688             goto err_svq;
689         }
690 
691         vhost_vdpa_get_iova_range(vdpa_device_fd, &iova_range);
692         iova_tree = vhost_iova_tree_new(iova_range.first, iova_range.last);
693     }
694 
695     ncs = g_malloc0(sizeof(*ncs) * queue_pairs);
696 
697     for (i = 0; i < queue_pairs; i++) {
698         ncs[i] = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
699                                      vdpa_device_fd, i, 2, true, opts->x_svq,
700                                      iova_tree);
701         if (!ncs[i])
702             goto err;
703     }
704 
705     if (has_cvq) {
706         nc = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
707                                  vdpa_device_fd, i, 1, false,
708                                  opts->x_svq, iova_tree);
709         if (!nc)
710             goto err;
711     }
712 
713     /* iova_tree ownership belongs to last NetClientState */
714     g_steal_pointer(&iova_tree);
715     return 0;
716 
717 err:
718     if (i) {
719         for (i--; i >= 0; i--) {
720             qemu_del_net_client(ncs[i]);
721         }
722     }
723 
724 err_svq:
725     qemu_close(vdpa_device_fd);
726 
727     return -1;
728 }
729