1 /* 2 * vhost-vdpa 3 * 4 * Copyright(c) 2017-2018 Intel Corporation. 5 * Copyright(c) 2020 Red Hat, Inc. 6 * 7 * This work is licensed under the terms of the GNU GPL, version 2 or later. 8 * See the COPYING file in the top-level directory. 9 * 10 */ 11 12 #include "qemu/osdep.h" 13 #include <linux/vhost.h> 14 #include <linux/vfio.h> 15 #include <sys/eventfd.h> 16 #include <sys/ioctl.h> 17 #include "exec/target_page.h" 18 #include "hw/virtio/vhost.h" 19 #include "hw/virtio/vhost-backend.h" 20 #include "hw/virtio/virtio-net.h" 21 #include "hw/virtio/vhost-shadow-virtqueue.h" 22 #include "hw/virtio/vhost-vdpa.h" 23 #include "exec/address-spaces.h" 24 #include "migration/blocker.h" 25 #include "qemu/cutils.h" 26 #include "qemu/main-loop.h" 27 #include "trace.h" 28 #include "qapi/error.h" 29 30 /* 31 * Return one past the end of the end of section. Be careful with uint64_t 32 * conversions! 33 */ 34 static Int128 vhost_vdpa_section_end(const MemoryRegionSection *section, 35 int page_mask) 36 { 37 Int128 llend = int128_make64(section->offset_within_address_space); 38 llend = int128_add(llend, section->size); 39 llend = int128_and(llend, int128_exts64(page_mask)); 40 41 return llend; 42 } 43 44 static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section, 45 uint64_t iova_min, 46 uint64_t iova_max, 47 int page_mask) 48 { 49 Int128 llend; 50 51 if ((!memory_region_is_ram(section->mr) && 52 !memory_region_is_iommu(section->mr)) || 53 memory_region_is_protected(section->mr) || 54 /* vhost-vDPA doesn't allow MMIO to be mapped */ 55 memory_region_is_ram_device(section->mr)) { 56 return true; 57 } 58 59 if (section->offset_within_address_space < iova_min) { 60 error_report("RAM section out of device range (min=0x%" PRIx64 61 ", addr=0x%" HWADDR_PRIx ")", 62 iova_min, section->offset_within_address_space); 63 return true; 64 } 65 /* 66 * While using vIOMMU, sometimes the section will be larger than iova_max, 67 * but the memory that actually maps is smaller, so move the check to 68 * function vhost_vdpa_iommu_map_notify(). That function will use the actual 69 * size that maps to the kernel 70 */ 71 72 if (!memory_region_is_iommu(section->mr)) { 73 llend = vhost_vdpa_section_end(section, page_mask); 74 if (int128_gt(llend, int128_make64(iova_max))) { 75 error_report("RAM section out of device range (max=0x%" PRIx64 76 ", end addr=0x%" PRIx64 ")", 77 iova_max, int128_get64(llend)); 78 return true; 79 } 80 } 81 82 return false; 83 } 84 85 /* 86 * The caller must set asid = 0 if the device does not support asid. 87 * This is not an ABI break since it is set to 0 by the initializer anyway. 88 */ 89 int vhost_vdpa_dma_map(struct vhost_vdpa *v, uint32_t asid, hwaddr iova, 90 hwaddr size, void *vaddr, bool readonly) 91 { 92 struct vhost_msg_v2 msg = {}; 93 int fd = v->device_fd; 94 int ret = 0; 95 96 msg.type = v->msg_type; 97 msg.asid = asid; 98 msg.iotlb.iova = iova; 99 msg.iotlb.size = size; 100 msg.iotlb.uaddr = (uint64_t)(uintptr_t)vaddr; 101 msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW; 102 msg.iotlb.type = VHOST_IOTLB_UPDATE; 103 104 trace_vhost_vdpa_dma_map(v, fd, msg.type, msg.asid, msg.iotlb.iova, 105 msg.iotlb.size, msg.iotlb.uaddr, msg.iotlb.perm, 106 msg.iotlb.type); 107 108 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 109 error_report("failed to write, fd=%d, errno=%d (%s)", 110 fd, errno, strerror(errno)); 111 return -EIO ; 112 } 113 114 return ret; 115 } 116 117 /* 118 * The caller must set asid = 0 if the device does not support asid. 119 * This is not an ABI break since it is set to 0 by the initializer anyway. 120 */ 121 int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, uint32_t asid, hwaddr iova, 122 hwaddr size) 123 { 124 struct vhost_msg_v2 msg = {}; 125 int fd = v->device_fd; 126 int ret = 0; 127 128 msg.type = v->msg_type; 129 msg.asid = asid; 130 msg.iotlb.iova = iova; 131 msg.iotlb.size = size; 132 msg.iotlb.type = VHOST_IOTLB_INVALIDATE; 133 134 trace_vhost_vdpa_dma_unmap(v, fd, msg.type, msg.asid, msg.iotlb.iova, 135 msg.iotlb.size, msg.iotlb.type); 136 137 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 138 error_report("failed to write, fd=%d, errno=%d (%s)", 139 fd, errno, strerror(errno)); 140 return -EIO ; 141 } 142 143 return ret; 144 } 145 146 static void vhost_vdpa_listener_begin_batch(struct vhost_vdpa *v) 147 { 148 int fd = v->device_fd; 149 struct vhost_msg_v2 msg = { 150 .type = v->msg_type, 151 .iotlb.type = VHOST_IOTLB_BATCH_BEGIN, 152 }; 153 154 trace_vhost_vdpa_listener_begin_batch(v, fd, msg.type, msg.iotlb.type); 155 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 156 error_report("failed to write, fd=%d, errno=%d (%s)", 157 fd, errno, strerror(errno)); 158 } 159 } 160 161 static void vhost_vdpa_iotlb_batch_begin_once(struct vhost_vdpa *v) 162 { 163 if (v->dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH) && 164 !v->iotlb_batch_begin_sent) { 165 vhost_vdpa_listener_begin_batch(v); 166 } 167 168 v->iotlb_batch_begin_sent = true; 169 } 170 171 static void vhost_vdpa_listener_commit(MemoryListener *listener) 172 { 173 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 174 struct vhost_dev *dev = v->dev; 175 struct vhost_msg_v2 msg = {}; 176 int fd = v->device_fd; 177 178 if (!(dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH))) { 179 return; 180 } 181 182 if (!v->iotlb_batch_begin_sent) { 183 return; 184 } 185 186 msg.type = v->msg_type; 187 msg.iotlb.type = VHOST_IOTLB_BATCH_END; 188 189 trace_vhost_vdpa_listener_commit(v, fd, msg.type, msg.iotlb.type); 190 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { 191 error_report("failed to write, fd=%d, errno=%d (%s)", 192 fd, errno, strerror(errno)); 193 } 194 195 v->iotlb_batch_begin_sent = false; 196 } 197 198 static void vhost_vdpa_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 199 { 200 struct vdpa_iommu *iommu = container_of(n, struct vdpa_iommu, n); 201 202 hwaddr iova = iotlb->iova + iommu->iommu_offset; 203 struct vhost_vdpa *v = iommu->dev; 204 void *vaddr; 205 int ret; 206 Int128 llend; 207 208 if (iotlb->target_as != &address_space_memory) { 209 error_report("Wrong target AS \"%s\", only system memory is allowed", 210 iotlb->target_as->name ? iotlb->target_as->name : "none"); 211 return; 212 } 213 RCU_READ_LOCK_GUARD(); 214 /* check if RAM section out of device range */ 215 llend = int128_add(int128_makes64(iotlb->addr_mask), int128_makes64(iova)); 216 if (int128_gt(llend, int128_make64(v->iova_range.last))) { 217 error_report("RAM section out of device range (max=0x%" PRIx64 218 ", end addr=0x%" PRIx64 ")", 219 v->iova_range.last, int128_get64(llend)); 220 return; 221 } 222 223 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 224 bool read_only; 225 226 if (!memory_get_xlat_addr(iotlb, &vaddr, NULL, &read_only, NULL)) { 227 return; 228 } 229 ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova, 230 iotlb->addr_mask + 1, vaddr, read_only); 231 if (ret) { 232 error_report("vhost_vdpa_dma_map(%p, 0x%" HWADDR_PRIx ", " 233 "0x%" HWADDR_PRIx ", %p) = %d (%m)", 234 v, iova, iotlb->addr_mask + 1, vaddr, ret); 235 } 236 } else { 237 ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova, 238 iotlb->addr_mask + 1); 239 if (ret) { 240 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 241 "0x%" HWADDR_PRIx ") = %d (%m)", 242 v, iova, iotlb->addr_mask + 1, ret); 243 } 244 } 245 } 246 247 static void vhost_vdpa_iommu_region_add(MemoryListener *listener, 248 MemoryRegionSection *section) 249 { 250 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 251 252 struct vdpa_iommu *iommu; 253 Int128 end; 254 int iommu_idx; 255 IOMMUMemoryRegion *iommu_mr; 256 int ret; 257 258 iommu_mr = IOMMU_MEMORY_REGION(section->mr); 259 260 iommu = g_malloc0(sizeof(*iommu)); 261 end = int128_add(int128_make64(section->offset_within_region), 262 section->size); 263 end = int128_sub(end, int128_one()); 264 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 265 MEMTXATTRS_UNSPECIFIED); 266 iommu->iommu_mr = iommu_mr; 267 iommu_notifier_init(&iommu->n, vhost_vdpa_iommu_map_notify, 268 IOMMU_NOTIFIER_IOTLB_EVENTS, 269 section->offset_within_region, 270 int128_get64(end), 271 iommu_idx); 272 iommu->iommu_offset = section->offset_within_address_space - 273 section->offset_within_region; 274 iommu->dev = v; 275 276 ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL); 277 if (ret) { 278 g_free(iommu); 279 return; 280 } 281 282 QLIST_INSERT_HEAD(&v->iommu_list, iommu, iommu_next); 283 memory_region_iommu_replay(iommu->iommu_mr, &iommu->n); 284 285 return; 286 } 287 288 static void vhost_vdpa_iommu_region_del(MemoryListener *listener, 289 MemoryRegionSection *section) 290 { 291 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 292 293 struct vdpa_iommu *iommu; 294 295 QLIST_FOREACH(iommu, &v->iommu_list, iommu_next) 296 { 297 if (MEMORY_REGION(iommu->iommu_mr) == section->mr && 298 iommu->n.start == section->offset_within_region) { 299 memory_region_unregister_iommu_notifier(section->mr, &iommu->n); 300 QLIST_REMOVE(iommu, iommu_next); 301 g_free(iommu); 302 break; 303 } 304 } 305 } 306 307 static void vhost_vdpa_listener_region_add(MemoryListener *listener, 308 MemoryRegionSection *section) 309 { 310 DMAMap mem_region = {}; 311 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 312 hwaddr iova; 313 Int128 llend, llsize; 314 void *vaddr; 315 int ret; 316 int page_size = qemu_target_page_size(); 317 int page_mask = -page_size; 318 319 if (vhost_vdpa_listener_skipped_section(section, v->iova_range.first, 320 v->iova_range.last, page_mask)) { 321 return; 322 } 323 if (memory_region_is_iommu(section->mr)) { 324 vhost_vdpa_iommu_region_add(listener, section); 325 return; 326 } 327 328 if (unlikely((section->offset_within_address_space & ~page_mask) != 329 (section->offset_within_region & ~page_mask))) { 330 trace_vhost_vdpa_listener_region_add_unaligned(v, section->mr->name, 331 section->offset_within_address_space & ~page_mask, 332 section->offset_within_region & ~page_mask); 333 return; 334 } 335 336 iova = ROUND_UP(section->offset_within_address_space, page_size); 337 llend = vhost_vdpa_section_end(section, page_mask); 338 if (int128_ge(int128_make64(iova), llend)) { 339 return; 340 } 341 342 memory_region_ref(section->mr); 343 344 /* Here we assume that memory_region_is_ram(section->mr)==true */ 345 346 vaddr = memory_region_get_ram_ptr(section->mr) + 347 section->offset_within_region + 348 (iova - section->offset_within_address_space); 349 350 trace_vhost_vdpa_listener_region_add(v, iova, int128_get64(llend), 351 vaddr, section->readonly); 352 353 llsize = int128_sub(llend, int128_make64(iova)); 354 if (v->shadow_data) { 355 int r; 356 357 mem_region.translated_addr = (hwaddr)(uintptr_t)vaddr, 358 mem_region.size = int128_get64(llsize) - 1, 359 mem_region.perm = IOMMU_ACCESS_FLAG(true, section->readonly), 360 361 r = vhost_iova_tree_map_alloc(v->iova_tree, &mem_region); 362 if (unlikely(r != IOVA_OK)) { 363 error_report("Can't allocate a mapping (%d)", r); 364 goto fail; 365 } 366 367 iova = mem_region.iova; 368 } 369 370 vhost_vdpa_iotlb_batch_begin_once(v); 371 ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova, 372 int128_get64(llsize), vaddr, section->readonly); 373 if (ret) { 374 error_report("vhost vdpa map fail!"); 375 goto fail_map; 376 } 377 378 return; 379 380 fail_map: 381 if (v->shadow_data) { 382 vhost_iova_tree_remove(v->iova_tree, mem_region); 383 } 384 385 fail: 386 /* 387 * On the initfn path, store the first error in the container so we 388 * can gracefully fail. Runtime, there's not much we can do other 389 * than throw a hardware error. 390 */ 391 error_report("vhost-vdpa: DMA mapping failed, unable to continue"); 392 return; 393 394 } 395 396 static void vhost_vdpa_listener_region_del(MemoryListener *listener, 397 MemoryRegionSection *section) 398 { 399 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); 400 hwaddr iova; 401 Int128 llend, llsize; 402 int ret; 403 int page_size = qemu_target_page_size(); 404 int page_mask = -page_size; 405 406 if (vhost_vdpa_listener_skipped_section(section, v->iova_range.first, 407 v->iova_range.last, page_mask)) { 408 return; 409 } 410 if (memory_region_is_iommu(section->mr)) { 411 vhost_vdpa_iommu_region_del(listener, section); 412 } 413 414 if (unlikely((section->offset_within_address_space & ~page_mask) != 415 (section->offset_within_region & ~page_mask))) { 416 trace_vhost_vdpa_listener_region_del_unaligned(v, section->mr->name, 417 section->offset_within_address_space & ~page_mask, 418 section->offset_within_region & ~page_mask); 419 return; 420 } 421 422 iova = ROUND_UP(section->offset_within_address_space, page_size); 423 llend = vhost_vdpa_section_end(section, page_mask); 424 425 trace_vhost_vdpa_listener_region_del(v, iova, 426 int128_get64(int128_sub(llend, int128_one()))); 427 428 if (int128_ge(int128_make64(iova), llend)) { 429 return; 430 } 431 432 llsize = int128_sub(llend, int128_make64(iova)); 433 434 if (v->shadow_data) { 435 const DMAMap *result; 436 const void *vaddr = memory_region_get_ram_ptr(section->mr) + 437 section->offset_within_region + 438 (iova - section->offset_within_address_space); 439 DMAMap mem_region = { 440 .translated_addr = (hwaddr)(uintptr_t)vaddr, 441 .size = int128_get64(llsize) - 1, 442 }; 443 444 result = vhost_iova_tree_find_iova(v->iova_tree, &mem_region); 445 if (!result) { 446 /* The memory listener map wasn't mapped */ 447 return; 448 } 449 iova = result->iova; 450 vhost_iova_tree_remove(v->iova_tree, *result); 451 } 452 vhost_vdpa_iotlb_batch_begin_once(v); 453 /* 454 * The unmap ioctl doesn't accept a full 64-bit. need to check it 455 */ 456 if (int128_eq(llsize, int128_2_64())) { 457 llsize = int128_rshift(llsize, 1); 458 ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova, 459 int128_get64(llsize)); 460 461 if (ret) { 462 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 463 "0x%" HWADDR_PRIx ") = %d (%m)", 464 v, iova, int128_get64(llsize), ret); 465 } 466 iova += int128_get64(llsize); 467 } 468 ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova, 469 int128_get64(llsize)); 470 471 if (ret) { 472 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", " 473 "0x%" HWADDR_PRIx ") = %d (%m)", 474 v, iova, int128_get64(llsize), ret); 475 } 476 477 memory_region_unref(section->mr); 478 } 479 /* 480 * IOTLB API is used by vhost-vdpa which requires incremental updating 481 * of the mapping. So we can not use generic vhost memory listener which 482 * depends on the addnop(). 483 */ 484 static const MemoryListener vhost_vdpa_memory_listener = { 485 .name = "vhost-vdpa", 486 .commit = vhost_vdpa_listener_commit, 487 .region_add = vhost_vdpa_listener_region_add, 488 .region_del = vhost_vdpa_listener_region_del, 489 }; 490 491 static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request, 492 void *arg) 493 { 494 struct vhost_vdpa *v = dev->opaque; 495 int fd = v->device_fd; 496 int ret; 497 498 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 499 500 ret = ioctl(fd, request, arg); 501 return ret < 0 ? -errno : ret; 502 } 503 504 static int vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status) 505 { 506 uint8_t s; 507 int ret; 508 509 trace_vhost_vdpa_add_status(dev, status); 510 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s); 511 if (ret < 0) { 512 return ret; 513 } 514 515 s |= status; 516 517 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s); 518 if (ret < 0) { 519 return ret; 520 } 521 522 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s); 523 if (ret < 0) { 524 return ret; 525 } 526 527 if (!(s & status)) { 528 return -EIO; 529 } 530 531 return 0; 532 } 533 534 int vhost_vdpa_get_iova_range(int fd, struct vhost_vdpa_iova_range *iova_range) 535 { 536 int ret = ioctl(fd, VHOST_VDPA_GET_IOVA_RANGE, iova_range); 537 538 return ret < 0 ? -errno : 0; 539 } 540 541 /* 542 * The use of this function is for requests that only need to be 543 * applied once. Typically such request occurs at the beginning 544 * of operation, and before setting up queues. It should not be 545 * used for request that performs operation until all queues are 546 * set, which would need to check dev->vq_index_end instead. 547 */ 548 static bool vhost_vdpa_first_dev(struct vhost_dev *dev) 549 { 550 struct vhost_vdpa *v = dev->opaque; 551 552 return v->index == 0; 553 } 554 555 static int vhost_vdpa_get_dev_features(struct vhost_dev *dev, 556 uint64_t *features) 557 { 558 int ret; 559 560 ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features); 561 trace_vhost_vdpa_get_features(dev, *features); 562 return ret; 563 } 564 565 static void vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v) 566 { 567 g_autoptr(GPtrArray) shadow_vqs = NULL; 568 569 shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free); 570 for (unsigned n = 0; n < hdev->nvqs; ++n) { 571 VhostShadowVirtqueue *svq; 572 573 svq = vhost_svq_new(v->shadow_vq_ops, v->shadow_vq_ops_opaque); 574 g_ptr_array_add(shadow_vqs, svq); 575 } 576 577 v->shadow_vqs = g_steal_pointer(&shadow_vqs); 578 } 579 580 static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp) 581 { 582 struct vhost_vdpa *v; 583 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 584 trace_vhost_vdpa_init(dev, opaque); 585 int ret; 586 587 v = opaque; 588 v->dev = dev; 589 dev->opaque = opaque ; 590 v->listener = vhost_vdpa_memory_listener; 591 v->msg_type = VHOST_IOTLB_MSG_V2; 592 vhost_vdpa_init_svq(dev, v); 593 594 error_propagate(&dev->migration_blocker, v->migration_blocker); 595 if (!vhost_vdpa_first_dev(dev)) { 596 return 0; 597 } 598 599 /* 600 * If dev->shadow_vqs_enabled at initialization that means the device has 601 * been started with x-svq=on, so don't block migration 602 */ 603 if (dev->migration_blocker == NULL && !v->shadow_vqs_enabled) { 604 /* We don't have dev->features yet */ 605 uint64_t features; 606 ret = vhost_vdpa_get_dev_features(dev, &features); 607 if (unlikely(ret)) { 608 error_setg_errno(errp, -ret, "Could not get device features"); 609 return ret; 610 } 611 vhost_svq_valid_features(features, &dev->migration_blocker); 612 } 613 614 /* 615 * Similar to VFIO, we end up pinning all guest memory and have to 616 * disable discarding of RAM. 617 */ 618 ret = ram_block_discard_disable(true); 619 if (ret) { 620 error_report("Cannot set discarding of RAM broken"); 621 return ret; 622 } 623 624 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 625 VIRTIO_CONFIG_S_DRIVER); 626 627 return 0; 628 } 629 630 static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev, 631 int queue_index) 632 { 633 size_t page_size = qemu_real_host_page_size(); 634 struct vhost_vdpa *v = dev->opaque; 635 VirtIODevice *vdev = dev->vdev; 636 VhostVDPAHostNotifier *n; 637 638 n = &v->notifier[queue_index]; 639 640 if (n->addr) { 641 virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, false); 642 object_unparent(OBJECT(&n->mr)); 643 munmap(n->addr, page_size); 644 n->addr = NULL; 645 } 646 } 647 648 static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int queue_index) 649 { 650 size_t page_size = qemu_real_host_page_size(); 651 struct vhost_vdpa *v = dev->opaque; 652 VirtIODevice *vdev = dev->vdev; 653 VhostVDPAHostNotifier *n; 654 int fd = v->device_fd; 655 void *addr; 656 char *name; 657 658 vhost_vdpa_host_notifier_uninit(dev, queue_index); 659 660 n = &v->notifier[queue_index]; 661 662 addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd, 663 queue_index * page_size); 664 if (addr == MAP_FAILED) { 665 goto err; 666 } 667 668 name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]", 669 v, queue_index); 670 memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name, 671 page_size, addr); 672 g_free(name); 673 674 if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, true)) { 675 object_unparent(OBJECT(&n->mr)); 676 munmap(addr, page_size); 677 goto err; 678 } 679 n->addr = addr; 680 681 return 0; 682 683 err: 684 return -1; 685 } 686 687 static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n) 688 { 689 int i; 690 691 /* 692 * Pack all the changes to the memory regions in a single 693 * transaction to avoid a few updating of the address space 694 * topology. 695 */ 696 memory_region_transaction_begin(); 697 698 for (i = dev->vq_index; i < dev->vq_index + n; i++) { 699 vhost_vdpa_host_notifier_uninit(dev, i); 700 } 701 702 memory_region_transaction_commit(); 703 } 704 705 static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev) 706 { 707 struct vhost_vdpa *v = dev->opaque; 708 int i; 709 710 if (v->shadow_vqs_enabled) { 711 /* FIXME SVQ is not compatible with host notifiers mr */ 712 return; 713 } 714 715 /* 716 * Pack all the changes to the memory regions in a single 717 * transaction to avoid a few updating of the address space 718 * topology. 719 */ 720 memory_region_transaction_begin(); 721 722 for (i = dev->vq_index; i < dev->vq_index + dev->nvqs; i++) { 723 if (vhost_vdpa_host_notifier_init(dev, i)) { 724 vhost_vdpa_host_notifiers_uninit(dev, i - dev->vq_index); 725 break; 726 } 727 } 728 729 memory_region_transaction_commit(); 730 } 731 732 static void vhost_vdpa_svq_cleanup(struct vhost_dev *dev) 733 { 734 struct vhost_vdpa *v = dev->opaque; 735 size_t idx; 736 737 for (idx = 0; idx < v->shadow_vqs->len; ++idx) { 738 vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, idx)); 739 } 740 g_ptr_array_free(v->shadow_vqs, true); 741 } 742 743 static int vhost_vdpa_cleanup(struct vhost_dev *dev) 744 { 745 struct vhost_vdpa *v; 746 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 747 v = dev->opaque; 748 trace_vhost_vdpa_cleanup(dev, v); 749 if (vhost_vdpa_first_dev(dev)) { 750 ram_block_discard_disable(false); 751 } 752 753 vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); 754 memory_listener_unregister(&v->listener); 755 vhost_vdpa_svq_cleanup(dev); 756 757 dev->opaque = NULL; 758 759 return 0; 760 } 761 762 static int vhost_vdpa_memslots_limit(struct vhost_dev *dev) 763 { 764 trace_vhost_vdpa_memslots_limit(dev, INT_MAX); 765 return INT_MAX; 766 } 767 768 static int vhost_vdpa_set_mem_table(struct vhost_dev *dev, 769 struct vhost_memory *mem) 770 { 771 if (!vhost_vdpa_first_dev(dev)) { 772 return 0; 773 } 774 775 trace_vhost_vdpa_set_mem_table(dev, mem->nregions, mem->padding); 776 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_MEM_TABLE) && 777 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_REGIONS)) { 778 int i; 779 for (i = 0; i < mem->nregions; i++) { 780 trace_vhost_vdpa_dump_regions(dev, i, 781 mem->regions[i].guest_phys_addr, 782 mem->regions[i].memory_size, 783 mem->regions[i].userspace_addr, 784 mem->regions[i].flags_padding); 785 } 786 } 787 if (mem->padding) { 788 return -EINVAL; 789 } 790 791 return 0; 792 } 793 794 static int vhost_vdpa_set_features(struct vhost_dev *dev, 795 uint64_t features) 796 { 797 struct vhost_vdpa *v = dev->opaque; 798 int ret; 799 800 if (!vhost_vdpa_first_dev(dev)) { 801 return 0; 802 } 803 804 if (v->shadow_vqs_enabled) { 805 if ((v->acked_features ^ features) == BIT_ULL(VHOST_F_LOG_ALL)) { 806 /* 807 * QEMU is just trying to enable or disable logging. SVQ handles 808 * this sepparately, so no need to forward this. 809 */ 810 v->acked_features = features; 811 return 0; 812 } 813 814 v->acked_features = features; 815 816 /* We must not ack _F_LOG if SVQ is enabled */ 817 features &= ~BIT_ULL(VHOST_F_LOG_ALL); 818 } 819 820 trace_vhost_vdpa_set_features(dev, features); 821 ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features); 822 if (ret) { 823 return ret; 824 } 825 826 return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK); 827 } 828 829 static int vhost_vdpa_set_backend_cap(struct vhost_dev *dev) 830 { 831 uint64_t features; 832 uint64_t f = 0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2 | 833 0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH | 834 0x1ULL << VHOST_BACKEND_F_IOTLB_ASID | 835 0x1ULL << VHOST_BACKEND_F_SUSPEND; 836 int r; 837 838 if (vhost_vdpa_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) { 839 return -EFAULT; 840 } 841 842 features &= f; 843 844 if (vhost_vdpa_first_dev(dev)) { 845 r = vhost_vdpa_call(dev, VHOST_SET_BACKEND_FEATURES, &features); 846 if (r) { 847 return -EFAULT; 848 } 849 } 850 851 dev->backend_cap = features; 852 853 return 0; 854 } 855 856 static int vhost_vdpa_get_device_id(struct vhost_dev *dev, 857 uint32_t *device_id) 858 { 859 int ret; 860 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id); 861 trace_vhost_vdpa_get_device_id(dev, *device_id); 862 return ret; 863 } 864 865 static int vhost_vdpa_reset_device(struct vhost_dev *dev) 866 { 867 struct vhost_vdpa *v = dev->opaque; 868 int ret; 869 uint8_t status = 0; 870 871 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status); 872 trace_vhost_vdpa_reset_device(dev); 873 v->suspended = false; 874 return ret; 875 } 876 877 static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx) 878 { 879 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs); 880 881 trace_vhost_vdpa_get_vq_index(dev, idx, idx); 882 return idx; 883 } 884 885 int vhost_vdpa_set_vring_ready(struct vhost_vdpa *v, unsigned idx) 886 { 887 struct vhost_dev *dev = v->dev; 888 struct vhost_vring_state state = { 889 .index = idx, 890 .num = 1, 891 }; 892 int r = vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state); 893 894 trace_vhost_vdpa_set_vring_ready(dev, idx, r); 895 return r; 896 } 897 898 static int vhost_vdpa_set_config_call(struct vhost_dev *dev, 899 int fd) 900 { 901 trace_vhost_vdpa_set_config_call(dev, fd); 902 return vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG_CALL, &fd); 903 } 904 905 static void vhost_vdpa_dump_config(struct vhost_dev *dev, const uint8_t *config, 906 uint32_t config_len) 907 { 908 int b, len; 909 char line[QEMU_HEXDUMP_LINE_LEN]; 910 911 for (b = 0; b < config_len; b += 16) { 912 len = config_len - b; 913 qemu_hexdump_line(line, b, config, len, false); 914 trace_vhost_vdpa_dump_config(dev, line); 915 } 916 } 917 918 static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data, 919 uint32_t offset, uint32_t size, 920 uint32_t flags) 921 { 922 struct vhost_vdpa_config *config; 923 int ret; 924 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 925 926 trace_vhost_vdpa_set_config(dev, offset, size, flags); 927 config = g_malloc(size + config_size); 928 config->off = offset; 929 config->len = size; 930 memcpy(config->buf, data, size); 931 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_CONFIG) && 932 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) { 933 vhost_vdpa_dump_config(dev, data, size); 934 } 935 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, config); 936 g_free(config); 937 return ret; 938 } 939 940 static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config, 941 uint32_t config_len, Error **errp) 942 { 943 struct vhost_vdpa_config *v_config; 944 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf); 945 int ret; 946 947 trace_vhost_vdpa_get_config(dev, config, config_len); 948 v_config = g_malloc(config_len + config_size); 949 v_config->len = config_len; 950 v_config->off = 0; 951 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, v_config); 952 memcpy(config, v_config->buf, config_len); 953 g_free(v_config); 954 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_GET_CONFIG) && 955 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) { 956 vhost_vdpa_dump_config(dev, config, config_len); 957 } 958 return ret; 959 } 960 961 static int vhost_vdpa_set_dev_vring_base(struct vhost_dev *dev, 962 struct vhost_vring_state *ring) 963 { 964 trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num); 965 return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring); 966 } 967 968 static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev, 969 struct vhost_vring_file *file) 970 { 971 trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd); 972 return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file); 973 } 974 975 static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev, 976 struct vhost_vring_file *file) 977 { 978 trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd); 979 return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file); 980 } 981 982 static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev, 983 struct vhost_vring_addr *addr) 984 { 985 trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags, 986 addr->desc_user_addr, addr->used_user_addr, 987 addr->avail_user_addr, 988 addr->log_guest_addr); 989 990 return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr); 991 992 } 993 994 /** 995 * Set the shadow virtqueue descriptors to the device 996 * 997 * @dev: The vhost device model 998 * @svq: The shadow virtqueue 999 * @idx: The index of the virtqueue in the vhost device 1000 * @errp: Error 1001 * 1002 * Note that this function does not rewind kick file descriptor if cannot set 1003 * call one. 1004 */ 1005 static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev, 1006 VhostShadowVirtqueue *svq, unsigned idx, 1007 Error **errp) 1008 { 1009 struct vhost_vring_file file = { 1010 .index = dev->vq_index + idx, 1011 }; 1012 const EventNotifier *event_notifier = &svq->hdev_kick; 1013 int r; 1014 1015 r = event_notifier_init(&svq->hdev_kick, 0); 1016 if (r != 0) { 1017 error_setg_errno(errp, -r, "Couldn't create kick event notifier"); 1018 goto err_init_hdev_kick; 1019 } 1020 1021 r = event_notifier_init(&svq->hdev_call, 0); 1022 if (r != 0) { 1023 error_setg_errno(errp, -r, "Couldn't create call event notifier"); 1024 goto err_init_hdev_call; 1025 } 1026 1027 file.fd = event_notifier_get_fd(event_notifier); 1028 r = vhost_vdpa_set_vring_dev_kick(dev, &file); 1029 if (unlikely(r != 0)) { 1030 error_setg_errno(errp, -r, "Can't set device kick fd"); 1031 goto err_init_set_dev_fd; 1032 } 1033 1034 event_notifier = &svq->hdev_call; 1035 file.fd = event_notifier_get_fd(event_notifier); 1036 r = vhost_vdpa_set_vring_dev_call(dev, &file); 1037 if (unlikely(r != 0)) { 1038 error_setg_errno(errp, -r, "Can't set device call fd"); 1039 goto err_init_set_dev_fd; 1040 } 1041 1042 return 0; 1043 1044 err_init_set_dev_fd: 1045 event_notifier_set_handler(&svq->hdev_call, NULL); 1046 1047 err_init_hdev_call: 1048 event_notifier_cleanup(&svq->hdev_kick); 1049 1050 err_init_hdev_kick: 1051 return r; 1052 } 1053 1054 /** 1055 * Unmap a SVQ area in the device 1056 */ 1057 static void vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr addr) 1058 { 1059 const DMAMap needle = { 1060 .translated_addr = addr, 1061 }; 1062 const DMAMap *result = vhost_iova_tree_find_iova(v->iova_tree, &needle); 1063 hwaddr size; 1064 int r; 1065 1066 if (unlikely(!result)) { 1067 error_report("Unable to find SVQ address to unmap"); 1068 return; 1069 } 1070 1071 size = ROUND_UP(result->size, qemu_real_host_page_size()); 1072 r = vhost_vdpa_dma_unmap(v, v->address_space_id, result->iova, size); 1073 if (unlikely(r < 0)) { 1074 error_report("Unable to unmap SVQ vring: %s (%d)", g_strerror(-r), -r); 1075 return; 1076 } 1077 1078 vhost_iova_tree_remove(v->iova_tree, *result); 1079 } 1080 1081 static void vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev, 1082 const VhostShadowVirtqueue *svq) 1083 { 1084 struct vhost_vdpa *v = dev->opaque; 1085 struct vhost_vring_addr svq_addr; 1086 1087 vhost_svq_get_vring_addr(svq, &svq_addr); 1088 1089 vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr); 1090 1091 vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr); 1092 } 1093 1094 /** 1095 * Map the SVQ area in the device 1096 * 1097 * @v: Vhost-vdpa device 1098 * @needle: The area to search iova 1099 * @errorp: Error pointer 1100 */ 1101 static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa *v, DMAMap *needle, 1102 Error **errp) 1103 { 1104 int r; 1105 1106 r = vhost_iova_tree_map_alloc(v->iova_tree, needle); 1107 if (unlikely(r != IOVA_OK)) { 1108 error_setg(errp, "Cannot allocate iova (%d)", r); 1109 return false; 1110 } 1111 1112 r = vhost_vdpa_dma_map(v, v->address_space_id, needle->iova, 1113 needle->size + 1, 1114 (void *)(uintptr_t)needle->translated_addr, 1115 needle->perm == IOMMU_RO); 1116 if (unlikely(r != 0)) { 1117 error_setg_errno(errp, -r, "Cannot map region to device"); 1118 vhost_iova_tree_remove(v->iova_tree, *needle); 1119 } 1120 1121 return r == 0; 1122 } 1123 1124 /** 1125 * Map the shadow virtqueue rings in the device 1126 * 1127 * @dev: The vhost device 1128 * @svq: The shadow virtqueue 1129 * @addr: Assigned IOVA addresses 1130 * @errp: Error pointer 1131 */ 1132 static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev, 1133 const VhostShadowVirtqueue *svq, 1134 struct vhost_vring_addr *addr, 1135 Error **errp) 1136 { 1137 ERRP_GUARD(); 1138 DMAMap device_region, driver_region; 1139 struct vhost_vring_addr svq_addr; 1140 struct vhost_vdpa *v = dev->opaque; 1141 size_t device_size = vhost_svq_device_area_size(svq); 1142 size_t driver_size = vhost_svq_driver_area_size(svq); 1143 size_t avail_offset; 1144 bool ok; 1145 1146 vhost_svq_get_vring_addr(svq, &svq_addr); 1147 1148 driver_region = (DMAMap) { 1149 .translated_addr = svq_addr.desc_user_addr, 1150 .size = driver_size - 1, 1151 .perm = IOMMU_RO, 1152 }; 1153 ok = vhost_vdpa_svq_map_ring(v, &driver_region, errp); 1154 if (unlikely(!ok)) { 1155 error_prepend(errp, "Cannot create vq driver region: "); 1156 return false; 1157 } 1158 addr->desc_user_addr = driver_region.iova; 1159 avail_offset = svq_addr.avail_user_addr - svq_addr.desc_user_addr; 1160 addr->avail_user_addr = driver_region.iova + avail_offset; 1161 1162 device_region = (DMAMap) { 1163 .translated_addr = svq_addr.used_user_addr, 1164 .size = device_size - 1, 1165 .perm = IOMMU_RW, 1166 }; 1167 ok = vhost_vdpa_svq_map_ring(v, &device_region, errp); 1168 if (unlikely(!ok)) { 1169 error_prepend(errp, "Cannot create vq device region: "); 1170 vhost_vdpa_svq_unmap_ring(v, driver_region.translated_addr); 1171 } 1172 addr->used_user_addr = device_region.iova; 1173 1174 return ok; 1175 } 1176 1177 static bool vhost_vdpa_svq_setup(struct vhost_dev *dev, 1178 VhostShadowVirtqueue *svq, unsigned idx, 1179 Error **errp) 1180 { 1181 uint16_t vq_index = dev->vq_index + idx; 1182 struct vhost_vring_state s = { 1183 .index = vq_index, 1184 }; 1185 int r; 1186 1187 r = vhost_vdpa_set_dev_vring_base(dev, &s); 1188 if (unlikely(r)) { 1189 error_setg_errno(errp, -r, "Cannot set vring base"); 1190 return false; 1191 } 1192 1193 r = vhost_vdpa_svq_set_fds(dev, svq, idx, errp); 1194 return r == 0; 1195 } 1196 1197 static bool vhost_vdpa_svqs_start(struct vhost_dev *dev) 1198 { 1199 struct vhost_vdpa *v = dev->opaque; 1200 Error *err = NULL; 1201 unsigned i; 1202 1203 if (!v->shadow_vqs_enabled) { 1204 return true; 1205 } 1206 1207 for (i = 0; i < v->shadow_vqs->len; ++i) { 1208 VirtQueue *vq = virtio_get_queue(dev->vdev, dev->vq_index + i); 1209 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i); 1210 struct vhost_vring_addr addr = { 1211 .index = dev->vq_index + i, 1212 }; 1213 int r; 1214 bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err); 1215 if (unlikely(!ok)) { 1216 goto err; 1217 } 1218 1219 vhost_svq_start(svq, dev->vdev, vq, v->iova_tree); 1220 ok = vhost_vdpa_svq_map_rings(dev, svq, &addr, &err); 1221 if (unlikely(!ok)) { 1222 goto err_map; 1223 } 1224 1225 /* Override vring GPA set by vhost subsystem */ 1226 r = vhost_vdpa_set_vring_dev_addr(dev, &addr); 1227 if (unlikely(r != 0)) { 1228 error_setg_errno(&err, -r, "Cannot set device address"); 1229 goto err_set_addr; 1230 } 1231 } 1232 1233 return true; 1234 1235 err_set_addr: 1236 vhost_vdpa_svq_unmap_rings(dev, g_ptr_array_index(v->shadow_vqs, i)); 1237 1238 err_map: 1239 vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, i)); 1240 1241 err: 1242 error_reportf_err(err, "Cannot setup SVQ %u: ", i); 1243 for (unsigned j = 0; j < i; ++j) { 1244 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, j); 1245 vhost_vdpa_svq_unmap_rings(dev, svq); 1246 vhost_svq_stop(svq); 1247 } 1248 1249 return false; 1250 } 1251 1252 static void vhost_vdpa_svqs_stop(struct vhost_dev *dev) 1253 { 1254 struct vhost_vdpa *v = dev->opaque; 1255 1256 if (!v->shadow_vqs_enabled) { 1257 return; 1258 } 1259 1260 for (unsigned i = 0; i < v->shadow_vqs->len; ++i) { 1261 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i); 1262 1263 vhost_svq_stop(svq); 1264 vhost_vdpa_svq_unmap_rings(dev, svq); 1265 1266 event_notifier_cleanup(&svq->hdev_kick); 1267 event_notifier_cleanup(&svq->hdev_call); 1268 } 1269 } 1270 1271 static void vhost_vdpa_suspend(struct vhost_dev *dev) 1272 { 1273 struct vhost_vdpa *v = dev->opaque; 1274 int r; 1275 1276 if (!vhost_vdpa_first_dev(dev)) { 1277 return; 1278 } 1279 1280 if (dev->backend_cap & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) { 1281 trace_vhost_vdpa_suspend(dev); 1282 r = ioctl(v->device_fd, VHOST_VDPA_SUSPEND); 1283 if (unlikely(r)) { 1284 error_report("Cannot suspend: %s(%d)", g_strerror(errno), errno); 1285 } else { 1286 v->suspended = true; 1287 return; 1288 } 1289 } 1290 1291 vhost_vdpa_reset_device(dev); 1292 } 1293 1294 static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) 1295 { 1296 struct vhost_vdpa *v = dev->opaque; 1297 bool ok; 1298 trace_vhost_vdpa_dev_start(dev, started); 1299 1300 if (started) { 1301 vhost_vdpa_host_notifiers_init(dev); 1302 ok = vhost_vdpa_svqs_start(dev); 1303 if (unlikely(!ok)) { 1304 return -1; 1305 } 1306 } else { 1307 vhost_vdpa_suspend(dev); 1308 vhost_vdpa_svqs_stop(dev); 1309 vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); 1310 } 1311 1312 if (dev->vq_index + dev->nvqs != dev->vq_index_end) { 1313 return 0; 1314 } 1315 1316 if (started) { 1317 if (vhost_dev_has_iommu(dev) && (v->shadow_vqs_enabled)) { 1318 error_report("SVQ can not work while IOMMU enable, please disable" 1319 "IOMMU and try again"); 1320 return -1; 1321 } 1322 memory_listener_register(&v->listener, dev->vdev->dma_as); 1323 1324 return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK); 1325 } 1326 1327 return 0; 1328 } 1329 1330 static void vhost_vdpa_reset_status(struct vhost_dev *dev) 1331 { 1332 struct vhost_vdpa *v = dev->opaque; 1333 1334 if (dev->vq_index + dev->nvqs != dev->vq_index_end) { 1335 return; 1336 } 1337 1338 vhost_vdpa_reset_device(dev); 1339 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | 1340 VIRTIO_CONFIG_S_DRIVER); 1341 memory_listener_unregister(&v->listener); 1342 } 1343 1344 static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base, 1345 struct vhost_log *log) 1346 { 1347 struct vhost_vdpa *v = dev->opaque; 1348 if (v->shadow_vqs_enabled || !vhost_vdpa_first_dev(dev)) { 1349 return 0; 1350 } 1351 1352 trace_vhost_vdpa_set_log_base(dev, base, log->size, log->refcnt, log->fd, 1353 log->log); 1354 return vhost_vdpa_call(dev, VHOST_SET_LOG_BASE, &base); 1355 } 1356 1357 static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev, 1358 struct vhost_vring_addr *addr) 1359 { 1360 struct vhost_vdpa *v = dev->opaque; 1361 1362 if (v->shadow_vqs_enabled) { 1363 /* 1364 * Device vring addr was set at device start. SVQ base is handled by 1365 * VirtQueue code. 1366 */ 1367 return 0; 1368 } 1369 1370 return vhost_vdpa_set_vring_dev_addr(dev, addr); 1371 } 1372 1373 static int vhost_vdpa_set_vring_num(struct vhost_dev *dev, 1374 struct vhost_vring_state *ring) 1375 { 1376 trace_vhost_vdpa_set_vring_num(dev, ring->index, ring->num); 1377 return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring); 1378 } 1379 1380 static int vhost_vdpa_set_vring_base(struct vhost_dev *dev, 1381 struct vhost_vring_state *ring) 1382 { 1383 struct vhost_vdpa *v = dev->opaque; 1384 1385 if (v->shadow_vqs_enabled) { 1386 /* 1387 * Device vring base was set at device start. SVQ base is handled by 1388 * VirtQueue code. 1389 */ 1390 return 0; 1391 } 1392 1393 return vhost_vdpa_set_dev_vring_base(dev, ring); 1394 } 1395 1396 static int vhost_vdpa_get_vring_base(struct vhost_dev *dev, 1397 struct vhost_vring_state *ring) 1398 { 1399 struct vhost_vdpa *v = dev->opaque; 1400 int ret; 1401 1402 if (v->shadow_vqs_enabled) { 1403 ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index); 1404 return 0; 1405 } 1406 1407 if (!v->suspended) { 1408 /* 1409 * Cannot trust in value returned by device, let vhost recover used 1410 * idx from guest. 1411 */ 1412 return -1; 1413 } 1414 1415 ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring); 1416 trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num); 1417 return ret; 1418 } 1419 1420 static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev, 1421 struct vhost_vring_file *file) 1422 { 1423 struct vhost_vdpa *v = dev->opaque; 1424 int vdpa_idx = file->index - dev->vq_index; 1425 1426 if (v->shadow_vqs_enabled) { 1427 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); 1428 vhost_svq_set_svq_kick_fd(svq, file->fd); 1429 return 0; 1430 } else { 1431 return vhost_vdpa_set_vring_dev_kick(dev, file); 1432 } 1433 } 1434 1435 static int vhost_vdpa_set_vring_call(struct vhost_dev *dev, 1436 struct vhost_vring_file *file) 1437 { 1438 struct vhost_vdpa *v = dev->opaque; 1439 int vdpa_idx = file->index - dev->vq_index; 1440 VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); 1441 1442 /* Remember last call fd because we can switch to SVQ anytime. */ 1443 vhost_svq_set_svq_call_fd(svq, file->fd); 1444 if (v->shadow_vqs_enabled) { 1445 return 0; 1446 } 1447 1448 return vhost_vdpa_set_vring_dev_call(dev, file); 1449 } 1450 1451 static int vhost_vdpa_get_features(struct vhost_dev *dev, 1452 uint64_t *features) 1453 { 1454 int ret = vhost_vdpa_get_dev_features(dev, features); 1455 1456 if (ret == 0) { 1457 /* Add SVQ logging capabilities */ 1458 *features |= BIT_ULL(VHOST_F_LOG_ALL); 1459 } 1460 1461 return ret; 1462 } 1463 1464 static int vhost_vdpa_set_owner(struct vhost_dev *dev) 1465 { 1466 if (!vhost_vdpa_first_dev(dev)) { 1467 return 0; 1468 } 1469 1470 trace_vhost_vdpa_set_owner(dev); 1471 return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL); 1472 } 1473 1474 static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev, 1475 struct vhost_vring_addr *addr, struct vhost_virtqueue *vq) 1476 { 1477 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); 1478 addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys; 1479 addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys; 1480 addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys; 1481 trace_vhost_vdpa_vq_get_addr(dev, vq, addr->desc_user_addr, 1482 addr->avail_user_addr, addr->used_user_addr); 1483 return 0; 1484 } 1485 1486 static bool vhost_vdpa_force_iommu(struct vhost_dev *dev) 1487 { 1488 return true; 1489 } 1490 1491 const VhostOps vdpa_ops = { 1492 .backend_type = VHOST_BACKEND_TYPE_VDPA, 1493 .vhost_backend_init = vhost_vdpa_init, 1494 .vhost_backend_cleanup = vhost_vdpa_cleanup, 1495 .vhost_set_log_base = vhost_vdpa_set_log_base, 1496 .vhost_set_vring_addr = vhost_vdpa_set_vring_addr, 1497 .vhost_set_vring_num = vhost_vdpa_set_vring_num, 1498 .vhost_set_vring_base = vhost_vdpa_set_vring_base, 1499 .vhost_get_vring_base = vhost_vdpa_get_vring_base, 1500 .vhost_set_vring_kick = vhost_vdpa_set_vring_kick, 1501 .vhost_set_vring_call = vhost_vdpa_set_vring_call, 1502 .vhost_get_features = vhost_vdpa_get_features, 1503 .vhost_set_backend_cap = vhost_vdpa_set_backend_cap, 1504 .vhost_set_owner = vhost_vdpa_set_owner, 1505 .vhost_set_vring_endian = NULL, 1506 .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit, 1507 .vhost_set_mem_table = vhost_vdpa_set_mem_table, 1508 .vhost_set_features = vhost_vdpa_set_features, 1509 .vhost_reset_device = vhost_vdpa_reset_device, 1510 .vhost_get_vq_index = vhost_vdpa_get_vq_index, 1511 .vhost_get_config = vhost_vdpa_get_config, 1512 .vhost_set_config = vhost_vdpa_set_config, 1513 .vhost_requires_shm_log = NULL, 1514 .vhost_migration_done = NULL, 1515 .vhost_net_set_mtu = NULL, 1516 .vhost_set_iotlb_callback = NULL, 1517 .vhost_send_device_iotlb_msg = NULL, 1518 .vhost_dev_start = vhost_vdpa_dev_start, 1519 .vhost_get_device_id = vhost_vdpa_get_device_id, 1520 .vhost_vq_get_addr = vhost_vdpa_vq_get_addr, 1521 .vhost_force_iommu = vhost_vdpa_force_iommu, 1522 .vhost_set_config_call = vhost_vdpa_set_config_call, 1523 .vhost_reset_status = vhost_vdpa_reset_status, 1524 }; 1525