1 /* 2 * generic functions used by VFIO devices 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Alex Williamson <alex.williamson@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Based on qemu-kvm device-assignment: 13 * Adapted for KVM by Qumranet. 14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) 15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) 16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) 17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) 18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) 19 */ 20 21 #include "qemu/osdep.h" 22 #include <sys/ioctl.h> 23 #ifdef CONFIG_KVM 24 #include <linux/kvm.h> 25 #endif 26 #include <linux/vfio.h> 27 28 #include "hw/vfio/vfio-common.h" 29 #include "hw/vfio/vfio.h" 30 #include "hw/vfio/pci.h" 31 #include "exec/address-spaces.h" 32 #include "exec/memory.h" 33 #include "exec/ram_addr.h" 34 #include "hw/hw.h" 35 #include "qemu/error-report.h" 36 #include "qemu/main-loop.h" 37 #include "qemu/range.h" 38 #include "sysemu/kvm.h" 39 #include "sysemu/reset.h" 40 #include "sysemu/runstate.h" 41 #include "trace.h" 42 #include "qapi/error.h" 43 #include "migration/migration.h" 44 #include "migration/misc.h" 45 #include "migration/blocker.h" 46 #include "migration/qemu-file.h" 47 #include "sysemu/tpm.h" 48 49 VFIODeviceList vfio_device_list = 50 QLIST_HEAD_INITIALIZER(vfio_device_list); 51 static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces = 52 QLIST_HEAD_INITIALIZER(vfio_address_spaces); 53 54 #ifdef CONFIG_KVM 55 /* 56 * We have a single VFIO pseudo device per KVM VM. Once created it lives 57 * for the life of the VM. Closing the file descriptor only drops our 58 * reference to it and the device's reference to kvm. Therefore once 59 * initialized, this file descriptor is only released on QEMU exit and 60 * we'll re-use it should another vfio device be attached before then. 61 */ 62 int vfio_kvm_device_fd = -1; 63 #endif 64 65 /* 66 * Device state interfaces 67 */ 68 69 bool vfio_mig_active(void) 70 { 71 VFIODevice *vbasedev; 72 73 if (QLIST_EMPTY(&vfio_device_list)) { 74 return false; 75 } 76 77 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 78 if (vbasedev->migration_blocker) { 79 return false; 80 } 81 } 82 return true; 83 } 84 85 static Error *multiple_devices_migration_blocker; 86 87 /* 88 * Multiple devices migration is allowed only if all devices support P2P 89 * migration. Single device migration is allowed regardless of P2P migration 90 * support. 91 */ 92 static bool vfio_multiple_devices_migration_is_supported(void) 93 { 94 VFIODevice *vbasedev; 95 unsigned int device_num = 0; 96 bool all_support_p2p = true; 97 98 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 99 if (vbasedev->migration) { 100 device_num++; 101 102 if (!(vbasedev->migration->mig_flags & VFIO_MIGRATION_P2P)) { 103 all_support_p2p = false; 104 } 105 } 106 } 107 108 return all_support_p2p || device_num <= 1; 109 } 110 111 int vfio_block_multiple_devices_migration(VFIODevice *vbasedev, Error **errp) 112 { 113 int ret; 114 115 if (vfio_multiple_devices_migration_is_supported()) { 116 return 0; 117 } 118 119 if (vbasedev->enable_migration == ON_OFF_AUTO_ON) { 120 error_setg(errp, "Multiple VFIO devices migration is supported only if " 121 "all of them support P2P migration"); 122 return -EINVAL; 123 } 124 125 if (multiple_devices_migration_blocker) { 126 return 0; 127 } 128 129 error_setg(&multiple_devices_migration_blocker, 130 "Multiple VFIO devices migration is supported only if all of " 131 "them support P2P migration"); 132 ret = migrate_add_blocker(multiple_devices_migration_blocker, errp); 133 if (ret < 0) { 134 error_free(multiple_devices_migration_blocker); 135 multiple_devices_migration_blocker = NULL; 136 } 137 138 return ret; 139 } 140 141 void vfio_unblock_multiple_devices_migration(void) 142 { 143 if (!multiple_devices_migration_blocker || 144 !vfio_multiple_devices_migration_is_supported()) { 145 return; 146 } 147 148 migrate_del_blocker(multiple_devices_migration_blocker); 149 error_free(multiple_devices_migration_blocker); 150 multiple_devices_migration_blocker = NULL; 151 } 152 153 bool vfio_viommu_preset(VFIODevice *vbasedev) 154 { 155 return vbasedev->container->space->as != &address_space_memory; 156 } 157 158 static void vfio_set_migration_error(int err) 159 { 160 MigrationState *ms = migrate_get_current(); 161 162 if (migration_is_setup_or_active(ms->state)) { 163 WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) { 164 if (ms->to_dst_file) { 165 qemu_file_set_error(ms->to_dst_file, err); 166 } 167 } 168 } 169 } 170 171 bool vfio_device_state_is_running(VFIODevice *vbasedev) 172 { 173 VFIOMigration *migration = vbasedev->migration; 174 175 return migration->device_state == VFIO_DEVICE_STATE_RUNNING || 176 migration->device_state == VFIO_DEVICE_STATE_RUNNING_P2P; 177 } 178 179 bool vfio_device_state_is_precopy(VFIODevice *vbasedev) 180 { 181 VFIOMigration *migration = vbasedev->migration; 182 183 return migration->device_state == VFIO_DEVICE_STATE_PRE_COPY || 184 migration->device_state == VFIO_DEVICE_STATE_PRE_COPY_P2P; 185 } 186 187 static bool vfio_devices_all_dirty_tracking(VFIOContainer *container) 188 { 189 VFIODevice *vbasedev; 190 MigrationState *ms = migrate_get_current(); 191 192 if (ms->state != MIGRATION_STATUS_ACTIVE && 193 ms->state != MIGRATION_STATUS_DEVICE) { 194 return false; 195 } 196 197 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 198 VFIOMigration *migration = vbasedev->migration; 199 200 if (!migration) { 201 return false; 202 } 203 204 if (vbasedev->pre_copy_dirty_page_tracking == ON_OFF_AUTO_OFF && 205 (vfio_device_state_is_running(vbasedev) || 206 vfio_device_state_is_precopy(vbasedev))) { 207 return false; 208 } 209 } 210 return true; 211 } 212 213 bool vfio_devices_all_device_dirty_tracking(VFIOContainer *container) 214 { 215 VFIODevice *vbasedev; 216 217 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 218 if (!vbasedev->dirty_pages_supported) { 219 return false; 220 } 221 } 222 223 return true; 224 } 225 226 /* 227 * Check if all VFIO devices are running and migration is active, which is 228 * essentially equivalent to the migration being in pre-copy phase. 229 */ 230 bool vfio_devices_all_running_and_mig_active(VFIOContainer *container) 231 { 232 VFIODevice *vbasedev; 233 234 if (!migration_is_active(migrate_get_current())) { 235 return false; 236 } 237 238 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 239 VFIOMigration *migration = vbasedev->migration; 240 241 if (!migration) { 242 return false; 243 } 244 245 if (vfio_device_state_is_running(vbasedev) || 246 vfio_device_state_is_precopy(vbasedev)) { 247 continue; 248 } else { 249 return false; 250 } 251 } 252 return true; 253 } 254 255 void vfio_host_win_add(VFIOContainer *container, hwaddr min_iova, 256 hwaddr max_iova, uint64_t iova_pgsizes) 257 { 258 VFIOHostDMAWindow *hostwin; 259 260 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 261 if (ranges_overlap(hostwin->min_iova, 262 hostwin->max_iova - hostwin->min_iova + 1, 263 min_iova, 264 max_iova - min_iova + 1)) { 265 hw_error("%s: Overlapped IOMMU are not enabled", __func__); 266 } 267 } 268 269 hostwin = g_malloc0(sizeof(*hostwin)); 270 271 hostwin->min_iova = min_iova; 272 hostwin->max_iova = max_iova; 273 hostwin->iova_pgsizes = iova_pgsizes; 274 QLIST_INSERT_HEAD(&container->hostwin_list, hostwin, hostwin_next); 275 } 276 277 int vfio_host_win_del(VFIOContainer *container, 278 hwaddr min_iova, hwaddr max_iova) 279 { 280 VFIOHostDMAWindow *hostwin; 281 282 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 283 if (hostwin->min_iova == min_iova && hostwin->max_iova == max_iova) { 284 QLIST_REMOVE(hostwin, hostwin_next); 285 g_free(hostwin); 286 return 0; 287 } 288 } 289 290 return -1; 291 } 292 293 static bool vfio_listener_skipped_section(MemoryRegionSection *section) 294 { 295 return (!memory_region_is_ram(section->mr) && 296 !memory_region_is_iommu(section->mr)) || 297 memory_region_is_protected(section->mr) || 298 /* 299 * Sizing an enabled 64-bit BAR can cause spurious mappings to 300 * addresses in the upper part of the 64-bit address space. These 301 * are never accessed by the CPU and beyond the address width of 302 * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width. 303 */ 304 section->offset_within_address_space & (1ULL << 63); 305 } 306 307 /* Called with rcu_read_lock held. */ 308 static bool vfio_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, 309 ram_addr_t *ram_addr, bool *read_only) 310 { 311 bool ret, mr_has_discard_manager; 312 313 ret = memory_get_xlat_addr(iotlb, vaddr, ram_addr, read_only, 314 &mr_has_discard_manager); 315 if (ret && mr_has_discard_manager) { 316 /* 317 * Malicious VMs might trigger discarding of IOMMU-mapped memory. The 318 * pages will remain pinned inside vfio until unmapped, resulting in a 319 * higher memory consumption than expected. If memory would get 320 * populated again later, there would be an inconsistency between pages 321 * pinned by vfio and pages seen by QEMU. This is the case until 322 * unmapped from the IOMMU (e.g., during device reset). 323 * 324 * With malicious guests, we really only care about pinning more memory 325 * than expected. RLIMIT_MEMLOCK set for the user/process can never be 326 * exceeded and can be used to mitigate this problem. 327 */ 328 warn_report_once("Using vfio with vIOMMUs and coordinated discarding of" 329 " RAM (e.g., virtio-mem) works, however, malicious" 330 " guests can trigger pinning of more memory than" 331 " intended via an IOMMU. It's possible to mitigate " 332 " by setting/adjusting RLIMIT_MEMLOCK."); 333 } 334 return ret; 335 } 336 337 static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 338 { 339 VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n); 340 VFIOContainer *container = giommu->container; 341 hwaddr iova = iotlb->iova + giommu->iommu_offset; 342 void *vaddr; 343 int ret; 344 345 trace_vfio_iommu_map_notify(iotlb->perm == IOMMU_NONE ? "UNMAP" : "MAP", 346 iova, iova + iotlb->addr_mask); 347 348 if (iotlb->target_as != &address_space_memory) { 349 error_report("Wrong target AS \"%s\", only system memory is allowed", 350 iotlb->target_as->name ? iotlb->target_as->name : "none"); 351 vfio_set_migration_error(-EINVAL); 352 return; 353 } 354 355 rcu_read_lock(); 356 357 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 358 bool read_only; 359 360 if (!vfio_get_xlat_addr(iotlb, &vaddr, NULL, &read_only)) { 361 goto out; 362 } 363 /* 364 * vaddr is only valid until rcu_read_unlock(). But after 365 * vfio_dma_map has set up the mapping the pages will be 366 * pinned by the kernel. This makes sure that the RAM backend 367 * of vaddr will always be there, even if the memory object is 368 * destroyed and its backing memory munmap-ed. 369 */ 370 ret = vfio_dma_map(container, iova, 371 iotlb->addr_mask + 1, vaddr, 372 read_only); 373 if (ret) { 374 error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", " 375 "0x%"HWADDR_PRIx", %p) = %d (%s)", 376 container, iova, 377 iotlb->addr_mask + 1, vaddr, ret, strerror(-ret)); 378 } 379 } else { 380 ret = vfio_dma_unmap(container, iova, iotlb->addr_mask + 1, iotlb); 381 if (ret) { 382 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 383 "0x%"HWADDR_PRIx") = %d (%s)", 384 container, iova, 385 iotlb->addr_mask + 1, ret, strerror(-ret)); 386 vfio_set_migration_error(ret); 387 } 388 } 389 out: 390 rcu_read_unlock(); 391 } 392 393 static void vfio_ram_discard_notify_discard(RamDiscardListener *rdl, 394 MemoryRegionSection *section) 395 { 396 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 397 listener); 398 const hwaddr size = int128_get64(section->size); 399 const hwaddr iova = section->offset_within_address_space; 400 int ret; 401 402 /* Unmap with a single call. */ 403 ret = vfio_dma_unmap(vrdl->container, iova, size , NULL); 404 if (ret) { 405 error_report("%s: vfio_dma_unmap() failed: %s", __func__, 406 strerror(-ret)); 407 } 408 } 409 410 static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl, 411 MemoryRegionSection *section) 412 { 413 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 414 listener); 415 const hwaddr end = section->offset_within_region + 416 int128_get64(section->size); 417 hwaddr start, next, iova; 418 void *vaddr; 419 int ret; 420 421 /* 422 * Map in (aligned within memory region) minimum granularity, so we can 423 * unmap in minimum granularity later. 424 */ 425 for (start = section->offset_within_region; start < end; start = next) { 426 next = ROUND_UP(start + 1, vrdl->granularity); 427 next = MIN(next, end); 428 429 iova = start - section->offset_within_region + 430 section->offset_within_address_space; 431 vaddr = memory_region_get_ram_ptr(section->mr) + start; 432 433 ret = vfio_dma_map(vrdl->container, iova, next - start, 434 vaddr, section->readonly); 435 if (ret) { 436 /* Rollback */ 437 vfio_ram_discard_notify_discard(rdl, section); 438 return ret; 439 } 440 } 441 return 0; 442 } 443 444 static void vfio_register_ram_discard_listener(VFIOContainer *container, 445 MemoryRegionSection *section) 446 { 447 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 448 VFIORamDiscardListener *vrdl; 449 450 /* Ignore some corner cases not relevant in practice. */ 451 g_assert(QEMU_IS_ALIGNED(section->offset_within_region, TARGET_PAGE_SIZE)); 452 g_assert(QEMU_IS_ALIGNED(section->offset_within_address_space, 453 TARGET_PAGE_SIZE)); 454 g_assert(QEMU_IS_ALIGNED(int128_get64(section->size), TARGET_PAGE_SIZE)); 455 456 vrdl = g_new0(VFIORamDiscardListener, 1); 457 vrdl->container = container; 458 vrdl->mr = section->mr; 459 vrdl->offset_within_address_space = section->offset_within_address_space; 460 vrdl->size = int128_get64(section->size); 461 vrdl->granularity = ram_discard_manager_get_min_granularity(rdm, 462 section->mr); 463 464 g_assert(vrdl->granularity && is_power_of_2(vrdl->granularity)); 465 g_assert(container->pgsizes && 466 vrdl->granularity >= 1ULL << ctz64(container->pgsizes)); 467 468 ram_discard_listener_init(&vrdl->listener, 469 vfio_ram_discard_notify_populate, 470 vfio_ram_discard_notify_discard, true); 471 ram_discard_manager_register_listener(rdm, &vrdl->listener, section); 472 QLIST_INSERT_HEAD(&container->vrdl_list, vrdl, next); 473 474 /* 475 * Sanity-check if we have a theoretically problematic setup where we could 476 * exceed the maximum number of possible DMA mappings over time. We assume 477 * that each mapped section in the same address space as a RamDiscardManager 478 * section consumes exactly one DMA mapping, with the exception of 479 * RamDiscardManager sections; i.e., we don't expect to have gIOMMU sections 480 * in the same address space as RamDiscardManager sections. 481 * 482 * We assume that each section in the address space consumes one memslot. 483 * We take the number of KVM memory slots as a best guess for the maximum 484 * number of sections in the address space we could have over time, 485 * also consuming DMA mappings. 486 */ 487 if (container->dma_max_mappings) { 488 unsigned int vrdl_count = 0, vrdl_mappings = 0, max_memslots = 512; 489 490 #ifdef CONFIG_KVM 491 if (kvm_enabled()) { 492 max_memslots = kvm_get_max_memslots(); 493 } 494 #endif 495 496 QLIST_FOREACH(vrdl, &container->vrdl_list, next) { 497 hwaddr start, end; 498 499 start = QEMU_ALIGN_DOWN(vrdl->offset_within_address_space, 500 vrdl->granularity); 501 end = ROUND_UP(vrdl->offset_within_address_space + vrdl->size, 502 vrdl->granularity); 503 vrdl_mappings += (end - start) / vrdl->granularity; 504 vrdl_count++; 505 } 506 507 if (vrdl_mappings + max_memslots - vrdl_count > 508 container->dma_max_mappings) { 509 warn_report("%s: possibly running out of DMA mappings. E.g., try" 510 " increasing the 'block-size' of virtio-mem devies." 511 " Maximum possible DMA mappings: %d, Maximum possible" 512 " memslots: %d", __func__, container->dma_max_mappings, 513 max_memslots); 514 } 515 } 516 } 517 518 static void vfio_unregister_ram_discard_listener(VFIOContainer *container, 519 MemoryRegionSection *section) 520 { 521 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 522 VFIORamDiscardListener *vrdl = NULL; 523 524 QLIST_FOREACH(vrdl, &container->vrdl_list, next) { 525 if (vrdl->mr == section->mr && 526 vrdl->offset_within_address_space == 527 section->offset_within_address_space) { 528 break; 529 } 530 } 531 532 if (!vrdl) { 533 hw_error("vfio: Trying to unregister missing RAM discard listener"); 534 } 535 536 ram_discard_manager_unregister_listener(rdm, &vrdl->listener); 537 QLIST_REMOVE(vrdl, next); 538 g_free(vrdl); 539 } 540 541 static VFIOHostDMAWindow *vfio_find_hostwin(VFIOContainer *container, 542 hwaddr iova, hwaddr end) 543 { 544 VFIOHostDMAWindow *hostwin; 545 bool hostwin_found = false; 546 547 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 548 if (hostwin->min_iova <= iova && end <= hostwin->max_iova) { 549 hostwin_found = true; 550 break; 551 } 552 } 553 554 return hostwin_found ? hostwin : NULL; 555 } 556 557 static bool vfio_known_safe_misalignment(MemoryRegionSection *section) 558 { 559 MemoryRegion *mr = section->mr; 560 561 if (!TPM_IS_CRB(mr->owner)) { 562 return false; 563 } 564 565 /* this is a known safe misaligned region, just trace for debug purpose */ 566 trace_vfio_known_safe_misalignment(memory_region_name(mr), 567 section->offset_within_address_space, 568 section->offset_within_region, 569 qemu_real_host_page_size()); 570 return true; 571 } 572 573 static bool vfio_listener_valid_section(MemoryRegionSection *section, 574 const char *name) 575 { 576 if (vfio_listener_skipped_section(section)) { 577 trace_vfio_listener_region_skip(name, 578 section->offset_within_address_space, 579 section->offset_within_address_space + 580 int128_get64(int128_sub(section->size, int128_one()))); 581 return false; 582 } 583 584 if (unlikely((section->offset_within_address_space & 585 ~qemu_real_host_page_mask()) != 586 (section->offset_within_region & ~qemu_real_host_page_mask()))) { 587 if (!vfio_known_safe_misalignment(section)) { 588 error_report("%s received unaligned region %s iova=0x%"PRIx64 589 " offset_within_region=0x%"PRIx64 590 " qemu_real_host_page_size=0x%"PRIxPTR, 591 __func__, memory_region_name(section->mr), 592 section->offset_within_address_space, 593 section->offset_within_region, 594 qemu_real_host_page_size()); 595 } 596 return false; 597 } 598 599 return true; 600 } 601 602 static bool vfio_get_section_iova_range(VFIOContainer *container, 603 MemoryRegionSection *section, 604 hwaddr *out_iova, hwaddr *out_end, 605 Int128 *out_llend) 606 { 607 Int128 llend; 608 hwaddr iova; 609 610 iova = REAL_HOST_PAGE_ALIGN(section->offset_within_address_space); 611 llend = int128_make64(section->offset_within_address_space); 612 llend = int128_add(llend, section->size); 613 llend = int128_and(llend, int128_exts64(qemu_real_host_page_mask())); 614 615 if (int128_ge(int128_make64(iova), llend)) { 616 return false; 617 } 618 619 *out_iova = iova; 620 *out_end = int128_get64(int128_sub(llend, int128_one())); 621 if (out_llend) { 622 *out_llend = llend; 623 } 624 return true; 625 } 626 627 static void vfio_listener_region_add(MemoryListener *listener, 628 MemoryRegionSection *section) 629 { 630 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 631 hwaddr iova, end; 632 Int128 llend, llsize; 633 void *vaddr; 634 int ret; 635 VFIOHostDMAWindow *hostwin; 636 Error *err = NULL; 637 638 if (!vfio_listener_valid_section(section, "region_add")) { 639 return; 640 } 641 642 if (!vfio_get_section_iova_range(container, section, &iova, &end, &llend)) { 643 if (memory_region_is_ram_device(section->mr)) { 644 trace_vfio_listener_region_add_no_dma_map( 645 memory_region_name(section->mr), 646 section->offset_within_address_space, 647 int128_getlo(section->size), 648 qemu_real_host_page_size()); 649 } 650 return; 651 } 652 653 if (vfio_container_add_section_window(container, section, &err)) { 654 goto fail; 655 } 656 657 hostwin = vfio_find_hostwin(container, iova, end); 658 if (!hostwin) { 659 error_setg(&err, "Container %p can't map guest IOVA region" 660 " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx, container, iova, end); 661 goto fail; 662 } 663 664 memory_region_ref(section->mr); 665 666 if (memory_region_is_iommu(section->mr)) { 667 VFIOGuestIOMMU *giommu; 668 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 669 int iommu_idx; 670 671 trace_vfio_listener_region_add_iommu(iova, end); 672 /* 673 * FIXME: For VFIO iommu types which have KVM acceleration to 674 * avoid bouncing all map/unmaps through qemu this way, this 675 * would be the right place to wire that up (tell the KVM 676 * device emulation the VFIO iommu handles to use). 677 */ 678 giommu = g_malloc0(sizeof(*giommu)); 679 giommu->iommu_mr = iommu_mr; 680 giommu->iommu_offset = section->offset_within_address_space - 681 section->offset_within_region; 682 giommu->container = container; 683 llend = int128_add(int128_make64(section->offset_within_region), 684 section->size); 685 llend = int128_sub(llend, int128_one()); 686 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 687 MEMTXATTRS_UNSPECIFIED); 688 iommu_notifier_init(&giommu->n, vfio_iommu_map_notify, 689 IOMMU_NOTIFIER_IOTLB_EVENTS, 690 section->offset_within_region, 691 int128_get64(llend), 692 iommu_idx); 693 694 ret = memory_region_iommu_set_page_size_mask(giommu->iommu_mr, 695 container->pgsizes, 696 &err); 697 if (ret) { 698 g_free(giommu); 699 goto fail; 700 } 701 702 ret = memory_region_register_iommu_notifier(section->mr, &giommu->n, 703 &err); 704 if (ret) { 705 g_free(giommu); 706 goto fail; 707 } 708 QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next); 709 memory_region_iommu_replay(giommu->iommu_mr, &giommu->n); 710 711 return; 712 } 713 714 /* Here we assume that memory_region_is_ram(section->mr)==true */ 715 716 /* 717 * For RAM memory regions with a RamDiscardManager, we only want to map the 718 * actually populated parts - and update the mapping whenever we're notified 719 * about changes. 720 */ 721 if (memory_region_has_ram_discard_manager(section->mr)) { 722 vfio_register_ram_discard_listener(container, section); 723 return; 724 } 725 726 vaddr = memory_region_get_ram_ptr(section->mr) + 727 section->offset_within_region + 728 (iova - section->offset_within_address_space); 729 730 trace_vfio_listener_region_add_ram(iova, end, vaddr); 731 732 llsize = int128_sub(llend, int128_make64(iova)); 733 734 if (memory_region_is_ram_device(section->mr)) { 735 hwaddr pgmask = (1ULL << ctz64(hostwin->iova_pgsizes)) - 1; 736 737 if ((iova & pgmask) || (int128_get64(llsize) & pgmask)) { 738 trace_vfio_listener_region_add_no_dma_map( 739 memory_region_name(section->mr), 740 section->offset_within_address_space, 741 int128_getlo(section->size), 742 pgmask + 1); 743 return; 744 } 745 } 746 747 ret = vfio_dma_map(container, iova, int128_get64(llsize), 748 vaddr, section->readonly); 749 if (ret) { 750 error_setg(&err, "vfio_dma_map(%p, 0x%"HWADDR_PRIx", " 751 "0x%"HWADDR_PRIx", %p) = %d (%s)", 752 container, iova, int128_get64(llsize), vaddr, ret, 753 strerror(-ret)); 754 if (memory_region_is_ram_device(section->mr)) { 755 /* Allow unexpected mappings not to be fatal for RAM devices */ 756 error_report_err(err); 757 return; 758 } 759 goto fail; 760 } 761 762 return; 763 764 fail: 765 if (memory_region_is_ram_device(section->mr)) { 766 error_reportf_err(err, "PCI p2p may not work: "); 767 return; 768 } 769 /* 770 * On the initfn path, store the first error in the container so we 771 * can gracefully fail. Runtime, there's not much we can do other 772 * than throw a hardware error. 773 */ 774 if (!container->initialized) { 775 if (!container->error) { 776 error_propagate_prepend(&container->error, err, 777 "Region %s: ", 778 memory_region_name(section->mr)); 779 } else { 780 error_free(err); 781 } 782 } else { 783 error_report_err(err); 784 hw_error("vfio: DMA mapping failed, unable to continue"); 785 } 786 } 787 788 static void vfio_listener_region_del(MemoryListener *listener, 789 MemoryRegionSection *section) 790 { 791 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 792 hwaddr iova, end; 793 Int128 llend, llsize; 794 int ret; 795 bool try_unmap = true; 796 797 if (!vfio_listener_valid_section(section, "region_del")) { 798 return; 799 } 800 801 if (memory_region_is_iommu(section->mr)) { 802 VFIOGuestIOMMU *giommu; 803 804 QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) { 805 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 806 giommu->n.start == section->offset_within_region) { 807 memory_region_unregister_iommu_notifier(section->mr, 808 &giommu->n); 809 QLIST_REMOVE(giommu, giommu_next); 810 g_free(giommu); 811 break; 812 } 813 } 814 815 /* 816 * FIXME: We assume the one big unmap below is adequate to 817 * remove any individual page mappings in the IOMMU which 818 * might have been copied into VFIO. This works for a page table 819 * based IOMMU where a big unmap flattens a large range of IO-PTEs. 820 * That may not be true for all IOMMU types. 821 */ 822 } 823 824 if (!vfio_get_section_iova_range(container, section, &iova, &end, &llend)) { 825 return; 826 } 827 828 llsize = int128_sub(llend, int128_make64(iova)); 829 830 trace_vfio_listener_region_del(iova, end); 831 832 if (memory_region_is_ram_device(section->mr)) { 833 hwaddr pgmask; 834 VFIOHostDMAWindow *hostwin; 835 836 hostwin = vfio_find_hostwin(container, iova, end); 837 assert(hostwin); /* or region_add() would have failed */ 838 839 pgmask = (1ULL << ctz64(hostwin->iova_pgsizes)) - 1; 840 try_unmap = !((iova & pgmask) || (int128_get64(llsize) & pgmask)); 841 } else if (memory_region_has_ram_discard_manager(section->mr)) { 842 vfio_unregister_ram_discard_listener(container, section); 843 /* Unregistering will trigger an unmap. */ 844 try_unmap = false; 845 } 846 847 if (try_unmap) { 848 if (int128_eq(llsize, int128_2_64())) { 849 /* The unmap ioctl doesn't accept a full 64-bit span. */ 850 llsize = int128_rshift(llsize, 1); 851 ret = vfio_dma_unmap(container, iova, int128_get64(llsize), NULL); 852 if (ret) { 853 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 854 "0x%"HWADDR_PRIx") = %d (%s)", 855 container, iova, int128_get64(llsize), ret, 856 strerror(-ret)); 857 } 858 iova += int128_get64(llsize); 859 } 860 ret = vfio_dma_unmap(container, iova, int128_get64(llsize), NULL); 861 if (ret) { 862 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 863 "0x%"HWADDR_PRIx") = %d (%s)", 864 container, iova, int128_get64(llsize), ret, 865 strerror(-ret)); 866 } 867 } 868 869 memory_region_unref(section->mr); 870 871 vfio_container_del_section_window(container, section); 872 } 873 874 typedef struct VFIODirtyRanges { 875 hwaddr min32; 876 hwaddr max32; 877 hwaddr min64; 878 hwaddr max64; 879 hwaddr minpci64; 880 hwaddr maxpci64; 881 } VFIODirtyRanges; 882 883 typedef struct VFIODirtyRangesListener { 884 VFIOContainer *container; 885 VFIODirtyRanges ranges; 886 MemoryListener listener; 887 } VFIODirtyRangesListener; 888 889 static bool vfio_section_is_vfio_pci(MemoryRegionSection *section, 890 VFIOContainer *container) 891 { 892 VFIOPCIDevice *pcidev; 893 VFIODevice *vbasedev; 894 Object *owner; 895 896 owner = memory_region_owner(section->mr); 897 898 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 899 if (vbasedev->type != VFIO_DEVICE_TYPE_PCI) { 900 continue; 901 } 902 pcidev = container_of(vbasedev, VFIOPCIDevice, vbasedev); 903 if (OBJECT(pcidev) == owner) { 904 return true; 905 } 906 } 907 908 return false; 909 } 910 911 static void vfio_dirty_tracking_update(MemoryListener *listener, 912 MemoryRegionSection *section) 913 { 914 VFIODirtyRangesListener *dirty = container_of(listener, 915 VFIODirtyRangesListener, 916 listener); 917 VFIODirtyRanges *range = &dirty->ranges; 918 hwaddr iova, end, *min, *max; 919 920 if (!vfio_listener_valid_section(section, "tracking_update") || 921 !vfio_get_section_iova_range(dirty->container, section, 922 &iova, &end, NULL)) { 923 return; 924 } 925 926 /* 927 * The address space passed to the dirty tracker is reduced to three ranges: 928 * one for 32-bit DMA ranges, one for 64-bit DMA ranges and one for the 929 * PCI 64-bit hole. 930 * 931 * The underlying reports of dirty will query a sub-interval of each of 932 * these ranges. 933 * 934 * The purpose of the three range handling is to handle known cases of big 935 * holes in the address space, like the x86 AMD 1T hole, and firmware (like 936 * OVMF) which may relocate the pci-hole64 to the end of the address space. 937 * The latter would otherwise generate large ranges for tracking, stressing 938 * the limits of supported hardware. The pci-hole32 will always be below 4G 939 * (overlapping or not) so it doesn't need special handling and is part of 940 * the 32-bit range. 941 * 942 * The alternative would be an IOVATree but that has a much bigger runtime 943 * overhead and unnecessary complexity. 944 */ 945 if (vfio_section_is_vfio_pci(section, dirty->container) && 946 iova >= UINT32_MAX) { 947 min = &range->minpci64; 948 max = &range->maxpci64; 949 } else { 950 min = (end <= UINT32_MAX) ? &range->min32 : &range->min64; 951 max = (end <= UINT32_MAX) ? &range->max32 : &range->max64; 952 } 953 if (*min > iova) { 954 *min = iova; 955 } 956 if (*max < end) { 957 *max = end; 958 } 959 960 trace_vfio_device_dirty_tracking_update(iova, end, *min, *max); 961 return; 962 } 963 964 static const MemoryListener vfio_dirty_tracking_listener = { 965 .name = "vfio-tracking", 966 .region_add = vfio_dirty_tracking_update, 967 }; 968 969 static void vfio_dirty_tracking_init(VFIOContainer *container, 970 VFIODirtyRanges *ranges) 971 { 972 VFIODirtyRangesListener dirty; 973 974 memset(&dirty, 0, sizeof(dirty)); 975 dirty.ranges.min32 = UINT32_MAX; 976 dirty.ranges.min64 = UINT64_MAX; 977 dirty.ranges.minpci64 = UINT64_MAX; 978 dirty.listener = vfio_dirty_tracking_listener; 979 dirty.container = container; 980 981 memory_listener_register(&dirty.listener, 982 container->space->as); 983 984 *ranges = dirty.ranges; 985 986 /* 987 * The memory listener is synchronous, and used to calculate the range 988 * to dirty tracking. Unregister it after we are done as we are not 989 * interested in any follow-up updates. 990 */ 991 memory_listener_unregister(&dirty.listener); 992 } 993 994 static void vfio_devices_dma_logging_stop(VFIOContainer *container) 995 { 996 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature), 997 sizeof(uint64_t))] = {}; 998 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 999 VFIODevice *vbasedev; 1000 1001 feature->argsz = sizeof(buf); 1002 feature->flags = VFIO_DEVICE_FEATURE_SET | 1003 VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP; 1004 1005 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 1006 if (!vbasedev->dirty_tracking) { 1007 continue; 1008 } 1009 1010 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 1011 warn_report("%s: Failed to stop DMA logging, err %d (%s)", 1012 vbasedev->name, -errno, strerror(errno)); 1013 } 1014 vbasedev->dirty_tracking = false; 1015 } 1016 } 1017 1018 static struct vfio_device_feature * 1019 vfio_device_feature_dma_logging_start_create(VFIOContainer *container, 1020 VFIODirtyRanges *tracking) 1021 { 1022 struct vfio_device_feature *feature; 1023 size_t feature_size; 1024 struct vfio_device_feature_dma_logging_control *control; 1025 struct vfio_device_feature_dma_logging_range *ranges; 1026 1027 feature_size = sizeof(struct vfio_device_feature) + 1028 sizeof(struct vfio_device_feature_dma_logging_control); 1029 feature = g_try_malloc0(feature_size); 1030 if (!feature) { 1031 errno = ENOMEM; 1032 return NULL; 1033 } 1034 feature->argsz = feature_size; 1035 feature->flags = VFIO_DEVICE_FEATURE_SET | 1036 VFIO_DEVICE_FEATURE_DMA_LOGGING_START; 1037 1038 control = (struct vfio_device_feature_dma_logging_control *)feature->data; 1039 control->page_size = qemu_real_host_page_size(); 1040 1041 /* 1042 * DMA logging uAPI guarantees to support at least a number of ranges that 1043 * fits into a single host kernel base page. 1044 */ 1045 control->num_ranges = !!tracking->max32 + !!tracking->max64 + 1046 !!tracking->maxpci64; 1047 ranges = g_try_new0(struct vfio_device_feature_dma_logging_range, 1048 control->num_ranges); 1049 if (!ranges) { 1050 g_free(feature); 1051 errno = ENOMEM; 1052 1053 return NULL; 1054 } 1055 1056 control->ranges = (__u64)(uintptr_t)ranges; 1057 if (tracking->max32) { 1058 ranges->iova = tracking->min32; 1059 ranges->length = (tracking->max32 - tracking->min32) + 1; 1060 ranges++; 1061 } 1062 if (tracking->max64) { 1063 ranges->iova = tracking->min64; 1064 ranges->length = (tracking->max64 - tracking->min64) + 1; 1065 ranges++; 1066 } 1067 if (tracking->maxpci64) { 1068 ranges->iova = tracking->minpci64; 1069 ranges->length = (tracking->maxpci64 - tracking->minpci64) + 1; 1070 } 1071 1072 trace_vfio_device_dirty_tracking_start(control->num_ranges, 1073 tracking->min32, tracking->max32, 1074 tracking->min64, tracking->max64, 1075 tracking->minpci64, tracking->maxpci64); 1076 1077 return feature; 1078 } 1079 1080 static void vfio_device_feature_dma_logging_start_destroy( 1081 struct vfio_device_feature *feature) 1082 { 1083 struct vfio_device_feature_dma_logging_control *control = 1084 (struct vfio_device_feature_dma_logging_control *)feature->data; 1085 struct vfio_device_feature_dma_logging_range *ranges = 1086 (struct vfio_device_feature_dma_logging_range *)(uintptr_t)control->ranges; 1087 1088 g_free(ranges); 1089 g_free(feature); 1090 } 1091 1092 static int vfio_devices_dma_logging_start(VFIOContainer *container) 1093 { 1094 struct vfio_device_feature *feature; 1095 VFIODirtyRanges ranges; 1096 VFIODevice *vbasedev; 1097 int ret = 0; 1098 1099 vfio_dirty_tracking_init(container, &ranges); 1100 feature = vfio_device_feature_dma_logging_start_create(container, 1101 &ranges); 1102 if (!feature) { 1103 return -errno; 1104 } 1105 1106 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 1107 if (vbasedev->dirty_tracking) { 1108 continue; 1109 } 1110 1111 ret = ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature); 1112 if (ret) { 1113 ret = -errno; 1114 error_report("%s: Failed to start DMA logging, err %d (%s)", 1115 vbasedev->name, ret, strerror(errno)); 1116 goto out; 1117 } 1118 vbasedev->dirty_tracking = true; 1119 } 1120 1121 out: 1122 if (ret) { 1123 vfio_devices_dma_logging_stop(container); 1124 } 1125 1126 vfio_device_feature_dma_logging_start_destroy(feature); 1127 1128 return ret; 1129 } 1130 1131 static void vfio_listener_log_global_start(MemoryListener *listener) 1132 { 1133 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1134 int ret; 1135 1136 if (vfio_devices_all_device_dirty_tracking(container)) { 1137 ret = vfio_devices_dma_logging_start(container); 1138 } else { 1139 ret = vfio_set_dirty_page_tracking(container, true); 1140 } 1141 1142 if (ret) { 1143 error_report("vfio: Could not start dirty page tracking, err: %d (%s)", 1144 ret, strerror(-ret)); 1145 vfio_set_migration_error(ret); 1146 } 1147 } 1148 1149 static void vfio_listener_log_global_stop(MemoryListener *listener) 1150 { 1151 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1152 int ret = 0; 1153 1154 if (vfio_devices_all_device_dirty_tracking(container)) { 1155 vfio_devices_dma_logging_stop(container); 1156 } else { 1157 ret = vfio_set_dirty_page_tracking(container, false); 1158 } 1159 1160 if (ret) { 1161 error_report("vfio: Could not stop dirty page tracking, err: %d (%s)", 1162 ret, strerror(-ret)); 1163 vfio_set_migration_error(ret); 1164 } 1165 } 1166 1167 static int vfio_device_dma_logging_report(VFIODevice *vbasedev, hwaddr iova, 1168 hwaddr size, void *bitmap) 1169 { 1170 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + 1171 sizeof(struct vfio_device_feature_dma_logging_report), 1172 sizeof(__u64))] = {}; 1173 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 1174 struct vfio_device_feature_dma_logging_report *report = 1175 (struct vfio_device_feature_dma_logging_report *)feature->data; 1176 1177 report->iova = iova; 1178 report->length = size; 1179 report->page_size = qemu_real_host_page_size(); 1180 report->bitmap = (__u64)(uintptr_t)bitmap; 1181 1182 feature->argsz = sizeof(buf); 1183 feature->flags = VFIO_DEVICE_FEATURE_GET | 1184 VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT; 1185 1186 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 1187 return -errno; 1188 } 1189 1190 return 0; 1191 } 1192 1193 int vfio_devices_query_dirty_bitmap(VFIOContainer *container, 1194 VFIOBitmap *vbmap, hwaddr iova, 1195 hwaddr size) 1196 { 1197 VFIODevice *vbasedev; 1198 int ret; 1199 1200 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 1201 ret = vfio_device_dma_logging_report(vbasedev, iova, size, 1202 vbmap->bitmap); 1203 if (ret) { 1204 error_report("%s: Failed to get DMA logging report, iova: " 1205 "0x%" HWADDR_PRIx ", size: 0x%" HWADDR_PRIx 1206 ", err: %d (%s)", 1207 vbasedev->name, iova, size, ret, strerror(-ret)); 1208 1209 return ret; 1210 } 1211 } 1212 1213 return 0; 1214 } 1215 1216 int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova, 1217 uint64_t size, ram_addr_t ram_addr) 1218 { 1219 bool all_device_dirty_tracking = 1220 vfio_devices_all_device_dirty_tracking(container); 1221 uint64_t dirty_pages; 1222 VFIOBitmap vbmap; 1223 int ret; 1224 1225 if (!container->dirty_pages_supported && !all_device_dirty_tracking) { 1226 cpu_physical_memory_set_dirty_range(ram_addr, size, 1227 tcg_enabled() ? DIRTY_CLIENTS_ALL : 1228 DIRTY_CLIENTS_NOCODE); 1229 return 0; 1230 } 1231 1232 ret = vfio_bitmap_alloc(&vbmap, size); 1233 if (ret) { 1234 return ret; 1235 } 1236 1237 if (all_device_dirty_tracking) { 1238 ret = vfio_devices_query_dirty_bitmap(container, &vbmap, iova, size); 1239 } else { 1240 ret = vfio_query_dirty_bitmap(container, &vbmap, iova, size); 1241 } 1242 1243 if (ret) { 1244 goto out; 1245 } 1246 1247 dirty_pages = cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, ram_addr, 1248 vbmap.pages); 1249 1250 trace_vfio_get_dirty_bitmap(container->fd, iova, size, vbmap.size, 1251 ram_addr, dirty_pages); 1252 out: 1253 g_free(vbmap.bitmap); 1254 1255 return ret; 1256 } 1257 1258 typedef struct { 1259 IOMMUNotifier n; 1260 VFIOGuestIOMMU *giommu; 1261 } vfio_giommu_dirty_notifier; 1262 1263 static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 1264 { 1265 vfio_giommu_dirty_notifier *gdn = container_of(n, 1266 vfio_giommu_dirty_notifier, n); 1267 VFIOGuestIOMMU *giommu = gdn->giommu; 1268 VFIOContainer *container = giommu->container; 1269 hwaddr iova = iotlb->iova + giommu->iommu_offset; 1270 ram_addr_t translated_addr; 1271 int ret = -EINVAL; 1272 1273 trace_vfio_iommu_map_dirty_notify(iova, iova + iotlb->addr_mask); 1274 1275 if (iotlb->target_as != &address_space_memory) { 1276 error_report("Wrong target AS \"%s\", only system memory is allowed", 1277 iotlb->target_as->name ? iotlb->target_as->name : "none"); 1278 goto out; 1279 } 1280 1281 rcu_read_lock(); 1282 if (vfio_get_xlat_addr(iotlb, NULL, &translated_addr, NULL)) { 1283 ret = vfio_get_dirty_bitmap(container, iova, iotlb->addr_mask + 1, 1284 translated_addr); 1285 if (ret) { 1286 error_report("vfio_iommu_map_dirty_notify(%p, 0x%"HWADDR_PRIx", " 1287 "0x%"HWADDR_PRIx") = %d (%s)", 1288 container, iova, iotlb->addr_mask + 1, ret, 1289 strerror(-ret)); 1290 } 1291 } 1292 rcu_read_unlock(); 1293 1294 out: 1295 if (ret) { 1296 vfio_set_migration_error(ret); 1297 } 1298 } 1299 1300 static int vfio_ram_discard_get_dirty_bitmap(MemoryRegionSection *section, 1301 void *opaque) 1302 { 1303 const hwaddr size = int128_get64(section->size); 1304 const hwaddr iova = section->offset_within_address_space; 1305 const ram_addr_t ram_addr = memory_region_get_ram_addr(section->mr) + 1306 section->offset_within_region; 1307 VFIORamDiscardListener *vrdl = opaque; 1308 1309 /* 1310 * Sync the whole mapped region (spanning multiple individual mappings) 1311 * in one go. 1312 */ 1313 return vfio_get_dirty_bitmap(vrdl->container, iova, size, ram_addr); 1314 } 1315 1316 static int vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainer *container, 1317 MemoryRegionSection *section) 1318 { 1319 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 1320 VFIORamDiscardListener *vrdl = NULL; 1321 1322 QLIST_FOREACH(vrdl, &container->vrdl_list, next) { 1323 if (vrdl->mr == section->mr && 1324 vrdl->offset_within_address_space == 1325 section->offset_within_address_space) { 1326 break; 1327 } 1328 } 1329 1330 if (!vrdl) { 1331 hw_error("vfio: Trying to sync missing RAM discard listener"); 1332 } 1333 1334 /* 1335 * We only want/can synchronize the bitmap for actually mapped parts - 1336 * which correspond to populated parts. Replay all populated parts. 1337 */ 1338 return ram_discard_manager_replay_populated(rdm, section, 1339 vfio_ram_discard_get_dirty_bitmap, 1340 &vrdl); 1341 } 1342 1343 static int vfio_sync_dirty_bitmap(VFIOContainer *container, 1344 MemoryRegionSection *section) 1345 { 1346 ram_addr_t ram_addr; 1347 1348 if (memory_region_is_iommu(section->mr)) { 1349 VFIOGuestIOMMU *giommu; 1350 1351 QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) { 1352 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 1353 giommu->n.start == section->offset_within_region) { 1354 Int128 llend; 1355 vfio_giommu_dirty_notifier gdn = { .giommu = giommu }; 1356 int idx = memory_region_iommu_attrs_to_index(giommu->iommu_mr, 1357 MEMTXATTRS_UNSPECIFIED); 1358 1359 llend = int128_add(int128_make64(section->offset_within_region), 1360 section->size); 1361 llend = int128_sub(llend, int128_one()); 1362 1363 iommu_notifier_init(&gdn.n, 1364 vfio_iommu_map_dirty_notify, 1365 IOMMU_NOTIFIER_MAP, 1366 section->offset_within_region, 1367 int128_get64(llend), 1368 idx); 1369 memory_region_iommu_replay(giommu->iommu_mr, &gdn.n); 1370 break; 1371 } 1372 } 1373 return 0; 1374 } else if (memory_region_has_ram_discard_manager(section->mr)) { 1375 return vfio_sync_ram_discard_listener_dirty_bitmap(container, section); 1376 } 1377 1378 ram_addr = memory_region_get_ram_addr(section->mr) + 1379 section->offset_within_region; 1380 1381 return vfio_get_dirty_bitmap(container, 1382 REAL_HOST_PAGE_ALIGN(section->offset_within_address_space), 1383 int128_get64(section->size), ram_addr); 1384 } 1385 1386 static void vfio_listener_log_sync(MemoryListener *listener, 1387 MemoryRegionSection *section) 1388 { 1389 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1390 int ret; 1391 1392 if (vfio_listener_skipped_section(section)) { 1393 return; 1394 } 1395 1396 if (vfio_devices_all_dirty_tracking(container)) { 1397 ret = vfio_sync_dirty_bitmap(container, section); 1398 if (ret) { 1399 error_report("vfio: Failed to sync dirty bitmap, err: %d (%s)", ret, 1400 strerror(-ret)); 1401 vfio_set_migration_error(ret); 1402 } 1403 } 1404 } 1405 1406 const MemoryListener vfio_memory_listener = { 1407 .name = "vfio", 1408 .region_add = vfio_listener_region_add, 1409 .region_del = vfio_listener_region_del, 1410 .log_global_start = vfio_listener_log_global_start, 1411 .log_global_stop = vfio_listener_log_global_stop, 1412 .log_sync = vfio_listener_log_sync, 1413 }; 1414 1415 void vfio_reset_handler(void *opaque) 1416 { 1417 VFIODevice *vbasedev; 1418 1419 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 1420 if (vbasedev->dev->realized) { 1421 vbasedev->ops->vfio_compute_needs_reset(vbasedev); 1422 } 1423 } 1424 1425 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 1426 if (vbasedev->dev->realized && vbasedev->needs_reset) { 1427 vbasedev->ops->vfio_hot_reset_multi(vbasedev); 1428 } 1429 } 1430 } 1431 1432 int vfio_kvm_device_add_fd(int fd, Error **errp) 1433 { 1434 #ifdef CONFIG_KVM 1435 struct kvm_device_attr attr = { 1436 .group = KVM_DEV_VFIO_FILE, 1437 .attr = KVM_DEV_VFIO_FILE_ADD, 1438 .addr = (uint64_t)(unsigned long)&fd, 1439 }; 1440 1441 if (!kvm_enabled()) { 1442 return 0; 1443 } 1444 1445 if (vfio_kvm_device_fd < 0) { 1446 struct kvm_create_device cd = { 1447 .type = KVM_DEV_TYPE_VFIO, 1448 }; 1449 1450 if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) { 1451 error_setg_errno(errp, errno, "Failed to create KVM VFIO device"); 1452 return -errno; 1453 } 1454 1455 vfio_kvm_device_fd = cd.fd; 1456 } 1457 1458 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1459 error_setg_errno(errp, errno, "Failed to add fd %d to KVM VFIO device", 1460 fd); 1461 return -errno; 1462 } 1463 #endif 1464 return 0; 1465 } 1466 1467 int vfio_kvm_device_del_fd(int fd, Error **errp) 1468 { 1469 #ifdef CONFIG_KVM 1470 struct kvm_device_attr attr = { 1471 .group = KVM_DEV_VFIO_FILE, 1472 .attr = KVM_DEV_VFIO_FILE_DEL, 1473 .addr = (uint64_t)(unsigned long)&fd, 1474 }; 1475 1476 if (vfio_kvm_device_fd < 0) { 1477 error_setg(errp, "KVM VFIO device isn't created yet"); 1478 return -EINVAL; 1479 } 1480 1481 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1482 error_setg_errno(errp, errno, 1483 "Failed to remove fd %d from KVM VFIO device", fd); 1484 return -errno; 1485 } 1486 #endif 1487 return 0; 1488 } 1489 1490 VFIOAddressSpace *vfio_get_address_space(AddressSpace *as) 1491 { 1492 VFIOAddressSpace *space; 1493 1494 QLIST_FOREACH(space, &vfio_address_spaces, list) { 1495 if (space->as == as) { 1496 return space; 1497 } 1498 } 1499 1500 /* No suitable VFIOAddressSpace, create a new one */ 1501 space = g_malloc0(sizeof(*space)); 1502 space->as = as; 1503 QLIST_INIT(&space->containers); 1504 1505 if (QLIST_EMPTY(&vfio_address_spaces)) { 1506 qemu_register_reset(vfio_reset_handler, NULL); 1507 } 1508 1509 QLIST_INSERT_HEAD(&vfio_address_spaces, space, list); 1510 1511 return space; 1512 } 1513 1514 void vfio_put_address_space(VFIOAddressSpace *space) 1515 { 1516 if (QLIST_EMPTY(&space->containers)) { 1517 QLIST_REMOVE(space, list); 1518 g_free(space); 1519 } 1520 if (QLIST_EMPTY(&vfio_address_spaces)) { 1521 qemu_unregister_reset(vfio_reset_handler, NULL); 1522 } 1523 } 1524 1525 struct vfio_device_info *vfio_get_device_info(int fd) 1526 { 1527 struct vfio_device_info *info; 1528 uint32_t argsz = sizeof(*info); 1529 1530 info = g_malloc0(argsz); 1531 1532 retry: 1533 info->argsz = argsz; 1534 1535 if (ioctl(fd, VFIO_DEVICE_GET_INFO, info)) { 1536 g_free(info); 1537 return NULL; 1538 } 1539 1540 if (info->argsz > argsz) { 1541 argsz = info->argsz; 1542 info = g_realloc(info, argsz); 1543 goto retry; 1544 } 1545 1546 return info; 1547 } 1548