1 /* 2 * vfio based device assignment support 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Alex Williamson <alex.williamson@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Based on qemu-kvm device-assignment: 13 * Adapted for KVM by Qumranet. 14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) 15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) 16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) 17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) 18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) 19 */ 20 21 #include <linux/vfio.h> 22 #include <sys/ioctl.h> 23 #include <sys/mman.h> 24 #include <sys/stat.h> 25 #include <sys/types.h> 26 #include <unistd.h> 27 28 #include "config.h" 29 #include "exec/address-spaces.h" 30 #include "exec/memory.h" 31 #include "hw/pci/msi.h" 32 #include "hw/pci/msix.h" 33 #include "hw/pci/pci.h" 34 #include "qemu-common.h" 35 #include "qemu/error-report.h" 36 #include "qemu/event_notifier.h" 37 #include "qemu/queue.h" 38 #include "qemu/range.h" 39 #include "sysemu/kvm.h" 40 #include "sysemu/sysemu.h" 41 #include "trace.h" 42 #include "hw/vfio/vfio.h" 43 #include "hw/vfio/vfio-common.h" 44 45 struct VFIOPCIDevice; 46 47 typedef struct VFIOQuirk { 48 MemoryRegion mem; 49 struct VFIOPCIDevice *vdev; 50 QLIST_ENTRY(VFIOQuirk) next; 51 struct { 52 uint32_t base_offset:TARGET_PAGE_BITS; 53 uint32_t address_offset:TARGET_PAGE_BITS; 54 uint32_t address_size:3; 55 uint32_t bar:3; 56 57 uint32_t address_match; 58 uint32_t address_mask; 59 60 uint32_t address_val:TARGET_PAGE_BITS; 61 uint32_t data_offset:TARGET_PAGE_BITS; 62 uint32_t data_size:3; 63 64 uint8_t flags; 65 uint8_t read_flags; 66 uint8_t write_flags; 67 } data; 68 } VFIOQuirk; 69 70 typedef struct VFIOBAR { 71 VFIORegion region; 72 bool ioport; 73 bool mem64; 74 QLIST_HEAD(, VFIOQuirk) quirks; 75 } VFIOBAR; 76 77 typedef struct VFIOVGARegion { 78 MemoryRegion mem; 79 off_t offset; 80 int nr; 81 QLIST_HEAD(, VFIOQuirk) quirks; 82 } VFIOVGARegion; 83 84 typedef struct VFIOVGA { 85 off_t fd_offset; 86 int fd; 87 VFIOVGARegion region[QEMU_PCI_VGA_NUM_REGIONS]; 88 } VFIOVGA; 89 90 typedef struct VFIOINTx { 91 bool pending; /* interrupt pending */ 92 bool kvm_accel; /* set when QEMU bypass through KVM enabled */ 93 uint8_t pin; /* which pin to pull for qemu_set_irq */ 94 EventNotifier interrupt; /* eventfd triggered on interrupt */ 95 EventNotifier unmask; /* eventfd for unmask on QEMU bypass */ 96 PCIINTxRoute route; /* routing info for QEMU bypass */ 97 uint32_t mmap_timeout; /* delay to re-enable mmaps after interrupt */ 98 QEMUTimer *mmap_timer; /* enable mmaps after periods w/o interrupts */ 99 } VFIOINTx; 100 101 typedef struct VFIOMSIVector { 102 /* 103 * Two interrupt paths are configured per vector. The first, is only used 104 * for interrupts injected via QEMU. This is typically the non-accel path, 105 * but may also be used when we want QEMU to handle masking and pending 106 * bits. The KVM path bypasses QEMU and is therefore higher performance, 107 * but requires masking at the device. virq is used to track the MSI route 108 * through KVM, thus kvm_interrupt is only available when virq is set to a 109 * valid (>= 0) value. 110 */ 111 EventNotifier interrupt; 112 EventNotifier kvm_interrupt; 113 struct VFIOPCIDevice *vdev; /* back pointer to device */ 114 int virq; 115 bool use; 116 } VFIOMSIVector; 117 118 enum { 119 VFIO_INT_NONE = 0, 120 VFIO_INT_INTx = 1, 121 VFIO_INT_MSI = 2, 122 VFIO_INT_MSIX = 3, 123 }; 124 125 /* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */ 126 typedef struct VFIOMSIXInfo { 127 uint8_t table_bar; 128 uint8_t pba_bar; 129 uint16_t entries; 130 uint32_t table_offset; 131 uint32_t pba_offset; 132 MemoryRegion mmap_mem; 133 void *mmap; 134 } VFIOMSIXInfo; 135 136 typedef struct VFIOPCIDevice { 137 PCIDevice pdev; 138 VFIODevice vbasedev; 139 VFIOINTx intx; 140 unsigned int config_size; 141 uint8_t *emulated_config_bits; /* QEMU emulated bits, little-endian */ 142 off_t config_offset; /* Offset of config space region within device fd */ 143 unsigned int rom_size; 144 off_t rom_offset; /* Offset of ROM region within device fd */ 145 void *rom; 146 int msi_cap_size; 147 VFIOMSIVector *msi_vectors; 148 VFIOMSIXInfo *msix; 149 int nr_vectors; /* Number of MSI/MSIX vectors currently in use */ 150 int interrupt; /* Current interrupt type */ 151 VFIOBAR bars[PCI_NUM_REGIONS - 1]; /* No ROM */ 152 VFIOVGA vga; /* 0xa0000, 0x3b0, 0x3c0 */ 153 PCIHostDeviceAddress host; 154 EventNotifier err_notifier; 155 EventNotifier req_notifier; 156 int (*resetfn)(struct VFIOPCIDevice *); 157 uint32_t features; 158 #define VFIO_FEATURE_ENABLE_VGA_BIT 0 159 #define VFIO_FEATURE_ENABLE_VGA (1 << VFIO_FEATURE_ENABLE_VGA_BIT) 160 #define VFIO_FEATURE_ENABLE_REQ_BIT 1 161 #define VFIO_FEATURE_ENABLE_REQ (1 << VFIO_FEATURE_ENABLE_REQ_BIT) 162 int32_t bootindex; 163 uint8_t pm_cap; 164 bool has_vga; 165 bool pci_aer; 166 bool req_enabled; 167 bool has_flr; 168 bool has_pm_reset; 169 bool rom_read_failed; 170 } VFIOPCIDevice; 171 172 typedef struct VFIORomBlacklistEntry { 173 uint16_t vendor_id; 174 uint16_t device_id; 175 } VFIORomBlacklistEntry; 176 177 /* 178 * List of device ids/vendor ids for which to disable 179 * option rom loading. This avoids the guest hangs during rom 180 * execution as noticed with the BCM 57810 card for lack of a 181 * more better way to handle such issues. 182 * The user can still override by specifying a romfile or 183 * rombar=1. 184 * Please see https://bugs.launchpad.net/qemu/+bug/1284874 185 * for an analysis of the 57810 card hang. When adding 186 * a new vendor id/device id combination below, please also add 187 * your card/environment details and information that could 188 * help in debugging to the bug tracking this issue 189 */ 190 static const VFIORomBlacklistEntry romblacklist[] = { 191 /* Broadcom BCM 57810 */ 192 { 0x14e4, 0x168e } 193 }; 194 195 #define MSIX_CAP_LENGTH 12 196 197 static void vfio_disable_interrupts(VFIOPCIDevice *vdev); 198 static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len); 199 static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr, 200 uint32_t val, int len); 201 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled); 202 203 /* 204 * Disabling BAR mmaping can be slow, but toggling it around INTx can 205 * also be a huge overhead. We try to get the best of both worlds by 206 * waiting until an interrupt to disable mmaps (subsequent transitions 207 * to the same state are effectively no overhead). If the interrupt has 208 * been serviced and the time gap is long enough, we re-enable mmaps for 209 * performance. This works well for things like graphics cards, which 210 * may not use their interrupt at all and are penalized to an unusable 211 * level by read/write BAR traps. Other devices, like NICs, have more 212 * regular interrupts and see much better latency by staying in non-mmap 213 * mode. We therefore set the default mmap_timeout such that a ping 214 * is just enough to keep the mmap disabled. Users can experiment with 215 * other options with the x-intx-mmap-timeout-ms parameter (a value of 216 * zero disables the timer). 217 */ 218 static void vfio_intx_mmap_enable(void *opaque) 219 { 220 VFIOPCIDevice *vdev = opaque; 221 222 if (vdev->intx.pending) { 223 timer_mod(vdev->intx.mmap_timer, 224 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout); 225 return; 226 } 227 228 vfio_mmap_set_enabled(vdev, true); 229 } 230 231 static void vfio_intx_interrupt(void *opaque) 232 { 233 VFIOPCIDevice *vdev = opaque; 234 235 if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) { 236 return; 237 } 238 239 trace_vfio_intx_interrupt(vdev->vbasedev.name, 'A' + vdev->intx.pin); 240 241 vdev->intx.pending = true; 242 pci_irq_assert(&vdev->pdev); 243 vfio_mmap_set_enabled(vdev, false); 244 if (vdev->intx.mmap_timeout) { 245 timer_mod(vdev->intx.mmap_timer, 246 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout); 247 } 248 } 249 250 static void vfio_eoi(VFIODevice *vbasedev) 251 { 252 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev); 253 254 if (!vdev->intx.pending) { 255 return; 256 } 257 258 trace_vfio_eoi(vbasedev->name); 259 260 vdev->intx.pending = false; 261 pci_irq_deassert(&vdev->pdev); 262 vfio_unmask_single_irqindex(vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 263 } 264 265 static void vfio_enable_intx_kvm(VFIOPCIDevice *vdev) 266 { 267 #ifdef CONFIG_KVM 268 struct kvm_irqfd irqfd = { 269 .fd = event_notifier_get_fd(&vdev->intx.interrupt), 270 .gsi = vdev->intx.route.irq, 271 .flags = KVM_IRQFD_FLAG_RESAMPLE, 272 }; 273 struct vfio_irq_set *irq_set; 274 int ret, argsz; 275 int32_t *pfd; 276 277 if (!VFIO_ALLOW_KVM_INTX || !kvm_irqfds_enabled() || 278 vdev->intx.route.mode != PCI_INTX_ENABLED || 279 !kvm_resamplefds_enabled()) { 280 return; 281 } 282 283 /* Get to a known interrupt state */ 284 qemu_set_fd_handler(irqfd.fd, NULL, NULL, vdev); 285 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 286 vdev->intx.pending = false; 287 pci_irq_deassert(&vdev->pdev); 288 289 /* Get an eventfd for resample/unmask */ 290 if (event_notifier_init(&vdev->intx.unmask, 0)) { 291 error_report("vfio: Error: event_notifier_init failed eoi"); 292 goto fail; 293 } 294 295 /* KVM triggers it, VFIO listens for it */ 296 irqfd.resamplefd = event_notifier_get_fd(&vdev->intx.unmask); 297 298 if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) { 299 error_report("vfio: Error: Failed to setup resample irqfd: %m"); 300 goto fail_irqfd; 301 } 302 303 argsz = sizeof(*irq_set) + sizeof(*pfd); 304 305 irq_set = g_malloc0(argsz); 306 irq_set->argsz = argsz; 307 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_UNMASK; 308 irq_set->index = VFIO_PCI_INTX_IRQ_INDEX; 309 irq_set->start = 0; 310 irq_set->count = 1; 311 pfd = (int32_t *)&irq_set->data; 312 313 *pfd = irqfd.resamplefd; 314 315 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set); 316 g_free(irq_set); 317 if (ret) { 318 error_report("vfio: Error: Failed to setup INTx unmask fd: %m"); 319 goto fail_vfio; 320 } 321 322 /* Let'em rip */ 323 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 324 325 vdev->intx.kvm_accel = true; 326 327 trace_vfio_enable_intx_kvm(vdev->vbasedev.name); 328 329 return; 330 331 fail_vfio: 332 irqfd.flags = KVM_IRQFD_FLAG_DEASSIGN; 333 kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd); 334 fail_irqfd: 335 event_notifier_cleanup(&vdev->intx.unmask); 336 fail: 337 qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev); 338 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 339 #endif 340 } 341 342 static void vfio_disable_intx_kvm(VFIOPCIDevice *vdev) 343 { 344 #ifdef CONFIG_KVM 345 struct kvm_irqfd irqfd = { 346 .fd = event_notifier_get_fd(&vdev->intx.interrupt), 347 .gsi = vdev->intx.route.irq, 348 .flags = KVM_IRQFD_FLAG_DEASSIGN, 349 }; 350 351 if (!vdev->intx.kvm_accel) { 352 return; 353 } 354 355 /* 356 * Get to a known state, hardware masked, QEMU ready to accept new 357 * interrupts, QEMU IRQ de-asserted. 358 */ 359 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 360 vdev->intx.pending = false; 361 pci_irq_deassert(&vdev->pdev); 362 363 /* Tell KVM to stop listening for an INTx irqfd */ 364 if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) { 365 error_report("vfio: Error: Failed to disable INTx irqfd: %m"); 366 } 367 368 /* We only need to close the eventfd for VFIO to cleanup the kernel side */ 369 event_notifier_cleanup(&vdev->intx.unmask); 370 371 /* QEMU starts listening for interrupt events. */ 372 qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev); 373 374 vdev->intx.kvm_accel = false; 375 376 /* If we've missed an event, let it re-fire through QEMU */ 377 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 378 379 trace_vfio_disable_intx_kvm(vdev->vbasedev.name); 380 #endif 381 } 382 383 static void vfio_update_irq(PCIDevice *pdev) 384 { 385 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev); 386 PCIINTxRoute route; 387 388 if (vdev->interrupt != VFIO_INT_INTx) { 389 return; 390 } 391 392 route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin); 393 394 if (!pci_intx_route_changed(&vdev->intx.route, &route)) { 395 return; /* Nothing changed */ 396 } 397 398 trace_vfio_update_irq(vdev->vbasedev.name, 399 vdev->intx.route.irq, route.irq); 400 401 vfio_disable_intx_kvm(vdev); 402 403 vdev->intx.route = route; 404 405 if (route.mode != PCI_INTX_ENABLED) { 406 return; 407 } 408 409 vfio_enable_intx_kvm(vdev); 410 411 /* Re-enable the interrupt in cased we missed an EOI */ 412 vfio_eoi(&vdev->vbasedev); 413 } 414 415 static int vfio_enable_intx(VFIOPCIDevice *vdev) 416 { 417 uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1); 418 int ret, argsz; 419 struct vfio_irq_set *irq_set; 420 int32_t *pfd; 421 422 if (!pin) { 423 return 0; 424 } 425 426 vfio_disable_interrupts(vdev); 427 428 vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */ 429 pci_config_set_interrupt_pin(vdev->pdev.config, pin); 430 431 #ifdef CONFIG_KVM 432 /* 433 * Only conditional to avoid generating error messages on platforms 434 * where we won't actually use the result anyway. 435 */ 436 if (kvm_irqfds_enabled() && kvm_resamplefds_enabled()) { 437 vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev, 438 vdev->intx.pin); 439 } 440 #endif 441 442 ret = event_notifier_init(&vdev->intx.interrupt, 0); 443 if (ret) { 444 error_report("vfio: Error: event_notifier_init failed"); 445 return ret; 446 } 447 448 argsz = sizeof(*irq_set) + sizeof(*pfd); 449 450 irq_set = g_malloc0(argsz); 451 irq_set->argsz = argsz; 452 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; 453 irq_set->index = VFIO_PCI_INTX_IRQ_INDEX; 454 irq_set->start = 0; 455 irq_set->count = 1; 456 pfd = (int32_t *)&irq_set->data; 457 458 *pfd = event_notifier_get_fd(&vdev->intx.interrupt); 459 qemu_set_fd_handler(*pfd, vfio_intx_interrupt, NULL, vdev); 460 461 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set); 462 g_free(irq_set); 463 if (ret) { 464 error_report("vfio: Error: Failed to setup INTx fd: %m"); 465 qemu_set_fd_handler(*pfd, NULL, NULL, vdev); 466 event_notifier_cleanup(&vdev->intx.interrupt); 467 return -errno; 468 } 469 470 vfio_enable_intx_kvm(vdev); 471 472 vdev->interrupt = VFIO_INT_INTx; 473 474 trace_vfio_enable_intx(vdev->vbasedev.name); 475 476 return 0; 477 } 478 479 static void vfio_disable_intx(VFIOPCIDevice *vdev) 480 { 481 int fd; 482 483 timer_del(vdev->intx.mmap_timer); 484 vfio_disable_intx_kvm(vdev); 485 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX); 486 vdev->intx.pending = false; 487 pci_irq_deassert(&vdev->pdev); 488 vfio_mmap_set_enabled(vdev, true); 489 490 fd = event_notifier_get_fd(&vdev->intx.interrupt); 491 qemu_set_fd_handler(fd, NULL, NULL, vdev); 492 event_notifier_cleanup(&vdev->intx.interrupt); 493 494 vdev->interrupt = VFIO_INT_NONE; 495 496 trace_vfio_disable_intx(vdev->vbasedev.name); 497 } 498 499 /* 500 * MSI/X 501 */ 502 static void vfio_msi_interrupt(void *opaque) 503 { 504 VFIOMSIVector *vector = opaque; 505 VFIOPCIDevice *vdev = vector->vdev; 506 int nr = vector - vdev->msi_vectors; 507 508 if (!event_notifier_test_and_clear(&vector->interrupt)) { 509 return; 510 } 511 512 #ifdef DEBUG_VFIO 513 MSIMessage msg; 514 515 if (vdev->interrupt == VFIO_INT_MSIX) { 516 msg = msix_get_message(&vdev->pdev, nr); 517 } else if (vdev->interrupt == VFIO_INT_MSI) { 518 msg = msi_get_message(&vdev->pdev, nr); 519 } else { 520 abort(); 521 } 522 523 trace_vfio_msi_interrupt(vdev->vbasedev.name, nr, msg.address, msg.data); 524 #endif 525 526 if (vdev->interrupt == VFIO_INT_MSIX) { 527 msix_notify(&vdev->pdev, nr); 528 } else if (vdev->interrupt == VFIO_INT_MSI) { 529 msi_notify(&vdev->pdev, nr); 530 } else { 531 error_report("vfio: MSI interrupt receieved, but not enabled?"); 532 } 533 } 534 535 static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix) 536 { 537 struct vfio_irq_set *irq_set; 538 int ret = 0, i, argsz; 539 int32_t *fds; 540 541 argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds)); 542 543 irq_set = g_malloc0(argsz); 544 irq_set->argsz = argsz; 545 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; 546 irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX; 547 irq_set->start = 0; 548 irq_set->count = vdev->nr_vectors; 549 fds = (int32_t *)&irq_set->data; 550 551 for (i = 0; i < vdev->nr_vectors; i++) { 552 int fd = -1; 553 554 /* 555 * MSI vs MSI-X - The guest has direct access to MSI mask and pending 556 * bits, therefore we always use the KVM signaling path when setup. 557 * MSI-X mask and pending bits are emulated, so we want to use the 558 * KVM signaling path only when configured and unmasked. 559 */ 560 if (vdev->msi_vectors[i].use) { 561 if (vdev->msi_vectors[i].virq < 0 || 562 (msix && msix_is_masked(&vdev->pdev, i))) { 563 fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt); 564 } else { 565 fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt); 566 } 567 } 568 569 fds[i] = fd; 570 } 571 572 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set); 573 574 g_free(irq_set); 575 576 return ret; 577 } 578 579 static void vfio_add_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage *msg, 580 bool msix) 581 { 582 int virq; 583 584 if ((msix && !VFIO_ALLOW_KVM_MSIX) || 585 (!msix && !VFIO_ALLOW_KVM_MSI) || !msg) { 586 return; 587 } 588 589 if (event_notifier_init(&vector->kvm_interrupt, 0)) { 590 return; 591 } 592 593 virq = kvm_irqchip_add_msi_route(kvm_state, *msg); 594 if (virq < 0) { 595 event_notifier_cleanup(&vector->kvm_interrupt); 596 return; 597 } 598 599 if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt, 600 NULL, virq) < 0) { 601 kvm_irqchip_release_virq(kvm_state, virq); 602 event_notifier_cleanup(&vector->kvm_interrupt); 603 return; 604 } 605 606 vector->virq = virq; 607 } 608 609 static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector) 610 { 611 kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt, 612 vector->virq); 613 kvm_irqchip_release_virq(kvm_state, vector->virq); 614 vector->virq = -1; 615 event_notifier_cleanup(&vector->kvm_interrupt); 616 } 617 618 static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg) 619 { 620 kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg); 621 } 622 623 static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr, 624 MSIMessage *msg, IOHandler *handler) 625 { 626 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev); 627 VFIOMSIVector *vector; 628 int ret; 629 630 trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr); 631 632 vector = &vdev->msi_vectors[nr]; 633 634 if (!vector->use) { 635 vector->vdev = vdev; 636 vector->virq = -1; 637 if (event_notifier_init(&vector->interrupt, 0)) { 638 error_report("vfio: Error: event_notifier_init failed"); 639 } 640 vector->use = true; 641 msix_vector_use(pdev, nr); 642 } 643 644 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt), 645 handler, NULL, vector); 646 647 /* 648 * Attempt to enable route through KVM irqchip, 649 * default to userspace handling if unavailable. 650 */ 651 if (vector->virq >= 0) { 652 if (!msg) { 653 vfio_remove_kvm_msi_virq(vector); 654 } else { 655 vfio_update_kvm_msi_virq(vector, *msg); 656 } 657 } else { 658 vfio_add_kvm_msi_virq(vector, msg, true); 659 } 660 661 /* 662 * We don't want to have the host allocate all possible MSI vectors 663 * for a device if they're not in use, so we shutdown and incrementally 664 * increase them as needed. 665 */ 666 if (vdev->nr_vectors < nr + 1) { 667 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX); 668 vdev->nr_vectors = nr + 1; 669 ret = vfio_enable_vectors(vdev, true); 670 if (ret) { 671 error_report("vfio: failed to enable vectors, %d", ret); 672 } 673 } else { 674 int argsz; 675 struct vfio_irq_set *irq_set; 676 int32_t *pfd; 677 678 argsz = sizeof(*irq_set) + sizeof(*pfd); 679 680 irq_set = g_malloc0(argsz); 681 irq_set->argsz = argsz; 682 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | 683 VFIO_IRQ_SET_ACTION_TRIGGER; 684 irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX; 685 irq_set->start = nr; 686 irq_set->count = 1; 687 pfd = (int32_t *)&irq_set->data; 688 689 if (vector->virq >= 0) { 690 *pfd = event_notifier_get_fd(&vector->kvm_interrupt); 691 } else { 692 *pfd = event_notifier_get_fd(&vector->interrupt); 693 } 694 695 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set); 696 g_free(irq_set); 697 if (ret) { 698 error_report("vfio: failed to modify vector, %d", ret); 699 } 700 } 701 702 return 0; 703 } 704 705 static int vfio_msix_vector_use(PCIDevice *pdev, 706 unsigned int nr, MSIMessage msg) 707 { 708 return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt); 709 } 710 711 static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr) 712 { 713 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev); 714 VFIOMSIVector *vector = &vdev->msi_vectors[nr]; 715 716 trace_vfio_msix_vector_release(vdev->vbasedev.name, nr); 717 718 /* 719 * There are still old guests that mask and unmask vectors on every 720 * interrupt. If we're using QEMU bypass with a KVM irqfd, leave all of 721 * the KVM setup in place, simply switch VFIO to use the non-bypass 722 * eventfd. We'll then fire the interrupt through QEMU and the MSI-X 723 * core will mask the interrupt and set pending bits, allowing it to 724 * be re-asserted on unmask. Nothing to do if already using QEMU mode. 725 */ 726 if (vector->virq >= 0) { 727 int argsz; 728 struct vfio_irq_set *irq_set; 729 int32_t *pfd; 730 731 argsz = sizeof(*irq_set) + sizeof(*pfd); 732 733 irq_set = g_malloc0(argsz); 734 irq_set->argsz = argsz; 735 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | 736 VFIO_IRQ_SET_ACTION_TRIGGER; 737 irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX; 738 irq_set->start = nr; 739 irq_set->count = 1; 740 pfd = (int32_t *)&irq_set->data; 741 742 *pfd = event_notifier_get_fd(&vector->interrupt); 743 744 ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set); 745 746 g_free(irq_set); 747 } 748 } 749 750 static void vfio_enable_msix(VFIOPCIDevice *vdev) 751 { 752 vfio_disable_interrupts(vdev); 753 754 vdev->msi_vectors = g_malloc0(vdev->msix->entries * sizeof(VFIOMSIVector)); 755 756 vdev->interrupt = VFIO_INT_MSIX; 757 758 /* 759 * Some communication channels between VF & PF or PF & fw rely on the 760 * physical state of the device and expect that enabling MSI-X from the 761 * guest enables the same on the host. When our guest is Linux, the 762 * guest driver call to pci_enable_msix() sets the enabling bit in the 763 * MSI-X capability, but leaves the vector table masked. We therefore 764 * can't rely on a vector_use callback (from request_irq() in the guest) 765 * to switch the physical device into MSI-X mode because that may come a 766 * long time after pci_enable_msix(). This code enables vector 0 with 767 * triggering to userspace, then immediately release the vector, leaving 768 * the physical device with no vectors enabled, but MSI-X enabled, just 769 * like the guest view. 770 */ 771 vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL); 772 vfio_msix_vector_release(&vdev->pdev, 0); 773 774 if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use, 775 vfio_msix_vector_release, NULL)) { 776 error_report("vfio: msix_set_vector_notifiers failed"); 777 } 778 779 trace_vfio_enable_msix(vdev->vbasedev.name); 780 } 781 782 static void vfio_enable_msi(VFIOPCIDevice *vdev) 783 { 784 int ret, i; 785 786 vfio_disable_interrupts(vdev); 787 788 vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev); 789 retry: 790 vdev->msi_vectors = g_malloc0(vdev->nr_vectors * sizeof(VFIOMSIVector)); 791 792 for (i = 0; i < vdev->nr_vectors; i++) { 793 VFIOMSIVector *vector = &vdev->msi_vectors[i]; 794 MSIMessage msg = msi_get_message(&vdev->pdev, i); 795 796 vector->vdev = vdev; 797 vector->virq = -1; 798 vector->use = true; 799 800 if (event_notifier_init(&vector->interrupt, 0)) { 801 error_report("vfio: Error: event_notifier_init failed"); 802 } 803 804 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt), 805 vfio_msi_interrupt, NULL, vector); 806 807 /* 808 * Attempt to enable route through KVM irqchip, 809 * default to userspace handling if unavailable. 810 */ 811 vfio_add_kvm_msi_virq(vector, &msg, false); 812 } 813 814 /* Set interrupt type prior to possible interrupts */ 815 vdev->interrupt = VFIO_INT_MSI; 816 817 ret = vfio_enable_vectors(vdev, false); 818 if (ret) { 819 if (ret < 0) { 820 error_report("vfio: Error: Failed to setup MSI fds: %m"); 821 } else if (ret != vdev->nr_vectors) { 822 error_report("vfio: Error: Failed to enable %d " 823 "MSI vectors, retry with %d", vdev->nr_vectors, ret); 824 } 825 826 for (i = 0; i < vdev->nr_vectors; i++) { 827 VFIOMSIVector *vector = &vdev->msi_vectors[i]; 828 if (vector->virq >= 0) { 829 vfio_remove_kvm_msi_virq(vector); 830 } 831 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt), 832 NULL, NULL, NULL); 833 event_notifier_cleanup(&vector->interrupt); 834 } 835 836 g_free(vdev->msi_vectors); 837 838 if (ret > 0 && ret != vdev->nr_vectors) { 839 vdev->nr_vectors = ret; 840 goto retry; 841 } 842 vdev->nr_vectors = 0; 843 844 /* 845 * Failing to setup MSI doesn't really fall within any specification. 846 * Let's try leaving interrupts disabled and hope the guest figures 847 * out to fall back to INTx for this device. 848 */ 849 error_report("vfio: Error: Failed to enable MSI"); 850 vdev->interrupt = VFIO_INT_NONE; 851 852 return; 853 } 854 855 trace_vfio_enable_msi(vdev->vbasedev.name, vdev->nr_vectors); 856 } 857 858 static void vfio_disable_msi_common(VFIOPCIDevice *vdev) 859 { 860 int i; 861 862 for (i = 0; i < vdev->nr_vectors; i++) { 863 VFIOMSIVector *vector = &vdev->msi_vectors[i]; 864 if (vdev->msi_vectors[i].use) { 865 if (vector->virq >= 0) { 866 vfio_remove_kvm_msi_virq(vector); 867 } 868 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt), 869 NULL, NULL, NULL); 870 event_notifier_cleanup(&vector->interrupt); 871 } 872 } 873 874 g_free(vdev->msi_vectors); 875 vdev->msi_vectors = NULL; 876 vdev->nr_vectors = 0; 877 vdev->interrupt = VFIO_INT_NONE; 878 879 vfio_enable_intx(vdev); 880 } 881 882 static void vfio_disable_msix(VFIOPCIDevice *vdev) 883 { 884 int i; 885 886 msix_unset_vector_notifiers(&vdev->pdev); 887 888 /* 889 * MSI-X will only release vectors if MSI-X is still enabled on the 890 * device, check through the rest and release it ourselves if necessary. 891 */ 892 for (i = 0; i < vdev->nr_vectors; i++) { 893 if (vdev->msi_vectors[i].use) { 894 vfio_msix_vector_release(&vdev->pdev, i); 895 msix_vector_unuse(&vdev->pdev, i); 896 } 897 } 898 899 if (vdev->nr_vectors) { 900 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX); 901 } 902 903 vfio_disable_msi_common(vdev); 904 905 trace_vfio_disable_msix(vdev->vbasedev.name); 906 } 907 908 static void vfio_disable_msi(VFIOPCIDevice *vdev) 909 { 910 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX); 911 vfio_disable_msi_common(vdev); 912 913 trace_vfio_disable_msi(vdev->vbasedev.name); 914 } 915 916 static void vfio_update_msi(VFIOPCIDevice *vdev) 917 { 918 int i; 919 920 for (i = 0; i < vdev->nr_vectors; i++) { 921 VFIOMSIVector *vector = &vdev->msi_vectors[i]; 922 MSIMessage msg; 923 924 if (!vector->use || vector->virq < 0) { 925 continue; 926 } 927 928 msg = msi_get_message(&vdev->pdev, i); 929 vfio_update_kvm_msi_virq(vector, msg); 930 } 931 } 932 933 static void vfio_pci_load_rom(VFIOPCIDevice *vdev) 934 { 935 struct vfio_region_info reg_info = { 936 .argsz = sizeof(reg_info), 937 .index = VFIO_PCI_ROM_REGION_INDEX 938 }; 939 uint64_t size; 940 off_t off = 0; 941 ssize_t bytes; 942 943 if (ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { 944 error_report("vfio: Error getting ROM info: %m"); 945 return; 946 } 947 948 trace_vfio_pci_load_rom(vdev->vbasedev.name, (unsigned long)reg_info.size, 949 (unsigned long)reg_info.offset, 950 (unsigned long)reg_info.flags); 951 952 vdev->rom_size = size = reg_info.size; 953 vdev->rom_offset = reg_info.offset; 954 955 if (!vdev->rom_size) { 956 vdev->rom_read_failed = true; 957 error_report("vfio-pci: Cannot read device rom at " 958 "%s", vdev->vbasedev.name); 959 error_printf("Device option ROM contents are probably invalid " 960 "(check dmesg).\nSkip option ROM probe with rombar=0, " 961 "or load from file with romfile=\n"); 962 return; 963 } 964 965 vdev->rom = g_malloc(size); 966 memset(vdev->rom, 0xff, size); 967 968 while (size) { 969 bytes = pread(vdev->vbasedev.fd, vdev->rom + off, 970 size, vdev->rom_offset + off); 971 if (bytes == 0) { 972 break; 973 } else if (bytes > 0) { 974 off += bytes; 975 size -= bytes; 976 } else { 977 if (errno == EINTR || errno == EAGAIN) { 978 continue; 979 } 980 error_report("vfio: Error reading device ROM: %m"); 981 break; 982 } 983 } 984 } 985 986 static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size) 987 { 988 VFIOPCIDevice *vdev = opaque; 989 union { 990 uint8_t byte; 991 uint16_t word; 992 uint32_t dword; 993 uint64_t qword; 994 } val; 995 uint64_t data = 0; 996 997 /* Load the ROM lazily when the guest tries to read it */ 998 if (unlikely(!vdev->rom && !vdev->rom_read_failed)) { 999 vfio_pci_load_rom(vdev); 1000 } 1001 1002 memcpy(&val, vdev->rom + addr, 1003 (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0); 1004 1005 switch (size) { 1006 case 1: 1007 data = val.byte; 1008 break; 1009 case 2: 1010 data = le16_to_cpu(val.word); 1011 break; 1012 case 4: 1013 data = le32_to_cpu(val.dword); 1014 break; 1015 default: 1016 hw_error("vfio: unsupported read size, %d bytes\n", size); 1017 break; 1018 } 1019 1020 trace_vfio_rom_read(vdev->vbasedev.name, addr, size, data); 1021 1022 return data; 1023 } 1024 1025 static void vfio_rom_write(void *opaque, hwaddr addr, 1026 uint64_t data, unsigned size) 1027 { 1028 } 1029 1030 static const MemoryRegionOps vfio_rom_ops = { 1031 .read = vfio_rom_read, 1032 .write = vfio_rom_write, 1033 .endianness = DEVICE_LITTLE_ENDIAN, 1034 }; 1035 1036 static bool vfio_blacklist_opt_rom(VFIOPCIDevice *vdev) 1037 { 1038 PCIDevice *pdev = &vdev->pdev; 1039 uint16_t vendor_id, device_id; 1040 int count = 0; 1041 1042 vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID); 1043 device_id = pci_get_word(pdev->config + PCI_DEVICE_ID); 1044 1045 while (count < ARRAY_SIZE(romblacklist)) { 1046 if (romblacklist[count].vendor_id == vendor_id && 1047 romblacklist[count].device_id == device_id) { 1048 return true; 1049 } 1050 count++; 1051 } 1052 1053 return false; 1054 } 1055 1056 static void vfio_pci_size_rom(VFIOPCIDevice *vdev) 1057 { 1058 uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK); 1059 off_t offset = vdev->config_offset + PCI_ROM_ADDRESS; 1060 DeviceState *dev = DEVICE(vdev); 1061 char name[32]; 1062 int fd = vdev->vbasedev.fd; 1063 1064 if (vdev->pdev.romfile || !vdev->pdev.rom_bar) { 1065 /* Since pci handles romfile, just print a message and return */ 1066 if (vfio_blacklist_opt_rom(vdev) && vdev->pdev.romfile) { 1067 error_printf("Warning : Device at %04x:%02x:%02x.%x " 1068 "is known to cause system instability issues during " 1069 "option rom execution. " 1070 "Proceeding anyway since user specified romfile\n", 1071 vdev->host.domain, vdev->host.bus, vdev->host.slot, 1072 vdev->host.function); 1073 } 1074 return; 1075 } 1076 1077 /* 1078 * Use the same size ROM BAR as the physical device. The contents 1079 * will get filled in later when the guest tries to read it. 1080 */ 1081 if (pread(fd, &orig, 4, offset) != 4 || 1082 pwrite(fd, &size, 4, offset) != 4 || 1083 pread(fd, &size, 4, offset) != 4 || 1084 pwrite(fd, &orig, 4, offset) != 4) { 1085 error_report("%s(%04x:%02x:%02x.%x) failed: %m", 1086 __func__, vdev->host.domain, vdev->host.bus, 1087 vdev->host.slot, vdev->host.function); 1088 return; 1089 } 1090 1091 size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1; 1092 1093 if (!size) { 1094 return; 1095 } 1096 1097 if (vfio_blacklist_opt_rom(vdev)) { 1098 if (dev->opts && qemu_opt_get(dev->opts, "rombar")) { 1099 error_printf("Warning : Device at %04x:%02x:%02x.%x " 1100 "is known to cause system instability issues during " 1101 "option rom execution. " 1102 "Proceeding anyway since user specified non zero value for " 1103 "rombar\n", 1104 vdev->host.domain, vdev->host.bus, vdev->host.slot, 1105 vdev->host.function); 1106 } else { 1107 error_printf("Warning : Rom loading for device at " 1108 "%04x:%02x:%02x.%x has been disabled due to " 1109 "system instability issues. " 1110 "Specify rombar=1 or romfile to force\n", 1111 vdev->host.domain, vdev->host.bus, vdev->host.slot, 1112 vdev->host.function); 1113 return; 1114 } 1115 } 1116 1117 trace_vfio_pci_size_rom(vdev->vbasedev.name, size); 1118 1119 snprintf(name, sizeof(name), "vfio[%04x:%02x:%02x.%x].rom", 1120 vdev->host.domain, vdev->host.bus, vdev->host.slot, 1121 vdev->host.function); 1122 1123 memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev), 1124 &vfio_rom_ops, vdev, name, size); 1125 1126 pci_register_bar(&vdev->pdev, PCI_ROM_SLOT, 1127 PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom); 1128 1129 vdev->pdev.has_rom = true; 1130 vdev->rom_read_failed = false; 1131 } 1132 1133 static void vfio_vga_write(void *opaque, hwaddr addr, 1134 uint64_t data, unsigned size) 1135 { 1136 VFIOVGARegion *region = opaque; 1137 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]); 1138 union { 1139 uint8_t byte; 1140 uint16_t word; 1141 uint32_t dword; 1142 uint64_t qword; 1143 } buf; 1144 off_t offset = vga->fd_offset + region->offset + addr; 1145 1146 switch (size) { 1147 case 1: 1148 buf.byte = data; 1149 break; 1150 case 2: 1151 buf.word = cpu_to_le16(data); 1152 break; 1153 case 4: 1154 buf.dword = cpu_to_le32(data); 1155 break; 1156 default: 1157 hw_error("vfio: unsupported write size, %d bytes", size); 1158 break; 1159 } 1160 1161 if (pwrite(vga->fd, &buf, size, offset) != size) { 1162 error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m", 1163 __func__, region->offset + addr, data, size); 1164 } 1165 1166 trace_vfio_vga_write(region->offset + addr, data, size); 1167 } 1168 1169 static uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size) 1170 { 1171 VFIOVGARegion *region = opaque; 1172 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]); 1173 union { 1174 uint8_t byte; 1175 uint16_t word; 1176 uint32_t dword; 1177 uint64_t qword; 1178 } buf; 1179 uint64_t data = 0; 1180 off_t offset = vga->fd_offset + region->offset + addr; 1181 1182 if (pread(vga->fd, &buf, size, offset) != size) { 1183 error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m", 1184 __func__, region->offset + addr, size); 1185 return (uint64_t)-1; 1186 } 1187 1188 switch (size) { 1189 case 1: 1190 data = buf.byte; 1191 break; 1192 case 2: 1193 data = le16_to_cpu(buf.word); 1194 break; 1195 case 4: 1196 data = le32_to_cpu(buf.dword); 1197 break; 1198 default: 1199 hw_error("vfio: unsupported read size, %d bytes", size); 1200 break; 1201 } 1202 1203 trace_vfio_vga_read(region->offset + addr, size, data); 1204 1205 return data; 1206 } 1207 1208 static const MemoryRegionOps vfio_vga_ops = { 1209 .read = vfio_vga_read, 1210 .write = vfio_vga_write, 1211 .endianness = DEVICE_LITTLE_ENDIAN, 1212 }; 1213 1214 /* 1215 * Device specific quirks 1216 */ 1217 1218 /* Is range1 fully contained within range2? */ 1219 static bool vfio_range_contained(uint64_t first1, uint64_t len1, 1220 uint64_t first2, uint64_t len2) { 1221 return (first1 >= first2 && first1 + len1 <= first2 + len2); 1222 } 1223 1224 static bool vfio_flags_enabled(uint8_t flags, uint8_t mask) 1225 { 1226 return (mask && (flags & mask) == mask); 1227 } 1228 1229 static uint64_t vfio_generic_window_quirk_read(void *opaque, 1230 hwaddr addr, unsigned size) 1231 { 1232 VFIOQuirk *quirk = opaque; 1233 VFIOPCIDevice *vdev = quirk->vdev; 1234 uint64_t data; 1235 1236 if (vfio_flags_enabled(quirk->data.flags, quirk->data.read_flags) && 1237 ranges_overlap(addr, size, 1238 quirk->data.data_offset, quirk->data.data_size)) { 1239 hwaddr offset = addr - quirk->data.data_offset; 1240 1241 if (!vfio_range_contained(addr, size, quirk->data.data_offset, 1242 quirk->data.data_size)) { 1243 hw_error("%s: window data read not fully contained: %s", 1244 __func__, memory_region_name(&quirk->mem)); 1245 } 1246 1247 data = vfio_pci_read_config(&vdev->pdev, 1248 quirk->data.address_val + offset, size); 1249 1250 trace_vfio_generic_window_quirk_read(memory_region_name(&quirk->mem), 1251 vdev->vbasedev.name, 1252 quirk->data.bar, 1253 addr, size, data); 1254 } else { 1255 data = vfio_region_read(&vdev->bars[quirk->data.bar].region, 1256 addr + quirk->data.base_offset, size); 1257 } 1258 1259 return data; 1260 } 1261 1262 static void vfio_generic_window_quirk_write(void *opaque, hwaddr addr, 1263 uint64_t data, unsigned size) 1264 { 1265 VFIOQuirk *quirk = opaque; 1266 VFIOPCIDevice *vdev = quirk->vdev; 1267 1268 if (ranges_overlap(addr, size, 1269 quirk->data.address_offset, quirk->data.address_size)) { 1270 1271 if (addr != quirk->data.address_offset) { 1272 hw_error("%s: offset write into address window: %s", 1273 __func__, memory_region_name(&quirk->mem)); 1274 } 1275 1276 if ((data & ~quirk->data.address_mask) == quirk->data.address_match) { 1277 quirk->data.flags |= quirk->data.write_flags | 1278 quirk->data.read_flags; 1279 quirk->data.address_val = data & quirk->data.address_mask; 1280 } else { 1281 quirk->data.flags &= ~(quirk->data.write_flags | 1282 quirk->data.read_flags); 1283 } 1284 } 1285 1286 if (vfio_flags_enabled(quirk->data.flags, quirk->data.write_flags) && 1287 ranges_overlap(addr, size, 1288 quirk->data.data_offset, quirk->data.data_size)) { 1289 hwaddr offset = addr - quirk->data.data_offset; 1290 1291 if (!vfio_range_contained(addr, size, quirk->data.data_offset, 1292 quirk->data.data_size)) { 1293 hw_error("%s: window data write not fully contained: %s", 1294 __func__, memory_region_name(&quirk->mem)); 1295 } 1296 1297 vfio_pci_write_config(&vdev->pdev, 1298 quirk->data.address_val + offset, data, size); 1299 trace_vfio_generic_window_quirk_write(memory_region_name(&quirk->mem), 1300 vdev->vbasedev.name, 1301 quirk->data.bar, 1302 addr, data, size); 1303 return; 1304 } 1305 1306 vfio_region_write(&vdev->bars[quirk->data.bar].region, 1307 addr + quirk->data.base_offset, data, size); 1308 } 1309 1310 static const MemoryRegionOps vfio_generic_window_quirk = { 1311 .read = vfio_generic_window_quirk_read, 1312 .write = vfio_generic_window_quirk_write, 1313 .endianness = DEVICE_LITTLE_ENDIAN, 1314 }; 1315 1316 static uint64_t vfio_generic_quirk_read(void *opaque, 1317 hwaddr addr, unsigned size) 1318 { 1319 VFIOQuirk *quirk = opaque; 1320 VFIOPCIDevice *vdev = quirk->vdev; 1321 hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK; 1322 hwaddr offset = quirk->data.address_match & ~TARGET_PAGE_MASK; 1323 uint64_t data; 1324 1325 if (vfio_flags_enabled(quirk->data.flags, quirk->data.read_flags) && 1326 ranges_overlap(addr, size, offset, quirk->data.address_mask + 1)) { 1327 if (!vfio_range_contained(addr, size, offset, 1328 quirk->data.address_mask + 1)) { 1329 hw_error("%s: read not fully contained: %s", 1330 __func__, memory_region_name(&quirk->mem)); 1331 } 1332 1333 data = vfio_pci_read_config(&vdev->pdev, addr - offset, size); 1334 1335 trace_vfio_generic_quirk_read(memory_region_name(&quirk->mem), 1336 vdev->vbasedev.name, quirk->data.bar, 1337 addr + base, size, data); 1338 } else { 1339 data = vfio_region_read(&vdev->bars[quirk->data.bar].region, 1340 addr + base, size); 1341 } 1342 1343 return data; 1344 } 1345 1346 static void vfio_generic_quirk_write(void *opaque, hwaddr addr, 1347 uint64_t data, unsigned size) 1348 { 1349 VFIOQuirk *quirk = opaque; 1350 VFIOPCIDevice *vdev = quirk->vdev; 1351 hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK; 1352 hwaddr offset = quirk->data.address_match & ~TARGET_PAGE_MASK; 1353 1354 if (vfio_flags_enabled(quirk->data.flags, quirk->data.write_flags) && 1355 ranges_overlap(addr, size, offset, quirk->data.address_mask + 1)) { 1356 if (!vfio_range_contained(addr, size, offset, 1357 quirk->data.address_mask + 1)) { 1358 hw_error("%s: write not fully contained: %s", 1359 __func__, memory_region_name(&quirk->mem)); 1360 } 1361 1362 vfio_pci_write_config(&vdev->pdev, addr - offset, data, size); 1363 1364 trace_vfio_generic_quirk_write(memory_region_name(&quirk->mem), 1365 vdev->vbasedev.name, quirk->data.bar, 1366 addr + base, data, size); 1367 } else { 1368 vfio_region_write(&vdev->bars[quirk->data.bar].region, 1369 addr + base, data, size); 1370 } 1371 } 1372 1373 static const MemoryRegionOps vfio_generic_quirk = { 1374 .read = vfio_generic_quirk_read, 1375 .write = vfio_generic_quirk_write, 1376 .endianness = DEVICE_LITTLE_ENDIAN, 1377 }; 1378 1379 #define PCI_VENDOR_ID_ATI 0x1002 1380 1381 /* 1382 * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR 1383 * through VGA register 0x3c3. On newer cards, the I/O port BAR is always 1384 * BAR4 (older cards like the X550 used BAR1, but we don't care to support 1385 * those). Note that on bare metal, a read of 0x3c3 doesn't always return the 1386 * I/O port BAR address. Originally this was coded to return the virtual BAR 1387 * address only if the physical register read returns the actual BAR address, 1388 * but users have reported greater success if we return the virtual address 1389 * unconditionally. 1390 */ 1391 static uint64_t vfio_ati_3c3_quirk_read(void *opaque, 1392 hwaddr addr, unsigned size) 1393 { 1394 VFIOQuirk *quirk = opaque; 1395 VFIOPCIDevice *vdev = quirk->vdev; 1396 uint64_t data = vfio_pci_read_config(&vdev->pdev, 1397 PCI_BASE_ADDRESS_0 + (4 * 4) + 1, 1398 size); 1399 trace_vfio_ati_3c3_quirk_read(data); 1400 1401 return data; 1402 } 1403 1404 static const MemoryRegionOps vfio_ati_3c3_quirk = { 1405 .read = vfio_ati_3c3_quirk_read, 1406 .endianness = DEVICE_LITTLE_ENDIAN, 1407 }; 1408 1409 static void vfio_vga_probe_ati_3c3_quirk(VFIOPCIDevice *vdev) 1410 { 1411 PCIDevice *pdev = &vdev->pdev; 1412 VFIOQuirk *quirk; 1413 1414 if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) { 1415 return; 1416 } 1417 1418 /* 1419 * As long as the BAR is >= 256 bytes it will be aligned such that the 1420 * lower byte is always zero. Filter out anything else, if it exists. 1421 */ 1422 if (!vdev->bars[4].ioport || vdev->bars[4].region.size < 256) { 1423 return; 1424 } 1425 1426 quirk = g_malloc0(sizeof(*quirk)); 1427 quirk->vdev = vdev; 1428 1429 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_ati_3c3_quirk, quirk, 1430 "vfio-ati-3c3-quirk", 1); 1431 memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem, 1432 3 /* offset 3 bytes from 0x3c0 */, &quirk->mem); 1433 1434 QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks, 1435 quirk, next); 1436 1437 trace_vfio_vga_probe_ati_3c3_quirk(vdev->vbasedev.name); 1438 } 1439 1440 /* 1441 * Newer ATI/AMD devices, including HD5450 and HD7850, have a window to PCI 1442 * config space through MMIO BAR2 at offset 0x4000. Nothing seems to access 1443 * the MMIO space directly, but a window to this space is provided through 1444 * I/O port BAR4. Offset 0x0 is the address register and offset 0x4 is the 1445 * data register. When the address is programmed to a range of 0x4000-0x4fff 1446 * PCI configuration space is available. Experimentation seems to indicate 1447 * that only read-only access is provided, but we drop writes when the window 1448 * is enabled to config space nonetheless. 1449 */ 1450 static void vfio_probe_ati_bar4_window_quirk(VFIOPCIDevice *vdev, int nr) 1451 { 1452 PCIDevice *pdev = &vdev->pdev; 1453 VFIOQuirk *quirk; 1454 1455 if (!vdev->has_vga || nr != 4 || 1456 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) { 1457 return; 1458 } 1459 1460 quirk = g_malloc0(sizeof(*quirk)); 1461 quirk->vdev = vdev; 1462 quirk->data.address_size = 4; 1463 quirk->data.data_offset = 4; 1464 quirk->data.data_size = 4; 1465 quirk->data.address_match = 0x4000; 1466 quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1; 1467 quirk->data.bar = nr; 1468 quirk->data.read_flags = quirk->data.write_flags = 1; 1469 1470 memory_region_init_io(&quirk->mem, OBJECT(vdev), 1471 &vfio_generic_window_quirk, quirk, 1472 "vfio-ati-bar4-window-quirk", 8); 1473 memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem, 1474 quirk->data.base_offset, &quirk->mem, 1); 1475 1476 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); 1477 1478 trace_vfio_probe_ati_bar4_window_quirk(vdev->vbasedev.name); 1479 } 1480 1481 #define PCI_VENDOR_ID_REALTEK 0x10ec 1482 1483 /* 1484 * RTL8168 devices have a backdoor that can access the MSI-X table. At BAR2 1485 * offset 0x70 there is a dword data register, offset 0x74 is a dword address 1486 * register. According to the Linux r8169 driver, the MSI-X table is addressed 1487 * when the "type" portion of the address register is set to 0x1. This appears 1488 * to be bits 16:30. Bit 31 is both a write indicator and some sort of 1489 * "address latched" indicator. Bits 12:15 are a mask field, which we can 1490 * ignore because the MSI-X table should always be accessed as a dword (full 1491 * mask). Bits 0:11 is offset within the type. 1492 * 1493 * Example trace: 1494 * 1495 * Read from MSI-X table offset 0 1496 * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x1f000, 4) // store read addr 1497 * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x8001f000 // latch 1498 * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x70, 4) = 0xfee00398 // read data 1499 * 1500 * Write 0xfee00000 to MSI-X table offset 0 1501 * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x70, 0xfee00000, 4) // write data 1502 * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x8001f000, 4) // do write 1503 * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x1f000 // complete 1504 */ 1505 1506 static uint64_t vfio_rtl8168_window_quirk_read(void *opaque, 1507 hwaddr addr, unsigned size) 1508 { 1509 VFIOQuirk *quirk = opaque; 1510 VFIOPCIDevice *vdev = quirk->vdev; 1511 1512 switch (addr) { 1513 case 4: /* address */ 1514 if (quirk->data.flags) { 1515 trace_vfio_rtl8168_window_quirk_read_fake( 1516 memory_region_name(&quirk->mem), 1517 vdev->vbasedev.name); 1518 1519 return quirk->data.address_match ^ 0x80000000U; 1520 } 1521 break; 1522 case 0: /* data */ 1523 if (quirk->data.flags) { 1524 uint64_t val; 1525 1526 trace_vfio_rtl8168_window_quirk_read_table( 1527 memory_region_name(&quirk->mem), 1528 vdev->vbasedev.name); 1529 1530 if (!(vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX)) { 1531 return 0; 1532 } 1533 1534 memory_region_dispatch_read(&vdev->pdev.msix_table_mmio, 1535 (hwaddr)(quirk->data.address_match 1536 & 0xfff), 1537 &val, 1538 size, 1539 MEMTXATTRS_UNSPECIFIED); 1540 return val; 1541 } 1542 } 1543 1544 trace_vfio_rtl8168_window_quirk_read_direct(memory_region_name(&quirk->mem), 1545 vdev->vbasedev.name); 1546 1547 return vfio_region_read(&vdev->bars[quirk->data.bar].region, 1548 addr + 0x70, size); 1549 } 1550 1551 static void vfio_rtl8168_window_quirk_write(void *opaque, hwaddr addr, 1552 uint64_t data, unsigned size) 1553 { 1554 VFIOQuirk *quirk = opaque; 1555 VFIOPCIDevice *vdev = quirk->vdev; 1556 1557 switch (addr) { 1558 case 4: /* address */ 1559 if ((data & 0x7fff0000) == 0x10000) { 1560 if (data & 0x80000000U && 1561 vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX) { 1562 1563 trace_vfio_rtl8168_window_quirk_write_table( 1564 memory_region_name(&quirk->mem), 1565 vdev->vbasedev.name); 1566 1567 memory_region_dispatch_write(&vdev->pdev.msix_table_mmio, 1568 (hwaddr)(data & 0xfff), 1569 (uint64_t)quirk->data.address_mask, 1570 size, MEMTXATTRS_UNSPECIFIED); 1571 } 1572 1573 quirk->data.flags = 1; 1574 quirk->data.address_match = data; 1575 1576 return; 1577 } 1578 quirk->data.flags = 0; 1579 break; 1580 case 0: /* data */ 1581 quirk->data.address_mask = data; 1582 break; 1583 } 1584 1585 trace_vfio_rtl8168_window_quirk_write_direct( 1586 memory_region_name(&quirk->mem), 1587 vdev->vbasedev.name); 1588 1589 vfio_region_write(&vdev->bars[quirk->data.bar].region, 1590 addr + 0x70, data, size); 1591 } 1592 1593 static const MemoryRegionOps vfio_rtl8168_window_quirk = { 1594 .read = vfio_rtl8168_window_quirk_read, 1595 .write = vfio_rtl8168_window_quirk_write, 1596 .valid = { 1597 .min_access_size = 4, 1598 .max_access_size = 4, 1599 .unaligned = false, 1600 }, 1601 .endianness = DEVICE_LITTLE_ENDIAN, 1602 }; 1603 1604 static void vfio_probe_rtl8168_bar2_window_quirk(VFIOPCIDevice *vdev, int nr) 1605 { 1606 PCIDevice *pdev = &vdev->pdev; 1607 VFIOQuirk *quirk; 1608 1609 if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_REALTEK || 1610 pci_get_word(pdev->config + PCI_DEVICE_ID) != 0x8168 || nr != 2) { 1611 return; 1612 } 1613 1614 quirk = g_malloc0(sizeof(*quirk)); 1615 quirk->vdev = vdev; 1616 quirk->data.bar = nr; 1617 1618 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_rtl8168_window_quirk, 1619 quirk, "vfio-rtl8168-window-quirk", 8); 1620 memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem, 1621 0x70, &quirk->mem, 1); 1622 1623 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); 1624 1625 trace_vfio_probe_rtl8168_bar2_window_quirk(vdev->vbasedev.name); 1626 } 1627 /* 1628 * Trap the BAR2 MMIO window to config space as well. 1629 */ 1630 static void vfio_probe_ati_bar2_4000_quirk(VFIOPCIDevice *vdev, int nr) 1631 { 1632 PCIDevice *pdev = &vdev->pdev; 1633 VFIOQuirk *quirk; 1634 1635 /* Only enable on newer devices where BAR2 is 64bit */ 1636 if (!vdev->has_vga || nr != 2 || !vdev->bars[2].mem64 || 1637 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) { 1638 return; 1639 } 1640 1641 quirk = g_malloc0(sizeof(*quirk)); 1642 quirk->vdev = vdev; 1643 quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1; 1644 quirk->data.address_match = 0x4000; 1645 quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1; 1646 quirk->data.bar = nr; 1647 1648 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_generic_quirk, quirk, 1649 "vfio-ati-bar2-4000-quirk", 1650 TARGET_PAGE_ALIGN(quirk->data.address_mask + 1)); 1651 memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem, 1652 quirk->data.address_match & TARGET_PAGE_MASK, 1653 &quirk->mem, 1); 1654 1655 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); 1656 1657 trace_vfio_probe_ati_bar2_4000_quirk(vdev->vbasedev.name); 1658 } 1659 1660 /* 1661 * Older ATI/AMD cards like the X550 have a similar window to that above. 1662 * I/O port BAR1 provides a window to a mirror of PCI config space located 1663 * in BAR2 at offset 0xf00. We don't care to support such older cards, but 1664 * note it for future reference. 1665 */ 1666 1667 #define PCI_VENDOR_ID_NVIDIA 0x10de 1668 1669 /* 1670 * Nvidia has several different methods to get to config space, the 1671 * nouveu project has several of these documented here: 1672 * https://github.com/pathscale/envytools/tree/master/hwdocs 1673 * 1674 * The first quirk is actually not documented in envytools and is found 1675 * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]). This is an 1676 * NV46 chipset. The backdoor uses the legacy VGA I/O ports to access 1677 * the mirror of PCI config space found at BAR0 offset 0x1800. The access 1678 * sequence first writes 0x338 to I/O port 0x3d4. The target offset is 1679 * then written to 0x3d0. Finally 0x538 is written for a read and 0x738 1680 * is written for a write to 0x3d4. The BAR0 offset is then accessible 1681 * through 0x3d0. This quirk doesn't seem to be necessary on newer cards 1682 * that use the I/O port BAR5 window but it doesn't hurt to leave it. 1683 */ 1684 enum { 1685 NV_3D0_NONE = 0, 1686 NV_3D0_SELECT, 1687 NV_3D0_WINDOW, 1688 NV_3D0_READ, 1689 NV_3D0_WRITE, 1690 }; 1691 1692 static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque, 1693 hwaddr addr, unsigned size) 1694 { 1695 VFIOQuirk *quirk = opaque; 1696 VFIOPCIDevice *vdev = quirk->vdev; 1697 PCIDevice *pdev = &vdev->pdev; 1698 uint64_t data = vfio_vga_read(&vdev->vga.region[QEMU_PCI_VGA_IO_HI], 1699 addr + quirk->data.base_offset, size); 1700 1701 if (quirk->data.flags == NV_3D0_READ && addr == quirk->data.data_offset) { 1702 data = vfio_pci_read_config(pdev, quirk->data.address_val, size); 1703 trace_vfio_nvidia_3d0_quirk_read(size, data); 1704 } 1705 1706 quirk->data.flags = NV_3D0_NONE; 1707 1708 return data; 1709 } 1710 1711 static void vfio_nvidia_3d0_quirk_write(void *opaque, hwaddr addr, 1712 uint64_t data, unsigned size) 1713 { 1714 VFIOQuirk *quirk = opaque; 1715 VFIOPCIDevice *vdev = quirk->vdev; 1716 PCIDevice *pdev = &vdev->pdev; 1717 1718 switch (quirk->data.flags) { 1719 case NV_3D0_NONE: 1720 if (addr == quirk->data.address_offset && data == 0x338) { 1721 quirk->data.flags = NV_3D0_SELECT; 1722 } 1723 break; 1724 case NV_3D0_SELECT: 1725 quirk->data.flags = NV_3D0_NONE; 1726 if (addr == quirk->data.data_offset && 1727 (data & ~quirk->data.address_mask) == quirk->data.address_match) { 1728 quirk->data.flags = NV_3D0_WINDOW; 1729 quirk->data.address_val = data & quirk->data.address_mask; 1730 } 1731 break; 1732 case NV_3D0_WINDOW: 1733 quirk->data.flags = NV_3D0_NONE; 1734 if (addr == quirk->data.address_offset) { 1735 if (data == 0x538) { 1736 quirk->data.flags = NV_3D0_READ; 1737 } else if (data == 0x738) { 1738 quirk->data.flags = NV_3D0_WRITE; 1739 } 1740 } 1741 break; 1742 case NV_3D0_WRITE: 1743 quirk->data.flags = NV_3D0_NONE; 1744 if (addr == quirk->data.data_offset) { 1745 vfio_pci_write_config(pdev, quirk->data.address_val, data, size); 1746 trace_vfio_nvidia_3d0_quirk_write(data, size); 1747 return; 1748 } 1749 break; 1750 } 1751 1752 vfio_vga_write(&vdev->vga.region[QEMU_PCI_VGA_IO_HI], 1753 addr + quirk->data.base_offset, data, size); 1754 } 1755 1756 static const MemoryRegionOps vfio_nvidia_3d0_quirk = { 1757 .read = vfio_nvidia_3d0_quirk_read, 1758 .write = vfio_nvidia_3d0_quirk_write, 1759 .endianness = DEVICE_LITTLE_ENDIAN, 1760 }; 1761 1762 static void vfio_vga_probe_nvidia_3d0_quirk(VFIOPCIDevice *vdev) 1763 { 1764 PCIDevice *pdev = &vdev->pdev; 1765 VFIOQuirk *quirk; 1766 1767 if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA || 1768 !vdev->bars[1].region.size) { 1769 return; 1770 } 1771 1772 quirk = g_malloc0(sizeof(*quirk)); 1773 quirk->vdev = vdev; 1774 quirk->data.base_offset = 0x10; 1775 quirk->data.address_offset = 4; 1776 quirk->data.address_size = 2; 1777 quirk->data.address_match = 0x1800; 1778 quirk->data.address_mask = PCI_CONFIG_SPACE_SIZE - 1; 1779 quirk->data.data_offset = 0; 1780 quirk->data.data_size = 4; 1781 1782 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_nvidia_3d0_quirk, 1783 quirk, "vfio-nvidia-3d0-quirk", 6); 1784 memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem, 1785 quirk->data.base_offset, &quirk->mem); 1786 1787 QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks, 1788 quirk, next); 1789 1790 trace_vfio_vga_probe_nvidia_3d0_quirk(vdev->vbasedev.name); 1791 } 1792 1793 /* 1794 * The second quirk is documented in envytools. The I/O port BAR5 is just 1795 * a set of address/data ports to the MMIO BARs. The BAR we care about is 1796 * again BAR0. This backdoor is apparently a bit newer than the one above 1797 * so we need to not only trap 256 bytes @0x1800, but all of PCI config 1798 * space, including extended space is available at the 4k @0x88000. 1799 */ 1800 enum { 1801 NV_BAR5_ADDRESS = 0x1, 1802 NV_BAR5_ENABLE = 0x2, 1803 NV_BAR5_MASTER = 0x4, 1804 NV_BAR5_VALID = 0x7, 1805 }; 1806 1807 static void vfio_nvidia_bar5_window_quirk_write(void *opaque, hwaddr addr, 1808 uint64_t data, unsigned size) 1809 { 1810 VFIOQuirk *quirk = opaque; 1811 1812 switch (addr) { 1813 case 0x0: 1814 if (data & 0x1) { 1815 quirk->data.flags |= NV_BAR5_MASTER; 1816 } else { 1817 quirk->data.flags &= ~NV_BAR5_MASTER; 1818 } 1819 break; 1820 case 0x4: 1821 if (data & 0x1) { 1822 quirk->data.flags |= NV_BAR5_ENABLE; 1823 } else { 1824 quirk->data.flags &= ~NV_BAR5_ENABLE; 1825 } 1826 break; 1827 case 0x8: 1828 if (quirk->data.flags & NV_BAR5_MASTER) { 1829 if ((data & ~0xfff) == 0x88000) { 1830 quirk->data.flags |= NV_BAR5_ADDRESS; 1831 quirk->data.address_val = data & 0xfff; 1832 } else if ((data & ~0xff) == 0x1800) { 1833 quirk->data.flags |= NV_BAR5_ADDRESS; 1834 quirk->data.address_val = data & 0xff; 1835 } else { 1836 quirk->data.flags &= ~NV_BAR5_ADDRESS; 1837 } 1838 } 1839 break; 1840 } 1841 1842 vfio_generic_window_quirk_write(opaque, addr, data, size); 1843 } 1844 1845 static const MemoryRegionOps vfio_nvidia_bar5_window_quirk = { 1846 .read = vfio_generic_window_quirk_read, 1847 .write = vfio_nvidia_bar5_window_quirk_write, 1848 .valid.min_access_size = 4, 1849 .endianness = DEVICE_LITTLE_ENDIAN, 1850 }; 1851 1852 static void vfio_probe_nvidia_bar5_window_quirk(VFIOPCIDevice *vdev, int nr) 1853 { 1854 PCIDevice *pdev = &vdev->pdev; 1855 VFIOQuirk *quirk; 1856 1857 if (!vdev->has_vga || nr != 5 || 1858 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) { 1859 return; 1860 } 1861 1862 quirk = g_malloc0(sizeof(*quirk)); 1863 quirk->vdev = vdev; 1864 quirk->data.read_flags = quirk->data.write_flags = NV_BAR5_VALID; 1865 quirk->data.address_offset = 0x8; 1866 quirk->data.address_size = 0; /* actually 4, but avoids generic code */ 1867 quirk->data.data_offset = 0xc; 1868 quirk->data.data_size = 4; 1869 quirk->data.bar = nr; 1870 1871 memory_region_init_io(&quirk->mem, OBJECT(vdev), 1872 &vfio_nvidia_bar5_window_quirk, quirk, 1873 "vfio-nvidia-bar5-window-quirk", 16); 1874 memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem, 1875 0, &quirk->mem, 1); 1876 1877 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); 1878 1879 trace_vfio_probe_nvidia_bar5_window_quirk(vdev->vbasedev.name); 1880 } 1881 1882 static void vfio_nvidia_88000_quirk_write(void *opaque, hwaddr addr, 1883 uint64_t data, unsigned size) 1884 { 1885 VFIOQuirk *quirk = opaque; 1886 VFIOPCIDevice *vdev = quirk->vdev; 1887 PCIDevice *pdev = &vdev->pdev; 1888 hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK; 1889 1890 vfio_generic_quirk_write(opaque, addr, data, size); 1891 1892 /* 1893 * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the 1894 * MSI capability ID register. Both the ID and next register are 1895 * read-only, so we allow writes covering either of those to real hw. 1896 * NB - only fixed for the 0x88000 MMIO window. 1897 */ 1898 if ((pdev->cap_present & QEMU_PCI_CAP_MSI) && 1899 vfio_range_contained(addr, size, pdev->msi_cap, PCI_MSI_FLAGS)) { 1900 vfio_region_write(&vdev->bars[quirk->data.bar].region, 1901 addr + base, data, size); 1902 } 1903 } 1904 1905 static const MemoryRegionOps vfio_nvidia_88000_quirk = { 1906 .read = vfio_generic_quirk_read, 1907 .write = vfio_nvidia_88000_quirk_write, 1908 .endianness = DEVICE_LITTLE_ENDIAN, 1909 }; 1910 1911 /* 1912 * Finally, BAR0 itself. We want to redirect any accesses to either 1913 * 0x1800 or 0x88000 through the PCI config space access functions. 1914 * 1915 * NB - quirk at a page granularity or else they don't seem to work when 1916 * BARs are mmap'd 1917 * 1918 * Here's offset 0x88000... 1919 */ 1920 static void vfio_probe_nvidia_bar0_88000_quirk(VFIOPCIDevice *vdev, int nr) 1921 { 1922 PCIDevice *pdev = &vdev->pdev; 1923 VFIOQuirk *quirk; 1924 uint16_t vendor, class; 1925 1926 vendor = pci_get_word(pdev->config + PCI_VENDOR_ID); 1927 class = pci_get_word(pdev->config + PCI_CLASS_DEVICE); 1928 1929 if (nr != 0 || vendor != PCI_VENDOR_ID_NVIDIA || 1930 class != PCI_CLASS_DISPLAY_VGA) { 1931 return; 1932 } 1933 1934 quirk = g_malloc0(sizeof(*quirk)); 1935 quirk->vdev = vdev; 1936 quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1; 1937 quirk->data.address_match = 0x88000; 1938 quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1; 1939 quirk->data.bar = nr; 1940 1941 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_nvidia_88000_quirk, 1942 quirk, "vfio-nvidia-bar0-88000-quirk", 1943 TARGET_PAGE_ALIGN(quirk->data.address_mask + 1)); 1944 memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem, 1945 quirk->data.address_match & TARGET_PAGE_MASK, 1946 &quirk->mem, 1); 1947 1948 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); 1949 1950 trace_vfio_probe_nvidia_bar0_88000_quirk(vdev->vbasedev.name); 1951 } 1952 1953 /* 1954 * And here's the same for BAR0 offset 0x1800... 1955 */ 1956 static void vfio_probe_nvidia_bar0_1800_quirk(VFIOPCIDevice *vdev, int nr) 1957 { 1958 PCIDevice *pdev = &vdev->pdev; 1959 VFIOQuirk *quirk; 1960 1961 if (!vdev->has_vga || nr != 0 || 1962 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) { 1963 return; 1964 } 1965 1966 /* Log the chipset ID */ 1967 trace_vfio_probe_nvidia_bar0_1800_quirk_id( 1968 (unsigned int)(vfio_region_read(&vdev->bars[0].region, 0, 4) >> 20) 1969 & 0xff); 1970 1971 quirk = g_malloc0(sizeof(*quirk)); 1972 quirk->vdev = vdev; 1973 quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1; 1974 quirk->data.address_match = 0x1800; 1975 quirk->data.address_mask = PCI_CONFIG_SPACE_SIZE - 1; 1976 quirk->data.bar = nr; 1977 1978 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_generic_quirk, quirk, 1979 "vfio-nvidia-bar0-1800-quirk", 1980 TARGET_PAGE_ALIGN(quirk->data.address_mask + 1)); 1981 memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem, 1982 quirk->data.address_match & TARGET_PAGE_MASK, 1983 &quirk->mem, 1); 1984 1985 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); 1986 1987 trace_vfio_probe_nvidia_bar0_1800_quirk(vdev->vbasedev.name); 1988 } 1989 1990 /* 1991 * TODO - Some Nvidia devices provide config access to their companion HDA 1992 * device and even to their parent bridge via these config space mirrors. 1993 * Add quirks for those regions. 1994 */ 1995 1996 /* 1997 * Common quirk probe entry points. 1998 */ 1999 static void vfio_vga_quirk_setup(VFIOPCIDevice *vdev) 2000 { 2001 vfio_vga_probe_ati_3c3_quirk(vdev); 2002 vfio_vga_probe_nvidia_3d0_quirk(vdev); 2003 } 2004 2005 static void vfio_vga_quirk_teardown(VFIOPCIDevice *vdev) 2006 { 2007 VFIOQuirk *quirk; 2008 int i; 2009 2010 for (i = 0; i < ARRAY_SIZE(vdev->vga.region); i++) { 2011 QLIST_FOREACH(quirk, &vdev->vga.region[i].quirks, next) { 2012 memory_region_del_subregion(&vdev->vga.region[i].mem, &quirk->mem); 2013 } 2014 } 2015 } 2016 2017 static void vfio_vga_quirk_free(VFIOPCIDevice *vdev) 2018 { 2019 int i; 2020 2021 for (i = 0; i < ARRAY_SIZE(vdev->vga.region); i++) { 2022 while (!QLIST_EMPTY(&vdev->vga.region[i].quirks)) { 2023 VFIOQuirk *quirk = QLIST_FIRST(&vdev->vga.region[i].quirks); 2024 object_unparent(OBJECT(&quirk->mem)); 2025 QLIST_REMOVE(quirk, next); 2026 g_free(quirk); 2027 } 2028 } 2029 } 2030 2031 static void vfio_bar_quirk_setup(VFIOPCIDevice *vdev, int nr) 2032 { 2033 vfio_probe_ati_bar4_window_quirk(vdev, nr); 2034 vfio_probe_ati_bar2_4000_quirk(vdev, nr); 2035 vfio_probe_nvidia_bar5_window_quirk(vdev, nr); 2036 vfio_probe_nvidia_bar0_88000_quirk(vdev, nr); 2037 vfio_probe_nvidia_bar0_1800_quirk(vdev, nr); 2038 vfio_probe_rtl8168_bar2_window_quirk(vdev, nr); 2039 } 2040 2041 static void vfio_bar_quirk_teardown(VFIOPCIDevice *vdev, int nr) 2042 { 2043 VFIOBAR *bar = &vdev->bars[nr]; 2044 VFIOQuirk *quirk; 2045 2046 QLIST_FOREACH(quirk, &bar->quirks, next) { 2047 memory_region_del_subregion(&bar->region.mem, &quirk->mem); 2048 } 2049 } 2050 2051 static void vfio_bar_quirk_free(VFIOPCIDevice *vdev, int nr) 2052 { 2053 VFIOBAR *bar = &vdev->bars[nr]; 2054 2055 while (!QLIST_EMPTY(&bar->quirks)) { 2056 VFIOQuirk *quirk = QLIST_FIRST(&bar->quirks); 2057 object_unparent(OBJECT(&quirk->mem)); 2058 QLIST_REMOVE(quirk, next); 2059 g_free(quirk); 2060 } 2061 } 2062 2063 /* 2064 * PCI config space 2065 */ 2066 static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len) 2067 { 2068 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev); 2069 uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val; 2070 2071 memcpy(&emu_bits, vdev->emulated_config_bits + addr, len); 2072 emu_bits = le32_to_cpu(emu_bits); 2073 2074 if (emu_bits) { 2075 emu_val = pci_default_read_config(pdev, addr, len); 2076 } 2077 2078 if (~emu_bits & (0xffffffffU >> (32 - len * 8))) { 2079 ssize_t ret; 2080 2081 ret = pread(vdev->vbasedev.fd, &phys_val, len, 2082 vdev->config_offset + addr); 2083 if (ret != len) { 2084 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x) failed: %m", 2085 __func__, vdev->host.domain, vdev->host.bus, 2086 vdev->host.slot, vdev->host.function, addr, len); 2087 return -errno; 2088 } 2089 phys_val = le32_to_cpu(phys_val); 2090 } 2091 2092 val = (emu_val & emu_bits) | (phys_val & ~emu_bits); 2093 2094 trace_vfio_pci_read_config(vdev->vbasedev.name, addr, len, val); 2095 2096 return val; 2097 } 2098 2099 static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr, 2100 uint32_t val, int len) 2101 { 2102 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev); 2103 uint32_t val_le = cpu_to_le32(val); 2104 2105 trace_vfio_pci_write_config(vdev->vbasedev.name, addr, val, len); 2106 2107 /* Write everything to VFIO, let it filter out what we can't write */ 2108 if (pwrite(vdev->vbasedev.fd, &val_le, len, vdev->config_offset + addr) 2109 != len) { 2110 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x, 0x%x) failed: %m", 2111 __func__, vdev->host.domain, vdev->host.bus, 2112 vdev->host.slot, vdev->host.function, addr, val, len); 2113 } 2114 2115 /* MSI/MSI-X Enabling/Disabling */ 2116 if (pdev->cap_present & QEMU_PCI_CAP_MSI && 2117 ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) { 2118 int is_enabled, was_enabled = msi_enabled(pdev); 2119 2120 pci_default_write_config(pdev, addr, val, len); 2121 2122 is_enabled = msi_enabled(pdev); 2123 2124 if (!was_enabled) { 2125 if (is_enabled) { 2126 vfio_enable_msi(vdev); 2127 } 2128 } else { 2129 if (!is_enabled) { 2130 vfio_disable_msi(vdev); 2131 } else { 2132 vfio_update_msi(vdev); 2133 } 2134 } 2135 } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX && 2136 ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) { 2137 int is_enabled, was_enabled = msix_enabled(pdev); 2138 2139 pci_default_write_config(pdev, addr, val, len); 2140 2141 is_enabled = msix_enabled(pdev); 2142 2143 if (!was_enabled && is_enabled) { 2144 vfio_enable_msix(vdev); 2145 } else if (was_enabled && !is_enabled) { 2146 vfio_disable_msix(vdev); 2147 } 2148 } else { 2149 /* Write everything to QEMU to keep emulated bits correct */ 2150 pci_default_write_config(pdev, addr, val, len); 2151 } 2152 } 2153 2154 /* 2155 * Interrupt setup 2156 */ 2157 static void vfio_disable_interrupts(VFIOPCIDevice *vdev) 2158 { 2159 /* 2160 * More complicated than it looks. Disabling MSI/X transitions the 2161 * device to INTx mode (if supported). Therefore we need to first 2162 * disable MSI/X and then cleanup by disabling INTx. 2163 */ 2164 if (vdev->interrupt == VFIO_INT_MSIX) { 2165 vfio_disable_msix(vdev); 2166 } else if (vdev->interrupt == VFIO_INT_MSI) { 2167 vfio_disable_msi(vdev); 2168 } 2169 2170 if (vdev->interrupt == VFIO_INT_INTx) { 2171 vfio_disable_intx(vdev); 2172 } 2173 } 2174 2175 static int vfio_setup_msi(VFIOPCIDevice *vdev, int pos) 2176 { 2177 uint16_t ctrl; 2178 bool msi_64bit, msi_maskbit; 2179 int ret, entries; 2180 2181 if (pread(vdev->vbasedev.fd, &ctrl, sizeof(ctrl), 2182 vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) { 2183 return -errno; 2184 } 2185 ctrl = le16_to_cpu(ctrl); 2186 2187 msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT); 2188 msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT); 2189 entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1); 2190 2191 trace_vfio_setup_msi(vdev->vbasedev.name, pos); 2192 2193 ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit); 2194 if (ret < 0) { 2195 if (ret == -ENOTSUP) { 2196 return 0; 2197 } 2198 error_report("vfio: msi_init failed"); 2199 return ret; 2200 } 2201 vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0); 2202 2203 return 0; 2204 } 2205 2206 /* 2207 * We don't have any control over how pci_add_capability() inserts 2208 * capabilities into the chain. In order to setup MSI-X we need a 2209 * MemoryRegion for the BAR. In order to setup the BAR and not 2210 * attempt to mmap the MSI-X table area, which VFIO won't allow, we 2211 * need to first look for where the MSI-X table lives. So we 2212 * unfortunately split MSI-X setup across two functions. 2213 */ 2214 static int vfio_early_setup_msix(VFIOPCIDevice *vdev) 2215 { 2216 uint8_t pos; 2217 uint16_t ctrl; 2218 uint32_t table, pba; 2219 int fd = vdev->vbasedev.fd; 2220 2221 pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX); 2222 if (!pos) { 2223 return 0; 2224 } 2225 2226 if (pread(fd, &ctrl, sizeof(ctrl), 2227 vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) { 2228 return -errno; 2229 } 2230 2231 if (pread(fd, &table, sizeof(table), 2232 vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) { 2233 return -errno; 2234 } 2235 2236 if (pread(fd, &pba, sizeof(pba), 2237 vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) { 2238 return -errno; 2239 } 2240 2241 ctrl = le16_to_cpu(ctrl); 2242 table = le32_to_cpu(table); 2243 pba = le32_to_cpu(pba); 2244 2245 vdev->msix = g_malloc0(sizeof(*(vdev->msix))); 2246 vdev->msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK; 2247 vdev->msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK; 2248 vdev->msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK; 2249 vdev->msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK; 2250 vdev->msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 2251 2252 /* 2253 * Test the size of the pba_offset variable and catch if it extends outside 2254 * of the specified BAR. If it is the case, we need to apply a hardware 2255 * specific quirk if the device is known or we have a broken configuration. 2256 */ 2257 if (vdev->msix->pba_offset >= 2258 vdev->bars[vdev->msix->pba_bar].region.size) { 2259 2260 PCIDevice *pdev = &vdev->pdev; 2261 uint16_t vendor = pci_get_word(pdev->config + PCI_VENDOR_ID); 2262 uint16_t device = pci_get_word(pdev->config + PCI_DEVICE_ID); 2263 2264 /* 2265 * Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5 2266 * adapters. The T5 hardware returns an incorrect value of 0x8000 for 2267 * the VF PBA offset while the BAR itself is only 8k. The correct value 2268 * is 0x1000, so we hard code that here. 2269 */ 2270 if (vendor == PCI_VENDOR_ID_CHELSIO && (device & 0xff00) == 0x5800) { 2271 vdev->msix->pba_offset = 0x1000; 2272 } else { 2273 error_report("vfio: Hardware reports invalid configuration, " 2274 "MSIX PBA outside of specified BAR"); 2275 return -EINVAL; 2276 } 2277 } 2278 2279 trace_vfio_early_setup_msix(vdev->vbasedev.name, pos, 2280 vdev->msix->table_bar, 2281 vdev->msix->table_offset, 2282 vdev->msix->entries); 2283 2284 return 0; 2285 } 2286 2287 static int vfio_setup_msix(VFIOPCIDevice *vdev, int pos) 2288 { 2289 int ret; 2290 2291 ret = msix_init(&vdev->pdev, vdev->msix->entries, 2292 &vdev->bars[vdev->msix->table_bar].region.mem, 2293 vdev->msix->table_bar, vdev->msix->table_offset, 2294 &vdev->bars[vdev->msix->pba_bar].region.mem, 2295 vdev->msix->pba_bar, vdev->msix->pba_offset, pos); 2296 if (ret < 0) { 2297 if (ret == -ENOTSUP) { 2298 return 0; 2299 } 2300 error_report("vfio: msix_init failed"); 2301 return ret; 2302 } 2303 2304 return 0; 2305 } 2306 2307 static void vfio_teardown_msi(VFIOPCIDevice *vdev) 2308 { 2309 msi_uninit(&vdev->pdev); 2310 2311 if (vdev->msix) { 2312 msix_uninit(&vdev->pdev, 2313 &vdev->bars[vdev->msix->table_bar].region.mem, 2314 &vdev->bars[vdev->msix->pba_bar].region.mem); 2315 } 2316 } 2317 2318 /* 2319 * Resource setup 2320 */ 2321 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled) 2322 { 2323 int i; 2324 2325 for (i = 0; i < PCI_ROM_SLOT; i++) { 2326 VFIOBAR *bar = &vdev->bars[i]; 2327 2328 if (!bar->region.size) { 2329 continue; 2330 } 2331 2332 memory_region_set_enabled(&bar->region.mmap_mem, enabled); 2333 if (vdev->msix && vdev->msix->table_bar == i) { 2334 memory_region_set_enabled(&vdev->msix->mmap_mem, enabled); 2335 } 2336 } 2337 } 2338 2339 static void vfio_unregister_bar(VFIOPCIDevice *vdev, int nr) 2340 { 2341 VFIOBAR *bar = &vdev->bars[nr]; 2342 2343 if (!bar->region.size) { 2344 return; 2345 } 2346 2347 vfio_bar_quirk_teardown(vdev, nr); 2348 2349 memory_region_del_subregion(&bar->region.mem, &bar->region.mmap_mem); 2350 2351 if (vdev->msix && vdev->msix->table_bar == nr) { 2352 memory_region_del_subregion(&bar->region.mem, &vdev->msix->mmap_mem); 2353 } 2354 } 2355 2356 static void vfio_unmap_bar(VFIOPCIDevice *vdev, int nr) 2357 { 2358 VFIOBAR *bar = &vdev->bars[nr]; 2359 2360 if (!bar->region.size) { 2361 return; 2362 } 2363 2364 vfio_bar_quirk_free(vdev, nr); 2365 2366 munmap(bar->region.mmap, memory_region_size(&bar->region.mmap_mem)); 2367 2368 if (vdev->msix && vdev->msix->table_bar == nr) { 2369 munmap(vdev->msix->mmap, memory_region_size(&vdev->msix->mmap_mem)); 2370 } 2371 } 2372 2373 static void vfio_map_bar(VFIOPCIDevice *vdev, int nr) 2374 { 2375 VFIOBAR *bar = &vdev->bars[nr]; 2376 uint64_t size = bar->region.size; 2377 char name[64]; 2378 uint32_t pci_bar; 2379 uint8_t type; 2380 int ret; 2381 2382 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */ 2383 if (!size) { 2384 return; 2385 } 2386 2387 snprintf(name, sizeof(name), "VFIO %04x:%02x:%02x.%x BAR %d", 2388 vdev->host.domain, vdev->host.bus, vdev->host.slot, 2389 vdev->host.function, nr); 2390 2391 /* Determine what type of BAR this is for registration */ 2392 ret = pread(vdev->vbasedev.fd, &pci_bar, sizeof(pci_bar), 2393 vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr)); 2394 if (ret != sizeof(pci_bar)) { 2395 error_report("vfio: Failed to read BAR %d (%m)", nr); 2396 return; 2397 } 2398 2399 pci_bar = le32_to_cpu(pci_bar); 2400 bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO); 2401 bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64); 2402 type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK : 2403 ~PCI_BASE_ADDRESS_MEM_MASK); 2404 2405 /* A "slow" read/write mapping underlies all BARs */ 2406 memory_region_init_io(&bar->region.mem, OBJECT(vdev), &vfio_region_ops, 2407 bar, name, size); 2408 pci_register_bar(&vdev->pdev, nr, type, &bar->region.mem); 2409 2410 /* 2411 * We can't mmap areas overlapping the MSIX vector table, so we 2412 * potentially insert a direct-mapped subregion before and after it. 2413 */ 2414 if (vdev->msix && vdev->msix->table_bar == nr) { 2415 size = vdev->msix->table_offset & qemu_real_host_page_mask; 2416 } 2417 2418 strncat(name, " mmap", sizeof(name) - strlen(name) - 1); 2419 if (vfio_mmap_region(OBJECT(vdev), &bar->region, &bar->region.mem, 2420 &bar->region.mmap_mem, &bar->region.mmap, 2421 size, 0, name)) { 2422 error_report("%s unsupported. Performance may be slow", name); 2423 } 2424 2425 if (vdev->msix && vdev->msix->table_bar == nr) { 2426 uint64_t start; 2427 2428 start = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset + 2429 (vdev->msix->entries * 2430 PCI_MSIX_ENTRY_SIZE)); 2431 2432 size = start < bar->region.size ? bar->region.size - start : 0; 2433 strncat(name, " msix-hi", sizeof(name) - strlen(name) - 1); 2434 /* VFIOMSIXInfo contains another MemoryRegion for this mapping */ 2435 if (vfio_mmap_region(OBJECT(vdev), &bar->region, &bar->region.mem, 2436 &vdev->msix->mmap_mem, 2437 &vdev->msix->mmap, size, start, name)) { 2438 error_report("%s unsupported. Performance may be slow", name); 2439 } 2440 } 2441 2442 vfio_bar_quirk_setup(vdev, nr); 2443 } 2444 2445 static void vfio_map_bars(VFIOPCIDevice *vdev) 2446 { 2447 int i; 2448 2449 for (i = 0; i < PCI_ROM_SLOT; i++) { 2450 vfio_map_bar(vdev, i); 2451 } 2452 2453 if (vdev->has_vga) { 2454 memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_MEM].mem, 2455 OBJECT(vdev), &vfio_vga_ops, 2456 &vdev->vga.region[QEMU_PCI_VGA_MEM], 2457 "vfio-vga-mmio@0xa0000", 2458 QEMU_PCI_VGA_MEM_SIZE); 2459 memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_IO_LO].mem, 2460 OBJECT(vdev), &vfio_vga_ops, 2461 &vdev->vga.region[QEMU_PCI_VGA_IO_LO], 2462 "vfio-vga-io@0x3b0", 2463 QEMU_PCI_VGA_IO_LO_SIZE); 2464 memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem, 2465 OBJECT(vdev), &vfio_vga_ops, 2466 &vdev->vga.region[QEMU_PCI_VGA_IO_HI], 2467 "vfio-vga-io@0x3c0", 2468 QEMU_PCI_VGA_IO_HI_SIZE); 2469 2470 pci_register_vga(&vdev->pdev, &vdev->vga.region[QEMU_PCI_VGA_MEM].mem, 2471 &vdev->vga.region[QEMU_PCI_VGA_IO_LO].mem, 2472 &vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem); 2473 vfio_vga_quirk_setup(vdev); 2474 } 2475 } 2476 2477 static void vfio_unregister_bars(VFIOPCIDevice *vdev) 2478 { 2479 int i; 2480 2481 for (i = 0; i < PCI_ROM_SLOT; i++) { 2482 vfio_unregister_bar(vdev, i); 2483 } 2484 2485 if (vdev->has_vga) { 2486 vfio_vga_quirk_teardown(vdev); 2487 pci_unregister_vga(&vdev->pdev); 2488 } 2489 } 2490 2491 static void vfio_unmap_bars(VFIOPCIDevice *vdev) 2492 { 2493 int i; 2494 2495 for (i = 0; i < PCI_ROM_SLOT; i++) { 2496 vfio_unmap_bar(vdev, i); 2497 } 2498 2499 if (vdev->has_vga) { 2500 vfio_vga_quirk_free(vdev); 2501 } 2502 } 2503 2504 /* 2505 * General setup 2506 */ 2507 static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos) 2508 { 2509 uint8_t tmp, next = 0xff; 2510 2511 for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp; 2512 tmp = pdev->config[tmp + 1]) { 2513 if (tmp > pos && tmp < next) { 2514 next = tmp; 2515 } 2516 } 2517 2518 return next - pos; 2519 } 2520 2521 static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask) 2522 { 2523 pci_set_word(buf, (pci_get_word(buf) & ~mask) | val); 2524 } 2525 2526 static void vfio_add_emulated_word(VFIOPCIDevice *vdev, int pos, 2527 uint16_t val, uint16_t mask) 2528 { 2529 vfio_set_word_bits(vdev->pdev.config + pos, val, mask); 2530 vfio_set_word_bits(vdev->pdev.wmask + pos, ~mask, mask); 2531 vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask); 2532 } 2533 2534 static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask) 2535 { 2536 pci_set_long(buf, (pci_get_long(buf) & ~mask) | val); 2537 } 2538 2539 static void vfio_add_emulated_long(VFIOPCIDevice *vdev, int pos, 2540 uint32_t val, uint32_t mask) 2541 { 2542 vfio_set_long_bits(vdev->pdev.config + pos, val, mask); 2543 vfio_set_long_bits(vdev->pdev.wmask + pos, ~mask, mask); 2544 vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask); 2545 } 2546 2547 static int vfio_setup_pcie_cap(VFIOPCIDevice *vdev, int pos, uint8_t size) 2548 { 2549 uint16_t flags; 2550 uint8_t type; 2551 2552 flags = pci_get_word(vdev->pdev.config + pos + PCI_CAP_FLAGS); 2553 type = (flags & PCI_EXP_FLAGS_TYPE) >> 4; 2554 2555 if (type != PCI_EXP_TYPE_ENDPOINT && 2556 type != PCI_EXP_TYPE_LEG_END && 2557 type != PCI_EXP_TYPE_RC_END) { 2558 2559 error_report("vfio: Assignment of PCIe type 0x%x " 2560 "devices is not currently supported", type); 2561 return -EINVAL; 2562 } 2563 2564 if (!pci_bus_is_express(vdev->pdev.bus)) { 2565 /* 2566 * Use express capability as-is on PCI bus. It doesn't make much 2567 * sense to even expose, but some drivers (ex. tg3) depend on it 2568 * and guests don't seem to be particular about it. We'll need 2569 * to revist this or force express devices to express buses if we 2570 * ever expose an IOMMU to the guest. 2571 */ 2572 } else if (pci_bus_is_root(vdev->pdev.bus)) { 2573 /* 2574 * On a Root Complex bus Endpoints become Root Complex Integrated 2575 * Endpoints, which changes the type and clears the LNK & LNK2 fields. 2576 */ 2577 if (type == PCI_EXP_TYPE_ENDPOINT) { 2578 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS, 2579 PCI_EXP_TYPE_RC_END << 4, 2580 PCI_EXP_FLAGS_TYPE); 2581 2582 /* Link Capabilities, Status, and Control goes away */ 2583 if (size > PCI_EXP_LNKCTL) { 2584 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0); 2585 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0); 2586 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0); 2587 2588 #ifndef PCI_EXP_LNKCAP2 2589 #define PCI_EXP_LNKCAP2 44 2590 #endif 2591 #ifndef PCI_EXP_LNKSTA2 2592 #define PCI_EXP_LNKSTA2 50 2593 #endif 2594 /* Link 2 Capabilities, Status, and Control goes away */ 2595 if (size > PCI_EXP_LNKCAP2) { 2596 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0); 2597 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0); 2598 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0); 2599 } 2600 } 2601 2602 } else if (type == PCI_EXP_TYPE_LEG_END) { 2603 /* 2604 * Legacy endpoints don't belong on the root complex. Windows 2605 * seems to be happier with devices if we skip the capability. 2606 */ 2607 return 0; 2608 } 2609 2610 } else { 2611 /* 2612 * Convert Root Complex Integrated Endpoints to regular endpoints. 2613 * These devices don't support LNK/LNK2 capabilities, so make them up. 2614 */ 2615 if (type == PCI_EXP_TYPE_RC_END) { 2616 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS, 2617 PCI_EXP_TYPE_ENDPOINT << 4, 2618 PCI_EXP_FLAGS_TYPE); 2619 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 2620 PCI_EXP_LNK_MLW_1 | PCI_EXP_LNK_LS_25, ~0); 2621 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0); 2622 } 2623 2624 /* Mark the Link Status bits as emulated to allow virtual negotiation */ 2625 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 2626 pci_get_word(vdev->pdev.config + pos + 2627 PCI_EXP_LNKSTA), 2628 PCI_EXP_LNKCAP_MLW | PCI_EXP_LNKCAP_SLS); 2629 } 2630 2631 pos = pci_add_capability(&vdev->pdev, PCI_CAP_ID_EXP, pos, size); 2632 if (pos >= 0) { 2633 vdev->pdev.exp.exp_cap = pos; 2634 } 2635 2636 return pos; 2637 } 2638 2639 static void vfio_check_pcie_flr(VFIOPCIDevice *vdev, uint8_t pos) 2640 { 2641 uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP); 2642 2643 if (cap & PCI_EXP_DEVCAP_FLR) { 2644 trace_vfio_check_pcie_flr(vdev->vbasedev.name); 2645 vdev->has_flr = true; 2646 } 2647 } 2648 2649 static void vfio_check_pm_reset(VFIOPCIDevice *vdev, uint8_t pos) 2650 { 2651 uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL); 2652 2653 if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) { 2654 trace_vfio_check_pm_reset(vdev->vbasedev.name); 2655 vdev->has_pm_reset = true; 2656 } 2657 } 2658 2659 static void vfio_check_af_flr(VFIOPCIDevice *vdev, uint8_t pos) 2660 { 2661 uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP); 2662 2663 if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) { 2664 trace_vfio_check_af_flr(vdev->vbasedev.name); 2665 vdev->has_flr = true; 2666 } 2667 } 2668 2669 static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos) 2670 { 2671 PCIDevice *pdev = &vdev->pdev; 2672 uint8_t cap_id, next, size; 2673 int ret; 2674 2675 cap_id = pdev->config[pos]; 2676 next = pdev->config[pos + 1]; 2677 2678 /* 2679 * If it becomes important to configure capabilities to their actual 2680 * size, use this as the default when it's something we don't recognize. 2681 * Since QEMU doesn't actually handle many of the config accesses, 2682 * exact size doesn't seem worthwhile. 2683 */ 2684 size = vfio_std_cap_max_size(pdev, pos); 2685 2686 /* 2687 * pci_add_capability always inserts the new capability at the head 2688 * of the chain. Therefore to end up with a chain that matches the 2689 * physical device, we insert from the end by making this recursive. 2690 * This is also why we pre-caclulate size above as cached config space 2691 * will be changed as we unwind the stack. 2692 */ 2693 if (next) { 2694 ret = vfio_add_std_cap(vdev, next); 2695 if (ret) { 2696 return ret; 2697 } 2698 } else { 2699 /* Begin the rebuild, use QEMU emulated list bits */ 2700 pdev->config[PCI_CAPABILITY_LIST] = 0; 2701 vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff; 2702 vdev->emulated_config_bits[PCI_STATUS] |= PCI_STATUS_CAP_LIST; 2703 } 2704 2705 /* Use emulated next pointer to allow dropping caps */ 2706 pci_set_byte(vdev->emulated_config_bits + pos + 1, 0xff); 2707 2708 switch (cap_id) { 2709 case PCI_CAP_ID_MSI: 2710 ret = vfio_setup_msi(vdev, pos); 2711 break; 2712 case PCI_CAP_ID_EXP: 2713 vfio_check_pcie_flr(vdev, pos); 2714 ret = vfio_setup_pcie_cap(vdev, pos, size); 2715 break; 2716 case PCI_CAP_ID_MSIX: 2717 ret = vfio_setup_msix(vdev, pos); 2718 break; 2719 case PCI_CAP_ID_PM: 2720 vfio_check_pm_reset(vdev, pos); 2721 vdev->pm_cap = pos; 2722 ret = pci_add_capability(pdev, cap_id, pos, size); 2723 break; 2724 case PCI_CAP_ID_AF: 2725 vfio_check_af_flr(vdev, pos); 2726 ret = pci_add_capability(pdev, cap_id, pos, size); 2727 break; 2728 default: 2729 ret = pci_add_capability(pdev, cap_id, pos, size); 2730 break; 2731 } 2732 2733 if (ret < 0) { 2734 error_report("vfio: %04x:%02x:%02x.%x Error adding PCI capability " 2735 "0x%x[0x%x]@0x%x: %d", vdev->host.domain, 2736 vdev->host.bus, vdev->host.slot, vdev->host.function, 2737 cap_id, size, pos, ret); 2738 return ret; 2739 } 2740 2741 return 0; 2742 } 2743 2744 static int vfio_add_capabilities(VFIOPCIDevice *vdev) 2745 { 2746 PCIDevice *pdev = &vdev->pdev; 2747 2748 if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) || 2749 !pdev->config[PCI_CAPABILITY_LIST]) { 2750 return 0; /* Nothing to add */ 2751 } 2752 2753 return vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST]); 2754 } 2755 2756 static void vfio_pci_pre_reset(VFIOPCIDevice *vdev) 2757 { 2758 PCIDevice *pdev = &vdev->pdev; 2759 uint16_t cmd; 2760 2761 vfio_disable_interrupts(vdev); 2762 2763 /* Make sure the device is in D0 */ 2764 if (vdev->pm_cap) { 2765 uint16_t pmcsr; 2766 uint8_t state; 2767 2768 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2); 2769 state = pmcsr & PCI_PM_CTRL_STATE_MASK; 2770 if (state) { 2771 pmcsr &= ~PCI_PM_CTRL_STATE_MASK; 2772 vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2); 2773 /* vfio handles the necessary delay here */ 2774 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2); 2775 state = pmcsr & PCI_PM_CTRL_STATE_MASK; 2776 if (state) { 2777 error_report("vfio: Unable to power on device, stuck in D%d", 2778 state); 2779 } 2780 } 2781 } 2782 2783 /* 2784 * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master. 2785 * Also put INTx Disable in known state. 2786 */ 2787 cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2); 2788 cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER | 2789 PCI_COMMAND_INTX_DISABLE); 2790 vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2); 2791 } 2792 2793 static void vfio_pci_post_reset(VFIOPCIDevice *vdev) 2794 { 2795 vfio_enable_intx(vdev); 2796 } 2797 2798 static bool vfio_pci_host_match(PCIHostDeviceAddress *host1, 2799 PCIHostDeviceAddress *host2) 2800 { 2801 return (host1->domain == host2->domain && host1->bus == host2->bus && 2802 host1->slot == host2->slot && host1->function == host2->function); 2803 } 2804 2805 static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single) 2806 { 2807 VFIOGroup *group; 2808 struct vfio_pci_hot_reset_info *info; 2809 struct vfio_pci_dependent_device *devices; 2810 struct vfio_pci_hot_reset *reset; 2811 int32_t *fds; 2812 int ret, i, count; 2813 bool multi = false; 2814 2815 trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi"); 2816 2817 vfio_pci_pre_reset(vdev); 2818 vdev->vbasedev.needs_reset = false; 2819 2820 info = g_malloc0(sizeof(*info)); 2821 info->argsz = sizeof(*info); 2822 2823 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info); 2824 if (ret && errno != ENOSPC) { 2825 ret = -errno; 2826 if (!vdev->has_pm_reset) { 2827 error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, " 2828 "no available reset mechanism.", vdev->host.domain, 2829 vdev->host.bus, vdev->host.slot, vdev->host.function); 2830 } 2831 goto out_single; 2832 } 2833 2834 count = info->count; 2835 info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices))); 2836 info->argsz = sizeof(*info) + (count * sizeof(*devices)); 2837 devices = &info->devices[0]; 2838 2839 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info); 2840 if (ret) { 2841 ret = -errno; 2842 error_report("vfio: hot reset info failed: %m"); 2843 goto out_single; 2844 } 2845 2846 trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name); 2847 2848 /* Verify that we have all the groups required */ 2849 for (i = 0; i < info->count; i++) { 2850 PCIHostDeviceAddress host; 2851 VFIOPCIDevice *tmp; 2852 VFIODevice *vbasedev_iter; 2853 2854 host.domain = devices[i].segment; 2855 host.bus = devices[i].bus; 2856 host.slot = PCI_SLOT(devices[i].devfn); 2857 host.function = PCI_FUNC(devices[i].devfn); 2858 2859 trace_vfio_pci_hot_reset_dep_devices(host.domain, 2860 host.bus, host.slot, host.function, devices[i].group_id); 2861 2862 if (vfio_pci_host_match(&host, &vdev->host)) { 2863 continue; 2864 } 2865 2866 QLIST_FOREACH(group, &vfio_group_list, next) { 2867 if (group->groupid == devices[i].group_id) { 2868 break; 2869 } 2870 } 2871 2872 if (!group) { 2873 if (!vdev->has_pm_reset) { 2874 error_report("vfio: Cannot reset device %s, " 2875 "depends on group %d which is not owned.", 2876 vdev->vbasedev.name, devices[i].group_id); 2877 } 2878 ret = -EPERM; 2879 goto out; 2880 } 2881 2882 /* Prep dependent devices for reset and clear our marker. */ 2883 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) { 2884 if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) { 2885 continue; 2886 } 2887 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev); 2888 if (vfio_pci_host_match(&host, &tmp->host)) { 2889 if (single) { 2890 ret = -EINVAL; 2891 goto out_single; 2892 } 2893 vfio_pci_pre_reset(tmp); 2894 tmp->vbasedev.needs_reset = false; 2895 multi = true; 2896 break; 2897 } 2898 } 2899 } 2900 2901 if (!single && !multi) { 2902 ret = -EINVAL; 2903 goto out_single; 2904 } 2905 2906 /* Determine how many group fds need to be passed */ 2907 count = 0; 2908 QLIST_FOREACH(group, &vfio_group_list, next) { 2909 for (i = 0; i < info->count; i++) { 2910 if (group->groupid == devices[i].group_id) { 2911 count++; 2912 break; 2913 } 2914 } 2915 } 2916 2917 reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds))); 2918 reset->argsz = sizeof(*reset) + (count * sizeof(*fds)); 2919 fds = &reset->group_fds[0]; 2920 2921 /* Fill in group fds */ 2922 QLIST_FOREACH(group, &vfio_group_list, next) { 2923 for (i = 0; i < info->count; i++) { 2924 if (group->groupid == devices[i].group_id) { 2925 fds[reset->count++] = group->fd; 2926 break; 2927 } 2928 } 2929 } 2930 2931 /* Bus reset! */ 2932 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset); 2933 g_free(reset); 2934 2935 trace_vfio_pci_hot_reset_result(vdev->vbasedev.name, 2936 ret ? "%m" : "Success"); 2937 2938 out: 2939 /* Re-enable INTx on affected devices */ 2940 for (i = 0; i < info->count; i++) { 2941 PCIHostDeviceAddress host; 2942 VFIOPCIDevice *tmp; 2943 VFIODevice *vbasedev_iter; 2944 2945 host.domain = devices[i].segment; 2946 host.bus = devices[i].bus; 2947 host.slot = PCI_SLOT(devices[i].devfn); 2948 host.function = PCI_FUNC(devices[i].devfn); 2949 2950 if (vfio_pci_host_match(&host, &vdev->host)) { 2951 continue; 2952 } 2953 2954 QLIST_FOREACH(group, &vfio_group_list, next) { 2955 if (group->groupid == devices[i].group_id) { 2956 break; 2957 } 2958 } 2959 2960 if (!group) { 2961 break; 2962 } 2963 2964 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) { 2965 if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) { 2966 continue; 2967 } 2968 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev); 2969 if (vfio_pci_host_match(&host, &tmp->host)) { 2970 vfio_pci_post_reset(tmp); 2971 break; 2972 } 2973 } 2974 } 2975 out_single: 2976 vfio_pci_post_reset(vdev); 2977 g_free(info); 2978 2979 return ret; 2980 } 2981 2982 /* 2983 * We want to differentiate hot reset of mulitple in-use devices vs hot reset 2984 * of a single in-use device. VFIO_DEVICE_RESET will already handle the case 2985 * of doing hot resets when there is only a single device per bus. The in-use 2986 * here refers to how many VFIODevices are affected. A hot reset that affects 2987 * multiple devices, but only a single in-use device, means that we can call 2988 * it from our bus ->reset() callback since the extent is effectively a single 2989 * device. This allows us to make use of it in the hotplug path. When there 2990 * are multiple in-use devices, we can only trigger the hot reset during a 2991 * system reset and thus from our reset handler. We separate _one vs _multi 2992 * here so that we don't overlap and do a double reset on the system reset 2993 * path where both our reset handler and ->reset() callback are used. Calling 2994 * _one() will only do a hot reset for the one in-use devices case, calling 2995 * _multi() will do nothing if a _one() would have been sufficient. 2996 */ 2997 static int vfio_pci_hot_reset_one(VFIOPCIDevice *vdev) 2998 { 2999 return vfio_pci_hot_reset(vdev, true); 3000 } 3001 3002 static int vfio_pci_hot_reset_multi(VFIODevice *vbasedev) 3003 { 3004 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev); 3005 return vfio_pci_hot_reset(vdev, false); 3006 } 3007 3008 static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev) 3009 { 3010 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev); 3011 if (!vbasedev->reset_works || (!vdev->has_flr && vdev->has_pm_reset)) { 3012 vbasedev->needs_reset = true; 3013 } 3014 } 3015 3016 static VFIODeviceOps vfio_pci_ops = { 3017 .vfio_compute_needs_reset = vfio_pci_compute_needs_reset, 3018 .vfio_hot_reset_multi = vfio_pci_hot_reset_multi, 3019 .vfio_eoi = vfio_eoi, 3020 }; 3021 3022 static int vfio_populate_device(VFIOPCIDevice *vdev) 3023 { 3024 VFIODevice *vbasedev = &vdev->vbasedev; 3025 struct vfio_region_info reg_info = { .argsz = sizeof(reg_info) }; 3026 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) }; 3027 int i, ret = -1; 3028 3029 /* Sanity check device */ 3030 if (!(vbasedev->flags & VFIO_DEVICE_FLAGS_PCI)) { 3031 error_report("vfio: Um, this isn't a PCI device"); 3032 goto error; 3033 } 3034 3035 if (vbasedev->num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) { 3036 error_report("vfio: unexpected number of io regions %u", 3037 vbasedev->num_regions); 3038 goto error; 3039 } 3040 3041 if (vbasedev->num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) { 3042 error_report("vfio: unexpected number of irqs %u", vbasedev->num_irqs); 3043 goto error; 3044 } 3045 3046 for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) { 3047 reg_info.index = i; 3048 3049 ret = ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, ®_info); 3050 if (ret) { 3051 error_report("vfio: Error getting region %d info: %m", i); 3052 goto error; 3053 } 3054 3055 trace_vfio_populate_device_region(vbasedev->name, i, 3056 (unsigned long)reg_info.size, 3057 (unsigned long)reg_info.offset, 3058 (unsigned long)reg_info.flags); 3059 3060 vdev->bars[i].region.vbasedev = vbasedev; 3061 vdev->bars[i].region.flags = reg_info.flags; 3062 vdev->bars[i].region.size = reg_info.size; 3063 vdev->bars[i].region.fd_offset = reg_info.offset; 3064 vdev->bars[i].region.nr = i; 3065 QLIST_INIT(&vdev->bars[i].quirks); 3066 } 3067 3068 reg_info.index = VFIO_PCI_CONFIG_REGION_INDEX; 3069 3070 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_REGION_INFO, ®_info); 3071 if (ret) { 3072 error_report("vfio: Error getting config info: %m"); 3073 goto error; 3074 } 3075 3076 trace_vfio_populate_device_config(vdev->vbasedev.name, 3077 (unsigned long)reg_info.size, 3078 (unsigned long)reg_info.offset, 3079 (unsigned long)reg_info.flags); 3080 3081 vdev->config_size = reg_info.size; 3082 if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) { 3083 vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS; 3084 } 3085 vdev->config_offset = reg_info.offset; 3086 3087 if ((vdev->features & VFIO_FEATURE_ENABLE_VGA) && 3088 vbasedev->num_regions > VFIO_PCI_VGA_REGION_INDEX) { 3089 struct vfio_region_info vga_info = { 3090 .argsz = sizeof(vga_info), 3091 .index = VFIO_PCI_VGA_REGION_INDEX, 3092 }; 3093 3094 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_REGION_INFO, &vga_info); 3095 if (ret) { 3096 error_report( 3097 "vfio: Device does not support requested feature x-vga"); 3098 goto error; 3099 } 3100 3101 if (!(vga_info.flags & VFIO_REGION_INFO_FLAG_READ) || 3102 !(vga_info.flags & VFIO_REGION_INFO_FLAG_WRITE) || 3103 vga_info.size < 0xbffff + 1) { 3104 error_report("vfio: Unexpected VGA info, flags 0x%lx, size 0x%lx", 3105 (unsigned long)vga_info.flags, 3106 (unsigned long)vga_info.size); 3107 goto error; 3108 } 3109 3110 vdev->vga.fd_offset = vga_info.offset; 3111 vdev->vga.fd = vdev->vbasedev.fd; 3112 3113 vdev->vga.region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE; 3114 vdev->vga.region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM; 3115 QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_MEM].quirks); 3116 3117 vdev->vga.region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE; 3118 vdev->vga.region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO; 3119 QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_IO_LO].quirks); 3120 3121 vdev->vga.region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE; 3122 vdev->vga.region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI; 3123 QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks); 3124 3125 vdev->has_vga = true; 3126 } 3127 3128 irq_info.index = VFIO_PCI_ERR_IRQ_INDEX; 3129 3130 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info); 3131 if (ret) { 3132 /* This can fail for an old kernel or legacy PCI dev */ 3133 trace_vfio_populate_device_get_irq_info_failure(); 3134 ret = 0; 3135 } else if (irq_info.count == 1) { 3136 vdev->pci_aer = true; 3137 } else { 3138 error_report("vfio: %s " 3139 "Could not enable error recovery for the device", 3140 vbasedev->name); 3141 } 3142 3143 error: 3144 return ret; 3145 } 3146 3147 static void vfio_put_device(VFIOPCIDevice *vdev) 3148 { 3149 g_free(vdev->vbasedev.name); 3150 if (vdev->msix) { 3151 object_unparent(OBJECT(&vdev->msix->mmap_mem)); 3152 g_free(vdev->msix); 3153 vdev->msix = NULL; 3154 } 3155 vfio_put_base_device(&vdev->vbasedev); 3156 } 3157 3158 static void vfio_err_notifier_handler(void *opaque) 3159 { 3160 VFIOPCIDevice *vdev = opaque; 3161 3162 if (!event_notifier_test_and_clear(&vdev->err_notifier)) { 3163 return; 3164 } 3165 3166 /* 3167 * TBD. Retrieve the error details and decide what action 3168 * needs to be taken. One of the actions could be to pass 3169 * the error to the guest and have the guest driver recover 3170 * from the error. This requires that PCIe capabilities be 3171 * exposed to the guest. For now, we just terminate the 3172 * guest to contain the error. 3173 */ 3174 3175 error_report("%s(%04x:%02x:%02x.%x) Unrecoverable error detected. " 3176 "Please collect any data possible and then kill the guest", 3177 __func__, vdev->host.domain, vdev->host.bus, 3178 vdev->host.slot, vdev->host.function); 3179 3180 vm_stop(RUN_STATE_INTERNAL_ERROR); 3181 } 3182 3183 /* 3184 * Registers error notifier for devices supporting error recovery. 3185 * If we encounter a failure in this function, we report an error 3186 * and continue after disabling error recovery support for the 3187 * device. 3188 */ 3189 static void vfio_register_err_notifier(VFIOPCIDevice *vdev) 3190 { 3191 int ret; 3192 int argsz; 3193 struct vfio_irq_set *irq_set; 3194 int32_t *pfd; 3195 3196 if (!vdev->pci_aer) { 3197 return; 3198 } 3199 3200 if (event_notifier_init(&vdev->err_notifier, 0)) { 3201 error_report("vfio: Unable to init event notifier for error detection"); 3202 vdev->pci_aer = false; 3203 return; 3204 } 3205 3206 argsz = sizeof(*irq_set) + sizeof(*pfd); 3207 3208 irq_set = g_malloc0(argsz); 3209 irq_set->argsz = argsz; 3210 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | 3211 VFIO_IRQ_SET_ACTION_TRIGGER; 3212 irq_set->index = VFIO_PCI_ERR_IRQ_INDEX; 3213 irq_set->start = 0; 3214 irq_set->count = 1; 3215 pfd = (int32_t *)&irq_set->data; 3216 3217 *pfd = event_notifier_get_fd(&vdev->err_notifier); 3218 qemu_set_fd_handler(*pfd, vfio_err_notifier_handler, NULL, vdev); 3219 3220 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set); 3221 if (ret) { 3222 error_report("vfio: Failed to set up error notification"); 3223 qemu_set_fd_handler(*pfd, NULL, NULL, vdev); 3224 event_notifier_cleanup(&vdev->err_notifier); 3225 vdev->pci_aer = false; 3226 } 3227 g_free(irq_set); 3228 } 3229 3230 static void vfio_unregister_err_notifier(VFIOPCIDevice *vdev) 3231 { 3232 int argsz; 3233 struct vfio_irq_set *irq_set; 3234 int32_t *pfd; 3235 int ret; 3236 3237 if (!vdev->pci_aer) { 3238 return; 3239 } 3240 3241 argsz = sizeof(*irq_set) + sizeof(*pfd); 3242 3243 irq_set = g_malloc0(argsz); 3244 irq_set->argsz = argsz; 3245 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | 3246 VFIO_IRQ_SET_ACTION_TRIGGER; 3247 irq_set->index = VFIO_PCI_ERR_IRQ_INDEX; 3248 irq_set->start = 0; 3249 irq_set->count = 1; 3250 pfd = (int32_t *)&irq_set->data; 3251 *pfd = -1; 3252 3253 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set); 3254 if (ret) { 3255 error_report("vfio: Failed to de-assign error fd: %m"); 3256 } 3257 g_free(irq_set); 3258 qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier), 3259 NULL, NULL, vdev); 3260 event_notifier_cleanup(&vdev->err_notifier); 3261 } 3262 3263 static void vfio_req_notifier_handler(void *opaque) 3264 { 3265 VFIOPCIDevice *vdev = opaque; 3266 3267 if (!event_notifier_test_and_clear(&vdev->req_notifier)) { 3268 return; 3269 } 3270 3271 qdev_unplug(&vdev->pdev.qdev, NULL); 3272 } 3273 3274 static void vfio_register_req_notifier(VFIOPCIDevice *vdev) 3275 { 3276 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info), 3277 .index = VFIO_PCI_REQ_IRQ_INDEX }; 3278 int argsz; 3279 struct vfio_irq_set *irq_set; 3280 int32_t *pfd; 3281 3282 if (!(vdev->features & VFIO_FEATURE_ENABLE_REQ)) { 3283 return; 3284 } 3285 3286 if (ioctl(vdev->vbasedev.fd, 3287 VFIO_DEVICE_GET_IRQ_INFO, &irq_info) < 0 || irq_info.count < 1) { 3288 return; 3289 } 3290 3291 if (event_notifier_init(&vdev->req_notifier, 0)) { 3292 error_report("vfio: Unable to init event notifier for device request"); 3293 return; 3294 } 3295 3296 argsz = sizeof(*irq_set) + sizeof(*pfd); 3297 3298 irq_set = g_malloc0(argsz); 3299 irq_set->argsz = argsz; 3300 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | 3301 VFIO_IRQ_SET_ACTION_TRIGGER; 3302 irq_set->index = VFIO_PCI_REQ_IRQ_INDEX; 3303 irq_set->start = 0; 3304 irq_set->count = 1; 3305 pfd = (int32_t *)&irq_set->data; 3306 3307 *pfd = event_notifier_get_fd(&vdev->req_notifier); 3308 qemu_set_fd_handler(*pfd, vfio_req_notifier_handler, NULL, vdev); 3309 3310 if (ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set)) { 3311 error_report("vfio: Failed to set up device request notification"); 3312 qemu_set_fd_handler(*pfd, NULL, NULL, vdev); 3313 event_notifier_cleanup(&vdev->req_notifier); 3314 } else { 3315 vdev->req_enabled = true; 3316 } 3317 3318 g_free(irq_set); 3319 } 3320 3321 static void vfio_unregister_req_notifier(VFIOPCIDevice *vdev) 3322 { 3323 int argsz; 3324 struct vfio_irq_set *irq_set; 3325 int32_t *pfd; 3326 3327 if (!vdev->req_enabled) { 3328 return; 3329 } 3330 3331 argsz = sizeof(*irq_set) + sizeof(*pfd); 3332 3333 irq_set = g_malloc0(argsz); 3334 irq_set->argsz = argsz; 3335 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | 3336 VFIO_IRQ_SET_ACTION_TRIGGER; 3337 irq_set->index = VFIO_PCI_REQ_IRQ_INDEX; 3338 irq_set->start = 0; 3339 irq_set->count = 1; 3340 pfd = (int32_t *)&irq_set->data; 3341 *pfd = -1; 3342 3343 if (ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set)) { 3344 error_report("vfio: Failed to de-assign device request fd: %m"); 3345 } 3346 g_free(irq_set); 3347 qemu_set_fd_handler(event_notifier_get_fd(&vdev->req_notifier), 3348 NULL, NULL, vdev); 3349 event_notifier_cleanup(&vdev->req_notifier); 3350 3351 vdev->req_enabled = false; 3352 } 3353 3354 /* 3355 * AMD Radeon PCI config reset, based on Linux: 3356 * drivers/gpu/drm/radeon/ci_smc.c:ci_is_smc_running() 3357 * drivers/gpu/drm/radeon/radeon_device.c:radeon_pci_config_reset 3358 * drivers/gpu/drm/radeon/ci_smc.c:ci_reset_smc() 3359 * drivers/gpu/drm/radeon/ci_smc.c:ci_stop_smc_clock() 3360 * IDs: include/drm/drm_pciids.h 3361 * Registers: http://cgit.freedesktop.org/~agd5f/linux/commit/?id=4e2aa447f6f0 3362 * 3363 * Bonaire and Hawaii GPUs do not respond to a bus reset. This is a bug in the 3364 * hardware that should be fixed on future ASICs. The symptom of this is that 3365 * once the accerlated driver loads, Windows guests will bsod on subsequent 3366 * attmpts to load the driver, such as after VM reset or shutdown/restart. To 3367 * work around this, we do an AMD specific PCI config reset, followed by an SMC 3368 * reset. The PCI config reset only works if SMC firmware is running, so we 3369 * have a dependency on the state of the device as to whether this reset will 3370 * be effective. There are still cases where we won't be able to kick the 3371 * device into working, but this greatly improves the usability overall. The 3372 * config reset magic is relatively common on AMD GPUs, but the setup and SMC 3373 * poking is largely ASIC specific. 3374 */ 3375 static bool vfio_radeon_smc_is_running(VFIOPCIDevice *vdev) 3376 { 3377 uint32_t clk, pc_c; 3378 3379 /* 3380 * Registers 200h and 204h are index and data registers for accessing 3381 * indirect configuration registers within the device. 3382 */ 3383 vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4); 3384 clk = vfio_region_read(&vdev->bars[5].region, 0x204, 4); 3385 vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000370, 4); 3386 pc_c = vfio_region_read(&vdev->bars[5].region, 0x204, 4); 3387 3388 return (!(clk & 1) && (0x20100 <= pc_c)); 3389 } 3390 3391 /* 3392 * The scope of a config reset is controlled by a mode bit in the misc register 3393 * and a fuse, exposed as a bit in another register. The fuse is the default 3394 * (0 = GFX, 1 = whole GPU), the misc bit is a toggle, with the forumula 3395 * scope = !(misc ^ fuse), where the resulting scope is defined the same as 3396 * the fuse. A truth table therefore tells us that if misc == fuse, we need 3397 * to flip the value of the bit in the misc register. 3398 */ 3399 static void vfio_radeon_set_gfx_only_reset(VFIOPCIDevice *vdev) 3400 { 3401 uint32_t misc, fuse; 3402 bool a, b; 3403 3404 vfio_region_write(&vdev->bars[5].region, 0x200, 0xc00c0000, 4); 3405 fuse = vfio_region_read(&vdev->bars[5].region, 0x204, 4); 3406 b = fuse & 64; 3407 3408 vfio_region_write(&vdev->bars[5].region, 0x200, 0xc0000010, 4); 3409 misc = vfio_region_read(&vdev->bars[5].region, 0x204, 4); 3410 a = misc & 2; 3411 3412 if (a == b) { 3413 vfio_region_write(&vdev->bars[5].region, 0x204, misc ^ 2, 4); 3414 vfio_region_read(&vdev->bars[5].region, 0x204, 4); /* flush */ 3415 } 3416 } 3417 3418 static int vfio_radeon_reset(VFIOPCIDevice *vdev) 3419 { 3420 PCIDevice *pdev = &vdev->pdev; 3421 int i, ret = 0; 3422 uint32_t data; 3423 3424 /* Defer to a kernel implemented reset */ 3425 if (vdev->vbasedev.reset_works) { 3426 return -ENODEV; 3427 } 3428 3429 /* Enable only memory BAR access */ 3430 vfio_pci_write_config(pdev, PCI_COMMAND, PCI_COMMAND_MEMORY, 2); 3431 3432 /* Reset only works if SMC firmware is loaded and running */ 3433 if (!vfio_radeon_smc_is_running(vdev)) { 3434 ret = -EINVAL; 3435 goto out; 3436 } 3437 3438 /* Make sure only the GFX function is reset */ 3439 vfio_radeon_set_gfx_only_reset(vdev); 3440 3441 /* AMD PCI config reset */ 3442 vfio_pci_write_config(pdev, 0x7c, 0x39d5e86b, 4); 3443 usleep(100); 3444 3445 /* Read back the memory size to make sure we're out of reset */ 3446 for (i = 0; i < 100000; i++) { 3447 if (vfio_region_read(&vdev->bars[5].region, 0x5428, 4) != 0xffffffff) { 3448 break; 3449 } 3450 usleep(1); 3451 } 3452 3453 /* Reset SMC */ 3454 vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000000, 4); 3455 data = vfio_region_read(&vdev->bars[5].region, 0x204, 4); 3456 data |= 1; 3457 vfio_region_write(&vdev->bars[5].region, 0x204, data, 4); 3458 3459 /* Disable SMC clock */ 3460 vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4); 3461 data = vfio_region_read(&vdev->bars[5].region, 0x204, 4); 3462 data |= 1; 3463 vfio_region_write(&vdev->bars[5].region, 0x204, data, 4); 3464 3465 out: 3466 /* Restore PCI command register */ 3467 vfio_pci_write_config(pdev, PCI_COMMAND, 0, 2); 3468 3469 return ret; 3470 } 3471 3472 static void vfio_setup_resetfn(VFIOPCIDevice *vdev) 3473 { 3474 PCIDevice *pdev = &vdev->pdev; 3475 uint16_t vendor, device; 3476 3477 vendor = pci_get_word(pdev->config + PCI_VENDOR_ID); 3478 device = pci_get_word(pdev->config + PCI_DEVICE_ID); 3479 3480 switch (vendor) { 3481 case 0x1002: 3482 switch (device) { 3483 /* Bonaire */ 3484 case 0x6649: /* Bonaire [FirePro W5100] */ 3485 case 0x6650: 3486 case 0x6651: 3487 case 0x6658: /* Bonaire XTX [Radeon R7 260X] */ 3488 case 0x665c: /* Bonaire XT [Radeon HD 7790/8770 / R9 260 OEM] */ 3489 case 0x665d: /* Bonaire [Radeon R7 200 Series] */ 3490 /* Hawaii */ 3491 case 0x67A0: /* Hawaii XT GL [FirePro W9100] */ 3492 case 0x67A1: /* Hawaii PRO GL [FirePro W8100] */ 3493 case 0x67A2: 3494 case 0x67A8: 3495 case 0x67A9: 3496 case 0x67AA: 3497 case 0x67B0: /* Hawaii XT [Radeon R9 290X] */ 3498 case 0x67B1: /* Hawaii PRO [Radeon R9 290] */ 3499 case 0x67B8: 3500 case 0x67B9: 3501 case 0x67BA: 3502 case 0x67BE: 3503 vdev->resetfn = vfio_radeon_reset; 3504 break; 3505 } 3506 break; 3507 } 3508 } 3509 3510 static int vfio_initfn(PCIDevice *pdev) 3511 { 3512 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev); 3513 VFIODevice *vbasedev_iter; 3514 VFIOGroup *group; 3515 char path[PATH_MAX], iommu_group_path[PATH_MAX], *group_name; 3516 ssize_t len; 3517 struct stat st; 3518 int groupid; 3519 int ret; 3520 3521 /* Check that the host device exists */ 3522 snprintf(path, sizeof(path), 3523 "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/", 3524 vdev->host.domain, vdev->host.bus, vdev->host.slot, 3525 vdev->host.function); 3526 if (stat(path, &st) < 0) { 3527 error_report("vfio: error: no such host device: %s", path); 3528 return -errno; 3529 } 3530 3531 vdev->vbasedev.ops = &vfio_pci_ops; 3532 3533 vdev->vbasedev.type = VFIO_DEVICE_TYPE_PCI; 3534 vdev->vbasedev.name = g_strdup_printf("%04x:%02x:%02x.%01x", 3535 vdev->host.domain, vdev->host.bus, 3536 vdev->host.slot, vdev->host.function); 3537 3538 strncat(path, "iommu_group", sizeof(path) - strlen(path) - 1); 3539 3540 len = readlink(path, iommu_group_path, sizeof(path)); 3541 if (len <= 0 || len >= sizeof(path)) { 3542 error_report("vfio: error no iommu_group for device"); 3543 return len < 0 ? -errno : -ENAMETOOLONG; 3544 } 3545 3546 iommu_group_path[len] = 0; 3547 group_name = basename(iommu_group_path); 3548 3549 if (sscanf(group_name, "%d", &groupid) != 1) { 3550 error_report("vfio: error reading %s: %m", path); 3551 return -errno; 3552 } 3553 3554 trace_vfio_initfn(vdev->vbasedev.name, groupid); 3555 3556 group = vfio_get_group(groupid, pci_device_iommu_address_space(pdev)); 3557 if (!group) { 3558 error_report("vfio: failed to get group %d", groupid); 3559 return -ENOENT; 3560 } 3561 3562 snprintf(path, sizeof(path), "%04x:%02x:%02x.%01x", 3563 vdev->host.domain, vdev->host.bus, vdev->host.slot, 3564 vdev->host.function); 3565 3566 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) { 3567 if (strcmp(vbasedev_iter->name, vdev->vbasedev.name) == 0) { 3568 error_report("vfio: error: device %s is already attached", path); 3569 vfio_put_group(group); 3570 return -EBUSY; 3571 } 3572 } 3573 3574 ret = vfio_get_device(group, path, &vdev->vbasedev); 3575 if (ret) { 3576 error_report("vfio: failed to get device %s", path); 3577 vfio_put_group(group); 3578 return ret; 3579 } 3580 3581 ret = vfio_populate_device(vdev); 3582 if (ret) { 3583 return ret; 3584 } 3585 3586 /* Get a copy of config space */ 3587 ret = pread(vdev->vbasedev.fd, vdev->pdev.config, 3588 MIN(pci_config_size(&vdev->pdev), vdev->config_size), 3589 vdev->config_offset); 3590 if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) { 3591 ret = ret < 0 ? -errno : -EFAULT; 3592 error_report("vfio: Failed to read device config space"); 3593 return ret; 3594 } 3595 3596 /* vfio emulates a lot for us, but some bits need extra love */ 3597 vdev->emulated_config_bits = g_malloc0(vdev->config_size); 3598 3599 /* QEMU can choose to expose the ROM or not */ 3600 memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4); 3601 3602 /* QEMU can change multi-function devices to single function, or reverse */ 3603 vdev->emulated_config_bits[PCI_HEADER_TYPE] = 3604 PCI_HEADER_TYPE_MULTI_FUNCTION; 3605 3606 /* Restore or clear multifunction, this is always controlled by QEMU */ 3607 if (vdev->pdev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) { 3608 vdev->pdev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION; 3609 } else { 3610 vdev->pdev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION; 3611 } 3612 3613 /* 3614 * Clear host resource mapping info. If we choose not to register a 3615 * BAR, such as might be the case with the option ROM, we can get 3616 * confusing, unwritable, residual addresses from the host here. 3617 */ 3618 memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24); 3619 memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4); 3620 3621 vfio_pci_size_rom(vdev); 3622 3623 ret = vfio_early_setup_msix(vdev); 3624 if (ret) { 3625 return ret; 3626 } 3627 3628 vfio_map_bars(vdev); 3629 3630 ret = vfio_add_capabilities(vdev); 3631 if (ret) { 3632 goto out_teardown; 3633 } 3634 3635 /* QEMU emulates all of MSI & MSIX */ 3636 if (pdev->cap_present & QEMU_PCI_CAP_MSIX) { 3637 memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff, 3638 MSIX_CAP_LENGTH); 3639 } 3640 3641 if (pdev->cap_present & QEMU_PCI_CAP_MSI) { 3642 memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff, 3643 vdev->msi_cap_size); 3644 } 3645 3646 if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) { 3647 vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL, 3648 vfio_intx_mmap_enable, vdev); 3649 pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_update_irq); 3650 ret = vfio_enable_intx(vdev); 3651 if (ret) { 3652 goto out_teardown; 3653 } 3654 } 3655 3656 vfio_register_err_notifier(vdev); 3657 vfio_register_req_notifier(vdev); 3658 vfio_setup_resetfn(vdev); 3659 3660 return 0; 3661 3662 out_teardown: 3663 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL); 3664 vfio_teardown_msi(vdev); 3665 vfio_unregister_bars(vdev); 3666 return ret; 3667 } 3668 3669 static void vfio_instance_finalize(Object *obj) 3670 { 3671 PCIDevice *pci_dev = PCI_DEVICE(obj); 3672 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pci_dev); 3673 VFIOGroup *group = vdev->vbasedev.group; 3674 3675 vfio_unmap_bars(vdev); 3676 g_free(vdev->emulated_config_bits); 3677 g_free(vdev->rom); 3678 vfio_put_device(vdev); 3679 vfio_put_group(group); 3680 } 3681 3682 static void vfio_exitfn(PCIDevice *pdev) 3683 { 3684 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev); 3685 3686 vfio_unregister_req_notifier(vdev); 3687 vfio_unregister_err_notifier(vdev); 3688 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL); 3689 vfio_disable_interrupts(vdev); 3690 if (vdev->intx.mmap_timer) { 3691 timer_free(vdev->intx.mmap_timer); 3692 } 3693 vfio_teardown_msi(vdev); 3694 vfio_unregister_bars(vdev); 3695 } 3696 3697 static void vfio_pci_reset(DeviceState *dev) 3698 { 3699 PCIDevice *pdev = DO_UPCAST(PCIDevice, qdev, dev); 3700 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev); 3701 3702 trace_vfio_pci_reset(vdev->vbasedev.name); 3703 3704 vfio_pci_pre_reset(vdev); 3705 3706 if (vdev->resetfn && !vdev->resetfn(vdev)) { 3707 goto post_reset; 3708 } 3709 3710 if (vdev->vbasedev.reset_works && 3711 (vdev->has_flr || !vdev->has_pm_reset) && 3712 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) { 3713 trace_vfio_pci_reset_flr(vdev->vbasedev.name); 3714 goto post_reset; 3715 } 3716 3717 /* See if we can do our own bus reset */ 3718 if (!vfio_pci_hot_reset_one(vdev)) { 3719 goto post_reset; 3720 } 3721 3722 /* If nothing else works and the device supports PM reset, use it */ 3723 if (vdev->vbasedev.reset_works && vdev->has_pm_reset && 3724 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) { 3725 trace_vfio_pci_reset_pm(vdev->vbasedev.name); 3726 goto post_reset; 3727 } 3728 3729 post_reset: 3730 vfio_pci_post_reset(vdev); 3731 } 3732 3733 static void vfio_instance_init(Object *obj) 3734 { 3735 PCIDevice *pci_dev = PCI_DEVICE(obj); 3736 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, PCI_DEVICE(obj)); 3737 3738 device_add_bootindex_property(obj, &vdev->bootindex, 3739 "bootindex", NULL, 3740 &pci_dev->qdev, NULL); 3741 } 3742 3743 static Property vfio_pci_dev_properties[] = { 3744 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host), 3745 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice, 3746 intx.mmap_timeout, 1100), 3747 DEFINE_PROP_BIT("x-vga", VFIOPCIDevice, features, 3748 VFIO_FEATURE_ENABLE_VGA_BIT, false), 3749 DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features, 3750 VFIO_FEATURE_ENABLE_REQ_BIT, true), 3751 DEFINE_PROP_BOOL("x-mmap", VFIOPCIDevice, vbasedev.allow_mmap, true), 3752 /* 3753 * TODO - support passed fds... is this necessary? 3754 * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name), 3755 * DEFINE_PROP_STRING("vfiogroupfd, VFIOPCIDevice, vfiogroupfd_name), 3756 */ 3757 DEFINE_PROP_END_OF_LIST(), 3758 }; 3759 3760 static const VMStateDescription vfio_pci_vmstate = { 3761 .name = "vfio-pci", 3762 .unmigratable = 1, 3763 }; 3764 3765 static void vfio_pci_dev_class_init(ObjectClass *klass, void *data) 3766 { 3767 DeviceClass *dc = DEVICE_CLASS(klass); 3768 PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass); 3769 3770 dc->reset = vfio_pci_reset; 3771 dc->props = vfio_pci_dev_properties; 3772 dc->vmsd = &vfio_pci_vmstate; 3773 dc->desc = "VFIO-based PCI device assignment"; 3774 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 3775 pdc->init = vfio_initfn; 3776 pdc->exit = vfio_exitfn; 3777 pdc->config_read = vfio_pci_read_config; 3778 pdc->config_write = vfio_pci_write_config; 3779 pdc->is_express = 1; /* We might be */ 3780 } 3781 3782 static const TypeInfo vfio_pci_dev_info = { 3783 .name = "vfio-pci", 3784 .parent = TYPE_PCI_DEVICE, 3785 .instance_size = sizeof(VFIOPCIDevice), 3786 .class_init = vfio_pci_dev_class_init, 3787 .instance_init = vfio_instance_init, 3788 .instance_finalize = vfio_instance_finalize, 3789 }; 3790 3791 static void register_vfio_pci_dev_type(void) 3792 { 3793 type_register_static(&vfio_pci_dev_info); 3794 } 3795 3796 type_init(register_vfio_pci_dev_type) 3797