1 /* 2 * virtio-iommu device 3 * 4 * Copyright (c) 2020 Red Hat, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2 or later, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along with 16 * this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 */ 19 20 #include "qemu/osdep.h" 21 #include "qemu/log.h" 22 #include "qemu/iov.h" 23 #include "hw/qdev-properties.h" 24 #include "hw/virtio/virtio.h" 25 #include "sysemu/kvm.h" 26 #include "sysemu/reset.h" 27 #include "qapi/error.h" 28 #include "qemu/error-report.h" 29 #include "trace.h" 30 31 #include "standard-headers/linux/virtio_ids.h" 32 33 #include "hw/virtio/virtio-bus.h" 34 #include "hw/virtio/virtio-access.h" 35 #include "hw/virtio/virtio-iommu.h" 36 #include "hw/pci/pci_bus.h" 37 #include "hw/pci/pci.h" 38 39 /* Max size */ 40 #define VIOMMU_DEFAULT_QUEUE_SIZE 256 41 #define VIOMMU_PROBE_SIZE 512 42 43 typedef struct VirtIOIOMMUDomain { 44 uint32_t id; 45 bool bypass; 46 GTree *mappings; 47 QLIST_HEAD(, VirtIOIOMMUEndpoint) endpoint_list; 48 } VirtIOIOMMUDomain; 49 50 typedef struct VirtIOIOMMUEndpoint { 51 uint32_t id; 52 VirtIOIOMMUDomain *domain; 53 IOMMUMemoryRegion *iommu_mr; 54 QLIST_ENTRY(VirtIOIOMMUEndpoint) next; 55 } VirtIOIOMMUEndpoint; 56 57 typedef struct VirtIOIOMMUInterval { 58 uint64_t low; 59 uint64_t high; 60 } VirtIOIOMMUInterval; 61 62 typedef struct VirtIOIOMMUMapping { 63 uint64_t phys_addr; 64 uint32_t flags; 65 } VirtIOIOMMUMapping; 66 67 static inline uint16_t virtio_iommu_get_bdf(IOMMUDevice *dev) 68 { 69 return PCI_BUILD_BDF(pci_bus_num(dev->bus), dev->devfn); 70 } 71 72 static bool virtio_iommu_device_bypassed(IOMMUDevice *sdev) 73 { 74 uint32_t sid; 75 bool bypassed; 76 VirtIOIOMMU *s = sdev->viommu; 77 VirtIOIOMMUEndpoint *ep; 78 79 sid = virtio_iommu_get_bdf(sdev); 80 81 qemu_rec_mutex_lock(&s->mutex); 82 /* need to check bypass before system reset */ 83 if (!s->endpoints) { 84 bypassed = s->config.bypass; 85 goto unlock; 86 } 87 88 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); 89 if (!ep || !ep->domain) { 90 bypassed = s->config.bypass; 91 } else { 92 bypassed = ep->domain->bypass; 93 } 94 95 unlock: 96 qemu_rec_mutex_unlock(&s->mutex); 97 return bypassed; 98 } 99 100 /* Return whether the device is using IOMMU translation. */ 101 static bool virtio_iommu_switch_address_space(IOMMUDevice *sdev) 102 { 103 bool use_remapping; 104 105 assert(sdev); 106 107 use_remapping = !virtio_iommu_device_bypassed(sdev); 108 109 trace_virtio_iommu_switch_address_space(pci_bus_num(sdev->bus), 110 PCI_SLOT(sdev->devfn), 111 PCI_FUNC(sdev->devfn), 112 use_remapping); 113 114 /* Turn off first then on the other */ 115 if (use_remapping) { 116 memory_region_set_enabled(&sdev->bypass_mr, false); 117 memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), true); 118 } else { 119 memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), false); 120 memory_region_set_enabled(&sdev->bypass_mr, true); 121 } 122 123 return use_remapping; 124 } 125 126 static void virtio_iommu_switch_address_space_all(VirtIOIOMMU *s) 127 { 128 GHashTableIter iter; 129 IOMMUPciBus *iommu_pci_bus; 130 int i; 131 132 g_hash_table_iter_init(&iter, s->as_by_busptr); 133 while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) { 134 for (i = 0; i < PCI_DEVFN_MAX; i++) { 135 if (!iommu_pci_bus->pbdev[i]) { 136 continue; 137 } 138 virtio_iommu_switch_address_space(iommu_pci_bus->pbdev[i]); 139 } 140 } 141 } 142 143 /** 144 * The bus number is used for lookup when SID based operations occur. 145 * In that case we lazily populate the IOMMUPciBus array from the bus hash 146 * table. At the time the IOMMUPciBus is created (iommu_find_add_as), the bus 147 * numbers may not be always initialized yet. 148 */ 149 static IOMMUPciBus *iommu_find_iommu_pcibus(VirtIOIOMMU *s, uint8_t bus_num) 150 { 151 IOMMUPciBus *iommu_pci_bus = s->iommu_pcibus_by_bus_num[bus_num]; 152 153 if (!iommu_pci_bus) { 154 GHashTableIter iter; 155 156 g_hash_table_iter_init(&iter, s->as_by_busptr); 157 while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) { 158 if (pci_bus_num(iommu_pci_bus->bus) == bus_num) { 159 s->iommu_pcibus_by_bus_num[bus_num] = iommu_pci_bus; 160 return iommu_pci_bus; 161 } 162 } 163 return NULL; 164 } 165 return iommu_pci_bus; 166 } 167 168 static IOMMUMemoryRegion *virtio_iommu_mr(VirtIOIOMMU *s, uint32_t sid) 169 { 170 uint8_t bus_n, devfn; 171 IOMMUPciBus *iommu_pci_bus; 172 IOMMUDevice *dev; 173 174 bus_n = PCI_BUS_NUM(sid); 175 iommu_pci_bus = iommu_find_iommu_pcibus(s, bus_n); 176 if (iommu_pci_bus) { 177 devfn = sid & (PCI_DEVFN_MAX - 1); 178 dev = iommu_pci_bus->pbdev[devfn]; 179 if (dev) { 180 return &dev->iommu_mr; 181 } 182 } 183 return NULL; 184 } 185 186 static gint interval_cmp(gconstpointer a, gconstpointer b, gpointer user_data) 187 { 188 VirtIOIOMMUInterval *inta = (VirtIOIOMMUInterval *)a; 189 VirtIOIOMMUInterval *intb = (VirtIOIOMMUInterval *)b; 190 191 if (inta->high < intb->low) { 192 return -1; 193 } else if (intb->high < inta->low) { 194 return 1; 195 } else { 196 return 0; 197 } 198 } 199 200 static void virtio_iommu_notify_map(IOMMUMemoryRegion *mr, hwaddr virt_start, 201 hwaddr virt_end, hwaddr paddr, 202 uint32_t flags) 203 { 204 IOMMUTLBEvent event; 205 IOMMUAccessFlags perm = IOMMU_ACCESS_FLAG(flags & VIRTIO_IOMMU_MAP_F_READ, 206 flags & VIRTIO_IOMMU_MAP_F_WRITE); 207 208 if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_MAP) || 209 (flags & VIRTIO_IOMMU_MAP_F_MMIO) || !perm) { 210 return; 211 } 212 213 trace_virtio_iommu_notify_map(mr->parent_obj.name, virt_start, virt_end, 214 paddr, perm); 215 216 event.type = IOMMU_NOTIFIER_MAP; 217 event.entry.target_as = &address_space_memory; 218 event.entry.addr_mask = virt_end - virt_start; 219 event.entry.iova = virt_start; 220 event.entry.perm = perm; 221 event.entry.translated_addr = paddr; 222 223 memory_region_notify_iommu(mr, 0, event); 224 } 225 226 static void virtio_iommu_notify_unmap(IOMMUMemoryRegion *mr, hwaddr virt_start, 227 hwaddr virt_end) 228 { 229 IOMMUTLBEvent event; 230 uint64_t delta = virt_end - virt_start; 231 232 if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_UNMAP)) { 233 return; 234 } 235 236 trace_virtio_iommu_notify_unmap(mr->parent_obj.name, virt_start, virt_end); 237 238 event.type = IOMMU_NOTIFIER_UNMAP; 239 event.entry.target_as = &address_space_memory; 240 event.entry.perm = IOMMU_NONE; 241 event.entry.translated_addr = 0; 242 event.entry.addr_mask = delta; 243 event.entry.iova = virt_start; 244 245 if (delta == UINT64_MAX) { 246 memory_region_notify_iommu(mr, 0, event); 247 } 248 249 250 while (virt_start != virt_end + 1) { 251 uint64_t mask = dma_aligned_pow2_mask(virt_start, virt_end, 64); 252 253 event.entry.addr_mask = mask; 254 event.entry.iova = virt_start; 255 memory_region_notify_iommu(mr, 0, event); 256 virt_start += mask + 1; 257 } 258 } 259 260 static gboolean virtio_iommu_notify_unmap_cb(gpointer key, gpointer value, 261 gpointer data) 262 { 263 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; 264 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; 265 266 virtio_iommu_notify_unmap(mr, interval->low, interval->high); 267 268 return false; 269 } 270 271 static gboolean virtio_iommu_notify_map_cb(gpointer key, gpointer value, 272 gpointer data) 273 { 274 VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value; 275 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; 276 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; 277 278 virtio_iommu_notify_map(mr, interval->low, interval->high, 279 mapping->phys_addr, mapping->flags); 280 281 return false; 282 } 283 284 static void virtio_iommu_detach_endpoint_from_domain(VirtIOIOMMUEndpoint *ep) 285 { 286 VirtIOIOMMUDomain *domain = ep->domain; 287 IOMMUDevice *sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr); 288 289 if (!ep->domain) { 290 return; 291 } 292 g_tree_foreach(domain->mappings, virtio_iommu_notify_unmap_cb, 293 ep->iommu_mr); 294 QLIST_REMOVE(ep, next); 295 ep->domain = NULL; 296 virtio_iommu_switch_address_space(sdev); 297 } 298 299 static VirtIOIOMMUEndpoint *virtio_iommu_get_endpoint(VirtIOIOMMU *s, 300 uint32_t ep_id) 301 { 302 VirtIOIOMMUEndpoint *ep; 303 IOMMUMemoryRegion *mr; 304 305 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id)); 306 if (ep) { 307 return ep; 308 } 309 mr = virtio_iommu_mr(s, ep_id); 310 if (!mr) { 311 return NULL; 312 } 313 ep = g_malloc0(sizeof(*ep)); 314 ep->id = ep_id; 315 ep->iommu_mr = mr; 316 trace_virtio_iommu_get_endpoint(ep_id); 317 g_tree_insert(s->endpoints, GUINT_TO_POINTER(ep_id), ep); 318 return ep; 319 } 320 321 static void virtio_iommu_put_endpoint(gpointer data) 322 { 323 VirtIOIOMMUEndpoint *ep = (VirtIOIOMMUEndpoint *)data; 324 325 if (ep->domain) { 326 virtio_iommu_detach_endpoint_from_domain(ep); 327 } 328 329 trace_virtio_iommu_put_endpoint(ep->id); 330 g_free(ep); 331 } 332 333 static VirtIOIOMMUDomain *virtio_iommu_get_domain(VirtIOIOMMU *s, 334 uint32_t domain_id, 335 bool bypass) 336 { 337 VirtIOIOMMUDomain *domain; 338 339 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); 340 if (domain) { 341 if (domain->bypass != bypass) { 342 return NULL; 343 } 344 return domain; 345 } 346 domain = g_malloc0(sizeof(*domain)); 347 domain->id = domain_id; 348 domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp, 349 NULL, (GDestroyNotify)g_free, 350 (GDestroyNotify)g_free); 351 domain->bypass = bypass; 352 g_tree_insert(s->domains, GUINT_TO_POINTER(domain_id), domain); 353 QLIST_INIT(&domain->endpoint_list); 354 trace_virtio_iommu_get_domain(domain_id); 355 return domain; 356 } 357 358 static void virtio_iommu_put_domain(gpointer data) 359 { 360 VirtIOIOMMUDomain *domain = (VirtIOIOMMUDomain *)data; 361 VirtIOIOMMUEndpoint *iter, *tmp; 362 363 QLIST_FOREACH_SAFE(iter, &domain->endpoint_list, next, tmp) { 364 virtio_iommu_detach_endpoint_from_domain(iter); 365 } 366 g_tree_destroy(domain->mappings); 367 trace_virtio_iommu_put_domain(domain->id); 368 g_free(domain); 369 } 370 371 static AddressSpace *virtio_iommu_find_add_as(PCIBus *bus, void *opaque, 372 int devfn) 373 { 374 VirtIOIOMMU *s = opaque; 375 IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus); 376 static uint32_t mr_index; 377 IOMMUDevice *sdev; 378 379 if (!sbus) { 380 sbus = g_malloc0(sizeof(IOMMUPciBus) + 381 sizeof(IOMMUDevice *) * PCI_DEVFN_MAX); 382 sbus->bus = bus; 383 g_hash_table_insert(s->as_by_busptr, bus, sbus); 384 } 385 386 sdev = sbus->pbdev[devfn]; 387 if (!sdev) { 388 char *name = g_strdup_printf("%s-%d-%d", 389 TYPE_VIRTIO_IOMMU_MEMORY_REGION, 390 mr_index++, devfn); 391 sdev = sbus->pbdev[devfn] = g_new0(IOMMUDevice, 1); 392 393 sdev->viommu = s; 394 sdev->bus = bus; 395 sdev->devfn = devfn; 396 397 trace_virtio_iommu_init_iommu_mr(name); 398 399 memory_region_init(&sdev->root, OBJECT(s), name, UINT64_MAX); 400 address_space_init(&sdev->as, &sdev->root, TYPE_VIRTIO_IOMMU); 401 402 /* 403 * Build the IOMMU disabled container with aliases to the 404 * shared MRs. Note that aliasing to a shared memory region 405 * could help the memory API to detect same FlatViews so we 406 * can have devices to share the same FlatView when in bypass 407 * mode. (either by not configuring virtio-iommu driver or with 408 * "iommu=pt"). It will greatly reduce the total number of 409 * FlatViews of the system hence VM runs faster. 410 */ 411 memory_region_init_alias(&sdev->bypass_mr, OBJECT(s), 412 "system", get_system_memory(), 0, 413 memory_region_size(get_system_memory())); 414 415 memory_region_init_iommu(&sdev->iommu_mr, sizeof(sdev->iommu_mr), 416 TYPE_VIRTIO_IOMMU_MEMORY_REGION, 417 OBJECT(s), name, 418 UINT64_MAX); 419 420 /* 421 * Hook both the containers under the root container, we 422 * switch between iommu & bypass MRs by enable/disable 423 * corresponding sub-containers 424 */ 425 memory_region_add_subregion_overlap(&sdev->root, 0, 426 MEMORY_REGION(&sdev->iommu_mr), 427 0); 428 memory_region_add_subregion_overlap(&sdev->root, 0, 429 &sdev->bypass_mr, 0); 430 431 virtio_iommu_switch_address_space(sdev); 432 g_free(name); 433 } 434 return &sdev->as; 435 } 436 437 static int virtio_iommu_attach(VirtIOIOMMU *s, 438 struct virtio_iommu_req_attach *req) 439 { 440 uint32_t domain_id = le32_to_cpu(req->domain); 441 uint32_t ep_id = le32_to_cpu(req->endpoint); 442 uint32_t flags = le32_to_cpu(req->flags); 443 VirtIOIOMMUDomain *domain; 444 VirtIOIOMMUEndpoint *ep; 445 IOMMUDevice *sdev; 446 447 trace_virtio_iommu_attach(domain_id, ep_id); 448 449 if (flags & ~VIRTIO_IOMMU_ATTACH_F_BYPASS) { 450 return VIRTIO_IOMMU_S_INVAL; 451 } 452 453 ep = virtio_iommu_get_endpoint(s, ep_id); 454 if (!ep) { 455 return VIRTIO_IOMMU_S_NOENT; 456 } 457 458 if (ep->domain) { 459 VirtIOIOMMUDomain *previous_domain = ep->domain; 460 /* 461 * the device is already attached to a domain, 462 * detach it first 463 */ 464 virtio_iommu_detach_endpoint_from_domain(ep); 465 if (QLIST_EMPTY(&previous_domain->endpoint_list)) { 466 g_tree_remove(s->domains, GUINT_TO_POINTER(previous_domain->id)); 467 } 468 } 469 470 domain = virtio_iommu_get_domain(s, domain_id, 471 flags & VIRTIO_IOMMU_ATTACH_F_BYPASS); 472 if (!domain) { 473 /* Incompatible bypass flag */ 474 return VIRTIO_IOMMU_S_INVAL; 475 } 476 QLIST_INSERT_HEAD(&domain->endpoint_list, ep, next); 477 478 ep->domain = domain; 479 sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr); 480 virtio_iommu_switch_address_space(sdev); 481 482 /* Replay domain mappings on the associated memory region */ 483 g_tree_foreach(domain->mappings, virtio_iommu_notify_map_cb, 484 ep->iommu_mr); 485 486 return VIRTIO_IOMMU_S_OK; 487 } 488 489 static int virtio_iommu_detach(VirtIOIOMMU *s, 490 struct virtio_iommu_req_detach *req) 491 { 492 uint32_t domain_id = le32_to_cpu(req->domain); 493 uint32_t ep_id = le32_to_cpu(req->endpoint); 494 VirtIOIOMMUDomain *domain; 495 VirtIOIOMMUEndpoint *ep; 496 497 trace_virtio_iommu_detach(domain_id, ep_id); 498 499 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id)); 500 if (!ep) { 501 return VIRTIO_IOMMU_S_NOENT; 502 } 503 504 domain = ep->domain; 505 506 if (!domain || domain->id != domain_id) { 507 return VIRTIO_IOMMU_S_INVAL; 508 } 509 510 virtio_iommu_detach_endpoint_from_domain(ep); 511 512 if (QLIST_EMPTY(&domain->endpoint_list)) { 513 g_tree_remove(s->domains, GUINT_TO_POINTER(domain->id)); 514 } 515 return VIRTIO_IOMMU_S_OK; 516 } 517 518 static int virtio_iommu_map(VirtIOIOMMU *s, 519 struct virtio_iommu_req_map *req) 520 { 521 uint32_t domain_id = le32_to_cpu(req->domain); 522 uint64_t phys_start = le64_to_cpu(req->phys_start); 523 uint64_t virt_start = le64_to_cpu(req->virt_start); 524 uint64_t virt_end = le64_to_cpu(req->virt_end); 525 uint32_t flags = le32_to_cpu(req->flags); 526 VirtIOIOMMUDomain *domain; 527 VirtIOIOMMUInterval *interval; 528 VirtIOIOMMUMapping *mapping; 529 VirtIOIOMMUEndpoint *ep; 530 531 if (flags & ~VIRTIO_IOMMU_MAP_F_MASK) { 532 return VIRTIO_IOMMU_S_INVAL; 533 } 534 535 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); 536 if (!domain) { 537 return VIRTIO_IOMMU_S_NOENT; 538 } 539 540 if (domain->bypass) { 541 return VIRTIO_IOMMU_S_INVAL; 542 } 543 544 interval = g_malloc0(sizeof(*interval)); 545 546 interval->low = virt_start; 547 interval->high = virt_end; 548 549 mapping = g_tree_lookup(domain->mappings, (gpointer)interval); 550 if (mapping) { 551 g_free(interval); 552 return VIRTIO_IOMMU_S_INVAL; 553 } 554 555 trace_virtio_iommu_map(domain_id, virt_start, virt_end, phys_start, flags); 556 557 mapping = g_malloc0(sizeof(*mapping)); 558 mapping->phys_addr = phys_start; 559 mapping->flags = flags; 560 561 g_tree_insert(domain->mappings, interval, mapping); 562 563 QLIST_FOREACH(ep, &domain->endpoint_list, next) { 564 virtio_iommu_notify_map(ep->iommu_mr, virt_start, virt_end, phys_start, 565 flags); 566 } 567 568 return VIRTIO_IOMMU_S_OK; 569 } 570 571 static int virtio_iommu_unmap(VirtIOIOMMU *s, 572 struct virtio_iommu_req_unmap *req) 573 { 574 uint32_t domain_id = le32_to_cpu(req->domain); 575 uint64_t virt_start = le64_to_cpu(req->virt_start); 576 uint64_t virt_end = le64_to_cpu(req->virt_end); 577 VirtIOIOMMUMapping *iter_val; 578 VirtIOIOMMUInterval interval, *iter_key; 579 VirtIOIOMMUDomain *domain; 580 VirtIOIOMMUEndpoint *ep; 581 int ret = VIRTIO_IOMMU_S_OK; 582 583 trace_virtio_iommu_unmap(domain_id, virt_start, virt_end); 584 585 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); 586 if (!domain) { 587 return VIRTIO_IOMMU_S_NOENT; 588 } 589 590 if (domain->bypass) { 591 return VIRTIO_IOMMU_S_INVAL; 592 } 593 594 interval.low = virt_start; 595 interval.high = virt_end; 596 597 while (g_tree_lookup_extended(domain->mappings, &interval, 598 (void **)&iter_key, (void**)&iter_val)) { 599 uint64_t current_low = iter_key->low; 600 uint64_t current_high = iter_key->high; 601 602 if (interval.low <= current_low && interval.high >= current_high) { 603 QLIST_FOREACH(ep, &domain->endpoint_list, next) { 604 virtio_iommu_notify_unmap(ep->iommu_mr, current_low, 605 current_high); 606 } 607 g_tree_remove(domain->mappings, iter_key); 608 trace_virtio_iommu_unmap_done(domain_id, current_low, current_high); 609 } else { 610 ret = VIRTIO_IOMMU_S_RANGE; 611 break; 612 } 613 } 614 return ret; 615 } 616 617 static ssize_t virtio_iommu_fill_resv_mem_prop(VirtIOIOMMU *s, uint32_t ep, 618 uint8_t *buf, size_t free) 619 { 620 struct virtio_iommu_probe_resv_mem prop = {}; 621 size_t size = sizeof(prop), length = size - sizeof(prop.head), total; 622 int i; 623 624 total = size * s->nb_reserved_regions; 625 626 if (total > free) { 627 return -ENOSPC; 628 } 629 630 for (i = 0; i < s->nb_reserved_regions; i++) { 631 unsigned subtype = s->reserved_regions[i].type; 632 633 assert(subtype == VIRTIO_IOMMU_RESV_MEM_T_RESERVED || 634 subtype == VIRTIO_IOMMU_RESV_MEM_T_MSI); 635 prop.head.type = cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM); 636 prop.head.length = cpu_to_le16(length); 637 prop.subtype = subtype; 638 prop.start = cpu_to_le64(s->reserved_regions[i].low); 639 prop.end = cpu_to_le64(s->reserved_regions[i].high); 640 641 memcpy(buf, &prop, size); 642 643 trace_virtio_iommu_fill_resv_property(ep, prop.subtype, 644 prop.start, prop.end); 645 buf += size; 646 } 647 return total; 648 } 649 650 /** 651 * virtio_iommu_probe - Fill the probe request buffer with 652 * the properties the device is able to return 653 */ 654 static int virtio_iommu_probe(VirtIOIOMMU *s, 655 struct virtio_iommu_req_probe *req, 656 uint8_t *buf) 657 { 658 uint32_t ep_id = le32_to_cpu(req->endpoint); 659 size_t free = VIOMMU_PROBE_SIZE; 660 ssize_t count; 661 662 if (!virtio_iommu_mr(s, ep_id)) { 663 return VIRTIO_IOMMU_S_NOENT; 664 } 665 666 count = virtio_iommu_fill_resv_mem_prop(s, ep_id, buf, free); 667 if (count < 0) { 668 return VIRTIO_IOMMU_S_INVAL; 669 } 670 buf += count; 671 free -= count; 672 673 return VIRTIO_IOMMU_S_OK; 674 } 675 676 static int virtio_iommu_iov_to_req(struct iovec *iov, 677 unsigned int iov_cnt, 678 void *req, size_t req_sz) 679 { 680 size_t sz, payload_sz = req_sz - sizeof(struct virtio_iommu_req_tail); 681 682 sz = iov_to_buf(iov, iov_cnt, 0, req, payload_sz); 683 if (unlikely(sz != payload_sz)) { 684 return VIRTIO_IOMMU_S_INVAL; 685 } 686 return 0; 687 } 688 689 #define virtio_iommu_handle_req(__req) \ 690 static int virtio_iommu_handle_ ## __req(VirtIOIOMMU *s, \ 691 struct iovec *iov, \ 692 unsigned int iov_cnt) \ 693 { \ 694 struct virtio_iommu_req_ ## __req req; \ 695 int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req)); \ 696 \ 697 return ret ? ret : virtio_iommu_ ## __req(s, &req); \ 698 } 699 700 virtio_iommu_handle_req(attach) 701 virtio_iommu_handle_req(detach) 702 virtio_iommu_handle_req(map) 703 virtio_iommu_handle_req(unmap) 704 705 static int virtio_iommu_handle_probe(VirtIOIOMMU *s, 706 struct iovec *iov, 707 unsigned int iov_cnt, 708 uint8_t *buf) 709 { 710 struct virtio_iommu_req_probe req; 711 int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req)); 712 713 return ret ? ret : virtio_iommu_probe(s, &req, buf); 714 } 715 716 static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq) 717 { 718 VirtIOIOMMU *s = VIRTIO_IOMMU(vdev); 719 struct virtio_iommu_req_head head; 720 struct virtio_iommu_req_tail tail = {}; 721 size_t output_size = sizeof(tail), sz; 722 VirtQueueElement *elem; 723 unsigned int iov_cnt; 724 struct iovec *iov; 725 void *buf = NULL; 726 727 for (;;) { 728 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 729 if (!elem) { 730 return; 731 } 732 733 if (iov_size(elem->in_sg, elem->in_num) < sizeof(tail) || 734 iov_size(elem->out_sg, elem->out_num) < sizeof(head)) { 735 virtio_error(vdev, "virtio-iommu bad head/tail size"); 736 virtqueue_detach_element(vq, elem, 0); 737 g_free(elem); 738 break; 739 } 740 741 iov_cnt = elem->out_num; 742 iov = elem->out_sg; 743 sz = iov_to_buf(iov, iov_cnt, 0, &head, sizeof(head)); 744 if (unlikely(sz != sizeof(head))) { 745 tail.status = VIRTIO_IOMMU_S_DEVERR; 746 goto out; 747 } 748 qemu_rec_mutex_lock(&s->mutex); 749 switch (head.type) { 750 case VIRTIO_IOMMU_T_ATTACH: 751 tail.status = virtio_iommu_handle_attach(s, iov, iov_cnt); 752 break; 753 case VIRTIO_IOMMU_T_DETACH: 754 tail.status = virtio_iommu_handle_detach(s, iov, iov_cnt); 755 break; 756 case VIRTIO_IOMMU_T_MAP: 757 tail.status = virtio_iommu_handle_map(s, iov, iov_cnt); 758 break; 759 case VIRTIO_IOMMU_T_UNMAP: 760 tail.status = virtio_iommu_handle_unmap(s, iov, iov_cnt); 761 break; 762 case VIRTIO_IOMMU_T_PROBE: 763 { 764 struct virtio_iommu_req_tail *ptail; 765 766 output_size = s->config.probe_size + sizeof(tail); 767 buf = g_malloc0(output_size); 768 769 ptail = (struct virtio_iommu_req_tail *) 770 (buf + s->config.probe_size); 771 ptail->status = virtio_iommu_handle_probe(s, iov, iov_cnt, buf); 772 break; 773 } 774 default: 775 tail.status = VIRTIO_IOMMU_S_UNSUPP; 776 } 777 qemu_rec_mutex_unlock(&s->mutex); 778 779 out: 780 sz = iov_from_buf(elem->in_sg, elem->in_num, 0, 781 buf ? buf : &tail, output_size); 782 assert(sz == output_size); 783 784 virtqueue_push(vq, elem, sz); 785 virtio_notify(vdev, vq); 786 g_free(elem); 787 g_free(buf); 788 buf = NULL; 789 } 790 } 791 792 static void virtio_iommu_report_fault(VirtIOIOMMU *viommu, uint8_t reason, 793 int flags, uint32_t endpoint, 794 uint64_t address) 795 { 796 VirtIODevice *vdev = &viommu->parent_obj; 797 VirtQueue *vq = viommu->event_vq; 798 struct virtio_iommu_fault fault; 799 VirtQueueElement *elem; 800 size_t sz; 801 802 memset(&fault, 0, sizeof(fault)); 803 fault.reason = reason; 804 fault.flags = cpu_to_le32(flags); 805 fault.endpoint = cpu_to_le32(endpoint); 806 fault.address = cpu_to_le64(address); 807 808 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 809 810 if (!elem) { 811 error_report_once( 812 "no buffer available in event queue to report event"); 813 return; 814 } 815 816 if (iov_size(elem->in_sg, elem->in_num) < sizeof(fault)) { 817 virtio_error(vdev, "error buffer of wrong size"); 818 virtqueue_detach_element(vq, elem, 0); 819 g_free(elem); 820 return; 821 } 822 823 sz = iov_from_buf(elem->in_sg, elem->in_num, 0, 824 &fault, sizeof(fault)); 825 assert(sz == sizeof(fault)); 826 827 trace_virtio_iommu_report_fault(reason, flags, endpoint, address); 828 virtqueue_push(vq, elem, sz); 829 virtio_notify(vdev, vq); 830 g_free(elem); 831 832 } 833 834 static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr, 835 IOMMUAccessFlags flag, 836 int iommu_idx) 837 { 838 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); 839 VirtIOIOMMUInterval interval, *mapping_key; 840 VirtIOIOMMUMapping *mapping_value; 841 VirtIOIOMMU *s = sdev->viommu; 842 bool read_fault, write_fault; 843 VirtIOIOMMUEndpoint *ep; 844 uint32_t sid, flags; 845 bool bypass_allowed; 846 bool found; 847 int i; 848 849 interval.low = addr; 850 interval.high = addr + 1; 851 852 IOMMUTLBEntry entry = { 853 .target_as = &address_space_memory, 854 .iova = addr, 855 .translated_addr = addr, 856 .addr_mask = (1 << ctz32(s->config.page_size_mask)) - 1, 857 .perm = IOMMU_NONE, 858 }; 859 860 bypass_allowed = s->config.bypass; 861 862 sid = virtio_iommu_get_bdf(sdev); 863 864 trace_virtio_iommu_translate(mr->parent_obj.name, sid, addr, flag); 865 qemu_rec_mutex_lock(&s->mutex); 866 867 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); 868 869 if (bypass_allowed) 870 assert(ep && ep->domain && !ep->domain->bypass); 871 872 if (!ep) { 873 if (!bypass_allowed) { 874 error_report_once("%s sid=%d is not known!!", __func__, sid); 875 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_UNKNOWN, 876 VIRTIO_IOMMU_FAULT_F_ADDRESS, 877 sid, addr); 878 } else { 879 entry.perm = flag; 880 } 881 goto unlock; 882 } 883 884 for (i = 0; i < s->nb_reserved_regions; i++) { 885 ReservedRegion *reg = &s->reserved_regions[i]; 886 887 if (addr >= reg->low && addr <= reg->high) { 888 switch (reg->type) { 889 case VIRTIO_IOMMU_RESV_MEM_T_MSI: 890 entry.perm = flag; 891 break; 892 case VIRTIO_IOMMU_RESV_MEM_T_RESERVED: 893 default: 894 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, 895 VIRTIO_IOMMU_FAULT_F_ADDRESS, 896 sid, addr); 897 break; 898 } 899 goto unlock; 900 } 901 } 902 903 if (!ep->domain) { 904 if (!bypass_allowed) { 905 error_report_once("%s %02x:%02x.%01x not attached to any domain", 906 __func__, PCI_BUS_NUM(sid), 907 PCI_SLOT(sid), PCI_FUNC(sid)); 908 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_DOMAIN, 909 VIRTIO_IOMMU_FAULT_F_ADDRESS, 910 sid, addr); 911 } else { 912 entry.perm = flag; 913 } 914 goto unlock; 915 } else if (ep->domain->bypass) { 916 entry.perm = flag; 917 goto unlock; 918 } 919 920 found = g_tree_lookup_extended(ep->domain->mappings, (gpointer)(&interval), 921 (void **)&mapping_key, 922 (void **)&mapping_value); 923 if (!found) { 924 error_report_once("%s no mapping for 0x%"PRIx64" for sid=%d", 925 __func__, addr, sid); 926 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, 927 VIRTIO_IOMMU_FAULT_F_ADDRESS, 928 sid, addr); 929 goto unlock; 930 } 931 932 read_fault = (flag & IOMMU_RO) && 933 !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_READ); 934 write_fault = (flag & IOMMU_WO) && 935 !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_WRITE); 936 937 flags = read_fault ? VIRTIO_IOMMU_FAULT_F_READ : 0; 938 flags |= write_fault ? VIRTIO_IOMMU_FAULT_F_WRITE : 0; 939 if (flags) { 940 error_report_once("%s permission error on 0x%"PRIx64"(%d): allowed=%d", 941 __func__, addr, flag, mapping_value->flags); 942 flags |= VIRTIO_IOMMU_FAULT_F_ADDRESS; 943 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, 944 flags | VIRTIO_IOMMU_FAULT_F_ADDRESS, 945 sid, addr); 946 goto unlock; 947 } 948 entry.translated_addr = addr - mapping_key->low + mapping_value->phys_addr; 949 entry.perm = flag; 950 trace_virtio_iommu_translate_out(addr, entry.translated_addr, sid); 951 952 unlock: 953 qemu_rec_mutex_unlock(&s->mutex); 954 return entry; 955 } 956 957 static void virtio_iommu_get_config(VirtIODevice *vdev, uint8_t *config_data) 958 { 959 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); 960 struct virtio_iommu_config *dev_config = &dev->config; 961 struct virtio_iommu_config *out_config = (void *)config_data; 962 963 out_config->page_size_mask = cpu_to_le64(dev_config->page_size_mask); 964 out_config->input_range.start = cpu_to_le64(dev_config->input_range.start); 965 out_config->input_range.end = cpu_to_le64(dev_config->input_range.end); 966 out_config->domain_range.start = cpu_to_le32(dev_config->domain_range.start); 967 out_config->domain_range.end = cpu_to_le32(dev_config->domain_range.end); 968 out_config->probe_size = cpu_to_le32(dev_config->probe_size); 969 out_config->bypass = dev_config->bypass; 970 971 trace_virtio_iommu_get_config(dev_config->page_size_mask, 972 dev_config->input_range.start, 973 dev_config->input_range.end, 974 dev_config->domain_range.start, 975 dev_config->domain_range.end, 976 dev_config->probe_size, 977 dev_config->bypass); 978 } 979 980 static void virtio_iommu_set_config(VirtIODevice *vdev, 981 const uint8_t *config_data) 982 { 983 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); 984 struct virtio_iommu_config *dev_config = &dev->config; 985 const struct virtio_iommu_config *in_config = (void *)config_data; 986 987 if (in_config->bypass != dev_config->bypass) { 988 if (!virtio_vdev_has_feature(vdev, VIRTIO_IOMMU_F_BYPASS_CONFIG)) { 989 virtio_error(vdev, "cannot set config.bypass"); 990 return; 991 } else if (in_config->bypass != 0 && in_config->bypass != 1) { 992 virtio_error(vdev, "invalid config.bypass value '%u'", 993 in_config->bypass); 994 return; 995 } 996 dev_config->bypass = in_config->bypass; 997 virtio_iommu_switch_address_space_all(dev); 998 } 999 1000 trace_virtio_iommu_set_config(in_config->bypass); 1001 } 1002 1003 static uint64_t virtio_iommu_get_features(VirtIODevice *vdev, uint64_t f, 1004 Error **errp) 1005 { 1006 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); 1007 1008 f |= dev->features; 1009 trace_virtio_iommu_get_features(f); 1010 return f; 1011 } 1012 1013 static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data) 1014 { 1015 guint ua = GPOINTER_TO_UINT(a); 1016 guint ub = GPOINTER_TO_UINT(b); 1017 return (ua > ub) - (ua < ub); 1018 } 1019 1020 static gboolean virtio_iommu_remap(gpointer key, gpointer value, gpointer data) 1021 { 1022 VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value; 1023 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; 1024 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; 1025 1026 trace_virtio_iommu_remap(mr->parent_obj.name, interval->low, interval->high, 1027 mapping->phys_addr); 1028 virtio_iommu_notify_map(mr, interval->low, interval->high, 1029 mapping->phys_addr, mapping->flags); 1030 return false; 1031 } 1032 1033 static void virtio_iommu_replay(IOMMUMemoryRegion *mr, IOMMUNotifier *n) 1034 { 1035 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); 1036 VirtIOIOMMU *s = sdev->viommu; 1037 uint32_t sid; 1038 VirtIOIOMMUEndpoint *ep; 1039 1040 sid = virtio_iommu_get_bdf(sdev); 1041 1042 qemu_rec_mutex_lock(&s->mutex); 1043 1044 if (!s->endpoints) { 1045 goto unlock; 1046 } 1047 1048 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); 1049 if (!ep || !ep->domain) { 1050 goto unlock; 1051 } 1052 1053 g_tree_foreach(ep->domain->mappings, virtio_iommu_remap, mr); 1054 1055 unlock: 1056 qemu_rec_mutex_unlock(&s->mutex); 1057 } 1058 1059 static int virtio_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu_mr, 1060 IOMMUNotifierFlag old, 1061 IOMMUNotifierFlag new, 1062 Error **errp) 1063 { 1064 if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) { 1065 error_setg(errp, "Virtio-iommu does not support dev-iotlb yet"); 1066 return -EINVAL; 1067 } 1068 1069 if (old == IOMMU_NOTIFIER_NONE) { 1070 trace_virtio_iommu_notify_flag_add(iommu_mr->parent_obj.name); 1071 } else if (new == IOMMU_NOTIFIER_NONE) { 1072 trace_virtio_iommu_notify_flag_del(iommu_mr->parent_obj.name); 1073 } 1074 return 0; 1075 } 1076 1077 /* 1078 * The default mask (TARGET_PAGE_MASK) is the smallest supported guest granule, 1079 * for example 0xfffffffffffff000. When an assigned device has page size 1080 * restrictions due to the hardware IOMMU configuration, apply this restriction 1081 * to the mask. 1082 */ 1083 static int virtio_iommu_set_page_size_mask(IOMMUMemoryRegion *mr, 1084 uint64_t new_mask, 1085 Error **errp) 1086 { 1087 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); 1088 VirtIOIOMMU *s = sdev->viommu; 1089 uint64_t cur_mask = s->config.page_size_mask; 1090 1091 trace_virtio_iommu_set_page_size_mask(mr->parent_obj.name, cur_mask, 1092 new_mask); 1093 1094 if ((cur_mask & new_mask) == 0) { 1095 error_setg(errp, "virtio-iommu page mask 0x%"PRIx64 1096 " is incompatible with mask 0x%"PRIx64, cur_mask, new_mask); 1097 return -1; 1098 } 1099 1100 /* 1101 * After the machine is finalized, we can't change the mask anymore. If by 1102 * chance the hotplugged device supports the same granule, we can still 1103 * accept it. Having a different masks is possible but the guest will use 1104 * sub-optimal block sizes, so warn about it. 1105 */ 1106 if (phase_check(PHASE_MACHINE_READY)) { 1107 int new_granule = ctz64(new_mask); 1108 int cur_granule = ctz64(cur_mask); 1109 1110 if (new_granule != cur_granule) { 1111 error_setg(errp, "virtio-iommu page mask 0x%"PRIx64 1112 " is incompatible with mask 0x%"PRIx64, cur_mask, 1113 new_mask); 1114 return -1; 1115 } else if (new_mask != cur_mask) { 1116 warn_report("virtio-iommu page mask 0x%"PRIx64 1117 " does not match 0x%"PRIx64, cur_mask, new_mask); 1118 } 1119 return 0; 1120 } 1121 1122 s->config.page_size_mask &= new_mask; 1123 return 0; 1124 } 1125 1126 static void virtio_iommu_system_reset(void *opaque) 1127 { 1128 VirtIOIOMMU *s = opaque; 1129 1130 trace_virtio_iommu_system_reset(); 1131 1132 /* 1133 * config.bypass is sticky across device reset, but should be restored on 1134 * system reset 1135 */ 1136 s->config.bypass = s->boot_bypass; 1137 virtio_iommu_switch_address_space_all(s); 1138 1139 } 1140 1141 static void virtio_iommu_device_realize(DeviceState *dev, Error **errp) 1142 { 1143 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1144 VirtIOIOMMU *s = VIRTIO_IOMMU(dev); 1145 1146 virtio_init(vdev, VIRTIO_ID_IOMMU, sizeof(struct virtio_iommu_config)); 1147 1148 memset(s->iommu_pcibus_by_bus_num, 0, sizeof(s->iommu_pcibus_by_bus_num)); 1149 1150 s->req_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, 1151 virtio_iommu_handle_command); 1152 s->event_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, NULL); 1153 1154 /* 1155 * config.bypass is needed to get initial address space early, such as 1156 * in vfio realize 1157 */ 1158 s->config.bypass = s->boot_bypass; 1159 s->config.page_size_mask = TARGET_PAGE_MASK; 1160 s->config.input_range.end = UINT64_MAX; 1161 s->config.domain_range.end = UINT32_MAX; 1162 s->config.probe_size = VIOMMU_PROBE_SIZE; 1163 1164 virtio_add_feature(&s->features, VIRTIO_RING_F_EVENT_IDX); 1165 virtio_add_feature(&s->features, VIRTIO_RING_F_INDIRECT_DESC); 1166 virtio_add_feature(&s->features, VIRTIO_F_VERSION_1); 1167 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_INPUT_RANGE); 1168 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_DOMAIN_RANGE); 1169 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MAP_UNMAP); 1170 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MMIO); 1171 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_PROBE); 1172 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_BYPASS_CONFIG); 1173 1174 qemu_rec_mutex_init(&s->mutex); 1175 1176 s->as_by_busptr = g_hash_table_new_full(NULL, NULL, NULL, g_free); 1177 1178 if (s->primary_bus) { 1179 pci_setup_iommu(s->primary_bus, virtio_iommu_find_add_as, s); 1180 } else { 1181 error_setg(errp, "VIRTIO-IOMMU is not attached to any PCI bus!"); 1182 } 1183 1184 qemu_register_reset(virtio_iommu_system_reset, s); 1185 } 1186 1187 static void virtio_iommu_device_unrealize(DeviceState *dev) 1188 { 1189 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1190 VirtIOIOMMU *s = VIRTIO_IOMMU(dev); 1191 1192 qemu_unregister_reset(virtio_iommu_system_reset, s); 1193 1194 g_hash_table_destroy(s->as_by_busptr); 1195 if (s->domains) { 1196 g_tree_destroy(s->domains); 1197 } 1198 if (s->endpoints) { 1199 g_tree_destroy(s->endpoints); 1200 } 1201 1202 qemu_rec_mutex_destroy(&s->mutex); 1203 1204 virtio_delete_queue(s->req_vq); 1205 virtio_delete_queue(s->event_vq); 1206 virtio_cleanup(vdev); 1207 } 1208 1209 static void virtio_iommu_device_reset(VirtIODevice *vdev) 1210 { 1211 VirtIOIOMMU *s = VIRTIO_IOMMU(vdev); 1212 1213 trace_virtio_iommu_device_reset(); 1214 1215 if (s->domains) { 1216 g_tree_destroy(s->domains); 1217 } 1218 if (s->endpoints) { 1219 g_tree_destroy(s->endpoints); 1220 } 1221 s->domains = g_tree_new_full((GCompareDataFunc)int_cmp, 1222 NULL, NULL, virtio_iommu_put_domain); 1223 s->endpoints = g_tree_new_full((GCompareDataFunc)int_cmp, 1224 NULL, NULL, virtio_iommu_put_endpoint); 1225 } 1226 1227 static void virtio_iommu_set_status(VirtIODevice *vdev, uint8_t status) 1228 { 1229 trace_virtio_iommu_device_status(status); 1230 } 1231 1232 static void virtio_iommu_instance_init(Object *obj) 1233 { 1234 } 1235 1236 #define VMSTATE_INTERVAL \ 1237 { \ 1238 .name = "interval", \ 1239 .version_id = 1, \ 1240 .minimum_version_id = 1, \ 1241 .fields = (VMStateField[]) { \ 1242 VMSTATE_UINT64(low, VirtIOIOMMUInterval), \ 1243 VMSTATE_UINT64(high, VirtIOIOMMUInterval), \ 1244 VMSTATE_END_OF_LIST() \ 1245 } \ 1246 } 1247 1248 #define VMSTATE_MAPPING \ 1249 { \ 1250 .name = "mapping", \ 1251 .version_id = 1, \ 1252 .minimum_version_id = 1, \ 1253 .fields = (VMStateField[]) { \ 1254 VMSTATE_UINT64(phys_addr, VirtIOIOMMUMapping),\ 1255 VMSTATE_UINT32(flags, VirtIOIOMMUMapping), \ 1256 VMSTATE_END_OF_LIST() \ 1257 }, \ 1258 } 1259 1260 static const VMStateDescription vmstate_interval_mapping[2] = { 1261 VMSTATE_MAPPING, /* value */ 1262 VMSTATE_INTERVAL /* key */ 1263 }; 1264 1265 static int domain_preload(void *opaque) 1266 { 1267 VirtIOIOMMUDomain *domain = opaque; 1268 1269 domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp, 1270 NULL, g_free, g_free); 1271 return 0; 1272 } 1273 1274 static const VMStateDescription vmstate_endpoint = { 1275 .name = "endpoint", 1276 .version_id = 1, 1277 .minimum_version_id = 1, 1278 .fields = (VMStateField[]) { 1279 VMSTATE_UINT32(id, VirtIOIOMMUEndpoint), 1280 VMSTATE_END_OF_LIST() 1281 } 1282 }; 1283 1284 static const VMStateDescription vmstate_domain = { 1285 .name = "domain", 1286 .version_id = 2, 1287 .minimum_version_id = 2, 1288 .pre_load = domain_preload, 1289 .fields = (VMStateField[]) { 1290 VMSTATE_UINT32(id, VirtIOIOMMUDomain), 1291 VMSTATE_GTREE_V(mappings, VirtIOIOMMUDomain, 1, 1292 vmstate_interval_mapping, 1293 VirtIOIOMMUInterval, VirtIOIOMMUMapping), 1294 VMSTATE_QLIST_V(endpoint_list, VirtIOIOMMUDomain, 1, 1295 vmstate_endpoint, VirtIOIOMMUEndpoint, next), 1296 VMSTATE_BOOL_V(bypass, VirtIOIOMMUDomain, 2), 1297 VMSTATE_END_OF_LIST() 1298 } 1299 }; 1300 1301 static gboolean reconstruct_endpoints(gpointer key, gpointer value, 1302 gpointer data) 1303 { 1304 VirtIOIOMMU *s = (VirtIOIOMMU *)data; 1305 VirtIOIOMMUDomain *d = (VirtIOIOMMUDomain *)value; 1306 VirtIOIOMMUEndpoint *iter; 1307 IOMMUMemoryRegion *mr; 1308 1309 QLIST_FOREACH(iter, &d->endpoint_list, next) { 1310 mr = virtio_iommu_mr(s, iter->id); 1311 assert(mr); 1312 1313 iter->domain = d; 1314 iter->iommu_mr = mr; 1315 g_tree_insert(s->endpoints, GUINT_TO_POINTER(iter->id), iter); 1316 } 1317 return false; /* continue the domain traversal */ 1318 } 1319 1320 static int iommu_post_load(void *opaque, int version_id) 1321 { 1322 VirtIOIOMMU *s = opaque; 1323 1324 g_tree_foreach(s->domains, reconstruct_endpoints, s); 1325 return 0; 1326 } 1327 1328 static const VMStateDescription vmstate_virtio_iommu_device = { 1329 .name = "virtio-iommu-device", 1330 .minimum_version_id = 2, 1331 .version_id = 2, 1332 .post_load = iommu_post_load, 1333 .fields = (VMStateField[]) { 1334 VMSTATE_GTREE_DIRECT_KEY_V(domains, VirtIOIOMMU, 2, 1335 &vmstate_domain, VirtIOIOMMUDomain), 1336 VMSTATE_UINT8_V(config.bypass, VirtIOIOMMU, 2), 1337 VMSTATE_END_OF_LIST() 1338 }, 1339 }; 1340 1341 static const VMStateDescription vmstate_virtio_iommu = { 1342 .name = "virtio-iommu", 1343 .minimum_version_id = 2, 1344 .priority = MIG_PRI_IOMMU, 1345 .version_id = 2, 1346 .fields = (VMStateField[]) { 1347 VMSTATE_VIRTIO_DEVICE, 1348 VMSTATE_END_OF_LIST() 1349 }, 1350 }; 1351 1352 static Property virtio_iommu_properties[] = { 1353 DEFINE_PROP_LINK("primary-bus", VirtIOIOMMU, primary_bus, "PCI", PCIBus *), 1354 DEFINE_PROP_BOOL("boot-bypass", VirtIOIOMMU, boot_bypass, true), 1355 DEFINE_PROP_END_OF_LIST(), 1356 }; 1357 1358 static void virtio_iommu_class_init(ObjectClass *klass, void *data) 1359 { 1360 DeviceClass *dc = DEVICE_CLASS(klass); 1361 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 1362 1363 device_class_set_props(dc, virtio_iommu_properties); 1364 dc->vmsd = &vmstate_virtio_iommu; 1365 1366 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1367 vdc->realize = virtio_iommu_device_realize; 1368 vdc->unrealize = virtio_iommu_device_unrealize; 1369 vdc->reset = virtio_iommu_device_reset; 1370 vdc->get_config = virtio_iommu_get_config; 1371 vdc->set_config = virtio_iommu_set_config; 1372 vdc->get_features = virtio_iommu_get_features; 1373 vdc->set_status = virtio_iommu_set_status; 1374 vdc->vmsd = &vmstate_virtio_iommu_device; 1375 } 1376 1377 static void virtio_iommu_memory_region_class_init(ObjectClass *klass, 1378 void *data) 1379 { 1380 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass); 1381 1382 imrc->translate = virtio_iommu_translate; 1383 imrc->replay = virtio_iommu_replay; 1384 imrc->notify_flag_changed = virtio_iommu_notify_flag_changed; 1385 imrc->iommu_set_page_size_mask = virtio_iommu_set_page_size_mask; 1386 } 1387 1388 static const TypeInfo virtio_iommu_info = { 1389 .name = TYPE_VIRTIO_IOMMU, 1390 .parent = TYPE_VIRTIO_DEVICE, 1391 .instance_size = sizeof(VirtIOIOMMU), 1392 .instance_init = virtio_iommu_instance_init, 1393 .class_init = virtio_iommu_class_init, 1394 }; 1395 1396 static const TypeInfo virtio_iommu_memory_region_info = { 1397 .parent = TYPE_IOMMU_MEMORY_REGION, 1398 .name = TYPE_VIRTIO_IOMMU_MEMORY_REGION, 1399 .class_init = virtio_iommu_memory_region_class_init, 1400 }; 1401 1402 static void virtio_register_types(void) 1403 { 1404 type_register_static(&virtio_iommu_info); 1405 type_register_static(&virtio_iommu_memory_region_info); 1406 } 1407 1408 type_init(virtio_register_types) 1409