1 /* 2 * DMA memory preregistration 3 * 4 * Authors: 5 * Alexey Kardashevskiy <aik@ozlabs.ru> 6 * 7 * This work is licensed under the terms of the GNU GPL, version 2. See 8 * the COPYING file in the top-level directory. 9 */ 10 11 #include "qemu/osdep.h" 12 #include <sys/ioctl.h> 13 #include <linux/vfio.h> 14 #ifdef CONFIG_KVM 15 #include <linux/kvm.h> 16 #endif 17 #include "sysemu/kvm.h" 18 #include "exec/address-spaces.h" 19 20 #include "hw/vfio/vfio-common.h" 21 #include "hw/hw.h" 22 #include "exec/ram_addr.h" 23 #include "qemu/error-report.h" 24 #include "qapi/error.h" 25 #include "trace.h" 26 27 typedef struct VFIOSpaprContainer { 28 VFIOContainer container; 29 MemoryListener prereg_listener; 30 QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list; 31 } VFIOSpaprContainer; 32 33 static bool vfio_prereg_listener_skipped_section(MemoryRegionSection *section) 34 { 35 if (memory_region_is_iommu(section->mr)) { 36 hw_error("Cannot possibly preregister IOMMU memory"); 37 } 38 39 return !memory_region_is_ram(section->mr) || 40 memory_region_is_ram_device(section->mr); 41 } 42 43 static void *vfio_prereg_gpa_to_vaddr(MemoryRegionSection *section, hwaddr gpa) 44 { 45 return memory_region_get_ram_ptr(section->mr) + 46 section->offset_within_region + 47 (gpa - section->offset_within_address_space); 48 } 49 50 static void vfio_prereg_listener_region_add(MemoryListener *listener, 51 MemoryRegionSection *section) 52 { 53 VFIOSpaprContainer *scontainer = container_of(listener, VFIOSpaprContainer, 54 prereg_listener); 55 VFIOContainer *container = &scontainer->container; 56 VFIOContainerBase *bcontainer = &container->bcontainer; 57 const hwaddr gpa = section->offset_within_address_space; 58 hwaddr end; 59 int ret; 60 hwaddr page_mask = qemu_real_host_page_mask(); 61 struct vfio_iommu_spapr_register_memory reg = { 62 .argsz = sizeof(reg), 63 .flags = 0, 64 }; 65 66 if (vfio_prereg_listener_skipped_section(section)) { 67 trace_vfio_prereg_listener_region_add_skip( 68 section->offset_within_address_space, 69 section->offset_within_address_space + 70 int128_get64(int128_sub(section->size, int128_one()))); 71 return; 72 } 73 74 if (unlikely((section->offset_within_address_space & ~page_mask) || 75 (section->offset_within_region & ~page_mask) || 76 (int128_get64(section->size) & ~page_mask))) { 77 error_report("%s received unaligned region", __func__); 78 return; 79 } 80 81 end = section->offset_within_address_space + int128_get64(section->size); 82 if (gpa >= end) { 83 return; 84 } 85 86 memory_region_ref(section->mr); 87 88 reg.vaddr = (uintptr_t) vfio_prereg_gpa_to_vaddr(section, gpa); 89 reg.size = end - gpa; 90 91 ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_REGISTER_MEMORY, ®); 92 trace_vfio_prereg_register(reg.vaddr, reg.size, ret ? -errno : 0); 93 if (ret) { 94 /* 95 * On the initfn path, store the first error in the container so we 96 * can gracefully fail. Runtime, there's not much we can do other 97 * than throw a hardware error. 98 */ 99 if (!bcontainer->initialized) { 100 if (!bcontainer->error) { 101 error_setg_errno(&bcontainer->error, -ret, 102 "Memory registering failed"); 103 } 104 } else { 105 hw_error("vfio: Memory registering failed, unable to continue"); 106 } 107 } 108 } 109 110 static void vfio_prereg_listener_region_del(MemoryListener *listener, 111 MemoryRegionSection *section) 112 { 113 VFIOSpaprContainer *scontainer = container_of(listener, VFIOSpaprContainer, 114 prereg_listener); 115 VFIOContainer *container = &scontainer->container; 116 const hwaddr gpa = section->offset_within_address_space; 117 hwaddr end; 118 int ret; 119 hwaddr page_mask = qemu_real_host_page_mask(); 120 struct vfio_iommu_spapr_register_memory reg = { 121 .argsz = sizeof(reg), 122 .flags = 0, 123 }; 124 125 if (vfio_prereg_listener_skipped_section(section)) { 126 trace_vfio_prereg_listener_region_del_skip( 127 section->offset_within_address_space, 128 section->offset_within_address_space + 129 int128_get64(int128_sub(section->size, int128_one()))); 130 return; 131 } 132 133 if (unlikely((section->offset_within_address_space & ~page_mask) || 134 (section->offset_within_region & ~page_mask) || 135 (int128_get64(section->size) & ~page_mask))) { 136 error_report("%s received unaligned region", __func__); 137 return; 138 } 139 140 end = section->offset_within_address_space + int128_get64(section->size); 141 if (gpa >= end) { 142 return; 143 } 144 145 reg.vaddr = (uintptr_t) vfio_prereg_gpa_to_vaddr(section, gpa); 146 reg.size = end - gpa; 147 148 ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY, ®); 149 trace_vfio_prereg_unregister(reg.vaddr, reg.size, ret ? -errno : 0); 150 } 151 152 static const MemoryListener vfio_prereg_listener = { 153 .name = "vfio-pre-reg", 154 .region_add = vfio_prereg_listener_region_add, 155 .region_del = vfio_prereg_listener_region_del, 156 }; 157 158 static void vfio_host_win_add(VFIOSpaprContainer *scontainer, hwaddr min_iova, 159 hwaddr max_iova, uint64_t iova_pgsizes) 160 { 161 VFIOHostDMAWindow *hostwin; 162 163 QLIST_FOREACH(hostwin, &scontainer->hostwin_list, hostwin_next) { 164 if (ranges_overlap(hostwin->min_iova, 165 hostwin->max_iova - hostwin->min_iova + 1, 166 min_iova, 167 max_iova - min_iova + 1)) { 168 hw_error("%s: Overlapped IOMMU are not enabled", __func__); 169 } 170 } 171 172 hostwin = g_malloc0(sizeof(*hostwin)); 173 174 hostwin->min_iova = min_iova; 175 hostwin->max_iova = max_iova; 176 hostwin->iova_pgsizes = iova_pgsizes; 177 QLIST_INSERT_HEAD(&scontainer->hostwin_list, hostwin, hostwin_next); 178 } 179 180 static int vfio_host_win_del(VFIOSpaprContainer *scontainer, 181 hwaddr min_iova, hwaddr max_iova) 182 { 183 VFIOHostDMAWindow *hostwin; 184 185 QLIST_FOREACH(hostwin, &scontainer->hostwin_list, hostwin_next) { 186 if (hostwin->min_iova == min_iova && hostwin->max_iova == max_iova) { 187 QLIST_REMOVE(hostwin, hostwin_next); 188 g_free(hostwin); 189 return 0; 190 } 191 } 192 193 return -1; 194 } 195 196 static VFIOHostDMAWindow *vfio_find_hostwin(VFIOSpaprContainer *container, 197 hwaddr iova, hwaddr end) 198 { 199 VFIOHostDMAWindow *hostwin; 200 bool hostwin_found = false; 201 202 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 203 if (hostwin->min_iova <= iova && end <= hostwin->max_iova) { 204 hostwin_found = true; 205 break; 206 } 207 } 208 209 return hostwin_found ? hostwin : NULL; 210 } 211 212 static int vfio_spapr_remove_window(VFIOContainer *container, 213 hwaddr offset_within_address_space) 214 { 215 struct vfio_iommu_spapr_tce_remove remove = { 216 .argsz = sizeof(remove), 217 .start_addr = offset_within_address_space, 218 }; 219 int ret; 220 221 ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_TCE_REMOVE, &remove); 222 if (ret) { 223 error_report("Failed to remove window at %"PRIx64, 224 (uint64_t)remove.start_addr); 225 return -errno; 226 } 227 228 trace_vfio_spapr_remove_window(offset_within_address_space); 229 230 return 0; 231 } 232 233 static int vfio_spapr_create_window(VFIOContainer *container, 234 MemoryRegionSection *section, 235 hwaddr *pgsize) 236 { 237 int ret = 0; 238 VFIOContainerBase *bcontainer = &container->bcontainer; 239 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 240 uint64_t pagesize = memory_region_iommu_get_min_page_size(iommu_mr), pgmask; 241 unsigned entries, bits_total, bits_per_level, max_levels; 242 struct vfio_iommu_spapr_tce_create create = { .argsz = sizeof(create) }; 243 long rampagesize = qemu_minrampagesize(); 244 245 /* 246 * The host might not support the guest supported IOMMU page size, 247 * so we will use smaller physical IOMMU pages to back them. 248 */ 249 if (pagesize > rampagesize) { 250 pagesize = rampagesize; 251 } 252 pgmask = bcontainer->pgsizes & (pagesize | (pagesize - 1)); 253 pagesize = pgmask ? (1ULL << (63 - clz64(pgmask))) : 0; 254 if (!pagesize) { 255 error_report("Host doesn't support page size 0x%"PRIx64 256 ", the supported mask is 0x%lx", 257 memory_region_iommu_get_min_page_size(iommu_mr), 258 bcontainer->pgsizes); 259 return -EINVAL; 260 } 261 262 /* 263 * FIXME: For VFIO iommu types which have KVM acceleration to 264 * avoid bouncing all map/unmaps through qemu this way, this 265 * would be the right place to wire that up (tell the KVM 266 * device emulation the VFIO iommu handles to use). 267 */ 268 create.window_size = int128_get64(section->size); 269 create.page_shift = ctz64(pagesize); 270 /* 271 * SPAPR host supports multilevel TCE tables. We try to guess optimal 272 * levels number and if this fails (for example due to the host memory 273 * fragmentation), we increase levels. The DMA address structure is: 274 * rrrrrrrr rxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx iiiiiiii 275 * where: 276 * r = reserved (bits >= 55 are reserved in the existing hardware) 277 * i = IOMMU page offset (64K in this example) 278 * x = bits to index a TCE which can be split to equal chunks to index 279 * within the level. 280 * The aim is to split "x" to smaller possible number of levels. 281 */ 282 entries = create.window_size >> create.page_shift; 283 /* bits_total is number of "x" needed */ 284 bits_total = ctz64(entries * sizeof(uint64_t)); 285 /* 286 * bits_per_level is a safe guess of how much we can allocate per level: 287 * 8 is the current minimum for CONFIG_FORCE_MAX_ZONEORDER and MAX_ORDER 288 * is usually bigger than that. 289 * Below we look at qemu_real_host_page_size as TCEs are allocated from 290 * system pages. 291 */ 292 bits_per_level = ctz64(qemu_real_host_page_size()) + 8; 293 create.levels = bits_total / bits_per_level; 294 if (bits_total % bits_per_level) { 295 ++create.levels; 296 } 297 max_levels = (64 - create.page_shift) / ctz64(qemu_real_host_page_size()); 298 for ( ; create.levels <= max_levels; ++create.levels) { 299 ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_TCE_CREATE, &create); 300 if (!ret) { 301 break; 302 } 303 } 304 if (ret) { 305 error_report("Failed to create a window, ret = %d (%m)", ret); 306 return -errno; 307 } 308 309 if (create.start_addr != section->offset_within_address_space) { 310 vfio_spapr_remove_window(container, create.start_addr); 311 312 error_report("Host doesn't support DMA window at %"HWADDR_PRIx", must be %"PRIx64, 313 section->offset_within_address_space, 314 (uint64_t)create.start_addr); 315 return -EINVAL; 316 } 317 trace_vfio_spapr_create_window(create.page_shift, 318 create.levels, 319 create.window_size, 320 create.start_addr); 321 *pgsize = pagesize; 322 323 return 0; 324 } 325 326 static int 327 vfio_spapr_container_add_section_window(VFIOContainerBase *bcontainer, 328 MemoryRegionSection *section, 329 Error **errp) 330 { 331 VFIOContainer *container = container_of(bcontainer, VFIOContainer, 332 bcontainer); 333 VFIOSpaprContainer *scontainer = container_of(container, VFIOSpaprContainer, 334 container); 335 VFIOHostDMAWindow *hostwin; 336 hwaddr pgsize = 0; 337 int ret; 338 339 /* 340 * VFIO_SPAPR_TCE_IOMMU supports a single host window between 341 * [dma32_window_start, dma32_window_size), we need to ensure 342 * the section fall in this range. 343 */ 344 if (container->iommu_type == VFIO_SPAPR_TCE_IOMMU) { 345 hwaddr iova, end; 346 347 iova = section->offset_within_address_space; 348 end = iova + int128_get64(section->size) - 1; 349 350 if (!vfio_find_hostwin(scontainer, iova, end)) { 351 error_setg(errp, "Container %p can't map guest IOVA region" 352 " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx, container, 353 iova, end); 354 return -EINVAL; 355 } 356 return 0; 357 } 358 359 if (container->iommu_type != VFIO_SPAPR_TCE_v2_IOMMU) { 360 return 0; 361 } 362 363 /* For now intersections are not allowed, we may relax this later */ 364 QLIST_FOREACH(hostwin, &scontainer->hostwin_list, hostwin_next) { 365 if (ranges_overlap(hostwin->min_iova, 366 hostwin->max_iova - hostwin->min_iova + 1, 367 section->offset_within_address_space, 368 int128_get64(section->size))) { 369 error_setg(errp, 370 "region [0x%"PRIx64",0x%"PRIx64"] overlaps with existing" 371 "host DMA window [0x%"PRIx64",0x%"PRIx64"]", 372 section->offset_within_address_space, 373 section->offset_within_address_space + 374 int128_get64(section->size) - 1, 375 hostwin->min_iova, hostwin->max_iova); 376 return -EINVAL; 377 } 378 } 379 380 ret = vfio_spapr_create_window(container, section, &pgsize); 381 if (ret) { 382 error_setg_errno(errp, -ret, "Failed to create SPAPR window"); 383 return ret; 384 } 385 386 vfio_host_win_add(scontainer, section->offset_within_address_space, 387 section->offset_within_address_space + 388 int128_get64(section->size) - 1, pgsize); 389 #ifdef CONFIG_KVM 390 if (kvm_enabled()) { 391 VFIOGroup *group; 392 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 393 struct kvm_vfio_spapr_tce param; 394 struct kvm_device_attr attr = { 395 .group = KVM_DEV_VFIO_GROUP, 396 .attr = KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE, 397 .addr = (uint64_t)(unsigned long)¶m, 398 }; 399 400 if (!memory_region_iommu_get_attr(iommu_mr, IOMMU_ATTR_SPAPR_TCE_FD, 401 ¶m.tablefd)) { 402 QLIST_FOREACH(group, &container->group_list, container_next) { 403 param.groupfd = group->fd; 404 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 405 error_setg_errno(errp, errno, 406 "vfio: failed GROUP_SET_SPAPR_TCE for " 407 "KVM VFIO device %d and group fd %d", 408 param.tablefd, param.groupfd); 409 return -errno; 410 } 411 trace_vfio_spapr_group_attach(param.groupfd, param.tablefd); 412 } 413 } 414 } 415 #endif 416 return 0; 417 } 418 419 static void 420 vfio_spapr_container_del_section_window(VFIOContainerBase *bcontainer, 421 MemoryRegionSection *section) 422 { 423 VFIOContainer *container = container_of(bcontainer, VFIOContainer, 424 bcontainer); 425 VFIOSpaprContainer *scontainer = container_of(container, VFIOSpaprContainer, 426 container); 427 428 if (container->iommu_type != VFIO_SPAPR_TCE_v2_IOMMU) { 429 return; 430 } 431 432 vfio_spapr_remove_window(container, 433 section->offset_within_address_space); 434 if (vfio_host_win_del(scontainer, 435 section->offset_within_address_space, 436 section->offset_within_address_space + 437 int128_get64(section->size) - 1) < 0) { 438 hw_error("%s: Cannot delete missing window at %"HWADDR_PRIx, 439 __func__, section->offset_within_address_space); 440 } 441 } 442 443 static VFIOIOMMUOps vfio_iommu_spapr_ops; 444 445 static void setup_spapr_ops(VFIOContainerBase *bcontainer) 446 { 447 vfio_iommu_spapr_ops = *bcontainer->ops; 448 vfio_iommu_spapr_ops.add_window = vfio_spapr_container_add_section_window; 449 vfio_iommu_spapr_ops.del_window = vfio_spapr_container_del_section_window; 450 bcontainer->ops = &vfio_iommu_spapr_ops; 451 } 452 453 int vfio_spapr_container_init(VFIOContainer *container, Error **errp) 454 { 455 VFIOContainerBase *bcontainer = &container->bcontainer; 456 VFIOSpaprContainer *scontainer = container_of(container, VFIOSpaprContainer, 457 container); 458 struct vfio_iommu_spapr_tce_info info; 459 bool v2 = container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU; 460 int ret, fd = container->fd; 461 462 QLIST_INIT(&scontainer->hostwin_list); 463 464 /* 465 * The host kernel code implementing VFIO_IOMMU_DISABLE is called 466 * when container fd is closed so we do not call it explicitly 467 * in this file. 468 */ 469 if (!v2) { 470 ret = ioctl(fd, VFIO_IOMMU_ENABLE); 471 if (ret) { 472 error_setg_errno(errp, errno, "failed to enable container"); 473 return -errno; 474 } 475 } else { 476 scontainer->prereg_listener = vfio_prereg_listener; 477 478 memory_listener_register(&scontainer->prereg_listener, 479 &address_space_memory); 480 if (bcontainer->error) { 481 ret = -1; 482 error_propagate_prepend(errp, bcontainer->error, 483 "RAM memory listener initialization failed: "); 484 goto listener_unregister_exit; 485 } 486 } 487 488 info.argsz = sizeof(info); 489 ret = ioctl(fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info); 490 if (ret) { 491 error_setg_errno(errp, errno, 492 "VFIO_IOMMU_SPAPR_TCE_GET_INFO failed"); 493 ret = -errno; 494 goto listener_unregister_exit; 495 } 496 497 if (v2) { 498 bcontainer->pgsizes = info.ddw.pgsizes; 499 /* 500 * There is a default window in just created container. 501 * To make region_add/del simpler, we better remove this 502 * window now and let those iommu_listener callbacks 503 * create/remove them when needed. 504 */ 505 ret = vfio_spapr_remove_window(container, info.dma32_window_start); 506 if (ret) { 507 error_setg_errno(errp, -ret, 508 "failed to remove existing window"); 509 goto listener_unregister_exit; 510 } 511 } else { 512 /* The default table uses 4K pages */ 513 bcontainer->pgsizes = 0x1000; 514 vfio_host_win_add(scontainer, info.dma32_window_start, 515 info.dma32_window_start + 516 info.dma32_window_size - 1, 517 0x1000); 518 } 519 520 setup_spapr_ops(bcontainer); 521 522 return 0; 523 524 listener_unregister_exit: 525 if (v2) { 526 memory_listener_unregister(&scontainer->prereg_listener); 527 } 528 return ret; 529 } 530 531 void vfio_spapr_container_deinit(VFIOContainer *container) 532 { 533 VFIOSpaprContainer *scontainer = container_of(container, VFIOSpaprContainer, 534 container); 535 VFIOHostDMAWindow *hostwin, *next; 536 537 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { 538 memory_listener_unregister(&scontainer->prereg_listener); 539 } 540 QLIST_FOREACH_SAFE(hostwin, &scontainer->hostwin_list, hostwin_next, 541 next) { 542 QLIST_REMOVE(hostwin, hostwin_next); 543 g_free(hostwin); 544 } 545 } 546