1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VFIO core 4 * 5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 * 8 * Derived from original vfio: 9 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 10 * Author: Tom Lyon, pugs@cisco.com 11 */ 12 13 #include <linux/cdev.h> 14 #include <linux/compat.h> 15 #include <linux/device.h> 16 #include <linux/fs.h> 17 #include <linux/idr.h> 18 #include <linux/iommu.h> 19 #ifdef CONFIG_HAVE_KVM 20 #include <linux/kvm_host.h> 21 #endif 22 #include <linux/list.h> 23 #include <linux/miscdevice.h> 24 #include <linux/module.h> 25 #include <linux/mutex.h> 26 #include <linux/pci.h> 27 #include <linux/rwsem.h> 28 #include <linux/sched.h> 29 #include <linux/slab.h> 30 #include <linux/stat.h> 31 #include <linux/string.h> 32 #include <linux/uaccess.h> 33 #include <linux/vfio.h> 34 #include <linux/wait.h> 35 #include <linux/sched/signal.h> 36 #include <linux/pm_runtime.h> 37 #include <linux/interval_tree.h> 38 #include <linux/iova_bitmap.h> 39 #include <linux/iommufd.h> 40 #include "vfio.h" 41 42 #define DRIVER_VERSION "0.3" 43 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 44 #define DRIVER_DESC "VFIO - User Level meta-driver" 45 46 static struct vfio { 47 struct class *device_class; 48 struct ida device_ida; 49 } vfio; 50 51 #ifdef CONFIG_VFIO_NOIOMMU 52 bool vfio_noiommu __read_mostly; 53 module_param_named(enable_unsafe_noiommu_mode, 54 vfio_noiommu, bool, S_IRUGO | S_IWUSR); 55 MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)"); 56 #endif 57 58 static DEFINE_XARRAY(vfio_device_set_xa); 59 60 int vfio_assign_device_set(struct vfio_device *device, void *set_id) 61 { 62 unsigned long idx = (unsigned long)set_id; 63 struct vfio_device_set *new_dev_set; 64 struct vfio_device_set *dev_set; 65 66 if (WARN_ON(!set_id)) 67 return -EINVAL; 68 69 /* 70 * Atomically acquire a singleton object in the xarray for this set_id 71 */ 72 xa_lock(&vfio_device_set_xa); 73 dev_set = xa_load(&vfio_device_set_xa, idx); 74 if (dev_set) 75 goto found_get_ref; 76 xa_unlock(&vfio_device_set_xa); 77 78 new_dev_set = kzalloc(sizeof(*new_dev_set), GFP_KERNEL); 79 if (!new_dev_set) 80 return -ENOMEM; 81 mutex_init(&new_dev_set->lock); 82 INIT_LIST_HEAD(&new_dev_set->device_list); 83 new_dev_set->set_id = set_id; 84 85 xa_lock(&vfio_device_set_xa); 86 dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set, 87 GFP_KERNEL); 88 if (!dev_set) { 89 dev_set = new_dev_set; 90 goto found_get_ref; 91 } 92 93 kfree(new_dev_set); 94 if (xa_is_err(dev_set)) { 95 xa_unlock(&vfio_device_set_xa); 96 return xa_err(dev_set); 97 } 98 99 found_get_ref: 100 dev_set->device_count++; 101 xa_unlock(&vfio_device_set_xa); 102 mutex_lock(&dev_set->lock); 103 device->dev_set = dev_set; 104 list_add_tail(&device->dev_set_list, &dev_set->device_list); 105 mutex_unlock(&dev_set->lock); 106 return 0; 107 } 108 EXPORT_SYMBOL_GPL(vfio_assign_device_set); 109 110 static void vfio_release_device_set(struct vfio_device *device) 111 { 112 struct vfio_device_set *dev_set = device->dev_set; 113 114 if (!dev_set) 115 return; 116 117 mutex_lock(&dev_set->lock); 118 list_del(&device->dev_set_list); 119 mutex_unlock(&dev_set->lock); 120 121 xa_lock(&vfio_device_set_xa); 122 if (!--dev_set->device_count) { 123 __xa_erase(&vfio_device_set_xa, 124 (unsigned long)dev_set->set_id); 125 mutex_destroy(&dev_set->lock); 126 kfree(dev_set); 127 } 128 xa_unlock(&vfio_device_set_xa); 129 } 130 131 unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set) 132 { 133 struct vfio_device *cur; 134 unsigned int open_count = 0; 135 136 lockdep_assert_held(&dev_set->lock); 137 138 list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 139 open_count += cur->open_count; 140 return open_count; 141 } 142 EXPORT_SYMBOL_GPL(vfio_device_set_open_count); 143 144 struct vfio_device * 145 vfio_find_device_in_devset(struct vfio_device_set *dev_set, 146 struct device *dev) 147 { 148 struct vfio_device *cur; 149 150 lockdep_assert_held(&dev_set->lock); 151 152 list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 153 if (cur->dev == dev) 154 return cur; 155 return NULL; 156 } 157 EXPORT_SYMBOL_GPL(vfio_find_device_in_devset); 158 159 /* 160 * Device objects - create, release, get, put, search 161 */ 162 /* Device reference always implies a group reference */ 163 void vfio_device_put_registration(struct vfio_device *device) 164 { 165 if (refcount_dec_and_test(&device->refcount)) 166 complete(&device->comp); 167 } 168 169 bool vfio_device_try_get_registration(struct vfio_device *device) 170 { 171 return refcount_inc_not_zero(&device->refcount); 172 } 173 174 /* 175 * VFIO driver API 176 */ 177 /* Release helper called by vfio_put_device() */ 178 static void vfio_device_release(struct device *dev) 179 { 180 struct vfio_device *device = 181 container_of(dev, struct vfio_device, device); 182 183 vfio_release_device_set(device); 184 ida_free(&vfio.device_ida, device->index); 185 186 if (device->ops->release) 187 device->ops->release(device); 188 189 kvfree(device); 190 } 191 192 static int vfio_init_device(struct vfio_device *device, struct device *dev, 193 const struct vfio_device_ops *ops); 194 195 /* 196 * Allocate and initialize vfio_device so it can be registered to vfio 197 * core. 198 * 199 * Drivers should use the wrapper vfio_alloc_device() for allocation. 200 * @size is the size of the structure to be allocated, including any 201 * private data used by the driver. 202 * 203 * Driver may provide an @init callback to cover device private data. 204 * 205 * Use vfio_put_device() to release the structure after success return. 206 */ 207 struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, 208 const struct vfio_device_ops *ops) 209 { 210 struct vfio_device *device; 211 int ret; 212 213 if (WARN_ON(size < sizeof(struct vfio_device))) 214 return ERR_PTR(-EINVAL); 215 216 device = kvzalloc(size, GFP_KERNEL); 217 if (!device) 218 return ERR_PTR(-ENOMEM); 219 220 ret = vfio_init_device(device, dev, ops); 221 if (ret) 222 goto out_free; 223 return device; 224 225 out_free: 226 kvfree(device); 227 return ERR_PTR(ret); 228 } 229 EXPORT_SYMBOL_GPL(_vfio_alloc_device); 230 231 /* 232 * Initialize a vfio_device so it can be registered to vfio core. 233 */ 234 static int vfio_init_device(struct vfio_device *device, struct device *dev, 235 const struct vfio_device_ops *ops) 236 { 237 int ret; 238 239 ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL); 240 if (ret < 0) { 241 dev_dbg(dev, "Error to alloc index\n"); 242 return ret; 243 } 244 245 device->index = ret; 246 init_completion(&device->comp); 247 device->dev = dev; 248 device->ops = ops; 249 250 if (ops->init) { 251 ret = ops->init(device); 252 if (ret) 253 goto out_uninit; 254 } 255 256 device_initialize(&device->device); 257 device->device.release = vfio_device_release; 258 device->device.class = vfio.device_class; 259 device->device.parent = device->dev; 260 return 0; 261 262 out_uninit: 263 vfio_release_device_set(device); 264 ida_free(&vfio.device_ida, device->index); 265 return ret; 266 } 267 268 static int __vfio_register_dev(struct vfio_device *device, 269 enum vfio_group_type type) 270 { 271 int ret; 272 273 if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) && 274 (!device->ops->bind_iommufd || 275 !device->ops->unbind_iommufd || 276 !device->ops->attach_ioas || 277 !device->ops->detach_ioas))) 278 return -EINVAL; 279 280 /* 281 * If the driver doesn't specify a set then the device is added to a 282 * singleton set just for itself. 283 */ 284 if (!device->dev_set) 285 vfio_assign_device_set(device, device); 286 287 ret = dev_set_name(&device->device, "vfio%d", device->index); 288 if (ret) 289 return ret; 290 291 ret = vfio_device_set_group(device, type); 292 if (ret) 293 return ret; 294 295 /* 296 * VFIO always sets IOMMU_CACHE because we offer no way for userspace to 297 * restore cache coherency. It has to be checked here because it is only 298 * valid for cases where we are using iommu groups. 299 */ 300 if (type == VFIO_IOMMU && !vfio_device_is_noiommu(device) && 301 !device_iommu_capable(device->dev, IOMMU_CAP_CACHE_COHERENCY)) { 302 ret = -EINVAL; 303 goto err_out; 304 } 305 306 ret = vfio_device_add(device); 307 if (ret) 308 goto err_out; 309 310 /* Refcounting can't start until the driver calls register */ 311 refcount_set(&device->refcount, 1); 312 313 vfio_device_group_register(device); 314 315 return 0; 316 err_out: 317 vfio_device_remove_group(device); 318 return ret; 319 } 320 321 int vfio_register_group_dev(struct vfio_device *device) 322 { 323 return __vfio_register_dev(device, VFIO_IOMMU); 324 } 325 EXPORT_SYMBOL_GPL(vfio_register_group_dev); 326 327 /* 328 * Register a virtual device without IOMMU backing. The user of this 329 * device must not be able to directly trigger unmediated DMA. 330 */ 331 int vfio_register_emulated_iommu_dev(struct vfio_device *device) 332 { 333 return __vfio_register_dev(device, VFIO_EMULATED_IOMMU); 334 } 335 EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev); 336 337 /* 338 * Decrement the device reference count and wait for the device to be 339 * removed. Open file descriptors for the device... */ 340 void vfio_unregister_group_dev(struct vfio_device *device) 341 { 342 unsigned int i = 0; 343 bool interrupted = false; 344 long rc; 345 346 /* 347 * Prevent new device opened by userspace via the 348 * VFIO_GROUP_GET_DEVICE_FD in the group path. 349 */ 350 vfio_device_group_unregister(device); 351 352 /* 353 * Balances vfio_device_add() in register path, also prevents 354 * new device opened by userspace in the cdev path. 355 */ 356 vfio_device_del(device); 357 358 vfio_device_put_registration(device); 359 rc = try_wait_for_completion(&device->comp); 360 while (rc <= 0) { 361 if (device->ops->request) 362 device->ops->request(device, i++); 363 364 if (interrupted) { 365 rc = wait_for_completion_timeout(&device->comp, 366 HZ * 10); 367 } else { 368 rc = wait_for_completion_interruptible_timeout( 369 &device->comp, HZ * 10); 370 if (rc < 0) { 371 interrupted = true; 372 dev_warn(device->dev, 373 "Device is currently in use, task" 374 " \"%s\" (%d) " 375 "blocked until device is released", 376 current->comm, task_pid_nr(current)); 377 } 378 } 379 } 380 381 /* Balances vfio_device_set_group in register path */ 382 vfio_device_remove_group(device); 383 } 384 EXPORT_SYMBOL_GPL(vfio_unregister_group_dev); 385 386 #ifdef CONFIG_HAVE_KVM 387 void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm) 388 { 389 void (*pfn)(struct kvm *kvm); 390 bool (*fn)(struct kvm *kvm); 391 bool ret; 392 393 lockdep_assert_held(&device->dev_set->lock); 394 395 if (!kvm) 396 return; 397 398 pfn = symbol_get(kvm_put_kvm); 399 if (WARN_ON(!pfn)) 400 return; 401 402 fn = symbol_get(kvm_get_kvm_safe); 403 if (WARN_ON(!fn)) { 404 symbol_put(kvm_put_kvm); 405 return; 406 } 407 408 ret = fn(kvm); 409 symbol_put(kvm_get_kvm_safe); 410 if (!ret) { 411 symbol_put(kvm_put_kvm); 412 return; 413 } 414 415 device->put_kvm = pfn; 416 device->kvm = kvm; 417 } 418 419 void vfio_device_put_kvm(struct vfio_device *device) 420 { 421 lockdep_assert_held(&device->dev_set->lock); 422 423 if (!device->kvm) 424 return; 425 426 if (WARN_ON(!device->put_kvm)) 427 goto clear; 428 429 device->put_kvm(device->kvm); 430 device->put_kvm = NULL; 431 symbol_put(kvm_put_kvm); 432 433 clear: 434 device->kvm = NULL; 435 } 436 #endif 437 438 /* true if the vfio_device has open_device() called but not close_device() */ 439 static bool vfio_assert_device_open(struct vfio_device *device) 440 { 441 return !WARN_ON_ONCE(!READ_ONCE(device->open_count)); 442 } 443 444 struct vfio_device_file * 445 vfio_allocate_device_file(struct vfio_device *device) 446 { 447 struct vfio_device_file *df; 448 449 df = kzalloc(sizeof(*df), GFP_KERNEL_ACCOUNT); 450 if (!df) 451 return ERR_PTR(-ENOMEM); 452 453 df->device = device; 454 spin_lock_init(&df->kvm_ref_lock); 455 456 return df; 457 } 458 459 static int vfio_df_device_first_open(struct vfio_device_file *df) 460 { 461 struct vfio_device *device = df->device; 462 struct iommufd_ctx *iommufd = df->iommufd; 463 int ret; 464 465 lockdep_assert_held(&device->dev_set->lock); 466 467 if (!try_module_get(device->dev->driver->owner)) 468 return -ENODEV; 469 470 if (iommufd) 471 ret = vfio_df_iommufd_bind(df); 472 else 473 ret = vfio_device_group_use_iommu(device); 474 if (ret) 475 goto err_module_put; 476 477 if (device->ops->open_device) { 478 ret = device->ops->open_device(device); 479 if (ret) 480 goto err_unuse_iommu; 481 } 482 return 0; 483 484 err_unuse_iommu: 485 if (iommufd) 486 vfio_df_iommufd_unbind(df); 487 else 488 vfio_device_group_unuse_iommu(device); 489 err_module_put: 490 module_put(device->dev->driver->owner); 491 return ret; 492 } 493 494 static void vfio_df_device_last_close(struct vfio_device_file *df) 495 { 496 struct vfio_device *device = df->device; 497 struct iommufd_ctx *iommufd = df->iommufd; 498 499 lockdep_assert_held(&device->dev_set->lock); 500 501 if (device->ops->close_device) 502 device->ops->close_device(device); 503 if (iommufd) 504 vfio_df_iommufd_unbind(df); 505 else 506 vfio_device_group_unuse_iommu(device); 507 module_put(device->dev->driver->owner); 508 } 509 510 int vfio_df_open(struct vfio_device_file *df) 511 { 512 struct vfio_device *device = df->device; 513 int ret = 0; 514 515 lockdep_assert_held(&device->dev_set->lock); 516 517 /* 518 * Only the group path allows the device to be opened multiple 519 * times. The device cdev path doesn't have a secure way for it. 520 */ 521 if (device->open_count != 0 && !df->group) 522 return -EINVAL; 523 524 device->open_count++; 525 if (device->open_count == 1) { 526 ret = vfio_df_device_first_open(df); 527 if (ret) 528 device->open_count--; 529 } 530 531 return ret; 532 } 533 534 void vfio_df_close(struct vfio_device_file *df) 535 { 536 struct vfio_device *device = df->device; 537 538 lockdep_assert_held(&device->dev_set->lock); 539 540 vfio_assert_device_open(device); 541 if (device->open_count == 1) 542 vfio_df_device_last_close(df); 543 device->open_count--; 544 } 545 546 /* 547 * Wrapper around pm_runtime_resume_and_get(). 548 * Return error code on failure or 0 on success. 549 */ 550 static inline int vfio_device_pm_runtime_get(struct vfio_device *device) 551 { 552 struct device *dev = device->dev; 553 554 if (dev->driver && dev->driver->pm) { 555 int ret; 556 557 ret = pm_runtime_resume_and_get(dev); 558 if (ret) { 559 dev_info_ratelimited(dev, 560 "vfio: runtime resume failed %d\n", ret); 561 return -EIO; 562 } 563 } 564 565 return 0; 566 } 567 568 /* 569 * Wrapper around pm_runtime_put(). 570 */ 571 static inline void vfio_device_pm_runtime_put(struct vfio_device *device) 572 { 573 struct device *dev = device->dev; 574 575 if (dev->driver && dev->driver->pm) 576 pm_runtime_put(dev); 577 } 578 579 /* 580 * VFIO Device fd 581 */ 582 static int vfio_device_fops_release(struct inode *inode, struct file *filep) 583 { 584 struct vfio_device_file *df = filep->private_data; 585 struct vfio_device *device = df->device; 586 587 if (df->group) 588 vfio_df_group_close(df); 589 else 590 vfio_df_unbind_iommufd(df); 591 592 vfio_device_put_registration(device); 593 594 kfree(df); 595 596 return 0; 597 } 598 599 /* 600 * vfio_mig_get_next_state - Compute the next step in the FSM 601 * @cur_fsm - The current state the device is in 602 * @new_fsm - The target state to reach 603 * @next_fsm - Pointer to the next step to get to new_fsm 604 * 605 * Return 0 upon success, otherwise -errno 606 * Upon success the next step in the state progression between cur_fsm and 607 * new_fsm will be set in next_fsm. 608 * 609 * This breaks down requests for combination transitions into smaller steps and 610 * returns the next step to get to new_fsm. The function may need to be called 611 * multiple times before reaching new_fsm. 612 * 613 */ 614 int vfio_mig_get_next_state(struct vfio_device *device, 615 enum vfio_device_mig_state cur_fsm, 616 enum vfio_device_mig_state new_fsm, 617 enum vfio_device_mig_state *next_fsm) 618 { 619 enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_PRE_COPY_P2P + 1 }; 620 /* 621 * The coding in this table requires the driver to implement the 622 * following FSM arcs: 623 * RESUMING -> STOP 624 * STOP -> RESUMING 625 * STOP -> STOP_COPY 626 * STOP_COPY -> STOP 627 * 628 * If P2P is supported then the driver must also implement these FSM 629 * arcs: 630 * RUNNING -> RUNNING_P2P 631 * RUNNING_P2P -> RUNNING 632 * RUNNING_P2P -> STOP 633 * STOP -> RUNNING_P2P 634 * 635 * If precopy is supported then the driver must support these additional 636 * FSM arcs: 637 * RUNNING -> PRE_COPY 638 * PRE_COPY -> RUNNING 639 * PRE_COPY -> STOP_COPY 640 * However, if precopy and P2P are supported together then the driver 641 * must support these additional arcs beyond the P2P arcs above: 642 * PRE_COPY -> RUNNING 643 * PRE_COPY -> PRE_COPY_P2P 644 * PRE_COPY_P2P -> PRE_COPY 645 * PRE_COPY_P2P -> RUNNING_P2P 646 * PRE_COPY_P2P -> STOP_COPY 647 * RUNNING -> PRE_COPY 648 * RUNNING_P2P -> PRE_COPY_P2P 649 * 650 * Without P2P and precopy the driver must implement: 651 * RUNNING -> STOP 652 * STOP -> RUNNING 653 * 654 * The coding will step through multiple states for some combination 655 * transitions; if all optional features are supported, this means the 656 * following ones: 657 * PRE_COPY -> PRE_COPY_P2P -> STOP_COPY 658 * PRE_COPY -> RUNNING -> RUNNING_P2P 659 * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP 660 * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP -> RESUMING 661 * PRE_COPY_P2P -> RUNNING_P2P -> RUNNING 662 * PRE_COPY_P2P -> RUNNING_P2P -> STOP 663 * PRE_COPY_P2P -> RUNNING_P2P -> STOP -> RESUMING 664 * RESUMING -> STOP -> RUNNING_P2P 665 * RESUMING -> STOP -> RUNNING_P2P -> PRE_COPY_P2P 666 * RESUMING -> STOP -> RUNNING_P2P -> RUNNING 667 * RESUMING -> STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 668 * RESUMING -> STOP -> STOP_COPY 669 * RUNNING -> RUNNING_P2P -> PRE_COPY_P2P 670 * RUNNING -> RUNNING_P2P -> STOP 671 * RUNNING -> RUNNING_P2P -> STOP -> RESUMING 672 * RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY 673 * RUNNING_P2P -> RUNNING -> PRE_COPY 674 * RUNNING_P2P -> STOP -> RESUMING 675 * RUNNING_P2P -> STOP -> STOP_COPY 676 * STOP -> RUNNING_P2P -> PRE_COPY_P2P 677 * STOP -> RUNNING_P2P -> RUNNING 678 * STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 679 * STOP_COPY -> STOP -> RESUMING 680 * STOP_COPY -> STOP -> RUNNING_P2P 681 * STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING 682 * 683 * The following transitions are blocked: 684 * STOP_COPY -> PRE_COPY 685 * STOP_COPY -> PRE_COPY_P2P 686 */ 687 static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = { 688 [VFIO_DEVICE_STATE_STOP] = { 689 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 690 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 691 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 692 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 693 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 694 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 695 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 696 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 697 }, 698 [VFIO_DEVICE_STATE_RUNNING] = { 699 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 700 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 701 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 702 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 703 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 704 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 705 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 706 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 707 }, 708 [VFIO_DEVICE_STATE_PRE_COPY] = { 709 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING, 710 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 711 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 712 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 713 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 714 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING, 715 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING, 716 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 717 }, 718 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = { 719 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 720 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 721 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 722 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 723 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 724 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 725 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 726 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 727 }, 728 [VFIO_DEVICE_STATE_STOP_COPY] = { 729 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 730 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 731 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 732 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 733 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 734 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 735 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 736 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 737 }, 738 [VFIO_DEVICE_STATE_RESUMING] = { 739 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 740 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 741 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_STOP, 742 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_STOP, 743 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 744 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 745 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 746 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 747 }, 748 [VFIO_DEVICE_STATE_RUNNING_P2P] = { 749 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 750 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 751 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING, 752 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 753 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 754 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 755 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 756 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 757 }, 758 [VFIO_DEVICE_STATE_ERROR] = { 759 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR, 760 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR, 761 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 762 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 763 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR, 764 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR, 765 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR, 766 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 767 }, 768 }; 769 770 static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = { 771 [VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY, 772 [VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY, 773 [VFIO_DEVICE_STATE_PRE_COPY] = 774 VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY, 775 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_MIGRATION_STOP_COPY | 776 VFIO_MIGRATION_P2P | 777 VFIO_MIGRATION_PRE_COPY, 778 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY, 779 [VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY, 780 [VFIO_DEVICE_STATE_RUNNING_P2P] = 781 VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P, 782 [VFIO_DEVICE_STATE_ERROR] = ~0U, 783 }; 784 785 if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 786 (state_flags_table[cur_fsm] & device->migration_flags) != 787 state_flags_table[cur_fsm])) 788 return -EINVAL; 789 790 if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 791 (state_flags_table[new_fsm] & device->migration_flags) != 792 state_flags_table[new_fsm]) 793 return -EINVAL; 794 795 /* 796 * Arcs touching optional and unsupported states are skipped over. The 797 * driver will instead see an arc from the original state to the next 798 * logical state, as per the above comment. 799 */ 800 *next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm]; 801 while ((state_flags_table[*next_fsm] & device->migration_flags) != 802 state_flags_table[*next_fsm]) 803 *next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm]; 804 805 return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL; 806 } 807 EXPORT_SYMBOL_GPL(vfio_mig_get_next_state); 808 809 /* 810 * Convert the drivers's struct file into a FD number and return it to userspace 811 */ 812 static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg, 813 struct vfio_device_feature_mig_state *mig) 814 { 815 int ret; 816 int fd; 817 818 fd = get_unused_fd_flags(O_CLOEXEC); 819 if (fd < 0) { 820 ret = fd; 821 goto out_fput; 822 } 823 824 mig->data_fd = fd; 825 if (copy_to_user(arg, mig, sizeof(*mig))) { 826 ret = -EFAULT; 827 goto out_put_unused; 828 } 829 fd_install(fd, filp); 830 return 0; 831 832 out_put_unused: 833 put_unused_fd(fd); 834 out_fput: 835 fput(filp); 836 return ret; 837 } 838 839 static int 840 vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device, 841 u32 flags, void __user *arg, 842 size_t argsz) 843 { 844 size_t minsz = 845 offsetofend(struct vfio_device_feature_mig_state, data_fd); 846 struct vfio_device_feature_mig_state mig; 847 struct file *filp = NULL; 848 int ret; 849 850 if (!device->mig_ops) 851 return -ENOTTY; 852 853 ret = vfio_check_feature(flags, argsz, 854 VFIO_DEVICE_FEATURE_SET | 855 VFIO_DEVICE_FEATURE_GET, 856 sizeof(mig)); 857 if (ret != 1) 858 return ret; 859 860 if (copy_from_user(&mig, arg, minsz)) 861 return -EFAULT; 862 863 if (flags & VFIO_DEVICE_FEATURE_GET) { 864 enum vfio_device_mig_state curr_state; 865 866 ret = device->mig_ops->migration_get_state(device, 867 &curr_state); 868 if (ret) 869 return ret; 870 mig.device_state = curr_state; 871 goto out_copy; 872 } 873 874 /* Handle the VFIO_DEVICE_FEATURE_SET */ 875 filp = device->mig_ops->migration_set_state(device, mig.device_state); 876 if (IS_ERR(filp) || !filp) 877 goto out_copy; 878 879 return vfio_ioct_mig_return_fd(filp, arg, &mig); 880 out_copy: 881 mig.data_fd = -1; 882 if (copy_to_user(arg, &mig, sizeof(mig))) 883 return -EFAULT; 884 if (IS_ERR(filp)) 885 return PTR_ERR(filp); 886 return 0; 887 } 888 889 static int 890 vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device, 891 u32 flags, void __user *arg, 892 size_t argsz) 893 { 894 struct vfio_device_feature_mig_data_size data_size = {}; 895 unsigned long stop_copy_length; 896 int ret; 897 898 if (!device->mig_ops) 899 return -ENOTTY; 900 901 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 902 sizeof(data_size)); 903 if (ret != 1) 904 return ret; 905 906 ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length); 907 if (ret) 908 return ret; 909 910 data_size.stop_copy_length = stop_copy_length; 911 if (copy_to_user(arg, &data_size, sizeof(data_size))) 912 return -EFAULT; 913 914 return 0; 915 } 916 917 static int vfio_ioctl_device_feature_migration(struct vfio_device *device, 918 u32 flags, void __user *arg, 919 size_t argsz) 920 { 921 struct vfio_device_feature_migration mig = { 922 .flags = device->migration_flags, 923 }; 924 int ret; 925 926 if (!device->mig_ops) 927 return -ENOTTY; 928 929 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 930 sizeof(mig)); 931 if (ret != 1) 932 return ret; 933 if (copy_to_user(arg, &mig, sizeof(mig))) 934 return -EFAULT; 935 return 0; 936 } 937 938 void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes, 939 u32 req_nodes) 940 { 941 struct interval_tree_node *prev, *curr, *comb_start, *comb_end; 942 unsigned long min_gap, curr_gap; 943 944 /* Special shortcut when a single range is required */ 945 if (req_nodes == 1) { 946 unsigned long last; 947 948 comb_start = interval_tree_iter_first(root, 0, ULONG_MAX); 949 950 /* Empty list */ 951 if (WARN_ON_ONCE(!comb_start)) 952 return; 953 954 curr = comb_start; 955 while (curr) { 956 last = curr->last; 957 prev = curr; 958 curr = interval_tree_iter_next(curr, 0, ULONG_MAX); 959 if (prev != comb_start) 960 interval_tree_remove(prev, root); 961 } 962 comb_start->last = last; 963 return; 964 } 965 966 /* Combine ranges which have the smallest gap */ 967 while (cur_nodes > req_nodes) { 968 prev = NULL; 969 min_gap = ULONG_MAX; 970 curr = interval_tree_iter_first(root, 0, ULONG_MAX); 971 while (curr) { 972 if (prev) { 973 curr_gap = curr->start - prev->last; 974 if (curr_gap < min_gap) { 975 min_gap = curr_gap; 976 comb_start = prev; 977 comb_end = curr; 978 } 979 } 980 prev = curr; 981 curr = interval_tree_iter_next(curr, 0, ULONG_MAX); 982 } 983 984 /* Empty list or no nodes to combine */ 985 if (WARN_ON_ONCE(min_gap == ULONG_MAX)) 986 break; 987 988 comb_start->last = comb_end->last; 989 interval_tree_remove(comb_end, root); 990 cur_nodes--; 991 } 992 } 993 EXPORT_SYMBOL_GPL(vfio_combine_iova_ranges); 994 995 /* Ranges should fit into a single kernel page */ 996 #define LOG_MAX_RANGES \ 997 (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range)) 998 999 static int 1000 vfio_ioctl_device_feature_logging_start(struct vfio_device *device, 1001 u32 flags, void __user *arg, 1002 size_t argsz) 1003 { 1004 size_t minsz = 1005 offsetofend(struct vfio_device_feature_dma_logging_control, 1006 ranges); 1007 struct vfio_device_feature_dma_logging_range __user *ranges; 1008 struct vfio_device_feature_dma_logging_control control; 1009 struct vfio_device_feature_dma_logging_range range; 1010 struct rb_root_cached root = RB_ROOT_CACHED; 1011 struct interval_tree_node *nodes; 1012 u64 iova_end; 1013 u32 nnodes; 1014 int i, ret; 1015 1016 if (!device->log_ops) 1017 return -ENOTTY; 1018 1019 ret = vfio_check_feature(flags, argsz, 1020 VFIO_DEVICE_FEATURE_SET, 1021 sizeof(control)); 1022 if (ret != 1) 1023 return ret; 1024 1025 if (copy_from_user(&control, arg, minsz)) 1026 return -EFAULT; 1027 1028 nnodes = control.num_ranges; 1029 if (!nnodes) 1030 return -EINVAL; 1031 1032 if (nnodes > LOG_MAX_RANGES) 1033 return -E2BIG; 1034 1035 ranges = u64_to_user_ptr(control.ranges); 1036 nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node), 1037 GFP_KERNEL); 1038 if (!nodes) 1039 return -ENOMEM; 1040 1041 for (i = 0; i < nnodes; i++) { 1042 if (copy_from_user(&range, &ranges[i], sizeof(range))) { 1043 ret = -EFAULT; 1044 goto end; 1045 } 1046 if (!IS_ALIGNED(range.iova, control.page_size) || 1047 !IS_ALIGNED(range.length, control.page_size)) { 1048 ret = -EINVAL; 1049 goto end; 1050 } 1051 1052 if (check_add_overflow(range.iova, range.length, &iova_end) || 1053 iova_end > ULONG_MAX) { 1054 ret = -EOVERFLOW; 1055 goto end; 1056 } 1057 1058 nodes[i].start = range.iova; 1059 nodes[i].last = range.iova + range.length - 1; 1060 if (interval_tree_iter_first(&root, nodes[i].start, 1061 nodes[i].last)) { 1062 /* Range overlapping */ 1063 ret = -EINVAL; 1064 goto end; 1065 } 1066 interval_tree_insert(nodes + i, &root); 1067 } 1068 1069 ret = device->log_ops->log_start(device, &root, nnodes, 1070 &control.page_size); 1071 if (ret) 1072 goto end; 1073 1074 if (copy_to_user(arg, &control, sizeof(control))) { 1075 ret = -EFAULT; 1076 device->log_ops->log_stop(device); 1077 } 1078 1079 end: 1080 kfree(nodes); 1081 return ret; 1082 } 1083 1084 static int 1085 vfio_ioctl_device_feature_logging_stop(struct vfio_device *device, 1086 u32 flags, void __user *arg, 1087 size_t argsz) 1088 { 1089 int ret; 1090 1091 if (!device->log_ops) 1092 return -ENOTTY; 1093 1094 ret = vfio_check_feature(flags, argsz, 1095 VFIO_DEVICE_FEATURE_SET, 0); 1096 if (ret != 1) 1097 return ret; 1098 1099 return device->log_ops->log_stop(device); 1100 } 1101 1102 static int vfio_device_log_read_and_clear(struct iova_bitmap *iter, 1103 unsigned long iova, size_t length, 1104 void *opaque) 1105 { 1106 struct vfio_device *device = opaque; 1107 1108 return device->log_ops->log_read_and_clear(device, iova, length, iter); 1109 } 1110 1111 static int 1112 vfio_ioctl_device_feature_logging_report(struct vfio_device *device, 1113 u32 flags, void __user *arg, 1114 size_t argsz) 1115 { 1116 size_t minsz = 1117 offsetofend(struct vfio_device_feature_dma_logging_report, 1118 bitmap); 1119 struct vfio_device_feature_dma_logging_report report; 1120 struct iova_bitmap *iter; 1121 u64 iova_end; 1122 int ret; 1123 1124 if (!device->log_ops) 1125 return -ENOTTY; 1126 1127 ret = vfio_check_feature(flags, argsz, 1128 VFIO_DEVICE_FEATURE_GET, 1129 sizeof(report)); 1130 if (ret != 1) 1131 return ret; 1132 1133 if (copy_from_user(&report, arg, minsz)) 1134 return -EFAULT; 1135 1136 if (report.page_size < SZ_4K || !is_power_of_2(report.page_size)) 1137 return -EINVAL; 1138 1139 if (check_add_overflow(report.iova, report.length, &iova_end) || 1140 iova_end > ULONG_MAX) 1141 return -EOVERFLOW; 1142 1143 iter = iova_bitmap_alloc(report.iova, report.length, 1144 report.page_size, 1145 u64_to_user_ptr(report.bitmap)); 1146 if (IS_ERR(iter)) 1147 return PTR_ERR(iter); 1148 1149 ret = iova_bitmap_for_each(iter, device, 1150 vfio_device_log_read_and_clear); 1151 1152 iova_bitmap_free(iter); 1153 return ret; 1154 } 1155 1156 static int vfio_ioctl_device_feature(struct vfio_device *device, 1157 struct vfio_device_feature __user *arg) 1158 { 1159 size_t minsz = offsetofend(struct vfio_device_feature, flags); 1160 struct vfio_device_feature feature; 1161 1162 if (copy_from_user(&feature, arg, minsz)) 1163 return -EFAULT; 1164 1165 if (feature.argsz < minsz) 1166 return -EINVAL; 1167 1168 /* Check unknown flags */ 1169 if (feature.flags & 1170 ~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET | 1171 VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE)) 1172 return -EINVAL; 1173 1174 /* GET & SET are mutually exclusive except with PROBE */ 1175 if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) && 1176 (feature.flags & VFIO_DEVICE_FEATURE_SET) && 1177 (feature.flags & VFIO_DEVICE_FEATURE_GET)) 1178 return -EINVAL; 1179 1180 switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) { 1181 case VFIO_DEVICE_FEATURE_MIGRATION: 1182 return vfio_ioctl_device_feature_migration( 1183 device, feature.flags, arg->data, 1184 feature.argsz - minsz); 1185 case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE: 1186 return vfio_ioctl_device_feature_mig_device_state( 1187 device, feature.flags, arg->data, 1188 feature.argsz - minsz); 1189 case VFIO_DEVICE_FEATURE_DMA_LOGGING_START: 1190 return vfio_ioctl_device_feature_logging_start( 1191 device, feature.flags, arg->data, 1192 feature.argsz - minsz); 1193 case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP: 1194 return vfio_ioctl_device_feature_logging_stop( 1195 device, feature.flags, arg->data, 1196 feature.argsz - minsz); 1197 case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT: 1198 return vfio_ioctl_device_feature_logging_report( 1199 device, feature.flags, arg->data, 1200 feature.argsz - minsz); 1201 case VFIO_DEVICE_FEATURE_MIG_DATA_SIZE: 1202 return vfio_ioctl_device_feature_migration_data_size( 1203 device, feature.flags, arg->data, 1204 feature.argsz - minsz); 1205 default: 1206 if (unlikely(!device->ops->device_feature)) 1207 return -EINVAL; 1208 return device->ops->device_feature(device, feature.flags, 1209 arg->data, 1210 feature.argsz - minsz); 1211 } 1212 } 1213 1214 static long vfio_device_fops_unl_ioctl(struct file *filep, 1215 unsigned int cmd, unsigned long arg) 1216 { 1217 struct vfio_device_file *df = filep->private_data; 1218 struct vfio_device *device = df->device; 1219 void __user *uptr = (void __user *)arg; 1220 int ret; 1221 1222 if (cmd == VFIO_DEVICE_BIND_IOMMUFD) 1223 return vfio_df_ioctl_bind_iommufd(df, uptr); 1224 1225 /* Paired with smp_store_release() following vfio_df_open() */ 1226 if (!smp_load_acquire(&df->access_granted)) 1227 return -EINVAL; 1228 1229 ret = vfio_device_pm_runtime_get(device); 1230 if (ret) 1231 return ret; 1232 1233 /* cdev only ioctls */ 1234 if (IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV) && !df->group) { 1235 switch (cmd) { 1236 case VFIO_DEVICE_ATTACH_IOMMUFD_PT: 1237 ret = vfio_df_ioctl_attach_pt(df, uptr); 1238 goto out; 1239 1240 case VFIO_DEVICE_DETACH_IOMMUFD_PT: 1241 ret = vfio_df_ioctl_detach_pt(df, uptr); 1242 goto out; 1243 } 1244 } 1245 1246 switch (cmd) { 1247 case VFIO_DEVICE_FEATURE: 1248 ret = vfio_ioctl_device_feature(device, uptr); 1249 break; 1250 1251 default: 1252 if (unlikely(!device->ops->ioctl)) 1253 ret = -EINVAL; 1254 else 1255 ret = device->ops->ioctl(device, cmd, arg); 1256 break; 1257 } 1258 out: 1259 vfio_device_pm_runtime_put(device); 1260 return ret; 1261 } 1262 1263 static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf, 1264 size_t count, loff_t *ppos) 1265 { 1266 struct vfio_device_file *df = filep->private_data; 1267 struct vfio_device *device = df->device; 1268 1269 /* Paired with smp_store_release() following vfio_df_open() */ 1270 if (!smp_load_acquire(&df->access_granted)) 1271 return -EINVAL; 1272 1273 if (unlikely(!device->ops->read)) 1274 return -EINVAL; 1275 1276 return device->ops->read(device, buf, count, ppos); 1277 } 1278 1279 static ssize_t vfio_device_fops_write(struct file *filep, 1280 const char __user *buf, 1281 size_t count, loff_t *ppos) 1282 { 1283 struct vfio_device_file *df = filep->private_data; 1284 struct vfio_device *device = df->device; 1285 1286 /* Paired with smp_store_release() following vfio_df_open() */ 1287 if (!smp_load_acquire(&df->access_granted)) 1288 return -EINVAL; 1289 1290 if (unlikely(!device->ops->write)) 1291 return -EINVAL; 1292 1293 return device->ops->write(device, buf, count, ppos); 1294 } 1295 1296 static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma) 1297 { 1298 struct vfio_device_file *df = filep->private_data; 1299 struct vfio_device *device = df->device; 1300 1301 /* Paired with smp_store_release() following vfio_df_open() */ 1302 if (!smp_load_acquire(&df->access_granted)) 1303 return -EINVAL; 1304 1305 if (unlikely(!device->ops->mmap)) 1306 return -EINVAL; 1307 1308 return device->ops->mmap(device, vma); 1309 } 1310 1311 const struct file_operations vfio_device_fops = { 1312 .owner = THIS_MODULE, 1313 .open = vfio_device_fops_cdev_open, 1314 .release = vfio_device_fops_release, 1315 .read = vfio_device_fops_read, 1316 .write = vfio_device_fops_write, 1317 .unlocked_ioctl = vfio_device_fops_unl_ioctl, 1318 .compat_ioctl = compat_ptr_ioctl, 1319 .mmap = vfio_device_fops_mmap, 1320 }; 1321 1322 static struct vfio_device *vfio_device_from_file(struct file *file) 1323 { 1324 struct vfio_device_file *df = file->private_data; 1325 1326 if (file->f_op != &vfio_device_fops) 1327 return NULL; 1328 return df->device; 1329 } 1330 1331 /** 1332 * vfio_file_is_valid - True if the file is valid vfio file 1333 * @file: VFIO group file or VFIO device file 1334 */ 1335 bool vfio_file_is_valid(struct file *file) 1336 { 1337 return vfio_group_from_file(file) || 1338 vfio_device_from_file(file); 1339 } 1340 EXPORT_SYMBOL_GPL(vfio_file_is_valid); 1341 1342 /** 1343 * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file 1344 * is always CPU cache coherent 1345 * @file: VFIO group file or VFIO device file 1346 * 1347 * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop 1348 * bit in DMA transactions. A return of false indicates that the user has 1349 * rights to access additional instructions such as wbinvd on x86. 1350 */ 1351 bool vfio_file_enforced_coherent(struct file *file) 1352 { 1353 struct vfio_device *device; 1354 struct vfio_group *group; 1355 1356 group = vfio_group_from_file(file); 1357 if (group) 1358 return vfio_group_enforced_coherent(group); 1359 1360 device = vfio_device_from_file(file); 1361 if (device) 1362 return device_iommu_capable(device->dev, 1363 IOMMU_CAP_ENFORCE_CACHE_COHERENCY); 1364 1365 return true; 1366 } 1367 EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent); 1368 1369 static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm) 1370 { 1371 struct vfio_device_file *df = file->private_data; 1372 1373 /* 1374 * The kvm is first recorded in the vfio_device_file, and will 1375 * be propagated to vfio_device::kvm when the file is bound to 1376 * iommufd successfully in the vfio device cdev path. 1377 */ 1378 spin_lock(&df->kvm_ref_lock); 1379 df->kvm = kvm; 1380 spin_unlock(&df->kvm_ref_lock); 1381 } 1382 1383 /** 1384 * vfio_file_set_kvm - Link a kvm with VFIO drivers 1385 * @file: VFIO group file or VFIO device file 1386 * @kvm: KVM to link 1387 * 1388 * When a VFIO device is first opened the KVM will be available in 1389 * device->kvm if one was associated with the file. 1390 */ 1391 void vfio_file_set_kvm(struct file *file, struct kvm *kvm) 1392 { 1393 struct vfio_group *group; 1394 1395 group = vfio_group_from_file(file); 1396 if (group) 1397 vfio_group_set_kvm(group, kvm); 1398 1399 if (vfio_device_from_file(file)) 1400 vfio_device_file_set_kvm(file, kvm); 1401 } 1402 EXPORT_SYMBOL_GPL(vfio_file_set_kvm); 1403 1404 /* 1405 * Sub-module support 1406 */ 1407 /* 1408 * Helper for managing a buffer of info chain capabilities, allocate or 1409 * reallocate a buffer with additional @size, filling in @id and @version 1410 * of the capability. A pointer to the new capability is returned. 1411 * 1412 * NB. The chain is based at the head of the buffer, so new entries are 1413 * added to the tail, vfio_info_cap_shift() should be called to fixup the 1414 * next offsets prior to copying to the user buffer. 1415 */ 1416 struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps, 1417 size_t size, u16 id, u16 version) 1418 { 1419 void *buf; 1420 struct vfio_info_cap_header *header, *tmp; 1421 1422 /* Ensure that the next capability struct will be aligned */ 1423 size = ALIGN(size, sizeof(u64)); 1424 1425 buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL); 1426 if (!buf) { 1427 kfree(caps->buf); 1428 caps->buf = NULL; 1429 caps->size = 0; 1430 return ERR_PTR(-ENOMEM); 1431 } 1432 1433 caps->buf = buf; 1434 header = buf + caps->size; 1435 1436 /* Eventually copied to user buffer, zero */ 1437 memset(header, 0, size); 1438 1439 header->id = id; 1440 header->version = version; 1441 1442 /* Add to the end of the capability chain */ 1443 for (tmp = buf; tmp->next; tmp = buf + tmp->next) 1444 ; /* nothing */ 1445 1446 tmp->next = caps->size; 1447 caps->size += size; 1448 1449 return header; 1450 } 1451 EXPORT_SYMBOL_GPL(vfio_info_cap_add); 1452 1453 void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset) 1454 { 1455 struct vfio_info_cap_header *tmp; 1456 void *buf = (void *)caps->buf; 1457 1458 /* Capability structs should start with proper alignment */ 1459 WARN_ON(!IS_ALIGNED(offset, sizeof(u64))); 1460 1461 for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset) 1462 tmp->next += offset; 1463 } 1464 EXPORT_SYMBOL(vfio_info_cap_shift); 1465 1466 int vfio_info_add_capability(struct vfio_info_cap *caps, 1467 struct vfio_info_cap_header *cap, size_t size) 1468 { 1469 struct vfio_info_cap_header *header; 1470 1471 header = vfio_info_cap_add(caps, size, cap->id, cap->version); 1472 if (IS_ERR(header)) 1473 return PTR_ERR(header); 1474 1475 memcpy(header + 1, cap + 1, size - sizeof(*header)); 1476 1477 return 0; 1478 } 1479 EXPORT_SYMBOL(vfio_info_add_capability); 1480 1481 int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs, 1482 int max_irq_type, size_t *data_size) 1483 { 1484 unsigned long minsz; 1485 size_t size; 1486 1487 minsz = offsetofend(struct vfio_irq_set, count); 1488 1489 if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) || 1490 (hdr->count >= (U32_MAX - hdr->start)) || 1491 (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK | 1492 VFIO_IRQ_SET_ACTION_TYPE_MASK))) 1493 return -EINVAL; 1494 1495 if (data_size) 1496 *data_size = 0; 1497 1498 if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs) 1499 return -EINVAL; 1500 1501 switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 1502 case VFIO_IRQ_SET_DATA_NONE: 1503 size = 0; 1504 break; 1505 case VFIO_IRQ_SET_DATA_BOOL: 1506 size = sizeof(uint8_t); 1507 break; 1508 case VFIO_IRQ_SET_DATA_EVENTFD: 1509 size = sizeof(int32_t); 1510 break; 1511 default: 1512 return -EINVAL; 1513 } 1514 1515 if (size) { 1516 if (hdr->argsz - minsz < hdr->count * size) 1517 return -EINVAL; 1518 1519 if (!data_size) 1520 return -EINVAL; 1521 1522 *data_size = hdr->count * size; 1523 } 1524 1525 return 0; 1526 } 1527 EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare); 1528 1529 /* 1530 * Pin contiguous user pages and return their associated host pages for local 1531 * domain only. 1532 * @device [in] : device 1533 * @iova [in] : starting IOVA of user pages to be pinned. 1534 * @npage [in] : count of pages to be pinned. This count should not 1535 * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 1536 * @prot [in] : protection flags 1537 * @pages[out] : array of host pages 1538 * Return error or number of pages pinned. 1539 * 1540 * A driver may only call this function if the vfio_device was created 1541 * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages(). 1542 */ 1543 int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, 1544 int npage, int prot, struct page **pages) 1545 { 1546 /* group->container cannot change while a vfio device is open */ 1547 if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device))) 1548 return -EINVAL; 1549 if (!device->ops->dma_unmap) 1550 return -EINVAL; 1551 if (vfio_device_has_container(device)) 1552 return vfio_device_container_pin_pages(device, iova, 1553 npage, prot, pages); 1554 if (device->iommufd_access) { 1555 int ret; 1556 1557 if (iova > ULONG_MAX) 1558 return -EINVAL; 1559 /* 1560 * VFIO ignores the sub page offset, npages is from the start of 1561 * a PAGE_SIZE chunk of IOVA. The caller is expected to recover 1562 * the sub page offset by doing: 1563 * pages[0] + (iova % PAGE_SIZE) 1564 */ 1565 ret = iommufd_access_pin_pages( 1566 device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE), 1567 npage * PAGE_SIZE, pages, 1568 (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0); 1569 if (ret) 1570 return ret; 1571 return npage; 1572 } 1573 return -EINVAL; 1574 } 1575 EXPORT_SYMBOL(vfio_pin_pages); 1576 1577 /* 1578 * Unpin contiguous host pages for local domain only. 1579 * @device [in] : device 1580 * @iova [in] : starting address of user pages to be unpinned. 1581 * @npage [in] : count of pages to be unpinned. This count should not 1582 * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 1583 */ 1584 void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage) 1585 { 1586 if (WARN_ON(!vfio_assert_device_open(device))) 1587 return; 1588 if (WARN_ON(!device->ops->dma_unmap)) 1589 return; 1590 1591 if (vfio_device_has_container(device)) { 1592 vfio_device_container_unpin_pages(device, iova, npage); 1593 return; 1594 } 1595 if (device->iommufd_access) { 1596 if (WARN_ON(iova > ULONG_MAX)) 1597 return; 1598 iommufd_access_unpin_pages(device->iommufd_access, 1599 ALIGN_DOWN(iova, PAGE_SIZE), 1600 npage * PAGE_SIZE); 1601 return; 1602 } 1603 } 1604 EXPORT_SYMBOL(vfio_unpin_pages); 1605 1606 /* 1607 * This interface allows the CPUs to perform some sort of virtual DMA on 1608 * behalf of the device. 1609 * 1610 * CPUs read/write from/into a range of IOVAs pointing to user space memory 1611 * into/from a kernel buffer. 1612 * 1613 * As the read/write of user space memory is conducted via the CPUs and is 1614 * not a real device DMA, it is not necessary to pin the user space memory. 1615 * 1616 * @device [in] : VFIO device 1617 * @iova [in] : base IOVA of a user space buffer 1618 * @data [in] : pointer to kernel buffer 1619 * @len [in] : kernel buffer length 1620 * @write : indicate read or write 1621 * Return error code on failure or 0 on success. 1622 */ 1623 int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data, 1624 size_t len, bool write) 1625 { 1626 if (!data || len <= 0 || !vfio_assert_device_open(device)) 1627 return -EINVAL; 1628 1629 if (vfio_device_has_container(device)) 1630 return vfio_device_container_dma_rw(device, iova, 1631 data, len, write); 1632 1633 if (device->iommufd_access) { 1634 unsigned int flags = 0; 1635 1636 if (iova > ULONG_MAX) 1637 return -EINVAL; 1638 1639 /* VFIO historically tries to auto-detect a kthread */ 1640 if (!current->mm) 1641 flags |= IOMMUFD_ACCESS_RW_KTHREAD; 1642 if (write) 1643 flags |= IOMMUFD_ACCESS_RW_WRITE; 1644 return iommufd_access_rw(device->iommufd_access, iova, data, 1645 len, flags); 1646 } 1647 return -EINVAL; 1648 } 1649 EXPORT_SYMBOL(vfio_dma_rw); 1650 1651 /* 1652 * Module/class support 1653 */ 1654 static int __init vfio_init(void) 1655 { 1656 int ret; 1657 1658 ida_init(&vfio.device_ida); 1659 1660 ret = vfio_group_init(); 1661 if (ret) 1662 return ret; 1663 1664 ret = vfio_virqfd_init(); 1665 if (ret) 1666 goto err_virqfd; 1667 1668 /* /sys/class/vfio-dev/vfioX */ 1669 vfio.device_class = class_create("vfio-dev"); 1670 if (IS_ERR(vfio.device_class)) { 1671 ret = PTR_ERR(vfio.device_class); 1672 goto err_dev_class; 1673 } 1674 1675 ret = vfio_cdev_init(vfio.device_class); 1676 if (ret) 1677 goto err_alloc_dev_chrdev; 1678 1679 pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n"); 1680 return 0; 1681 1682 err_alloc_dev_chrdev: 1683 class_destroy(vfio.device_class); 1684 vfio.device_class = NULL; 1685 err_dev_class: 1686 vfio_virqfd_exit(); 1687 err_virqfd: 1688 vfio_group_cleanup(); 1689 return ret; 1690 } 1691 1692 static void __exit vfio_cleanup(void) 1693 { 1694 ida_destroy(&vfio.device_ida); 1695 vfio_cdev_cleanup(); 1696 class_destroy(vfio.device_class); 1697 vfio.device_class = NULL; 1698 vfio_virqfd_exit(); 1699 vfio_group_cleanup(); 1700 xa_destroy(&vfio_device_set_xa); 1701 } 1702 1703 module_init(vfio_init); 1704 module_exit(vfio_cleanup); 1705 1706 MODULE_IMPORT_NS(IOMMUFD); 1707 MODULE_VERSION(DRIVER_VERSION); 1708 MODULE_LICENSE("GPL v2"); 1709 MODULE_AUTHOR(DRIVER_AUTHOR); 1710 MODULE_DESCRIPTION(DRIVER_DESC); 1711 MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce"); 1712