1 /* 2 * Copyright © 2012-2014 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <drm/drmP.h> 26 #include <drm/i915_drm.h> 27 #include "i915_drv.h" 28 #include "i915_trace.h" 29 #include "intel_drv.h" 30 #include <linux/mmu_context.h> 31 #include <linux/mmu_notifier.h> 32 #include <linux/swap.h> 33 34 struct i915_mm_struct { 35 struct mm_struct *mm; 36 struct drm_i915_private *i915; 37 struct i915_mmu_notifier *mn; 38 struct hlist_node node; 39 struct kref kref; 40 struct work_struct work; 41 }; 42 43 #if defined(CONFIG_MMU_NOTIFIER) 44 #include <linux/interval_tree.h> 45 46 struct i915_mmu_notifier { 47 spinlock_t lock; 48 struct hlist_node node; 49 struct mmu_notifier mn; 50 struct rb_root objects; 51 struct workqueue_struct *wq; 52 }; 53 54 struct i915_mmu_object { 55 struct i915_mmu_notifier *mn; 56 struct drm_i915_gem_object *obj; 57 struct interval_tree_node it; 58 struct list_head link; 59 struct work_struct work; 60 bool attached; 61 }; 62 63 static void wait_rendering(struct drm_i915_gem_object *obj) 64 { 65 struct drm_device *dev = obj->base.dev; 66 struct drm_i915_gem_request *requests[I915_NUM_ENGINES]; 67 int i, n; 68 69 if (!obj->active) 70 return; 71 72 n = 0; 73 for (i = 0; i < I915_NUM_ENGINES; i++) { 74 struct drm_i915_gem_request *req; 75 76 req = obj->last_read_req[i]; 77 if (req == NULL) 78 continue; 79 80 requests[n++] = i915_gem_request_reference(req); 81 } 82 83 mutex_unlock(&dev->struct_mutex); 84 85 for (i = 0; i < n; i++) 86 __i915_wait_request(requests[i], false, NULL, NULL); 87 88 mutex_lock(&dev->struct_mutex); 89 90 for (i = 0; i < n; i++) 91 i915_gem_request_unreference(requests[i]); 92 } 93 94 static void cancel_userptr(struct work_struct *work) 95 { 96 struct i915_mmu_object *mo = container_of(work, typeof(*mo), work); 97 struct drm_i915_gem_object *obj = mo->obj; 98 struct drm_device *dev = obj->base.dev; 99 100 mutex_lock(&dev->struct_mutex); 101 /* Cancel any active worker and force us to re-evaluate gup */ 102 obj->userptr.work = NULL; 103 104 if (obj->pages != NULL) { 105 struct drm_i915_private *dev_priv = to_i915(dev); 106 struct i915_vma *vma, *tmp; 107 bool was_interruptible; 108 109 wait_rendering(obj); 110 111 was_interruptible = dev_priv->mm.interruptible; 112 dev_priv->mm.interruptible = false; 113 114 list_for_each_entry_safe(vma, tmp, &obj->vma_list, obj_link) 115 WARN_ON(i915_vma_unbind(vma)); 116 WARN_ON(i915_gem_object_put_pages(obj)); 117 118 dev_priv->mm.interruptible = was_interruptible; 119 } 120 121 drm_gem_object_unreference(&obj->base); 122 mutex_unlock(&dev->struct_mutex); 123 } 124 125 static void add_object(struct i915_mmu_object *mo) 126 { 127 if (mo->attached) 128 return; 129 130 interval_tree_insert(&mo->it, &mo->mn->objects); 131 mo->attached = true; 132 } 133 134 static void del_object(struct i915_mmu_object *mo) 135 { 136 if (!mo->attached) 137 return; 138 139 interval_tree_remove(&mo->it, &mo->mn->objects); 140 mo->attached = false; 141 } 142 143 static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, 144 struct mm_struct *mm, 145 unsigned long start, 146 unsigned long end) 147 { 148 struct i915_mmu_notifier *mn = 149 container_of(_mn, struct i915_mmu_notifier, mn); 150 struct i915_mmu_object *mo; 151 struct interval_tree_node *it; 152 LIST_HEAD(cancelled); 153 154 if (RB_EMPTY_ROOT(&mn->objects)) 155 return; 156 157 /* interval ranges are inclusive, but invalidate range is exclusive */ 158 end--; 159 160 spin_lock(&mn->lock); 161 it = interval_tree_iter_first(&mn->objects, start, end); 162 while (it) { 163 /* The mmu_object is released late when destroying the 164 * GEM object so it is entirely possible to gain a 165 * reference on an object in the process of being freed 166 * since our serialisation is via the spinlock and not 167 * the struct_mutex - and consequently use it after it 168 * is freed and then double free it. To prevent that 169 * use-after-free we only acquire a reference on the 170 * object if it is not in the process of being destroyed. 171 */ 172 mo = container_of(it, struct i915_mmu_object, it); 173 if (kref_get_unless_zero(&mo->obj->base.refcount)) 174 queue_work(mn->wq, &mo->work); 175 176 list_add(&mo->link, &cancelled); 177 it = interval_tree_iter_next(it, start, end); 178 } 179 list_for_each_entry(mo, &cancelled, link) 180 del_object(mo); 181 spin_unlock(&mn->lock); 182 183 flush_workqueue(mn->wq); 184 } 185 186 static const struct mmu_notifier_ops i915_gem_userptr_notifier = { 187 .invalidate_range_start = i915_gem_userptr_mn_invalidate_range_start, 188 }; 189 190 static struct i915_mmu_notifier * 191 i915_mmu_notifier_create(struct mm_struct *mm) 192 { 193 struct i915_mmu_notifier *mn; 194 int ret; 195 196 mn = kmalloc(sizeof(*mn), M_DRM, GFP_KERNEL); 197 if (mn == NULL) 198 return ERR_PTR(-ENOMEM); 199 200 spin_lock_init(&mn->lock); 201 mn->mn.ops = &i915_gem_userptr_notifier; 202 mn->objects = RB_ROOT; 203 mn->wq = alloc_workqueue("i915-userptr-release", WQ_UNBOUND, 0); 204 if (mn->wq == NULL) { 205 kfree(mn); 206 return ERR_PTR(-ENOMEM); 207 } 208 209 /* Protected by mmap_sem (write-lock) */ 210 ret = __mmu_notifier_register(&mn->mn, mm); 211 if (ret) { 212 destroy_workqueue(mn->wq); 213 kfree(mn); 214 return ERR_PTR(ret); 215 } 216 217 return mn; 218 } 219 220 static void 221 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 222 { 223 struct i915_mmu_object *mo; 224 225 mo = obj->userptr.mmu_object; 226 if (mo == NULL) 227 return; 228 229 spin_lock(&mo->mn->lock); 230 del_object(mo); 231 spin_unlock(&mo->mn->lock); 232 kfree(mo); 233 234 obj->userptr.mmu_object = NULL; 235 } 236 237 static struct i915_mmu_notifier * 238 i915_mmu_notifier_find(struct i915_mm_struct *mm) 239 { 240 struct i915_mmu_notifier *mn = mm->mn; 241 242 mn = mm->mn; 243 if (mn) 244 return mn; 245 246 down_write(&mm->mm->mmap_sem); 247 mutex_lock(&mm->i915->mm_lock); 248 if ((mn = mm->mn) == NULL) { 249 mn = i915_mmu_notifier_create(mm->mm); 250 if (!IS_ERR(mn)) 251 mm->mn = mn; 252 } 253 mutex_unlock(&mm->i915->mm_lock); 254 up_write(&mm->mm->mmap_sem); 255 256 return mn; 257 } 258 259 static int 260 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 261 unsigned flags) 262 { 263 struct i915_mmu_notifier *mn; 264 struct i915_mmu_object *mo; 265 266 if (flags & I915_USERPTR_UNSYNCHRONIZED) 267 return capable(CAP_SYS_ADMIN) ? 0 : -EPERM; 268 269 if (WARN_ON(obj->userptr.mm == NULL)) 270 return -EINVAL; 271 272 mn = i915_mmu_notifier_find(obj->userptr.mm); 273 if (IS_ERR(mn)) 274 return PTR_ERR(mn); 275 276 mo = kzalloc(sizeof(*mo), GFP_KERNEL); 277 if (mo == NULL) 278 return -ENOMEM; 279 280 mo->mn = mn; 281 mo->obj = obj; 282 mo->it.start = obj->userptr.ptr; 283 mo->it.last = obj->userptr.ptr + obj->base.size - 1; 284 INIT_WORK(&mo->work, cancel_userptr); 285 286 obj->userptr.mmu_object = mo; 287 return 0; 288 } 289 290 static void 291 i915_mmu_notifier_free(struct i915_mmu_notifier *mn, 292 struct mm_struct *mm) 293 { 294 if (mn == NULL) 295 return; 296 297 mmu_notifier_unregister(&mn->mn, mm); 298 destroy_workqueue(mn->wq); 299 kfree(mn); 300 } 301 302 #else 303 304 static void 305 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 306 { 307 } 308 309 static int 310 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 311 unsigned flags) 312 { 313 if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0) 314 return -ENODEV; 315 316 if (!capable(CAP_SYS_ADMIN)) 317 return -EPERM; 318 319 return 0; 320 } 321 322 static void 323 i915_mmu_notifier_free(struct i915_mmu_notifier *mn, 324 struct mm_struct *mm) 325 { 326 } 327 328 #endif 329 330 static struct i915_mm_struct * 331 __i915_mm_struct_find(struct drm_i915_private *dev_priv, struct mm_struct *real) 332 { 333 struct i915_mm_struct *mm; 334 335 /* Protected by dev_priv->mm_lock */ 336 hash_for_each_possible(dev_priv->mm_structs, mm, node, (unsigned long)real) 337 if (mm->mm == real) 338 return mm; 339 340 return NULL; 341 } 342 343 static int 344 i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj) 345 { 346 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 347 struct i915_mm_struct *mm; 348 int ret = 0; 349 350 /* During release of the GEM object we hold the struct_mutex. This 351 * precludes us from calling mmput() at that time as that may be 352 * the last reference and so call exit_mmap(). exit_mmap() will 353 * attempt to reap the vma, and if we were holding a GTT mmap 354 * would then call drm_gem_vm_close() and attempt to reacquire 355 * the struct mutex. So in order to avoid that recursion, we have 356 * to defer releasing the mm reference until after we drop the 357 * struct_mutex, i.e. we need to schedule a worker to do the clean 358 * up. 359 */ 360 mutex_lock(&dev_priv->mm_lock); 361 mm = __i915_mm_struct_find(dev_priv, current->mm); 362 #if 0 363 if (mm == NULL) { 364 mm = kmalloc(sizeof(*mm), M_DRM, GFP_KERNEL); 365 if (mm == NULL) { 366 #endif 367 ret = -ENOMEM; 368 #if 0 369 goto out; 370 } 371 372 kref_init(&mm->kref); 373 mm->i915 = to_i915(obj->base.dev); 374 375 mm->mm = current->mm; 376 atomic_inc(¤t->mm->mm_count); 377 378 mm->mn = NULL; 379 380 /* Protected by dev_priv->mm_lock */ 381 hash_add(dev_priv->mm_structs, 382 &mm->node, (unsigned long)mm->mm); 383 } else 384 kref_get(&mm->kref); 385 386 obj->userptr.mm = mm; 387 out: 388 mutex_unlock(&dev_priv->mm_lock); 389 #endif 390 return ret; 391 } 392 393 static void 394 __i915_mm_struct_free__worker(struct work_struct *work) 395 { 396 struct i915_mm_struct *mm = container_of(work, typeof(*mm), work); 397 i915_mmu_notifier_free(mm->mn, mm->mm); 398 #if 0 399 mmdrop(mm->mm); 400 #endif 401 kfree(mm); 402 } 403 404 static void 405 __i915_mm_struct_free(struct kref *kref) 406 { 407 struct i915_mm_struct *mm = container_of(kref, typeof(*mm), kref); 408 409 /* Protected by dev_priv->mm_lock */ 410 hash_del(&mm->node); 411 mutex_unlock(&mm->i915->mm_lock); 412 413 INIT_WORK(&mm->work, __i915_mm_struct_free__worker); 414 schedule_work(&mm->work); 415 } 416 417 static void 418 i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj) 419 { 420 if (obj->userptr.mm == NULL) 421 return; 422 423 kref_put_mutex(&obj->userptr.mm->kref, 424 __i915_mm_struct_free, 425 &to_i915(obj->base.dev)->mm_lock); 426 obj->userptr.mm = NULL; 427 } 428 429 struct get_pages_work { 430 struct work_struct work; 431 struct drm_i915_gem_object *obj; 432 struct task_struct *task; 433 }; 434 435 #if IS_ENABLED(CONFIG_SWIOTLB) 436 #define swiotlb_active() swiotlb_nr_tbl() 437 #else 438 #define swiotlb_active() 0 439 #endif 440 441 #if 0 442 static int 443 st_set_pages(struct sg_table **st, struct page **pvec, int num_pages) 444 { 445 struct scatterlist *sg; 446 int ret, n; 447 448 *st = kmalloc(sizeof(**st), M_DRM, GFP_KERNEL); 449 if (*st == NULL) 450 return -ENOMEM; 451 452 if (swiotlb_active()) { 453 ret = sg_alloc_table(*st, num_pages, GFP_KERNEL); 454 if (ret) 455 goto err; 456 457 for_each_sg((*st)->sgl, sg, num_pages, n) 458 sg_set_page(sg, pvec[n], PAGE_SIZE, 0); 459 } else { 460 ret = sg_alloc_table_from_pages(*st, pvec, num_pages, 461 0, num_pages << PAGE_SHIFT, 462 GFP_KERNEL); 463 if (ret) 464 goto err; 465 } 466 467 return 0; 468 469 err: 470 kfree(*st); 471 *st = NULL; 472 return ret; 473 } 474 475 static int 476 __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj, 477 struct page **pvec, int num_pages) 478 { 479 int ret; 480 481 ret = st_set_pages(&obj->pages, pvec, num_pages); 482 if (ret) 483 return ret; 484 485 ret = i915_gem_gtt_prepare_object(obj); 486 if (ret) { 487 sg_free_table(obj->pages); 488 kfree(obj->pages); 489 obj->pages = NULL; 490 } 491 492 return ret; 493 } 494 #endif 495 496 static int 497 __i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, 498 bool value) 499 { 500 int ret = 0; 501 502 /* During mm_invalidate_range we need to cancel any userptr that 503 * overlaps the range being invalidated. Doing so requires the 504 * struct_mutex, and that risks recursion. In order to cause 505 * recursion, the user must alias the userptr address space with 506 * a GTT mmapping (possible with a MAP_FIXED) - then when we have 507 * to invalidate that mmaping, mm_invalidate_range is called with 508 * the userptr address *and* the struct_mutex held. To prevent that 509 * we set a flag under the i915_mmu_notifier spinlock to indicate 510 * whether this object is valid. 511 */ 512 #if defined(CONFIG_MMU_NOTIFIER) 513 if (obj->userptr.mmu_object == NULL) 514 return 0; 515 516 spin_lock(&obj->userptr.mmu_object->mn->lock); 517 /* In order to serialise get_pages with an outstanding 518 * cancel_userptr, we must drop the struct_mutex and try again. 519 */ 520 if (!value) 521 del_object(obj->userptr.mmu_object); 522 else if (!work_pending(&obj->userptr.mmu_object->work)) 523 add_object(obj->userptr.mmu_object); 524 else 525 ret = -EAGAIN; 526 spin_unlock(&obj->userptr.mmu_object->mn->lock); 527 #endif 528 529 return ret; 530 } 531 532 #if 0 533 static void 534 __i915_gem_userptr_get_pages_worker(struct work_struct *_work) 535 { 536 struct get_pages_work *work = container_of(_work, typeof(*work), work); 537 struct drm_i915_gem_object *obj = work->obj; 538 struct drm_device *dev = obj->base.dev; 539 const int npages = obj->base.size >> PAGE_SHIFT; 540 struct page **pvec; 541 int pinned, ret; 542 543 ret = -ENOMEM; 544 pinned = 0; 545 546 pvec = drm_malloc_gfp(npages, sizeof(struct page *), GFP_TEMPORARY); 547 if (pvec != NULL) { 548 struct mm_struct *mm = obj->userptr.mm->mm; 549 550 ret = -EFAULT; 551 if (atomic_inc_not_zero(&mm->mm_users)) { 552 down_read(&mm->mmap_sem); 553 while (pinned < npages) { 554 ret = get_user_pages_remote 555 (work->task, mm, 556 obj->userptr.ptr + pinned * PAGE_SIZE, 557 npages - pinned, 558 !obj->userptr.read_only, 0, 559 pvec + pinned, NULL); 560 if (ret < 0) 561 break; 562 563 pinned += ret; 564 } 565 up_read(&mm->mmap_sem); 566 mmput(mm); 567 } 568 } 569 570 mutex_lock(&dev->struct_mutex); 571 if (obj->userptr.work == &work->work) { 572 if (pinned == npages) { 573 ret = __i915_gem_userptr_set_pages(obj, pvec, npages); 574 if (ret == 0) { 575 list_add_tail(&obj->global_list, 576 &to_i915(dev)->mm.unbound_list); 577 obj->get_page.sg = obj->pages->sgl; 578 obj->get_page.last = 0; 579 pinned = 0; 580 } 581 } 582 obj->userptr.work = ERR_PTR(ret); 583 if (ret) 584 __i915_gem_userptr_set_active(obj, false); 585 } 586 587 obj->userptr.workers--; 588 drm_gem_object_unreference(&obj->base); 589 mutex_unlock(&dev->struct_mutex); 590 591 release_pages(pvec, pinned, 0); 592 drm_free_large(pvec); 593 594 put_task_struct(work->task); 595 kfree(work); 596 } 597 598 static int 599 __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj, 600 bool *active) 601 { 602 struct get_pages_work *work; 603 604 /* Spawn a worker so that we can acquire the 605 * user pages without holding our mutex. Access 606 * to the user pages requires mmap_sem, and we have 607 * a strict lock ordering of mmap_sem, struct_mutex - 608 * we already hold struct_mutex here and so cannot 609 * call gup without encountering a lock inversion. 610 * 611 * Userspace will keep on repeating the operation 612 * (thanks to EAGAIN) until either we hit the fast 613 * path or the worker completes. If the worker is 614 * cancelled or superseded, the task is still run 615 * but the results ignored. (This leads to 616 * complications that we may have a stray object 617 * refcount that we need to be wary of when 618 * checking for existing objects during creation.) 619 * If the worker encounters an error, it reports 620 * that error back to this function through 621 * obj->userptr.work = ERR_PTR. 622 */ 623 if (obj->userptr.workers >= I915_GEM_USERPTR_MAX_WORKERS) 624 return -EAGAIN; 625 626 work = kmalloc(sizeof(*work), GFP_KERNEL); 627 if (work == NULL) 628 return -ENOMEM; 629 630 obj->userptr.work = &work->work; 631 obj->userptr.workers++; 632 633 work->obj = obj; 634 drm_gem_object_reference(&obj->base); 635 636 work->task = current; 637 get_task_struct(work->task); 638 639 INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker); 640 schedule_work(&work->work); 641 642 *active = true; 643 return -EAGAIN; 644 } 645 #endif 646 647 static int 648 i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) 649 { 650 #if 0 651 const int num_pages = obj->base.size >> PAGE_SHIFT; 652 struct page **pvec; 653 int pinned, ret; 654 bool active; 655 656 /* If userspace should engineer that these pages are replaced in 657 * the vma between us binding this page into the GTT and completion 658 * of rendering... Their loss. If they change the mapping of their 659 * pages they need to create a new bo to point to the new vma. 660 * 661 * However, that still leaves open the possibility of the vma 662 * being copied upon fork. Which falls under the same userspace 663 * synchronisation issue as a regular bo, except that this time 664 * the process may not be expecting that a particular piece of 665 * memory is tied to the GPU. 666 * 667 * Fortunately, we can hook into the mmu_notifier in order to 668 * discard the page references prior to anything nasty happening 669 * to the vma (discard or cloning) which should prevent the more 670 * egregious cases from causing harm. 671 */ 672 if (IS_ERR(obj->userptr.work)) { 673 /* active flag will have been dropped already by the worker */ 674 ret = PTR_ERR(obj->userptr.work); 675 obj->userptr.work = NULL; 676 return ret; 677 } 678 if (obj->userptr.work) 679 /* active flag should still be held for the pending work */ 680 return -EAGAIN; 681 682 /* Let the mmu-notifier know that we have begun and need cancellation */ 683 ret = __i915_gem_userptr_set_active(obj, true); 684 if (ret) 685 return ret; 686 687 pvec = NULL; 688 pinned = 0; 689 if (obj->userptr.mm->mm == current->mm) { 690 pvec = drm_malloc_gfp(num_pages, sizeof(struct page *), 691 GFP_TEMPORARY); 692 if (pvec == NULL) { 693 __i915_gem_userptr_set_active(obj, false); 694 return -ENOMEM; 695 } 696 697 pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages, 698 !obj->userptr.read_only, pvec); 699 } 700 701 active = false; 702 if (pinned < 0) 703 ret = pinned, pinned = 0; 704 else if (pinned < num_pages) 705 ret = __i915_gem_userptr_get_pages_schedule(obj, &active); 706 else 707 ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages); 708 if (ret) { 709 __i915_gem_userptr_set_active(obj, active); 710 release_pages(pvec, pinned, 0); 711 } 712 drm_free_large(pvec); 713 return ret; 714 #else 715 return 0; 716 #endif /* 0 */ 717 } 718 719 static void 720 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) 721 { 722 struct sgt_iter sgt_iter; 723 struct page *page; 724 725 BUG_ON(obj->userptr.work != NULL); 726 __i915_gem_userptr_set_active(obj, false); 727 728 if (obj->madv != I915_MADV_WILLNEED) 729 obj->dirty = 0; 730 731 i915_gem_gtt_finish_object(obj); 732 733 for_each_sgt_page(page, sgt_iter, obj->pages) { 734 if (obj->dirty) 735 set_page_dirty(page); 736 737 mark_page_accessed(page); 738 #if 0 739 put_page(page); 740 #endif 741 } 742 obj->dirty = 0; 743 744 sg_free_table(obj->pages); 745 kfree(obj->pages); 746 } 747 748 static void 749 i915_gem_userptr_release(struct drm_i915_gem_object *obj) 750 { 751 i915_gem_userptr_release__mmu_notifier(obj); 752 i915_gem_userptr_release__mm_struct(obj); 753 } 754 755 static int 756 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj) 757 { 758 if (obj->userptr.mmu_object) 759 return 0; 760 761 return i915_gem_userptr_init__mmu_notifier(obj, 0); 762 } 763 764 static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { 765 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE, 766 .get_pages = i915_gem_userptr_get_pages, 767 .put_pages = i915_gem_userptr_put_pages, 768 .dmabuf_export = i915_gem_userptr_dmabuf_export, 769 .release = i915_gem_userptr_release, 770 }; 771 772 /** 773 * Creates a new mm object that wraps some normal memory from the process 774 * context - user memory. 775 * 776 * We impose several restrictions upon the memory being mapped 777 * into the GPU. 778 * 1. It must be page aligned (both start/end addresses, i.e ptr and size). 779 * 2. It must be normal system memory, not a pointer into another map of IO 780 * space (e.g. it must not be a GTT mmapping of another object). 781 * 3. We only allow a bo as large as we could in theory map into the GTT, 782 * that is we limit the size to the total size of the GTT. 783 * 4. The bo is marked as being snoopable. The backing pages are left 784 * accessible directly by the CPU, but reads and writes by the GPU may 785 * incur the cost of a snoop (unless you have an LLC architecture). 786 * 787 * Synchronisation between multiple users and the GPU is left to userspace 788 * through the normal set-domain-ioctl. The kernel will enforce that the 789 * GPU relinquishes the VMA before it is returned back to the system 790 * i.e. upon free(), munmap() or process termination. However, the userspace 791 * malloc() library may not immediately relinquish the VMA after free() and 792 * instead reuse it whilst the GPU is still reading and writing to the VMA. 793 * Caveat emptor. 794 * 795 * Also note, that the object created here is not currently a "first class" 796 * object, in that several ioctls are banned. These are the CPU access 797 * ioctls: mmap(), pwrite and pread. In practice, you are expected to use 798 * direct access via your pointer rather than use those ioctls. Another 799 * restriction is that we do not allow userptr surfaces to be pinned to the 800 * hardware and so we reject any attempt to create a framebuffer out of a 801 * userptr. 802 * 803 * If you think this is a good interface to use to pass GPU memory between 804 * drivers, please use dma-buf instead. In fact, wherever possible use 805 * dma-buf instead. 806 */ 807 int 808 i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 809 { 810 struct drm_i915_gem_userptr *args = data; 811 struct drm_i915_gem_object *obj; 812 int ret; 813 u32 handle; 814 815 if (!HAS_LLC(dev) && !HAS_SNOOP(dev)) { 816 /* We cannot support coherent userptr objects on hw without 817 * LLC and broken snooping. 818 */ 819 return -ENODEV; 820 } 821 822 if (args->flags & ~(I915_USERPTR_READ_ONLY | 823 I915_USERPTR_UNSYNCHRONIZED)) 824 return -EINVAL; 825 826 if (offset_in_page(args->user_ptr | args->user_size)) 827 return -EINVAL; 828 829 #if 0 830 if (!access_ok(args->flags & I915_USERPTR_READ_ONLY ? VERIFY_READ : VERIFY_WRITE, 831 (char __user *)(unsigned long)args->user_ptr, args->user_size)) 832 return -EFAULT; 833 #endif 834 835 if (args->flags & I915_USERPTR_READ_ONLY) { 836 /* On almost all of the current hw, we cannot tell the GPU that a 837 * page is readonly, so this is just a placeholder in the uAPI. 838 */ 839 return -ENODEV; 840 } 841 842 obj = i915_gem_object_alloc(dev); 843 if (obj == NULL) 844 return -ENOMEM; 845 846 drm_gem_private_object_init(dev, &obj->base, args->user_size); 847 i915_gem_object_init(obj, &i915_gem_userptr_ops); 848 obj->cache_level = I915_CACHE_LLC; 849 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 850 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 851 852 obj->userptr.ptr = args->user_ptr; 853 obj->userptr.read_only = !!(args->flags & I915_USERPTR_READ_ONLY); 854 855 /* And keep a pointer to the current->mm for resolving the user pages 856 * at binding. This means that we need to hook into the mmu_notifier 857 * in order to detect if the mmu is destroyed. 858 */ 859 ret = i915_gem_userptr_init__mm_struct(obj); 860 if (ret == 0) 861 ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags); 862 if (ret == 0) 863 ret = drm_gem_handle_create(file, &obj->base, &handle); 864 865 /* drop reference from allocate - handle holds it now */ 866 drm_gem_object_unreference_unlocked(&obj->base); 867 if (ret) 868 return ret; 869 870 args->handle = handle; 871 return 0; 872 } 873 874 void i915_gem_init_userptr(struct drm_i915_private *dev_priv) 875 { 876 lockinit(&dev_priv->mm_lock, "i915dmm", 0, LK_CANRECURSE); 877 hash_init(dev_priv->mm_structs); 878 } 879