1 /* 2 * Copyright © 2012-2014 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <drm/drmP.h> 26 #include <drm/i915_drm.h> 27 #include "i915_drv.h" 28 #include "i915_trace.h" 29 #include "intel_drv.h" 30 #include <linux/mmu_context.h> 31 #include <linux/mmu_notifier.h> 32 #include <linux/swap.h> 33 #include <linux/sched/mm.h> 34 35 struct i915_mm_struct { 36 struct mm_struct *mm; 37 struct drm_i915_private *i915; 38 struct i915_mmu_notifier *mn; 39 struct hlist_node node; 40 struct kref kref; 41 struct work_struct work; 42 }; 43 44 #if defined(CONFIG_MMU_NOTIFIER) 45 #include <linux/interval_tree.h> 46 47 struct i915_mmu_notifier { 48 spinlock_t lock; 49 struct hlist_node node; 50 struct mmu_notifier mn; 51 struct rb_root objects; 52 struct workqueue_struct *wq; 53 }; 54 55 struct i915_mmu_object { 56 struct i915_mmu_notifier *mn; 57 struct drm_i915_gem_object *obj; 58 struct interval_tree_node it; 59 struct list_head link; 60 struct work_struct work; 61 bool attached; 62 }; 63 64 static void cancel_userptr(struct work_struct *work) 65 { 66 struct i915_mmu_object *mo = container_of(work, typeof(*mo), work); 67 struct drm_i915_gem_object *obj = mo->obj; 68 struct work_struct *active; 69 70 /* Cancel any active worker and force us to re-evaluate gup */ 71 mutex_lock(&obj->mm.lock); 72 active = fetch_and_zero(&obj->userptr.work); 73 mutex_unlock(&obj->mm.lock); 74 if (active) 75 goto out; 76 77 i915_gem_object_wait(obj, I915_WAIT_ALL, MAX_SCHEDULE_TIMEOUT, NULL); 78 79 mutex_lock(&obj->base.dev->struct_mutex); 80 81 /* We are inside a kthread context and can't be interrupted */ 82 if (i915_gem_object_unbind(obj) == 0) 83 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 84 WARN_ONCE(obj->mm.pages, 85 "Failed to release pages: bind_count=%d, pages_pin_count=%d, pin_display=%d\n", 86 obj->bind_count, 87 atomic_read(&obj->mm.pages_pin_count), 88 obj->pin_display); 89 90 mutex_unlock(&obj->base.dev->struct_mutex); 91 92 out: 93 i915_gem_object_put(obj); 94 } 95 96 static void add_object(struct i915_mmu_object *mo) 97 { 98 if (mo->attached) 99 return; 100 101 interval_tree_insert(&mo->it, &mo->mn->objects); 102 mo->attached = true; 103 } 104 105 static void del_object(struct i915_mmu_object *mo) 106 { 107 if (!mo->attached) 108 return; 109 110 interval_tree_remove(&mo->it, &mo->mn->objects); 111 mo->attached = false; 112 } 113 114 static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, 115 struct mm_struct *mm, 116 unsigned long start, 117 unsigned long end) 118 { 119 struct i915_mmu_notifier *mn = 120 container_of(_mn, struct i915_mmu_notifier, mn); 121 struct i915_mmu_object *mo; 122 struct interval_tree_node *it; 123 LINUX_LIST_HEAD(cancelled); 124 125 if (RB_EMPTY_ROOT(&mn->objects)) 126 return; 127 128 /* interval ranges are inclusive, but invalidate range is exclusive */ 129 end--; 130 131 lockmgr(&mn->lock, LK_EXCLUSIVE); 132 it = interval_tree_iter_first(&mn->objects, start, end); 133 while (it) { 134 /* The mmu_object is released late when destroying the 135 * GEM object so it is entirely possible to gain a 136 * reference on an object in the process of being freed 137 * since our serialisation is via the spinlock and not 138 * the struct_mutex - and consequently use it after it 139 * is freed and then double free it. To prevent that 140 * use-after-free we only acquire a reference on the 141 * object if it is not in the process of being destroyed. 142 */ 143 mo = container_of(it, struct i915_mmu_object, it); 144 if (kref_get_unless_zero(&mo->obj->base.refcount)) 145 queue_work(mn->wq, &mo->work); 146 147 list_add(&mo->link, &cancelled); 148 it = interval_tree_iter_next(it, start, end); 149 } 150 list_for_each_entry(mo, &cancelled, link) 151 del_object(mo); 152 lockmgr(&mn->lock, LK_RELEASE); 153 154 if (!list_empty(&cancelled)) 155 flush_workqueue(mn->wq); 156 } 157 158 static const struct mmu_notifier_ops i915_gem_userptr_notifier = { 159 .invalidate_range_start = i915_gem_userptr_mn_invalidate_range_start, 160 }; 161 162 static struct i915_mmu_notifier * 163 i915_mmu_notifier_create(struct mm_struct *mm) 164 { 165 struct i915_mmu_notifier *mn; 166 int ret; 167 168 mn = kmalloc(sizeof(*mn), M_DRM, GFP_KERNEL); 169 if (mn == NULL) 170 return ERR_PTR(-ENOMEM); 171 172 spin_lock_init(&mn->lock); 173 mn->mn.ops = &i915_gem_userptr_notifier; 174 mn->objects = LINUX_RB_ROOT; 175 mn->wq = alloc_workqueue("i915-userptr-release", WQ_UNBOUND, 0); 176 if (mn->wq == NULL) { 177 kfree(mn); 178 return ERR_PTR(-ENOMEM); 179 } 180 181 /* Protected by mmap_sem (write-lock) */ 182 ret = __mmu_notifier_register(&mn->mn, mm); 183 if (ret) { 184 destroy_workqueue(mn->wq); 185 kfree(mn); 186 return ERR_PTR(ret); 187 } 188 189 return mn; 190 } 191 192 static void 193 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 194 { 195 struct i915_mmu_object *mo; 196 197 mo = obj->userptr.mmu_object; 198 if (mo == NULL) 199 return; 200 201 lockmgr(&mo->mn->lock, LK_EXCLUSIVE); 202 del_object(mo); 203 lockmgr(&mo->mn->lock, LK_RELEASE); 204 kfree(mo); 205 206 obj->userptr.mmu_object = NULL; 207 } 208 209 static struct i915_mmu_notifier * 210 i915_mmu_notifier_find(struct i915_mm_struct *mm) 211 { 212 struct i915_mmu_notifier *mn = mm->mn; 213 214 mn = mm->mn; 215 if (mn) 216 return mn; 217 218 down_write(&mm->mm->mmap_sem); 219 mutex_lock(&mm->i915->mm_lock); 220 if ((mn = mm->mn) == NULL) { 221 mn = i915_mmu_notifier_create(mm->mm); 222 if (!IS_ERR(mn)) 223 mm->mn = mn; 224 } 225 mutex_unlock(&mm->i915->mm_lock); 226 up_write(&mm->mm->mmap_sem); 227 228 return mn; 229 } 230 231 static int 232 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 233 unsigned flags) 234 { 235 struct i915_mmu_notifier *mn; 236 struct i915_mmu_object *mo; 237 238 if (flags & I915_USERPTR_UNSYNCHRONIZED) 239 return capable(CAP_SYS_ADMIN) ? 0 : -EPERM; 240 241 if (WARN_ON(obj->userptr.mm == NULL)) 242 return -EINVAL; 243 244 mn = i915_mmu_notifier_find(obj->userptr.mm); 245 if (IS_ERR(mn)) 246 return PTR_ERR(mn); 247 248 mo = kzalloc(sizeof(*mo), GFP_KERNEL); 249 if (mo == NULL) 250 return -ENOMEM; 251 252 mo->mn = mn; 253 mo->obj = obj; 254 mo->it.start = obj->userptr.ptr; 255 mo->it.last = obj->userptr.ptr + obj->base.size - 1; 256 INIT_WORK(&mo->work, cancel_userptr); 257 258 obj->userptr.mmu_object = mo; 259 return 0; 260 } 261 262 static void 263 i915_mmu_notifier_free(struct i915_mmu_notifier *mn, 264 struct mm_struct *mm) 265 { 266 if (mn == NULL) 267 return; 268 269 mmu_notifier_unregister(&mn->mn, mm); 270 destroy_workqueue(mn->wq); 271 kfree(mn); 272 } 273 274 #else 275 276 static void 277 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 278 { 279 } 280 281 static int 282 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 283 unsigned flags) 284 { 285 if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0) 286 return -ENODEV; 287 288 if (!capable(CAP_SYS_ADMIN)) 289 return -EPERM; 290 291 return 0; 292 } 293 294 static void 295 i915_mmu_notifier_free(struct i915_mmu_notifier *mn, 296 struct mm_struct *mm) 297 { 298 } 299 300 #endif 301 302 static struct i915_mm_struct * 303 __i915_mm_struct_find(struct drm_i915_private *dev_priv, struct mm_struct *real) 304 { 305 struct i915_mm_struct *mm; 306 307 /* Protected by dev_priv->mm_lock */ 308 hash_for_each_possible(dev_priv->mm_structs, mm, node, (unsigned long)real) 309 if (mm->mm == real) 310 return mm; 311 312 return NULL; 313 } 314 315 static int 316 i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj) 317 { 318 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 319 struct i915_mm_struct *mm; 320 int ret = 0; 321 322 /* During release of the GEM object we hold the struct_mutex. This 323 * precludes us from calling mmput() at that time as that may be 324 * the last reference and so call exit_mmap(). exit_mmap() will 325 * attempt to reap the vma, and if we were holding a GTT mmap 326 * would then call drm_gem_vm_close() and attempt to reacquire 327 * the struct mutex. So in order to avoid that recursion, we have 328 * to defer releasing the mm reference until after we drop the 329 * struct_mutex, i.e. we need to schedule a worker to do the clean 330 * up. 331 */ 332 mutex_lock(&dev_priv->mm_lock); 333 mm = __i915_mm_struct_find(dev_priv, current->mm); 334 #if 0 335 if (mm == NULL) { 336 mm = kmalloc(sizeof(*mm), M_DRM, GFP_KERNEL); 337 if (mm == NULL) { 338 #endif 339 ret = -ENOMEM; 340 #if 0 341 goto out; 342 } 343 344 kref_init(&mm->kref); 345 mm->i915 = to_i915(obj->base.dev); 346 347 mm->mm = current->mm; 348 mmgrab(current->mm); 349 350 mm->mn = NULL; 351 352 /* Protected by dev_priv->mm_lock */ 353 hash_add(dev_priv->mm_structs, 354 &mm->node, (unsigned long)mm->mm); 355 } else 356 kref_get(&mm->kref); 357 358 obj->userptr.mm = mm; 359 out: 360 mutex_unlock(&dev_priv->mm_lock); 361 #endif 362 return ret; 363 } 364 365 static void 366 __i915_mm_struct_free__worker(struct work_struct *work) 367 { 368 struct i915_mm_struct *mm = container_of(work, typeof(*mm), work); 369 i915_mmu_notifier_free(mm->mn, mm->mm); 370 #if 0 371 mmdrop(mm->mm); 372 #endif 373 kfree(mm); 374 } 375 376 static void 377 __i915_mm_struct_free(struct kref *kref) 378 { 379 struct i915_mm_struct *mm = container_of(kref, typeof(*mm), kref); 380 381 /* Protected by dev_priv->mm_lock */ 382 hash_del(&mm->node); 383 mutex_unlock(&mm->i915->mm_lock); 384 385 INIT_WORK(&mm->work, __i915_mm_struct_free__worker); 386 schedule_work(&mm->work); 387 } 388 389 #if 0 390 static bool noncontiguous_or_overlaps_ggtt(struct drm_i915_gem_object *obj, 391 struct mm_struct *mm) 392 { 393 const struct vm_operations_struct *gem_vm_ops = 394 obj->base.dev->driver->gem_vm_ops; 395 unsigned long addr = obj->userptr.ptr; 396 const unsigned long end = addr + obj->base.size; 397 struct vm_area_struct *vma; 398 399 /* Check for a contiguous set of vma covering the userptr, if any 400 * are absent, they will EFAULT. More importantly if any point back 401 * to a drm_i915_gem_object GTT mmaping, we may trigger a deadlock 402 * between the deferred gup of this userptr and the object being 403 * unbound calling invalidate_range -> cancel_userptr. 404 */ 405 for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) { 406 if (vma->vm_start > addr) /* gap */ 407 break; 408 409 if (vma->vm_ops == gem_vm_ops) /* GTT mmapping */ 410 break; 411 412 if (vma->vm_end >= end) 413 return false; 414 415 addr = vma->vm_end; 416 } 417 418 return true; 419 } 420 #endif 421 422 static void 423 i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj) 424 { 425 if (obj->userptr.mm == NULL) 426 return; 427 428 kref_put_mutex(&obj->userptr.mm->kref, 429 __i915_mm_struct_free, 430 &to_i915(obj->base.dev)->mm_lock); 431 obj->userptr.mm = NULL; 432 } 433 434 struct get_pages_work { 435 struct work_struct work; 436 struct drm_i915_gem_object *obj; 437 struct task_struct *task; 438 }; 439 440 #if IS_ENABLED(CONFIG_SWIOTLB) 441 #define swiotlb_active() swiotlb_nr_tbl() 442 #else 443 #define swiotlb_active() 0 444 #endif 445 446 #if 0 447 static int 448 st_set_pages(struct sg_table **st, struct page **pvec, int num_pages) 449 { 450 struct scatterlist *sg; 451 int ret, n; 452 453 *st = kmalloc(sizeof(**st), M_DRM, GFP_KERNEL); 454 if (*st == NULL) 455 return -ENOMEM; 456 457 if (swiotlb_active()) { 458 ret = sg_alloc_table(*st, num_pages, GFP_KERNEL); 459 if (ret) 460 goto err; 461 462 for_each_sg((*st)->sgl, sg, num_pages, n) 463 sg_set_page(sg, pvec[n], PAGE_SIZE, 0); 464 } else { 465 ret = sg_alloc_table_from_pages(*st, pvec, num_pages, 466 0, num_pages << PAGE_SHIFT, 467 GFP_KERNEL); 468 if (ret) 469 goto err; 470 } 471 472 return 0; 473 474 err: 475 kfree(*st); 476 *st = NULL; 477 return ret; 478 } 479 480 static struct sg_table * 481 __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj, 482 struct page **pvec, int num_pages) 483 { 484 struct sg_table *pages; 485 int ret; 486 487 ret = st_set_pages(&pages, pvec, num_pages); 488 if (ret) 489 return ERR_PTR(ret); 490 491 ret = i915_gem_gtt_prepare_pages(obj, pages); 492 if (ret) { 493 sg_free_table(pages); 494 kfree(pages); 495 return ERR_PTR(ret); 496 } 497 498 return pages; 499 } 500 #endif 501 502 static int 503 __i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, 504 bool value) 505 { 506 int ret = 0; 507 508 /* During mm_invalidate_range we need to cancel any userptr that 509 * overlaps the range being invalidated. Doing so requires the 510 * struct_mutex, and that risks recursion. In order to cause 511 * recursion, the user must alias the userptr address space with 512 * a GTT mmapping (possible with a MAP_FIXED) - then when we have 513 * to invalidate that mmaping, mm_invalidate_range is called with 514 * the userptr address *and* the struct_mutex held. To prevent that 515 * we set a flag under the i915_mmu_notifier spinlock to indicate 516 * whether this object is valid. 517 */ 518 #if defined(CONFIG_MMU_NOTIFIER) 519 if (obj->userptr.mmu_object == NULL) 520 return 0; 521 522 lockmgr(&obj->userptr.mmu_object->mn->lock, LK_EXCLUSIVE); 523 /* In order to serialise get_pages with an outstanding 524 * cancel_userptr, we must drop the struct_mutex and try again. 525 */ 526 if (!value) 527 del_object(obj->userptr.mmu_object); 528 else if (!work_pending(&obj->userptr.mmu_object->work)) 529 add_object(obj->userptr.mmu_object); 530 else 531 ret = -EAGAIN; 532 lockmgr(&obj->userptr.mmu_object->mn->lock, LK_RELEASE); 533 #endif 534 535 return ret; 536 } 537 538 #if 0 539 static void 540 __i915_gem_userptr_get_pages_worker(struct work_struct *_work) 541 { 542 struct get_pages_work *work = container_of(_work, typeof(*work), work); 543 struct drm_i915_gem_object *obj = work->obj; 544 const int npages = obj->base.size >> PAGE_SHIFT; 545 struct page **pvec; 546 int pinned, ret; 547 548 ret = -ENOMEM; 549 pinned = 0; 550 551 pvec = drm_malloc_gfp(npages, sizeof(struct page *), GFP_TEMPORARY); 552 if (pvec != NULL) { 553 struct mm_struct *mm = obj->userptr.mm->mm; 554 555 ret = -EFAULT; 556 if (mmget_not_zero(mm)) { 557 down_read(&mm->mmap_sem); 558 while (pinned < npages) { 559 ret = get_user_pages_remote 560 (work->task, mm, 561 obj->userptr.ptr + pinned * PAGE_SIZE, 562 npages - pinned, 563 !obj->userptr.read_only, 0, 564 pvec + pinned, NULL); 565 if (ret < 0) 566 break; 567 568 pinned += ret; 569 } 570 up_read(&mm->mmap_sem); 571 mmput(mm); 572 } 573 } 574 575 mutex_lock(&obj->mm.lock); 576 if (obj->userptr.work == &work->work) { 577 struct sg_table *pages = ERR_PTR(ret); 578 579 if (pinned == npages) { 580 pages = __i915_gem_userptr_set_pages(obj, pvec, npages); 581 if (!IS_ERR(pages)) { 582 __i915_gem_object_set_pages(obj, pages); 583 pinned = 0; 584 pages = NULL; 585 } 586 } 587 588 obj->userptr.work = ERR_CAST(pages); 589 if (IS_ERR(pages)) 590 __i915_gem_userptr_set_active(obj, false); 591 } 592 mutex_unlock(&obj->mm.lock); 593 594 release_pages(pvec, pinned, 0); 595 drm_free_large(pvec); 596 597 i915_gem_object_put(obj); 598 put_task_struct(work->task); 599 kfree(work); 600 } 601 602 static struct sg_table * 603 __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj) 604 { 605 struct get_pages_work *work; 606 607 /* Spawn a worker so that we can acquire the 608 * user pages without holding our mutex. Access 609 * to the user pages requires mmap_sem, and we have 610 * a strict lock ordering of mmap_sem, struct_mutex - 611 * we already hold struct_mutex here and so cannot 612 * call gup without encountering a lock inversion. 613 * 614 * Userspace will keep on repeating the operation 615 * (thanks to EAGAIN) until either we hit the fast 616 * path or the worker completes. If the worker is 617 * cancelled or superseded, the task is still run 618 * but the results ignored. (This leads to 619 * complications that we may have a stray object 620 * refcount that we need to be wary of when 621 * checking for existing objects during creation.) 622 * If the worker encounters an error, it reports 623 * that error back to this function through 624 * obj->userptr.work = ERR_PTR. 625 */ 626 work = kmalloc(sizeof(*work), GFP_KERNEL); 627 if (work == NULL) 628 return ERR_PTR(-ENOMEM); 629 630 obj->userptr.work = &work->work; 631 632 work->obj = i915_gem_object_get(obj); 633 634 work->task = current; 635 get_task_struct(work->task); 636 637 INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker); 638 schedule_work(&work->work); 639 640 return ERR_PTR(-EAGAIN); 641 } 642 #endif 643 644 static struct sg_table * 645 i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) 646 { 647 #if 0 648 const int num_pages = obj->base.size >> PAGE_SHIFT; 649 struct mm_struct *mm = obj->userptr.mm->mm; 650 struct page **pvec; 651 struct sg_table *pages; 652 bool active; 653 int pinned; 654 655 /* If userspace should engineer that these pages are replaced in 656 * the vma between us binding this page into the GTT and completion 657 * of rendering... Their loss. If they change the mapping of their 658 * pages they need to create a new bo to point to the new vma. 659 * 660 * However, that still leaves open the possibility of the vma 661 * being copied upon fork. Which falls under the same userspace 662 * synchronisation issue as a regular bo, except that this time 663 * the process may not be expecting that a particular piece of 664 * memory is tied to the GPU. 665 * 666 * Fortunately, we can hook into the mmu_notifier in order to 667 * discard the page references prior to anything nasty happening 668 * to the vma (discard or cloning) which should prevent the more 669 * egregious cases from causing harm. 670 */ 671 672 if (obj->userptr.work) { 673 /* active flag should still be held for the pending work */ 674 if (IS_ERR(obj->userptr.work)) 675 return ERR_CAST(obj->userptr.work); 676 else 677 return ERR_PTR(-EAGAIN); 678 } 679 680 pvec = NULL; 681 pinned = 0; 682 683 if (mm == current->mm) { 684 pvec = drm_malloc_gfp(num_pages, sizeof(struct page *), 685 GFP_TEMPORARY | 686 __GFP_NORETRY | 687 __GFP_NOWARN); 688 if (pvec) /* defer to worker if malloc fails */ 689 pinned = __get_user_pages_fast(obj->userptr.ptr, 690 num_pages, 691 !obj->userptr.read_only, 692 pvec); 693 } 694 695 active = false; 696 if (pinned < 0) { 697 pages = ERR_PTR(pinned); 698 pinned = 0; 699 } else if (pinned < num_pages) { 700 pages = __i915_gem_userptr_get_pages_schedule(obj); 701 active = pages == ERR_PTR(-EAGAIN); 702 } else { 703 pages = __i915_gem_userptr_set_pages(obj, pvec, num_pages); 704 active = !IS_ERR(pages); 705 } 706 if (active) 707 __i915_gem_userptr_set_active(obj, true); 708 709 if (IS_ERR(pages)) 710 release_pages(pvec, pinned, 0); 711 drm_free_large(pvec); 712 713 return pages; 714 #else 715 return NULL; 716 #endif /* 0 */ 717 } 718 719 static void 720 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj, 721 struct sg_table *pages) 722 { 723 struct sgt_iter sgt_iter; 724 struct page *page; 725 726 BUG_ON(obj->userptr.work != NULL); 727 __i915_gem_userptr_set_active(obj, false); 728 729 if (obj->mm.madv != I915_MADV_WILLNEED) 730 obj->mm.dirty = false; 731 732 i915_gem_gtt_finish_pages(obj, pages); 733 734 for_each_sgt_page(page, sgt_iter, pages) { 735 if (obj->mm.dirty) 736 set_page_dirty(page); 737 738 mark_page_accessed(page); 739 #if 0 740 put_page(page); 741 #endif 742 } 743 obj->mm.dirty = false; 744 745 sg_free_table(pages); 746 kfree(pages); 747 } 748 749 static void 750 i915_gem_userptr_release(struct drm_i915_gem_object *obj) 751 { 752 i915_gem_userptr_release__mmu_notifier(obj); 753 i915_gem_userptr_release__mm_struct(obj); 754 } 755 756 static int 757 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj) 758 { 759 if (obj->userptr.mmu_object) 760 return 0; 761 762 return i915_gem_userptr_init__mmu_notifier(obj, 0); 763 } 764 765 static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { 766 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | 767 I915_GEM_OBJECT_IS_SHRINKABLE, 768 .get_pages = i915_gem_userptr_get_pages, 769 .put_pages = i915_gem_userptr_put_pages, 770 .dmabuf_export = i915_gem_userptr_dmabuf_export, 771 .release = i915_gem_userptr_release, 772 }; 773 774 /** 775 * Creates a new mm object that wraps some normal memory from the process 776 * context - user memory. 777 * 778 * We impose several restrictions upon the memory being mapped 779 * into the GPU. 780 * 1. It must be page aligned (both start/end addresses, i.e ptr and size). 781 * 2. It must be normal system memory, not a pointer into another map of IO 782 * space (e.g. it must not be a GTT mmapping of another object). 783 * 3. We only allow a bo as large as we could in theory map into the GTT, 784 * that is we limit the size to the total size of the GTT. 785 * 4. The bo is marked as being snoopable. The backing pages are left 786 * accessible directly by the CPU, but reads and writes by the GPU may 787 * incur the cost of a snoop (unless you have an LLC architecture). 788 * 789 * Synchronisation between multiple users and the GPU is left to userspace 790 * through the normal set-domain-ioctl. The kernel will enforce that the 791 * GPU relinquishes the VMA before it is returned back to the system 792 * i.e. upon free(), munmap() or process termination. However, the userspace 793 * malloc() library may not immediately relinquish the VMA after free() and 794 * instead reuse it whilst the GPU is still reading and writing to the VMA. 795 * Caveat emptor. 796 * 797 * Also note, that the object created here is not currently a "first class" 798 * object, in that several ioctls are banned. These are the CPU access 799 * ioctls: mmap(), pwrite and pread. In practice, you are expected to use 800 * direct access via your pointer rather than use those ioctls. Another 801 * restriction is that we do not allow userptr surfaces to be pinned to the 802 * hardware and so we reject any attempt to create a framebuffer out of a 803 * userptr. 804 * 805 * If you think this is a good interface to use to pass GPU memory between 806 * drivers, please use dma-buf instead. In fact, wherever possible use 807 * dma-buf instead. 808 */ 809 int 810 i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 811 { 812 struct drm_i915_private *dev_priv = to_i915(dev); 813 struct drm_i915_gem_userptr *args = data; 814 struct drm_i915_gem_object *obj; 815 int ret; 816 u32 handle; 817 818 if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) { 819 /* We cannot support coherent userptr objects on hw without 820 * LLC and broken snooping. 821 */ 822 return -ENODEV; 823 } 824 825 if (args->flags & ~(I915_USERPTR_READ_ONLY | 826 I915_USERPTR_UNSYNCHRONIZED)) 827 return -EINVAL; 828 829 if (offset_in_page(args->user_ptr | args->user_size)) 830 return -EINVAL; 831 832 #if 0 833 if (!access_ok(args->flags & I915_USERPTR_READ_ONLY ? VERIFY_READ : VERIFY_WRITE, 834 (char __user *)(unsigned long)args->user_ptr, args->user_size)) 835 return -EFAULT; 836 #endif 837 838 if (args->flags & I915_USERPTR_READ_ONLY) { 839 /* On almost all of the current hw, we cannot tell the GPU that a 840 * page is readonly, so this is just a placeholder in the uAPI. 841 */ 842 return -ENODEV; 843 } 844 845 obj = i915_gem_object_alloc(dev_priv); 846 if (obj == NULL) 847 return -ENOMEM; 848 849 drm_gem_private_object_init(dev, &obj->base, args->user_size); 850 i915_gem_object_init(obj, &i915_gem_userptr_ops); 851 obj->cache_level = I915_CACHE_LLC; 852 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 853 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 854 855 obj->userptr.ptr = args->user_ptr; 856 obj->userptr.read_only = !!(args->flags & I915_USERPTR_READ_ONLY); 857 858 /* And keep a pointer to the current->mm for resolving the user pages 859 * at binding. This means that we need to hook into the mmu_notifier 860 * in order to detect if the mmu is destroyed. 861 */ 862 ret = i915_gem_userptr_init__mm_struct(obj); 863 if (ret == 0) 864 ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags); 865 if (ret == 0) 866 ret = drm_gem_handle_create(file, &obj->base, &handle); 867 868 /* drop reference from allocate - handle holds it now */ 869 i915_gem_object_put(obj); 870 if (ret) 871 return ret; 872 873 args->handle = handle; 874 return 0; 875 } 876 877 void i915_gem_init_userptr(struct drm_i915_private *dev_priv) 878 { 879 lockinit(&dev_priv->mm_lock, "i915dmm", 0, LK_CANRECURSE); 880 hash_init(dev_priv->mm_structs); 881 } 882