1 /* 2 * Copyright © 2012-2014 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <drm/drmP.h> 26 #include <drm/i915_drm.h> 27 #include "i915_drv.h" 28 #include "i915_trace.h" 29 #include "intel_drv.h" 30 #include <linux/mmu_context.h> 31 #include <linux/mmu_notifier.h> 32 #include <linux/mempolicy.h> 33 #include <linux/swap.h> 34 35 struct i915_mm_struct { 36 struct mm_struct *mm; 37 struct drm_i915_private *i915; 38 struct i915_mmu_notifier *mn; 39 struct hlist_node node; 40 struct kref kref; 41 struct work_struct work; 42 }; 43 44 #if defined(CONFIG_MMU_NOTIFIER) 45 #include <linux/interval_tree.h> 46 47 struct i915_mmu_notifier { 48 spinlock_t lock; 49 struct hlist_node node; 50 struct mmu_notifier mn; 51 struct rb_root objects; 52 struct workqueue_struct *wq; 53 }; 54 55 struct i915_mmu_object { 56 struct i915_mmu_notifier *mn; 57 struct drm_i915_gem_object *obj; 58 struct interval_tree_node it; 59 struct list_head link; 60 struct work_struct work; 61 bool attached; 62 }; 63 64 static void cancel_userptr(struct work_struct *work) 65 { 66 struct i915_mmu_object *mo = container_of(work, typeof(*mo), work); 67 struct drm_i915_gem_object *obj = mo->obj; 68 struct drm_device *dev = obj->base.dev; 69 70 i915_gem_object_wait(obj, I915_WAIT_ALL, MAX_SCHEDULE_TIMEOUT, NULL); 71 72 mutex_lock(&dev->struct_mutex); 73 /* Cancel any active worker and force us to re-evaluate gup */ 74 obj->userptr.work = NULL; 75 76 /* We are inside a kthread context and can't be interrupted */ 77 if (i915_gem_object_unbind(obj) == 0) 78 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 79 WARN_ONCE(obj->mm.pages, 80 "Failed to release pages: bind_count=%d, pages_pin_count=%d, pin_display=%d\n", 81 obj->bind_count, 82 atomic_read(&obj->mm.pages_pin_count), 83 obj->pin_display); 84 85 i915_gem_object_put(obj); 86 mutex_unlock(&dev->struct_mutex); 87 } 88 89 static void add_object(struct i915_mmu_object *mo) 90 { 91 if (mo->attached) 92 return; 93 94 interval_tree_insert(&mo->it, &mo->mn->objects); 95 mo->attached = true; 96 } 97 98 static void del_object(struct i915_mmu_object *mo) 99 { 100 if (!mo->attached) 101 return; 102 103 interval_tree_remove(&mo->it, &mo->mn->objects); 104 mo->attached = false; 105 } 106 107 static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, 108 struct mm_struct *mm, 109 unsigned long start, 110 unsigned long end) 111 { 112 struct i915_mmu_notifier *mn = 113 container_of(_mn, struct i915_mmu_notifier, mn); 114 struct i915_mmu_object *mo; 115 struct interval_tree_node *it; 116 LINUX_LIST_HEAD(cancelled); 117 118 if (RB_EMPTY_ROOT(&mn->objects)) 119 return; 120 121 /* interval ranges are inclusive, but invalidate range is exclusive */ 122 end--; 123 124 lockmgr(&mn->lock, LK_EXCLUSIVE); 125 it = interval_tree_iter_first(&mn->objects, start, end); 126 while (it) { 127 /* The mmu_object is released late when destroying the 128 * GEM object so it is entirely possible to gain a 129 * reference on an object in the process of being freed 130 * since our serialisation is via the spinlock and not 131 * the struct_mutex - and consequently use it after it 132 * is freed and then double free it. To prevent that 133 * use-after-free we only acquire a reference on the 134 * object if it is not in the process of being destroyed. 135 */ 136 mo = container_of(it, struct i915_mmu_object, it); 137 if (kref_get_unless_zero(&mo->obj->base.refcount)) 138 queue_work(mn->wq, &mo->work); 139 140 list_add(&mo->link, &cancelled); 141 it = interval_tree_iter_next(it, start, end); 142 } 143 list_for_each_entry(mo, &cancelled, link) 144 del_object(mo); 145 lockmgr(&mn->lock, LK_RELEASE); 146 147 flush_workqueue(mn->wq); 148 } 149 150 static const struct mmu_notifier_ops i915_gem_userptr_notifier = { 151 .invalidate_range_start = i915_gem_userptr_mn_invalidate_range_start, 152 }; 153 154 static struct i915_mmu_notifier * 155 i915_mmu_notifier_create(struct mm_struct *mm) 156 { 157 struct i915_mmu_notifier *mn; 158 int ret; 159 160 mn = kmalloc(sizeof(*mn), M_DRM, GFP_KERNEL); 161 if (mn == NULL) 162 return ERR_PTR(-ENOMEM); 163 164 spin_lock_init(&mn->lock); 165 mn->mn.ops = &i915_gem_userptr_notifier; 166 mn->objects = LINUX_RB_ROOT; 167 mn->wq = alloc_workqueue("i915-userptr-release", WQ_UNBOUND, 0); 168 if (mn->wq == NULL) { 169 kfree(mn); 170 return ERR_PTR(-ENOMEM); 171 } 172 173 /* Protected by mmap_sem (write-lock) */ 174 ret = __mmu_notifier_register(&mn->mn, mm); 175 if (ret) { 176 destroy_workqueue(mn->wq); 177 kfree(mn); 178 return ERR_PTR(ret); 179 } 180 181 return mn; 182 } 183 184 static void 185 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 186 { 187 struct i915_mmu_object *mo; 188 189 mo = obj->userptr.mmu_object; 190 if (mo == NULL) 191 return; 192 193 lockmgr(&mo->mn->lock, LK_EXCLUSIVE); 194 del_object(mo); 195 lockmgr(&mo->mn->lock, LK_RELEASE); 196 kfree(mo); 197 198 obj->userptr.mmu_object = NULL; 199 } 200 201 static struct i915_mmu_notifier * 202 i915_mmu_notifier_find(struct i915_mm_struct *mm) 203 { 204 struct i915_mmu_notifier *mn = mm->mn; 205 206 mn = mm->mn; 207 if (mn) 208 return mn; 209 210 down_write(&mm->mm->mmap_sem); 211 mutex_lock(&mm->i915->mm_lock); 212 if ((mn = mm->mn) == NULL) { 213 mn = i915_mmu_notifier_create(mm->mm); 214 if (!IS_ERR(mn)) 215 mm->mn = mn; 216 } 217 mutex_unlock(&mm->i915->mm_lock); 218 up_write(&mm->mm->mmap_sem); 219 220 return mn; 221 } 222 223 static int 224 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 225 unsigned flags) 226 { 227 struct i915_mmu_notifier *mn; 228 struct i915_mmu_object *mo; 229 230 if (flags & I915_USERPTR_UNSYNCHRONIZED) 231 return capable(CAP_SYS_ADMIN) ? 0 : -EPERM; 232 233 if (WARN_ON(obj->userptr.mm == NULL)) 234 return -EINVAL; 235 236 mn = i915_mmu_notifier_find(obj->userptr.mm); 237 if (IS_ERR(mn)) 238 return PTR_ERR(mn); 239 240 mo = kzalloc(sizeof(*mo), GFP_KERNEL); 241 if (mo == NULL) 242 return -ENOMEM; 243 244 mo->mn = mn; 245 mo->obj = obj; 246 mo->it.start = obj->userptr.ptr; 247 mo->it.last = obj->userptr.ptr + obj->base.size - 1; 248 INIT_WORK(&mo->work, cancel_userptr); 249 250 obj->userptr.mmu_object = mo; 251 return 0; 252 } 253 254 static void 255 i915_mmu_notifier_free(struct i915_mmu_notifier *mn, 256 struct mm_struct *mm) 257 { 258 if (mn == NULL) 259 return; 260 261 mmu_notifier_unregister(&mn->mn, mm); 262 destroy_workqueue(mn->wq); 263 kfree(mn); 264 } 265 266 #else 267 268 static void 269 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 270 { 271 } 272 273 static int 274 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 275 unsigned flags) 276 { 277 if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0) 278 return -ENODEV; 279 280 if (!capable(CAP_SYS_ADMIN)) 281 return -EPERM; 282 283 return 0; 284 } 285 286 static void 287 i915_mmu_notifier_free(struct i915_mmu_notifier *mn, 288 struct mm_struct *mm) 289 { 290 } 291 292 #endif 293 294 static struct i915_mm_struct * 295 __i915_mm_struct_find(struct drm_i915_private *dev_priv, struct mm_struct *real) 296 { 297 struct i915_mm_struct *mm; 298 299 /* Protected by dev_priv->mm_lock */ 300 hash_for_each_possible(dev_priv->mm_structs, mm, node, (unsigned long)real) 301 if (mm->mm == real) 302 return mm; 303 304 return NULL; 305 } 306 307 static int 308 i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj) 309 { 310 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 311 struct i915_mm_struct *mm; 312 int ret = 0; 313 314 /* During release of the GEM object we hold the struct_mutex. This 315 * precludes us from calling mmput() at that time as that may be 316 * the last reference and so call exit_mmap(). exit_mmap() will 317 * attempt to reap the vma, and if we were holding a GTT mmap 318 * would then call drm_gem_vm_close() and attempt to reacquire 319 * the struct mutex. So in order to avoid that recursion, we have 320 * to defer releasing the mm reference until after we drop the 321 * struct_mutex, i.e. we need to schedule a worker to do the clean 322 * up. 323 */ 324 mutex_lock(&dev_priv->mm_lock); 325 mm = __i915_mm_struct_find(dev_priv, current->mm); 326 if (mm == NULL) { 327 mm = kmalloc(sizeof(*mm), M_DRM, GFP_KERNEL); 328 if (mm == NULL) { 329 ret = -ENOMEM; 330 goto out; 331 } 332 333 kref_init(&mm->kref); 334 mm->i915 = to_i915(obj->base.dev); 335 336 mm->mm = current->mm; 337 atomic_inc(¤t->mm->mm_count); 338 339 mm->mn = NULL; 340 341 /* Protected by dev_priv->mm_lock */ 342 hash_add(dev_priv->mm_structs, 343 &mm->node, (unsigned long)mm->mm); 344 } else 345 kref_get(&mm->kref); 346 347 obj->userptr.mm = mm; 348 out: 349 mutex_unlock(&dev_priv->mm_lock); 350 return ret; 351 } 352 353 static void 354 __i915_mm_struct_free__worker(struct work_struct *work) 355 { 356 struct i915_mm_struct *mm = container_of(work, typeof(*mm), work); 357 i915_mmu_notifier_free(mm->mn, mm->mm); 358 #if 0 359 mmdrop(mm->mm); 360 #endif 361 kfree(mm); 362 } 363 364 static void 365 __i915_mm_struct_free(struct kref *kref) 366 { 367 struct i915_mm_struct *mm = container_of(kref, typeof(*mm), kref); 368 369 /* Protected by dev_priv->mm_lock */ 370 hash_del(&mm->node); 371 mutex_unlock(&mm->i915->mm_lock); 372 373 INIT_WORK(&mm->work, __i915_mm_struct_free__worker); 374 schedule_work(&mm->work); 375 } 376 377 static void 378 i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj) 379 { 380 if (obj->userptr.mm == NULL) 381 return; 382 383 kref_put_mutex(&obj->userptr.mm->kref, 384 __i915_mm_struct_free, 385 &to_i915(obj->base.dev)->mm_lock); 386 obj->userptr.mm = NULL; 387 } 388 389 struct get_pages_work { 390 struct work_struct work; 391 struct drm_i915_gem_object *obj; 392 struct task_struct *task; 393 }; 394 395 #if IS_ENABLED(CONFIG_SWIOTLB) 396 #define swiotlb_active() swiotlb_nr_tbl() 397 #else 398 #define swiotlb_active() 0 399 #endif 400 401 #if 0 402 static int 403 st_set_pages(struct sg_table **st, struct page **pvec, int num_pages) 404 { 405 struct scatterlist *sg; 406 int ret, n; 407 408 *st = kmalloc(sizeof(**st), M_DRM, GFP_KERNEL); 409 if (*st == NULL) 410 return -ENOMEM; 411 412 if (swiotlb_active()) { 413 ret = sg_alloc_table(*st, num_pages, GFP_KERNEL); 414 if (ret) 415 goto err; 416 417 for_each_sg((*st)->sgl, sg, num_pages, n) 418 sg_set_page(sg, pvec[n], PAGE_SIZE, 0); 419 } else { 420 ret = sg_alloc_table_from_pages(*st, pvec, num_pages, 421 0, num_pages << PAGE_SHIFT, 422 GFP_KERNEL); 423 if (ret) 424 goto err; 425 } 426 427 return 0; 428 429 err: 430 kfree(*st); 431 *st = NULL; 432 return ret; 433 } 434 435 static struct sg_table * 436 __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj, 437 struct page **pvec, int num_pages) 438 { 439 struct sg_table *pages; 440 int ret; 441 442 ret = st_set_pages(&pages, pvec, num_pages); 443 if (ret) 444 return ERR_PTR(ret); 445 446 ret = i915_gem_gtt_prepare_pages(obj, pages); 447 if (ret) { 448 sg_free_table(pages); 449 kfree(pages); 450 return ERR_PTR(ret); 451 } 452 453 return pages; 454 } 455 #endif 456 457 static int 458 __i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, 459 bool value) 460 { 461 int ret = 0; 462 463 /* During mm_invalidate_range we need to cancel any userptr that 464 * overlaps the range being invalidated. Doing so requires the 465 * struct_mutex, and that risks recursion. In order to cause 466 * recursion, the user must alias the userptr address space with 467 * a GTT mmapping (possible with a MAP_FIXED) - then when we have 468 * to invalidate that mmaping, mm_invalidate_range is called with 469 * the userptr address *and* the struct_mutex held. To prevent that 470 * we set a flag under the i915_mmu_notifier spinlock to indicate 471 * whether this object is valid. 472 */ 473 #if defined(CONFIG_MMU_NOTIFIER) 474 if (obj->userptr.mmu_object == NULL) 475 return 0; 476 477 lockmgr(&obj->userptr.mmu_object->mn->lock, LK_EXCLUSIVE); 478 /* In order to serialise get_pages with an outstanding 479 * cancel_userptr, we must drop the struct_mutex and try again. 480 */ 481 if (!value) 482 del_object(obj->userptr.mmu_object); 483 else if (!work_pending(&obj->userptr.mmu_object->work)) 484 add_object(obj->userptr.mmu_object); 485 else 486 ret = -EAGAIN; 487 lockmgr(&obj->userptr.mmu_object->mn->lock, LK_RELEASE); 488 #endif 489 490 return ret; 491 } 492 493 #if 0 494 static void 495 __i915_gem_userptr_get_pages_worker(struct work_struct *_work) 496 { 497 struct get_pages_work *work = container_of(_work, typeof(*work), work); 498 struct drm_i915_gem_object *obj = work->obj; 499 const int npages = obj->base.size >> PAGE_SHIFT; 500 struct page **pvec; 501 int pinned, ret; 502 503 ret = -ENOMEM; 504 pinned = 0; 505 506 pvec = drm_malloc_gfp(npages, sizeof(struct page *), GFP_TEMPORARY); 507 if (pvec != NULL) { 508 struct mm_struct *mm = obj->userptr.mm->mm; 509 unsigned int flags = 0; 510 511 if (!obj->userptr.read_only) 512 flags |= FOLL_WRITE; 513 514 ret = -EFAULT; 515 if (atomic_inc_not_zero(&mm->mm_users)) { 516 down_read(&mm->mmap_sem); 517 while (pinned < npages) { 518 ret = get_user_pages_remote 519 (work->task, mm, 520 obj->userptr.ptr + pinned * PAGE_SIZE, 521 npages - pinned, 522 flags, 523 pvec + pinned, NULL, NULL); 524 if (ret < 0) 525 break; 526 527 pinned += ret; 528 } 529 up_read(&mm->mmap_sem); 530 mmput(mm); 531 } 532 } 533 534 mutex_lock(&obj->mm.lock); 535 if (obj->userptr.work == &work->work) { 536 struct sg_table *pages = ERR_PTR(ret); 537 538 if (pinned == npages) { 539 pages = __i915_gem_userptr_set_pages(obj, pvec, npages); 540 if (!IS_ERR(pages)) { 541 __i915_gem_object_set_pages(obj, pages); 542 pinned = 0; 543 pages = NULL; 544 } 545 } 546 547 obj->userptr.work = ERR_CAST(pages); 548 } 549 mutex_unlock(&obj->mm.lock); 550 551 release_pages(pvec, pinned, 0); 552 drm_free_large(pvec); 553 554 i915_gem_object_put(obj); 555 put_task_struct(work->task); 556 kfree(work); 557 } 558 559 static struct sg_table * 560 __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj, 561 bool *active) 562 { 563 struct get_pages_work *work; 564 565 /* Spawn a worker so that we can acquire the 566 * user pages without holding our mutex. Access 567 * to the user pages requires mmap_sem, and we have 568 * a strict lock ordering of mmap_sem, struct_mutex - 569 * we already hold struct_mutex here and so cannot 570 * call gup without encountering a lock inversion. 571 * 572 * Userspace will keep on repeating the operation 573 * (thanks to EAGAIN) until either we hit the fast 574 * path or the worker completes. If the worker is 575 * cancelled or superseded, the task is still run 576 * but the results ignored. (This leads to 577 * complications that we may have a stray object 578 * refcount that we need to be wary of when 579 * checking for existing objects during creation.) 580 * If the worker encounters an error, it reports 581 * that error back to this function through 582 * obj->userptr.work = ERR_PTR. 583 */ 584 work = kmalloc(sizeof(*work), M_DRM, GFP_KERNEL); 585 if (work == NULL) 586 return ERR_PTR(-ENOMEM); 587 588 obj->userptr.work = &work->work; 589 590 work->obj = i915_gem_object_get(obj); 591 592 work->task = current; 593 get_task_struct(work->task); 594 595 INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker); 596 schedule_work(&work->work); 597 598 *active = true; 599 return ERR_PTR(-EAGAIN); 600 } 601 #endif 602 603 static struct sg_table * 604 i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) 605 { 606 #if 0 607 const int num_pages = obj->base.size >> PAGE_SHIFT; 608 struct page **pvec; 609 struct sg_table *pages; 610 int pinned, ret; 611 bool active; 612 613 /* If userspace should engineer that these pages are replaced in 614 * the vma between us binding this page into the GTT and completion 615 * of rendering... Their loss. If they change the mapping of their 616 * pages they need to create a new bo to point to the new vma. 617 * 618 * However, that still leaves open the possibility of the vma 619 * being copied upon fork. Which falls under the same userspace 620 * synchronisation issue as a regular bo, except that this time 621 * the process may not be expecting that a particular piece of 622 * memory is tied to the GPU. 623 * 624 * Fortunately, we can hook into the mmu_notifier in order to 625 * discard the page references prior to anything nasty happening 626 * to the vma (discard or cloning) which should prevent the more 627 * egregious cases from causing harm. 628 */ 629 630 if (obj->userptr.work) { 631 /* active flag should still be held for the pending work */ 632 if (IS_ERR(obj->userptr.work)) 633 return ERR_CAST(obj->userptr.work); 634 else 635 return ERR_PTR(-EAGAIN); 636 } 637 638 /* Let the mmu-notifier know that we have begun and need cancellation */ 639 ret = __i915_gem_userptr_set_active(obj, true); 640 if (ret) 641 return ERR_PTR(ret); 642 643 pvec = NULL; 644 pinned = 0; 645 if (obj->userptr.mm->mm == current->mm) { 646 pvec = drm_malloc_gfp(num_pages, sizeof(struct page *), 647 GFP_TEMPORARY); 648 if (pvec == NULL) { 649 __i915_gem_userptr_set_active(obj, false); 650 return ERR_PTR(-ENOMEM); 651 } 652 653 pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages, 654 !obj->userptr.read_only, pvec); 655 } 656 657 active = false; 658 if (pinned < 0) 659 pages = ERR_PTR(pinned), pinned = 0; 660 else if (pinned < num_pages) 661 pages = __i915_gem_userptr_get_pages_schedule(obj, &active); 662 else 663 pages = __i915_gem_userptr_set_pages(obj, pvec, num_pages); 664 if (IS_ERR(pages)) { 665 __i915_gem_userptr_set_active(obj, active); 666 release_pages(pvec, pinned, 0); 667 } 668 drm_free_large(pvec); 669 return pages; 670 #else 671 return ERR_PTR(-ENOMEM); 672 #endif /* 0 */ 673 } 674 675 static void 676 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj, 677 struct sg_table *pages) 678 { 679 struct sgt_iter sgt_iter; 680 struct page *page; 681 682 BUG_ON(obj->userptr.work != NULL); 683 __i915_gem_userptr_set_active(obj, false); 684 685 if (obj->mm.madv != I915_MADV_WILLNEED) 686 obj->mm.dirty = false; 687 688 i915_gem_gtt_finish_pages(obj, pages); 689 690 for_each_sgt_page(page, sgt_iter, pages) { 691 if (obj->mm.dirty) 692 set_page_dirty(page); 693 694 mark_page_accessed(page); 695 put_page(page); 696 } 697 obj->mm.dirty = false; 698 699 sg_free_table(pages); 700 kfree(pages); 701 } 702 703 static void 704 i915_gem_userptr_release(struct drm_i915_gem_object *obj) 705 { 706 i915_gem_userptr_release__mmu_notifier(obj); 707 i915_gem_userptr_release__mm_struct(obj); 708 } 709 710 static int 711 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj) 712 { 713 if (obj->userptr.mmu_object) 714 return 0; 715 716 return i915_gem_userptr_init__mmu_notifier(obj, 0); 717 } 718 719 static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { 720 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | 721 I915_GEM_OBJECT_IS_SHRINKABLE, 722 .get_pages = i915_gem_userptr_get_pages, 723 .put_pages = i915_gem_userptr_put_pages, 724 .dmabuf_export = i915_gem_userptr_dmabuf_export, 725 .release = i915_gem_userptr_release, 726 }; 727 728 /** 729 * Creates a new mm object that wraps some normal memory from the process 730 * context - user memory. 731 * 732 * We impose several restrictions upon the memory being mapped 733 * into the GPU. 734 * 1. It must be page aligned (both start/end addresses, i.e ptr and size). 735 * 2. It must be normal system memory, not a pointer into another map of IO 736 * space (e.g. it must not be a GTT mmapping of another object). 737 * 3. We only allow a bo as large as we could in theory map into the GTT, 738 * that is we limit the size to the total size of the GTT. 739 * 4. The bo is marked as being snoopable. The backing pages are left 740 * accessible directly by the CPU, but reads and writes by the GPU may 741 * incur the cost of a snoop (unless you have an LLC architecture). 742 * 743 * Synchronisation between multiple users and the GPU is left to userspace 744 * through the normal set-domain-ioctl. The kernel will enforce that the 745 * GPU relinquishes the VMA before it is returned back to the system 746 * i.e. upon free(), munmap() or process termination. However, the userspace 747 * malloc() library may not immediately relinquish the VMA after free() and 748 * instead reuse it whilst the GPU is still reading and writing to the VMA. 749 * Caveat emptor. 750 * 751 * Also note, that the object created here is not currently a "first class" 752 * object, in that several ioctls are banned. These are the CPU access 753 * ioctls: mmap(), pwrite and pread. In practice, you are expected to use 754 * direct access via your pointer rather than use those ioctls. Another 755 * restriction is that we do not allow userptr surfaces to be pinned to the 756 * hardware and so we reject any attempt to create a framebuffer out of a 757 * userptr. 758 * 759 * If you think this is a good interface to use to pass GPU memory between 760 * drivers, please use dma-buf instead. In fact, wherever possible use 761 * dma-buf instead. 762 */ 763 int 764 i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 765 { 766 struct drm_i915_private *dev_priv = to_i915(dev); 767 struct drm_i915_gem_userptr *args = data; 768 struct drm_i915_gem_object *obj; 769 int ret; 770 u32 handle; 771 772 if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) { 773 /* We cannot support coherent userptr objects on hw without 774 * LLC and broken snooping. 775 */ 776 return -ENODEV; 777 } 778 779 if (args->flags & ~(I915_USERPTR_READ_ONLY | 780 I915_USERPTR_UNSYNCHRONIZED)) 781 return -EINVAL; 782 783 if (offset_in_page(args->user_ptr | args->user_size)) 784 return -EINVAL; 785 786 #if 0 787 if (!access_ok(args->flags & I915_USERPTR_READ_ONLY ? VERIFY_READ : VERIFY_WRITE, 788 (char __user *)(unsigned long)args->user_ptr, args->user_size)) 789 return -EFAULT; 790 #endif 791 792 if (args->flags & I915_USERPTR_READ_ONLY) { 793 /* On almost all of the current hw, we cannot tell the GPU that a 794 * page is readonly, so this is just a placeholder in the uAPI. 795 */ 796 return -ENODEV; 797 } 798 799 obj = i915_gem_object_alloc(dev); 800 if (obj == NULL) 801 return -ENOMEM; 802 803 drm_gem_private_object_init(dev, &obj->base, args->user_size); 804 i915_gem_object_init(obj, &i915_gem_userptr_ops); 805 obj->cache_level = I915_CACHE_LLC; 806 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 807 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 808 809 obj->userptr.ptr = args->user_ptr; 810 obj->userptr.read_only = !!(args->flags & I915_USERPTR_READ_ONLY); 811 812 /* And keep a pointer to the current->mm for resolving the user pages 813 * at binding. This means that we need to hook into the mmu_notifier 814 * in order to detect if the mmu is destroyed. 815 */ 816 ret = i915_gem_userptr_init__mm_struct(obj); 817 if (ret == 0) 818 ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags); 819 if (ret == 0) 820 ret = drm_gem_handle_create(file, &obj->base, &handle); 821 822 /* drop reference from allocate - handle holds it now */ 823 i915_gem_object_put(obj); 824 if (ret) 825 return ret; 826 827 args->handle = handle; 828 return 0; 829 } 830 831 void i915_gem_init_userptr(struct drm_i915_private *dev_priv) 832 { 833 lockinit(&dev_priv->mm_lock, "i915dmm", 0, LK_CANRECURSE); 834 hash_init(dev_priv->mm_structs); 835 } 836