1 /* 2 * Copyright © 2012-2014 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <drm/drmP.h> 26 #include <drm/i915_drm.h> 27 #include "i915_drv.h" 28 #include "i915_trace.h" 29 #include "intel_drv.h" 30 #include <linux/mmu_context.h> 31 #include <linux/mmu_notifier.h> 32 #include <linux/swap.h> 33 34 struct i915_mm_struct { 35 struct mm_struct *mm; 36 struct drm_i915_private *i915; 37 struct i915_mmu_notifier *mn; 38 struct hlist_node node; 39 struct kref kref; 40 struct work_struct work; 41 }; 42 43 #if defined(CONFIG_MMU_NOTIFIER) 44 #include <linux/interval_tree.h> 45 46 struct i915_mmu_notifier { 47 spinlock_t lock; 48 struct hlist_node node; 49 struct mmu_notifier mn; 50 struct rb_root objects; 51 struct workqueue_struct *wq; 52 }; 53 54 struct i915_mmu_object { 55 struct i915_mmu_notifier *mn; 56 struct drm_i915_gem_object *obj; 57 struct interval_tree_node it; 58 struct list_head link; 59 struct work_struct work; 60 bool attached; 61 }; 62 63 static void wait_rendering(struct drm_i915_gem_object *obj) 64 { 65 unsigned long active = __I915_BO_ACTIVE(obj); 66 int idx; 67 68 for_each_active(active, idx) 69 i915_gem_active_wait_unlocked(&obj->last_read[idx], 70 0, NULL, NULL); 71 } 72 73 static void cancel_userptr(struct work_struct *work) 74 { 75 struct i915_mmu_object *mo = container_of(work, typeof(*mo), work); 76 struct drm_i915_gem_object *obj = mo->obj; 77 struct drm_device *dev = obj->base.dev; 78 79 wait_rendering(obj); 80 81 mutex_lock(&dev->struct_mutex); 82 /* Cancel any active worker and force us to re-evaluate gup */ 83 obj->userptr.work = NULL; 84 85 if (obj->pages != NULL) { 86 /* We are inside a kthread context and can't be interrupted */ 87 WARN_ON(i915_gem_object_unbind(obj)); 88 WARN_ON(i915_gem_object_put_pages(obj)); 89 } 90 91 i915_gem_object_put(obj); 92 mutex_unlock(&dev->struct_mutex); 93 } 94 95 static void add_object(struct i915_mmu_object *mo) 96 { 97 if (mo->attached) 98 return; 99 100 interval_tree_insert(&mo->it, &mo->mn->objects); 101 mo->attached = true; 102 } 103 104 static void del_object(struct i915_mmu_object *mo) 105 { 106 if (!mo->attached) 107 return; 108 109 interval_tree_remove(&mo->it, &mo->mn->objects); 110 mo->attached = false; 111 } 112 113 static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, 114 struct mm_struct *mm, 115 unsigned long start, 116 unsigned long end) 117 { 118 struct i915_mmu_notifier *mn = 119 container_of(_mn, struct i915_mmu_notifier, mn); 120 struct i915_mmu_object *mo; 121 struct interval_tree_node *it; 122 LIST_HEAD(cancelled); 123 124 if (RB_EMPTY_ROOT(&mn->objects)) 125 return; 126 127 /* interval ranges are inclusive, but invalidate range is exclusive */ 128 end--; 129 130 lockmgr(&mn->lock, LK_EXCLUSIVE); 131 it = interval_tree_iter_first(&mn->objects, start, end); 132 while (it) { 133 /* The mmu_object is released late when destroying the 134 * GEM object so it is entirely possible to gain a 135 * reference on an object in the process of being freed 136 * since our serialisation is via the spinlock and not 137 * the struct_mutex - and consequently use it after it 138 * is freed and then double free it. To prevent that 139 * use-after-free we only acquire a reference on the 140 * object if it is not in the process of being destroyed. 141 */ 142 mo = container_of(it, struct i915_mmu_object, it); 143 if (kref_get_unless_zero(&mo->obj->base.refcount)) 144 queue_work(mn->wq, &mo->work); 145 146 list_add(&mo->link, &cancelled); 147 it = interval_tree_iter_next(it, start, end); 148 } 149 list_for_each_entry(mo, &cancelled, link) 150 del_object(mo); 151 lockmgr(&mn->lock, LK_RELEASE); 152 153 flush_workqueue(mn->wq); 154 } 155 156 static const struct mmu_notifier_ops i915_gem_userptr_notifier = { 157 .invalidate_range_start = i915_gem_userptr_mn_invalidate_range_start, 158 }; 159 160 static struct i915_mmu_notifier * 161 i915_mmu_notifier_create(struct mm_struct *mm) 162 { 163 struct i915_mmu_notifier *mn; 164 int ret; 165 166 mn = kmalloc(sizeof(*mn), M_DRM, GFP_KERNEL); 167 if (mn == NULL) 168 return ERR_PTR(-ENOMEM); 169 170 spin_lock_init(&mn->lock); 171 mn->mn.ops = &i915_gem_userptr_notifier; 172 mn->objects = RB_ROOT; 173 mn->wq = alloc_workqueue("i915-userptr-release", WQ_UNBOUND, 0); 174 if (mn->wq == NULL) { 175 kfree(mn); 176 return ERR_PTR(-ENOMEM); 177 } 178 179 /* Protected by mmap_sem (write-lock) */ 180 ret = __mmu_notifier_register(&mn->mn, mm); 181 if (ret) { 182 destroy_workqueue(mn->wq); 183 kfree(mn); 184 return ERR_PTR(ret); 185 } 186 187 return mn; 188 } 189 190 static void 191 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 192 { 193 struct i915_mmu_object *mo; 194 195 mo = obj->userptr.mmu_object; 196 if (mo == NULL) 197 return; 198 199 lockmgr(&mo->mn->lock, LK_EXCLUSIVE); 200 del_object(mo); 201 lockmgr(&mo->mn->lock, LK_RELEASE); 202 kfree(mo); 203 204 obj->userptr.mmu_object = NULL; 205 } 206 207 static struct i915_mmu_notifier * 208 i915_mmu_notifier_find(struct i915_mm_struct *mm) 209 { 210 struct i915_mmu_notifier *mn = mm->mn; 211 212 mn = mm->mn; 213 if (mn) 214 return mn; 215 216 down_write(&mm->mm->mmap_sem); 217 mutex_lock(&mm->i915->mm_lock); 218 if ((mn = mm->mn) == NULL) { 219 mn = i915_mmu_notifier_create(mm->mm); 220 if (!IS_ERR(mn)) 221 mm->mn = mn; 222 } 223 mutex_unlock(&mm->i915->mm_lock); 224 up_write(&mm->mm->mmap_sem); 225 226 return mn; 227 } 228 229 static int 230 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 231 unsigned flags) 232 { 233 struct i915_mmu_notifier *mn; 234 struct i915_mmu_object *mo; 235 236 if (flags & I915_USERPTR_UNSYNCHRONIZED) 237 return capable(CAP_SYS_ADMIN) ? 0 : -EPERM; 238 239 if (WARN_ON(obj->userptr.mm == NULL)) 240 return -EINVAL; 241 242 mn = i915_mmu_notifier_find(obj->userptr.mm); 243 if (IS_ERR(mn)) 244 return PTR_ERR(mn); 245 246 mo = kzalloc(sizeof(*mo), GFP_KERNEL); 247 if (mo == NULL) 248 return -ENOMEM; 249 250 mo->mn = mn; 251 mo->obj = obj; 252 mo->it.start = obj->userptr.ptr; 253 mo->it.last = obj->userptr.ptr + obj->base.size - 1; 254 INIT_WORK(&mo->work, cancel_userptr); 255 256 obj->userptr.mmu_object = mo; 257 return 0; 258 } 259 260 static void 261 i915_mmu_notifier_free(struct i915_mmu_notifier *mn, 262 struct mm_struct *mm) 263 { 264 if (mn == NULL) 265 return; 266 267 mmu_notifier_unregister(&mn->mn, mm); 268 destroy_workqueue(mn->wq); 269 kfree(mn); 270 } 271 272 #else 273 274 static void 275 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 276 { 277 } 278 279 static int 280 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 281 unsigned flags) 282 { 283 if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0) 284 return -ENODEV; 285 286 if (!capable(CAP_SYS_ADMIN)) 287 return -EPERM; 288 289 return 0; 290 } 291 292 static void 293 i915_mmu_notifier_free(struct i915_mmu_notifier *mn, 294 struct mm_struct *mm) 295 { 296 } 297 298 #endif 299 300 static struct i915_mm_struct * 301 __i915_mm_struct_find(struct drm_i915_private *dev_priv, struct mm_struct *real) 302 { 303 struct i915_mm_struct *mm; 304 305 /* Protected by dev_priv->mm_lock */ 306 hash_for_each_possible(dev_priv->mm_structs, mm, node, (unsigned long)real) 307 if (mm->mm == real) 308 return mm; 309 310 return NULL; 311 } 312 313 static int 314 i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj) 315 { 316 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 317 struct i915_mm_struct *mm; 318 int ret = 0; 319 320 /* During release of the GEM object we hold the struct_mutex. This 321 * precludes us from calling mmput() at that time as that may be 322 * the last reference and so call exit_mmap(). exit_mmap() will 323 * attempt to reap the vma, and if we were holding a GTT mmap 324 * would then call drm_gem_vm_close() and attempt to reacquire 325 * the struct mutex. So in order to avoid that recursion, we have 326 * to defer releasing the mm reference until after we drop the 327 * struct_mutex, i.e. we need to schedule a worker to do the clean 328 * up. 329 */ 330 mutex_lock(&dev_priv->mm_lock); 331 mm = __i915_mm_struct_find(dev_priv, current->mm); 332 #if 0 333 if (mm == NULL) { 334 mm = kmalloc(sizeof(*mm), M_DRM, GFP_KERNEL); 335 if (mm == NULL) { 336 #endif 337 ret = -ENOMEM; 338 #if 0 339 goto out; 340 } 341 342 kref_init(&mm->kref); 343 mm->i915 = to_i915(obj->base.dev); 344 345 mm->mm = current->mm; 346 atomic_inc(¤t->mm->mm_count); 347 348 mm->mn = NULL; 349 350 /* Protected by dev_priv->mm_lock */ 351 hash_add(dev_priv->mm_structs, 352 &mm->node, (unsigned long)mm->mm); 353 } else 354 kref_get(&mm->kref); 355 356 obj->userptr.mm = mm; 357 out: 358 mutex_unlock(&dev_priv->mm_lock); 359 #endif 360 return ret; 361 } 362 363 static void 364 __i915_mm_struct_free__worker(struct work_struct *work) 365 { 366 struct i915_mm_struct *mm = container_of(work, typeof(*mm), work); 367 i915_mmu_notifier_free(mm->mn, mm->mm); 368 #if 0 369 mmdrop(mm->mm); 370 #endif 371 kfree(mm); 372 } 373 374 static void 375 __i915_mm_struct_free(struct kref *kref) 376 { 377 struct i915_mm_struct *mm = container_of(kref, typeof(*mm), kref); 378 379 /* Protected by dev_priv->mm_lock */ 380 hash_del(&mm->node); 381 mutex_unlock(&mm->i915->mm_lock); 382 383 INIT_WORK(&mm->work, __i915_mm_struct_free__worker); 384 schedule_work(&mm->work); 385 } 386 387 static void 388 i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj) 389 { 390 if (obj->userptr.mm == NULL) 391 return; 392 393 kref_put_mutex(&obj->userptr.mm->kref, 394 __i915_mm_struct_free, 395 &to_i915(obj->base.dev)->mm_lock); 396 obj->userptr.mm = NULL; 397 } 398 399 struct get_pages_work { 400 struct work_struct work; 401 struct drm_i915_gem_object *obj; 402 struct task_struct *task; 403 }; 404 405 #if IS_ENABLED(CONFIG_SWIOTLB) 406 #define swiotlb_active() swiotlb_nr_tbl() 407 #else 408 #define swiotlb_active() 0 409 #endif 410 411 #if 0 412 static int 413 st_set_pages(struct sg_table **st, struct page **pvec, int num_pages) 414 { 415 struct scatterlist *sg; 416 int ret, n; 417 418 *st = kmalloc(sizeof(**st), M_DRM, GFP_KERNEL); 419 if (*st == NULL) 420 return -ENOMEM; 421 422 if (swiotlb_active()) { 423 ret = sg_alloc_table(*st, num_pages, GFP_KERNEL); 424 if (ret) 425 goto err; 426 427 for_each_sg((*st)->sgl, sg, num_pages, n) 428 sg_set_page(sg, pvec[n], PAGE_SIZE, 0); 429 } else { 430 ret = sg_alloc_table_from_pages(*st, pvec, num_pages, 431 0, num_pages << PAGE_SHIFT, 432 GFP_KERNEL); 433 if (ret) 434 goto err; 435 } 436 437 return 0; 438 439 err: 440 kfree(*st); 441 *st = NULL; 442 return ret; 443 } 444 445 static int 446 __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj, 447 struct page **pvec, int num_pages) 448 { 449 int ret; 450 451 ret = st_set_pages(&obj->pages, pvec, num_pages); 452 if (ret) 453 return ret; 454 455 ret = i915_gem_gtt_prepare_object(obj); 456 if (ret) { 457 sg_free_table(obj->pages); 458 kfree(obj->pages); 459 obj->pages = NULL; 460 } 461 462 return ret; 463 } 464 #endif 465 466 static int 467 __i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, 468 bool value) 469 { 470 int ret = 0; 471 472 /* During mm_invalidate_range we need to cancel any userptr that 473 * overlaps the range being invalidated. Doing so requires the 474 * struct_mutex, and that risks recursion. In order to cause 475 * recursion, the user must alias the userptr address space with 476 * a GTT mmapping (possible with a MAP_FIXED) - then when we have 477 * to invalidate that mmaping, mm_invalidate_range is called with 478 * the userptr address *and* the struct_mutex held. To prevent that 479 * we set a flag under the i915_mmu_notifier spinlock to indicate 480 * whether this object is valid. 481 */ 482 #if defined(CONFIG_MMU_NOTIFIER) 483 if (obj->userptr.mmu_object == NULL) 484 return 0; 485 486 lockmgr(&obj->userptr.mmu_object->mn->lock, LK_EXCLUSIVE); 487 /* In order to serialise get_pages with an outstanding 488 * cancel_userptr, we must drop the struct_mutex and try again. 489 */ 490 if (!value) 491 del_object(obj->userptr.mmu_object); 492 else if (!work_pending(&obj->userptr.mmu_object->work)) 493 add_object(obj->userptr.mmu_object); 494 else 495 ret = -EAGAIN; 496 lockmgr(&obj->userptr.mmu_object->mn->lock, LK_RELEASE); 497 #endif 498 499 return ret; 500 } 501 502 #if 0 503 static void 504 __i915_gem_userptr_get_pages_worker(struct work_struct *_work) 505 { 506 struct get_pages_work *work = container_of(_work, typeof(*work), work); 507 struct drm_i915_gem_object *obj = work->obj; 508 struct drm_device *dev = obj->base.dev; 509 const int npages = obj->base.size >> PAGE_SHIFT; 510 struct page **pvec; 511 int pinned, ret; 512 513 ret = -ENOMEM; 514 pinned = 0; 515 516 pvec = drm_malloc_gfp(npages, sizeof(struct page *), GFP_TEMPORARY); 517 if (pvec != NULL) { 518 struct mm_struct *mm = obj->userptr.mm->mm; 519 520 ret = -EFAULT; 521 if (atomic_inc_not_zero(&mm->mm_users)) { 522 down_read(&mm->mmap_sem); 523 while (pinned < npages) { 524 ret = get_user_pages_remote 525 (work->task, mm, 526 obj->userptr.ptr + pinned * PAGE_SIZE, 527 npages - pinned, 528 !obj->userptr.read_only, 0, 529 pvec + pinned, NULL); 530 if (ret < 0) 531 break; 532 533 pinned += ret; 534 } 535 up_read(&mm->mmap_sem); 536 mmput(mm); 537 } 538 } 539 540 mutex_lock(&dev->struct_mutex); 541 if (obj->userptr.work == &work->work) { 542 if (pinned == npages) { 543 ret = __i915_gem_userptr_set_pages(obj, pvec, npages); 544 if (ret == 0) { 545 list_add_tail(&obj->global_list, 546 &to_i915(dev)->mm.unbound_list); 547 obj->get_page.sg = obj->pages->sgl; 548 obj->get_page.last = 0; 549 pinned = 0; 550 } 551 } 552 obj->userptr.work = ERR_PTR(ret); 553 } 554 555 obj->userptr.workers--; 556 i915_gem_object_put(obj); 557 mutex_unlock(&dev->struct_mutex); 558 559 release_pages(pvec, pinned, 0); 560 drm_free_large(pvec); 561 562 put_task_struct(work->task); 563 kfree(work); 564 } 565 566 static int 567 __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj, 568 bool *active) 569 { 570 struct get_pages_work *work; 571 572 /* Spawn a worker so that we can acquire the 573 * user pages without holding our mutex. Access 574 * to the user pages requires mmap_sem, and we have 575 * a strict lock ordering of mmap_sem, struct_mutex - 576 * we already hold struct_mutex here and so cannot 577 * call gup without encountering a lock inversion. 578 * 579 * Userspace will keep on repeating the operation 580 * (thanks to EAGAIN) until either we hit the fast 581 * path or the worker completes. If the worker is 582 * cancelled or superseded, the task is still run 583 * but the results ignored. (This leads to 584 * complications that we may have a stray object 585 * refcount that we need to be wary of when 586 * checking for existing objects during creation.) 587 * If the worker encounters an error, it reports 588 * that error back to this function through 589 * obj->userptr.work = ERR_PTR. 590 */ 591 if (obj->userptr.workers >= I915_GEM_USERPTR_MAX_WORKERS) 592 return -EAGAIN; 593 594 work = kmalloc(sizeof(*work), GFP_KERNEL); 595 if (work == NULL) 596 return -ENOMEM; 597 598 obj->userptr.work = &work->work; 599 obj->userptr.workers++; 600 601 work->obj = i915_gem_object_get(obj); 602 603 work->task = current; 604 get_task_struct(work->task); 605 606 INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker); 607 schedule_work(&work->work); 608 609 *active = true; 610 return -EAGAIN; 611 } 612 #endif 613 614 static int 615 i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) 616 { 617 #if 0 618 const int num_pages = obj->base.size >> PAGE_SHIFT; 619 struct page **pvec; 620 int pinned, ret; 621 bool active; 622 623 /* If userspace should engineer that these pages are replaced in 624 * the vma between us binding this page into the GTT and completion 625 * of rendering... Their loss. If they change the mapping of their 626 * pages they need to create a new bo to point to the new vma. 627 * 628 * However, that still leaves open the possibility of the vma 629 * being copied upon fork. Which falls under the same userspace 630 * synchronisation issue as a regular bo, except that this time 631 * the process may not be expecting that a particular piece of 632 * memory is tied to the GPU. 633 * 634 * Fortunately, we can hook into the mmu_notifier in order to 635 * discard the page references prior to anything nasty happening 636 * to the vma (discard or cloning) which should prevent the more 637 * egregious cases from causing harm. 638 */ 639 640 if (obj->userptr.work) { 641 /* active flag should still be held for the pending work */ 642 if (IS_ERR(obj->userptr.work)) 643 return PTR_ERR(obj->userptr.work); 644 else 645 return -EAGAIN; 646 } 647 648 /* Let the mmu-notifier know that we have begun and need cancellation */ 649 ret = __i915_gem_userptr_set_active(obj, true); 650 if (ret) 651 return ret; 652 653 pvec = NULL; 654 pinned = 0; 655 if (obj->userptr.mm->mm == current->mm) { 656 pvec = drm_malloc_gfp(num_pages, sizeof(struct page *), 657 GFP_TEMPORARY); 658 if (pvec == NULL) { 659 __i915_gem_userptr_set_active(obj, false); 660 return -ENOMEM; 661 } 662 663 pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages, 664 !obj->userptr.read_only, pvec); 665 } 666 667 active = false; 668 if (pinned < 0) 669 ret = pinned, pinned = 0; 670 else if (pinned < num_pages) 671 ret = __i915_gem_userptr_get_pages_schedule(obj, &active); 672 else 673 ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages); 674 if (ret) { 675 __i915_gem_userptr_set_active(obj, active); 676 release_pages(pvec, pinned, 0); 677 } 678 drm_free_large(pvec); 679 return ret; 680 #else 681 return 0; 682 #endif /* 0 */ 683 } 684 685 static void 686 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) 687 { 688 struct sgt_iter sgt_iter; 689 struct page *page; 690 691 BUG_ON(obj->userptr.work != NULL); 692 __i915_gem_userptr_set_active(obj, false); 693 694 if (obj->madv != I915_MADV_WILLNEED) 695 obj->dirty = 0; 696 697 i915_gem_gtt_finish_object(obj); 698 699 for_each_sgt_page(page, sgt_iter, obj->pages) { 700 if (obj->dirty) 701 set_page_dirty(page); 702 703 mark_page_accessed(page); 704 #if 0 705 put_page(page); 706 #endif 707 } 708 obj->dirty = 0; 709 710 sg_free_table(obj->pages); 711 kfree(obj->pages); 712 } 713 714 static void 715 i915_gem_userptr_release(struct drm_i915_gem_object *obj) 716 { 717 i915_gem_userptr_release__mmu_notifier(obj); 718 i915_gem_userptr_release__mm_struct(obj); 719 } 720 721 static int 722 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj) 723 { 724 if (obj->userptr.mmu_object) 725 return 0; 726 727 return i915_gem_userptr_init__mmu_notifier(obj, 0); 728 } 729 730 static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { 731 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE, 732 .get_pages = i915_gem_userptr_get_pages, 733 .put_pages = i915_gem_userptr_put_pages, 734 .dmabuf_export = i915_gem_userptr_dmabuf_export, 735 .release = i915_gem_userptr_release, 736 }; 737 738 /** 739 * Creates a new mm object that wraps some normal memory from the process 740 * context - user memory. 741 * 742 * We impose several restrictions upon the memory being mapped 743 * into the GPU. 744 * 1. It must be page aligned (both start/end addresses, i.e ptr and size). 745 * 2. It must be normal system memory, not a pointer into another map of IO 746 * space (e.g. it must not be a GTT mmapping of another object). 747 * 3. We only allow a bo as large as we could in theory map into the GTT, 748 * that is we limit the size to the total size of the GTT. 749 * 4. The bo is marked as being snoopable. The backing pages are left 750 * accessible directly by the CPU, but reads and writes by the GPU may 751 * incur the cost of a snoop (unless you have an LLC architecture). 752 * 753 * Synchronisation between multiple users and the GPU is left to userspace 754 * through the normal set-domain-ioctl. The kernel will enforce that the 755 * GPU relinquishes the VMA before it is returned back to the system 756 * i.e. upon free(), munmap() or process termination. However, the userspace 757 * malloc() library may not immediately relinquish the VMA after free() and 758 * instead reuse it whilst the GPU is still reading and writing to the VMA. 759 * Caveat emptor. 760 * 761 * Also note, that the object created here is not currently a "first class" 762 * object, in that several ioctls are banned. These are the CPU access 763 * ioctls: mmap(), pwrite and pread. In practice, you are expected to use 764 * direct access via your pointer rather than use those ioctls. Another 765 * restriction is that we do not allow userptr surfaces to be pinned to the 766 * hardware and so we reject any attempt to create a framebuffer out of a 767 * userptr. 768 * 769 * If you think this is a good interface to use to pass GPU memory between 770 * drivers, please use dma-buf instead. In fact, wherever possible use 771 * dma-buf instead. 772 */ 773 int 774 i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 775 { 776 struct drm_i915_gem_userptr *args = data; 777 struct drm_i915_gem_object *obj; 778 int ret; 779 u32 handle; 780 781 if (!HAS_LLC(dev) && !HAS_SNOOP(dev)) { 782 /* We cannot support coherent userptr objects on hw without 783 * LLC and broken snooping. 784 */ 785 return -ENODEV; 786 } 787 788 if (args->flags & ~(I915_USERPTR_READ_ONLY | 789 I915_USERPTR_UNSYNCHRONIZED)) 790 return -EINVAL; 791 792 if (offset_in_page(args->user_ptr | args->user_size)) 793 return -EINVAL; 794 795 #if 0 796 if (!access_ok(args->flags & I915_USERPTR_READ_ONLY ? VERIFY_READ : VERIFY_WRITE, 797 (char __user *)(unsigned long)args->user_ptr, args->user_size)) 798 return -EFAULT; 799 #endif 800 801 if (args->flags & I915_USERPTR_READ_ONLY) { 802 /* On almost all of the current hw, we cannot tell the GPU that a 803 * page is readonly, so this is just a placeholder in the uAPI. 804 */ 805 return -ENODEV; 806 } 807 808 obj = i915_gem_object_alloc(dev); 809 if (obj == NULL) 810 return -ENOMEM; 811 812 drm_gem_private_object_init(dev, &obj->base, args->user_size); 813 i915_gem_object_init(obj, &i915_gem_userptr_ops); 814 obj->cache_level = I915_CACHE_LLC; 815 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 816 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 817 818 obj->userptr.ptr = args->user_ptr; 819 obj->userptr.read_only = !!(args->flags & I915_USERPTR_READ_ONLY); 820 821 /* And keep a pointer to the current->mm for resolving the user pages 822 * at binding. This means that we need to hook into the mmu_notifier 823 * in order to detect if the mmu is destroyed. 824 */ 825 ret = i915_gem_userptr_init__mm_struct(obj); 826 if (ret == 0) 827 ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags); 828 if (ret == 0) 829 ret = drm_gem_handle_create(file, &obj->base, &handle); 830 831 /* drop reference from allocate - handle holds it now */ 832 i915_gem_object_put_unlocked(obj); 833 if (ret) 834 return ret; 835 836 args->handle = handle; 837 return 0; 838 } 839 840 void i915_gem_init_userptr(struct drm_i915_private *dev_priv) 841 { 842 lockinit(&dev_priv->mm_lock, "i915dmm", 0, LK_CANRECURSE); 843 hash_init(dev_priv->mm_structs); 844 } 845