1 /* 2 * Copyright © 2012-2014 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <drm/drmP.h> 26 #include <drm/i915_drm.h> 27 #include "i915_drv.h" 28 #include "i915_trace.h" 29 #include "intel_drv.h" 30 31 struct i915_mm_struct { 32 struct mm_struct *mm; 33 struct drm_device *dev; 34 struct i915_mmu_notifier *mn; 35 struct hlist_node node; 36 struct kref kref; 37 struct work_struct work; 38 }; 39 40 #if defined(CONFIG_MMU_NOTIFIER) 41 #include <linux/interval_tree.h> 42 43 struct i915_mmu_notifier { 44 spinlock_t lock; 45 struct hlist_node node; 46 struct mmu_notifier mn; 47 struct rb_root objects; 48 struct list_head linear; 49 unsigned long serial; 50 bool has_linear; 51 }; 52 53 struct i915_mmu_object { 54 struct i915_mmu_notifier *mn; 55 struct interval_tree_node it; 56 struct list_head link; 57 struct drm_i915_gem_object *obj; 58 bool is_linear; 59 }; 60 61 static unsigned long cancel_userptr(struct drm_i915_gem_object *obj) 62 { 63 struct drm_device *dev = obj->base.dev; 64 unsigned long end; 65 66 mutex_lock(&dev->struct_mutex); 67 /* Cancel any active worker and force us to re-evaluate gup */ 68 obj->userptr.work = NULL; 69 70 if (obj->pages != NULL) { 71 struct drm_i915_private *dev_priv = to_i915(dev); 72 struct i915_vma *vma, *tmp; 73 bool was_interruptible; 74 75 was_interruptible = dev_priv->mm.interruptible; 76 dev_priv->mm.interruptible = false; 77 78 list_for_each_entry_safe(vma, tmp, &obj->vma_list, vma_link) { 79 int ret = i915_vma_unbind(vma); 80 WARN_ON(ret && ret != -EIO); 81 } 82 WARN_ON(i915_gem_object_put_pages(obj)); 83 84 dev_priv->mm.interruptible = was_interruptible; 85 } 86 87 end = obj->userptr.ptr + obj->base.size; 88 89 drm_gem_object_unreference(&obj->base); 90 mutex_unlock(&dev->struct_mutex); 91 92 return end; 93 } 94 95 static void *invalidate_range__linear(struct i915_mmu_notifier *mn, 96 struct mm_struct *mm, 97 unsigned long start, 98 unsigned long end) 99 { 100 struct i915_mmu_object *mo; 101 unsigned long serial; 102 103 restart: 104 serial = mn->serial; 105 list_for_each_entry(mo, &mn->linear, link) { 106 struct drm_i915_gem_object *obj; 107 108 if (mo->it.last < start || mo->it.start > end) 109 continue; 110 111 obj = mo->obj; 112 113 if (!kref_get_unless_zero(&obj->base.refcount)) 114 continue; 115 116 spin_unlock(&mn->lock); 117 118 cancel_userptr(obj); 119 120 spin_lock(&mn->lock); 121 if (serial != mn->serial) 122 goto restart; 123 } 124 125 return NULL; 126 } 127 128 static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, 129 struct mm_struct *mm, 130 unsigned long start, 131 unsigned long end) 132 { 133 struct i915_mmu_notifier *mn = container_of(_mn, struct i915_mmu_notifier, mn); 134 struct interval_tree_node *it = NULL; 135 unsigned long next = start; 136 unsigned long serial = 0; 137 138 end--; /* interval ranges are inclusive, but invalidate range is exclusive */ 139 while (next < end) { 140 struct drm_i915_gem_object *obj = NULL; 141 142 spin_lock(&mn->lock); 143 if (mn->has_linear) 144 it = invalidate_range__linear(mn, mm, start, end); 145 else if (serial == mn->serial) 146 it = interval_tree_iter_next(it, next, end); 147 else 148 it = interval_tree_iter_first(&mn->objects, start, end); 149 if (it != NULL) { 150 obj = container_of(it, struct i915_mmu_object, it)->obj; 151 152 /* The mmu_object is released late when destroying the 153 * GEM object so it is entirely possible to gain a 154 * reference on an object in the process of being freed 155 * since our serialisation is via the spinlock and not 156 * the struct_mutex - and consequently use it after it 157 * is freed and then double free it. 158 */ 159 if (!kref_get_unless_zero(&obj->base.refcount)) { 160 spin_unlock(&mn->lock); 161 serial = 0; 162 continue; 163 } 164 165 serial = mn->serial; 166 } 167 spin_unlock(&mn->lock); 168 if (obj == NULL) 169 return; 170 171 next = cancel_userptr(obj); 172 } 173 } 174 175 static const struct mmu_notifier_ops i915_gem_userptr_notifier = { 176 .invalidate_range_start = i915_gem_userptr_mn_invalidate_range_start, 177 }; 178 179 static struct i915_mmu_notifier * 180 i915_mmu_notifier_create(struct mm_struct *mm) 181 { 182 struct i915_mmu_notifier *mn; 183 int ret; 184 185 mn = kmalloc(sizeof(*mn), GFP_KERNEL); 186 if (mn == NULL) 187 return ERR_PTR(-ENOMEM); 188 189 spin_lock_init(&mn->lock); 190 mn->mn.ops = &i915_gem_userptr_notifier; 191 mn->objects = RB_ROOT; 192 mn->serial = 1; 193 INIT_LIST_HEAD(&mn->linear); 194 mn->has_linear = false; 195 196 /* Protected by mmap_sem (write-lock) */ 197 ret = __mmu_notifier_register(&mn->mn, mm); 198 if (ret) { 199 kfree(mn); 200 return ERR_PTR(ret); 201 } 202 203 return mn; 204 } 205 206 static void __i915_mmu_notifier_update_serial(struct i915_mmu_notifier *mn) 207 { 208 if (++mn->serial == 0) 209 mn->serial = 1; 210 } 211 212 static int 213 i915_mmu_notifier_add(struct drm_device *dev, 214 struct i915_mmu_notifier *mn, 215 struct i915_mmu_object *mo) 216 { 217 struct interval_tree_node *it; 218 int ret = 0; 219 220 /* By this point we have already done a lot of expensive setup that 221 * we do not want to repeat just because the caller (e.g. X) has a 222 * signal pending (and partly because of that expensive setup, X 223 * using an interrupt timer is likely to get stuck in an EINTR loop). 224 */ 225 mutex_lock(&dev->struct_mutex); 226 227 /* Make sure we drop the final active reference (and thereby 228 * remove the objects from the interval tree) before we do 229 * the check for overlapping objects. 230 */ 231 i915_gem_retire_requests(dev); 232 233 spin_lock(&mn->lock); 234 it = interval_tree_iter_first(&mn->objects, 235 mo->it.start, mo->it.last); 236 if (it) { 237 struct drm_i915_gem_object *obj; 238 239 /* We only need to check the first object in the range as it 240 * either has cancelled gup work queued and we need to 241 * return back to the user to give time for the gup-workers 242 * to flush their object references upon which the object will 243 * be removed from the interval-tree, or the the range is 244 * still in use by another client and the overlap is invalid. 245 * 246 * If we do have an overlap, we cannot use the interval tree 247 * for fast range invalidation. 248 */ 249 250 obj = container_of(it, struct i915_mmu_object, it)->obj; 251 if (!obj->userptr.workers) 252 mn->has_linear = mo->is_linear = true; 253 else 254 ret = -EAGAIN; 255 } else 256 interval_tree_insert(&mo->it, &mn->objects); 257 258 if (ret == 0) { 259 list_add(&mo->link, &mn->linear); 260 __i915_mmu_notifier_update_serial(mn); 261 } 262 spin_unlock(&mn->lock); 263 mutex_unlock(&dev->struct_mutex); 264 265 return ret; 266 } 267 268 static bool i915_mmu_notifier_has_linear(struct i915_mmu_notifier *mn) 269 { 270 struct i915_mmu_object *mo; 271 272 list_for_each_entry(mo, &mn->linear, link) 273 if (mo->is_linear) 274 return true; 275 276 return false; 277 } 278 279 static void 280 i915_mmu_notifier_del(struct i915_mmu_notifier *mn, 281 struct i915_mmu_object *mo) 282 { 283 spin_lock(&mn->lock); 284 list_del(&mo->link); 285 if (mo->is_linear) 286 mn->has_linear = i915_mmu_notifier_has_linear(mn); 287 else 288 interval_tree_remove(&mo->it, &mn->objects); 289 __i915_mmu_notifier_update_serial(mn); 290 spin_unlock(&mn->lock); 291 } 292 293 static void 294 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 295 { 296 struct i915_mmu_object *mo; 297 298 mo = obj->userptr.mmu_object; 299 if (mo == NULL) 300 return; 301 302 i915_mmu_notifier_del(mo->mn, mo); 303 kfree(mo); 304 305 obj->userptr.mmu_object = NULL; 306 } 307 308 static struct i915_mmu_notifier * 309 i915_mmu_notifier_find(struct i915_mm_struct *mm) 310 { 311 struct i915_mmu_notifier *mn = mm->mn; 312 313 mn = mm->mn; 314 if (mn) 315 return mn; 316 317 down_write(&mm->mm->mmap_sem); 318 mutex_lock(&to_i915(mm->dev)->mm_lock); 319 if ((mn = mm->mn) == NULL) { 320 mn = i915_mmu_notifier_create(mm->mm); 321 if (!IS_ERR(mn)) 322 mm->mn = mn; 323 } 324 mutex_unlock(&to_i915(mm->dev)->mm_lock); 325 up_write(&mm->mm->mmap_sem); 326 327 return mn; 328 } 329 330 static int 331 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 332 unsigned flags) 333 { 334 struct i915_mmu_notifier *mn; 335 struct i915_mmu_object *mo; 336 int ret; 337 338 if (flags & I915_USERPTR_UNSYNCHRONIZED) 339 return capable(CAP_SYS_ADMIN) ? 0 : -EPERM; 340 341 if (WARN_ON(obj->userptr.mm == NULL)) 342 return -EINVAL; 343 344 mn = i915_mmu_notifier_find(obj->userptr.mm); 345 if (IS_ERR(mn)) 346 return PTR_ERR(mn); 347 348 mo = kzalloc(sizeof(*mo), GFP_KERNEL); 349 if (mo == NULL) 350 return -ENOMEM; 351 352 mo->mn = mn; 353 mo->it.start = obj->userptr.ptr; 354 mo->it.last = mo->it.start + obj->base.size - 1; 355 mo->obj = obj; 356 357 ret = i915_mmu_notifier_add(obj->base.dev, mn, mo); 358 if (ret) { 359 kfree(mo); 360 return ret; 361 } 362 363 obj->userptr.mmu_object = mo; 364 return 0; 365 } 366 367 static void 368 i915_mmu_notifier_free(struct i915_mmu_notifier *mn, 369 struct mm_struct *mm) 370 { 371 if (mn == NULL) 372 return; 373 374 mmu_notifier_unregister(&mn->mn, mm); 375 kfree(mn); 376 } 377 378 #else 379 380 #if 0 381 static void 382 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 383 { 384 } 385 386 static int 387 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 388 unsigned flags) 389 { 390 if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0) 391 return -ENODEV; 392 393 if (!capable(CAP_SYS_ADMIN)) 394 return -EPERM; 395 396 return 0; 397 } 398 399 static void 400 i915_mmu_notifier_free(struct i915_mmu_notifier *mn, 401 struct mm_struct *mm) 402 { 403 } 404 #endif 405 406 #endif 407 408 #if 0 409 static struct i915_mm_struct * 410 __i915_mm_struct_find(struct drm_i915_private *dev_priv, struct mm_struct *real) 411 { 412 struct i915_mm_struct *mm; 413 414 /* Protected by dev_priv->mm_lock */ 415 hash_for_each_possible(dev_priv->mm_structs, mm, node, (unsigned long)real) 416 if (mm->mm == real) 417 return mm; 418 419 return NULL; 420 } 421 422 static int 423 i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj) 424 { 425 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 426 struct i915_mm_struct *mm; 427 int ret = 0; 428 429 /* During release of the GEM object we hold the struct_mutex. This 430 * precludes us from calling mmput() at that time as that may be 431 * the last reference and so call exit_mmap(). exit_mmap() will 432 * attempt to reap the vma, and if we were holding a GTT mmap 433 * would then call drm_gem_vm_close() and attempt to reacquire 434 * the struct mutex. So in order to avoid that recursion, we have 435 * to defer releasing the mm reference until after we drop the 436 * struct_mutex, i.e. we need to schedule a worker to do the clean 437 * up. 438 */ 439 mutex_lock(&dev_priv->mm_lock); 440 mm = __i915_mm_struct_find(dev_priv, current->mm); 441 if (mm == NULL) { 442 mm = kmalloc(sizeof(*mm), GFP_KERNEL); 443 if (mm == NULL) { 444 ret = -ENOMEM; 445 goto out; 446 } 447 448 kref_init(&mm->kref); 449 mm->dev = obj->base.dev; 450 451 mm->mm = current->mm; 452 atomic_inc(¤t->mm->mm_count); 453 454 mm->mn = NULL; 455 456 /* Protected by dev_priv->mm_lock */ 457 hash_add(dev_priv->mm_structs, 458 &mm->node, (unsigned long)mm->mm); 459 } else 460 kref_get(&mm->kref); 461 462 obj->userptr.mm = mm; 463 out: 464 mutex_unlock(&dev_priv->mm_lock); 465 return ret; 466 } 467 468 static void 469 __i915_mm_struct_free__worker(struct work_struct *work) 470 { 471 struct i915_mm_struct *mm = container_of(work, typeof(*mm), work); 472 i915_mmu_notifier_free(mm->mn, mm->mm); 473 mmdrop(mm->mm); 474 kfree(mm); 475 } 476 477 static void 478 __i915_mm_struct_free(struct kref *kref) 479 { 480 struct i915_mm_struct *mm = container_of(kref, typeof(*mm), kref); 481 482 /* Protected by dev_priv->mm_lock */ 483 hash_del(&mm->node); 484 mutex_unlock(&to_i915(mm->dev)->mm_lock); 485 486 INIT_WORK(&mm->work, __i915_mm_struct_free__worker); 487 schedule_work(&mm->work); 488 } 489 490 static void 491 i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj) 492 { 493 if (obj->userptr.mm == NULL) 494 return; 495 496 kref_put_mutex(&obj->userptr.mm->kref, 497 __i915_mm_struct_free, 498 &to_i915(obj->base.dev)->mm_lock); 499 obj->userptr.mm = NULL; 500 } 501 #endif 502 503 struct get_pages_work { 504 struct work_struct work; 505 struct drm_i915_gem_object *obj; 506 struct task_struct *task; 507 }; 508 509 #if IS_ENABLED(CONFIG_SWIOTLB) 510 #define swiotlb_active() swiotlb_nr_tbl() 511 #else 512 #define swiotlb_active() 0 513 #endif 514 515 #if 0 516 static int 517 st_set_pages(struct sg_table **st, struct page **pvec, int num_pages) 518 { 519 struct scatterlist *sg; 520 int ret, n; 521 522 *st = kmalloc(sizeof(**st), GFP_KERNEL); 523 if (*st == NULL) 524 return -ENOMEM; 525 526 if (swiotlb_active()) { 527 ret = sg_alloc_table(*st, num_pages, GFP_KERNEL); 528 if (ret) 529 goto err; 530 531 for_each_sg((*st)->sgl, sg, num_pages, n) 532 sg_set_page(sg, pvec[n], PAGE_SIZE, 0); 533 } else { 534 ret = sg_alloc_table_from_pages(*st, pvec, num_pages, 535 0, num_pages << PAGE_SHIFT, 536 GFP_KERNEL); 537 if (ret) 538 goto err; 539 } 540 541 return 0; 542 543 err: 544 kfree(*st); 545 *st = NULL; 546 return ret; 547 } 548 549 static int 550 __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj, 551 struct page **pvec, int num_pages) 552 { 553 int ret; 554 555 ret = st_set_pages(&obj->pages, pvec, num_pages); 556 if (ret) 557 return ret; 558 559 ret = i915_gem_gtt_prepare_object(obj); 560 if (ret) { 561 sg_free_table(obj->pages); 562 kfree(obj->pages); 563 obj->pages = NULL; 564 } 565 566 return ret; 567 } 568 569 static void 570 __i915_gem_userptr_get_pages_worker(struct work_struct *_work) 571 { 572 struct get_pages_work *work = container_of(_work, typeof(*work), work); 573 struct drm_i915_gem_object *obj = work->obj; 574 struct drm_device *dev = obj->base.dev; 575 const int num_pages = obj->base.size >> PAGE_SHIFT; 576 struct page **pvec; 577 int pinned, ret; 578 579 ret = -ENOMEM; 580 pinned = 0; 581 582 pvec = kmalloc(num_pages*sizeof(struct page *), 583 GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); 584 if (pvec == NULL) 585 pvec = drm_malloc_ab(num_pages, sizeof(struct page *)); 586 if (pvec != NULL) { 587 struct mm_struct *mm = obj->userptr.mm->mm; 588 589 down_read(&mm->mmap_sem); 590 while (pinned < num_pages) { 591 ret = get_user_pages(work->task, mm, 592 obj->userptr.ptr + pinned * PAGE_SIZE, 593 num_pages - pinned, 594 !obj->userptr.read_only, 0, 595 pvec + pinned, NULL); 596 if (ret < 0) 597 break; 598 599 pinned += ret; 600 } 601 up_read(&mm->mmap_sem); 602 } 603 604 mutex_lock(&dev->struct_mutex); 605 if (obj->userptr.work != &work->work) { 606 ret = 0; 607 } else if (pinned == num_pages) { 608 ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages); 609 if (ret == 0) { 610 list_add_tail(&obj->global_list, &to_i915(dev)->mm.unbound_list); 611 obj->get_page.sg = obj->pages->sgl; 612 obj->get_page.last = 0; 613 614 pinned = 0; 615 } 616 } 617 618 obj->userptr.work = ERR_PTR(ret); 619 obj->userptr.workers--; 620 drm_gem_object_unreference(&obj->base); 621 mutex_unlock(&dev->struct_mutex); 622 623 release_pages(pvec, pinned, 0); 624 drm_free_large(pvec); 625 626 put_task_struct(work->task); 627 kfree(work); 628 } 629 630 static int 631 i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) 632 { 633 const int num_pages = obj->base.size >> PAGE_SHIFT; 634 struct page **pvec; 635 int pinned, ret; 636 637 /* If userspace should engineer that these pages are replaced in 638 * the vma between us binding this page into the GTT and completion 639 * of rendering... Their loss. If they change the mapping of their 640 * pages they need to create a new bo to point to the new vma. 641 * 642 * However, that still leaves open the possibility of the vma 643 * being copied upon fork. Which falls under the same userspace 644 * synchronisation issue as a regular bo, except that this time 645 * the process may not be expecting that a particular piece of 646 * memory is tied to the GPU. 647 * 648 * Fortunately, we can hook into the mmu_notifier in order to 649 * discard the page references prior to anything nasty happening 650 * to the vma (discard or cloning) which should prevent the more 651 * egregious cases from causing harm. 652 */ 653 654 pvec = NULL; 655 pinned = 0; 656 if (obj->userptr.mm->mm == current->mm) { 657 pvec = kmalloc(num_pages*sizeof(struct page *), 658 GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); 659 if (pvec == NULL) { 660 pvec = drm_malloc_ab(num_pages, sizeof(struct page *)); 661 if (pvec == NULL) 662 return -ENOMEM; 663 } 664 665 pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages, 666 !obj->userptr.read_only, pvec); 667 } 668 if (pinned < num_pages) { 669 if (pinned < 0) { 670 ret = pinned; 671 pinned = 0; 672 } else { 673 /* Spawn a worker so that we can acquire the 674 * user pages without holding our mutex. Access 675 * to the user pages requires mmap_sem, and we have 676 * a strict lock ordering of mmap_sem, struct_mutex - 677 * we already hold struct_mutex here and so cannot 678 * call gup without encountering a lock inversion. 679 * 680 * Userspace will keep on repeating the operation 681 * (thanks to EAGAIN) until either we hit the fast 682 * path or the worker completes. If the worker is 683 * cancelled or superseded, the task is still run 684 * but the results ignored. (This leads to 685 * complications that we may have a stray object 686 * refcount that we need to be wary of when 687 * checking for existing objects during creation.) 688 * If the worker encounters an error, it reports 689 * that error back to this function through 690 * obj->userptr.work = ERR_PTR. 691 */ 692 ret = -EAGAIN; 693 if (obj->userptr.work == NULL && 694 obj->userptr.workers < I915_GEM_USERPTR_MAX_WORKERS) { 695 struct get_pages_work *work; 696 697 work = kmalloc(sizeof(*work), GFP_KERNEL); 698 if (work != NULL) { 699 obj->userptr.work = &work->work; 700 obj->userptr.workers++; 701 702 work->obj = obj; 703 drm_gem_object_reference(&obj->base); 704 705 work->task = current; 706 get_task_struct(work->task); 707 708 INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker); 709 schedule_work(&work->work); 710 } else 711 ret = -ENOMEM; 712 } else { 713 if (IS_ERR(obj->userptr.work)) { 714 ret = PTR_ERR(obj->userptr.work); 715 obj->userptr.work = NULL; 716 } 717 } 718 } 719 } else { 720 ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages); 721 if (ret == 0) { 722 obj->userptr.work = NULL; 723 pinned = 0; 724 } 725 } 726 727 release_pages(pvec, pinned, 0); 728 drm_free_large(pvec); 729 return ret; 730 } 731 732 static void 733 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) 734 { 735 struct sg_page_iter sg_iter; 736 737 BUG_ON(obj->userptr.work != NULL); 738 739 if (obj->madv != I915_MADV_WILLNEED) 740 obj->dirty = 0; 741 742 i915_gem_gtt_finish_object(obj); 743 744 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { 745 struct page *page = sg_page_iter_page(&sg_iter); 746 747 if (obj->dirty) 748 set_page_dirty(page); 749 750 mark_page_accessed(page); 751 page_cache_release(page); 752 } 753 obj->dirty = 0; 754 755 sg_free_table(obj->pages); 756 kfree(obj->pages); 757 } 758 759 static void 760 i915_gem_userptr_release(struct drm_i915_gem_object *obj) 761 { 762 i915_gem_userptr_release__mmu_notifier(obj); 763 i915_gem_userptr_release__mm_struct(obj); 764 } 765 766 static int 767 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj) 768 { 769 if (obj->userptr.mmu_object) 770 return 0; 771 772 return i915_gem_userptr_init__mmu_notifier(obj, 0); 773 } 774 775 static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { 776 .dmabuf_export = i915_gem_userptr_dmabuf_export, 777 .get_pages = i915_gem_userptr_get_pages, 778 .put_pages = i915_gem_userptr_put_pages, 779 .release = i915_gem_userptr_release, 780 }; 781 782 /** 783 * Creates a new mm object that wraps some normal memory from the process 784 * context - user memory. 785 * 786 * We impose several restrictions upon the memory being mapped 787 * into the GPU. 788 * 1. It must be page aligned (both start/end addresses, i.e ptr and size). 789 * 2. It must be normal system memory, not a pointer into another map of IO 790 * space (e.g. it must not be a GTT mmapping of another object). 791 * 3. We only allow a bo as large as we could in theory map into the GTT, 792 * that is we limit the size to the total size of the GTT. 793 * 4. The bo is marked as being snoopable. The backing pages are left 794 * accessible directly by the CPU, but reads and writes by the GPU may 795 * incur the cost of a snoop (unless you have an LLC architecture). 796 * 797 * Synchronisation between multiple users and the GPU is left to userspace 798 * through the normal set-domain-ioctl. The kernel will enforce that the 799 * GPU relinquishes the VMA before it is returned back to the system 800 * i.e. upon free(), munmap() or process termination. However, the userspace 801 * malloc() library may not immediately relinquish the VMA after free() and 802 * instead reuse it whilst the GPU is still reading and writing to the VMA. 803 * Caveat emptor. 804 * 805 * Also note, that the object created here is not currently a "first class" 806 * object, in that several ioctls are banned. These are the CPU access 807 * ioctls: mmap(), pwrite and pread. In practice, you are expected to use 808 * direct access via your pointer rather than use those ioctls. 809 * 810 * If you think this is a good interface to use to pass GPU memory between 811 * drivers, please use dma-buf instead. In fact, wherever possible use 812 * dma-buf instead. 813 */ 814 int 815 i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 816 { 817 struct drm_i915_private *dev_priv = dev->dev_private; 818 struct drm_i915_gem_userptr *args = data; 819 struct drm_i915_gem_object *obj; 820 int ret; 821 u32 handle; 822 823 if (args->flags & ~(I915_USERPTR_READ_ONLY | 824 I915_USERPTR_UNSYNCHRONIZED)) 825 return -EINVAL; 826 827 if (offset_in_page(args->user_ptr | args->user_size)) 828 return -EINVAL; 829 830 if (args->user_size > dev_priv->gtt.base.total) 831 return -E2BIG; 832 833 if (!access_ok(args->flags & I915_USERPTR_READ_ONLY ? VERIFY_READ : VERIFY_WRITE, 834 (char __user *)(unsigned long)args->user_ptr, args->user_size)) 835 return -EFAULT; 836 837 if (args->flags & I915_USERPTR_READ_ONLY) { 838 /* On almost all of the current hw, we cannot tell the GPU that a 839 * page is readonly, so this is just a placeholder in the uAPI. 840 */ 841 return -ENODEV; 842 } 843 844 obj = i915_gem_object_alloc(dev); 845 if (obj == NULL) 846 return -ENOMEM; 847 848 drm_gem_private_object_init(dev, &obj->base, args->user_size); 849 i915_gem_object_init(obj, &i915_gem_userptr_ops); 850 obj->cache_level = I915_CACHE_LLC; 851 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 852 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 853 854 obj->userptr.ptr = args->user_ptr; 855 obj->userptr.read_only = !!(args->flags & I915_USERPTR_READ_ONLY); 856 857 /* And keep a pointer to the current->mm for resolving the user pages 858 * at binding. This means that we need to hook into the mmu_notifier 859 * in order to detect if the mmu is destroyed. 860 */ 861 ret = i915_gem_userptr_init__mm_struct(obj); 862 if (ret == 0) 863 ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags); 864 if (ret == 0) 865 ret = drm_gem_handle_create(file, &obj->base, &handle); 866 867 /* drop reference from allocate - handle holds it now */ 868 drm_gem_object_unreference_unlocked(&obj->base); 869 if (ret) 870 return ret; 871 872 args->handle = handle; 873 return 0; 874 } 875 #endif 876 877 int 878 i915_gem_init_userptr(struct drm_device *dev) 879 { 880 struct drm_i915_private *dev_priv = to_i915(dev); 881 lockinit(&dev_priv->mm_lock, "i915dmm", 0, LK_CANRECURSE); 882 #if 0 883 hash_init(dev_priv->mm_structs); 884 #endif 885 return 0; 886 } 887