1 /* 2 * Copyright © 2012-2014 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <drm/drmP.h> 26 #include <drm/i915_drm.h> 27 #include "i915_drv.h" 28 #include "i915_trace.h" 29 #include "intel_drv.h" 30 31 #if defined(CONFIG_MMU_NOTIFIER) 32 #include <linux/interval_tree.h> 33 34 struct i915_mmu_notifier { 35 spinlock_t lock; 36 struct hlist_node node; 37 struct mmu_notifier mn; 38 struct rb_root objects; 39 struct list_head linear; 40 struct drm_device *dev; 41 struct mm_struct *mm; 42 struct work_struct work; 43 unsigned long count; 44 unsigned long serial; 45 bool has_linear; 46 }; 47 48 struct i915_mmu_object { 49 struct i915_mmu_notifier *mmu; 50 struct interval_tree_node it; 51 struct list_head link; 52 struct drm_i915_gem_object *obj; 53 bool is_linear; 54 }; 55 56 static unsigned long cancel_userptr(struct drm_i915_gem_object *obj) 57 { 58 struct drm_device *dev = obj->base.dev; 59 unsigned long end; 60 61 mutex_lock(&dev->struct_mutex); 62 /* Cancel any active worker and force us to re-evaluate gup */ 63 obj->userptr.work = NULL; 64 65 if (obj->pages != NULL) { 66 struct drm_i915_private *dev_priv = to_i915(dev); 67 struct i915_vma *vma, *tmp; 68 bool was_interruptible; 69 70 was_interruptible = dev_priv->mm.interruptible; 71 dev_priv->mm.interruptible = false; 72 73 list_for_each_entry_safe(vma, tmp, &obj->vma_list, vma_link) { 74 int ret = i915_vma_unbind(vma); 75 WARN_ON(ret && ret != -EIO); 76 } 77 WARN_ON(i915_gem_object_put_pages(obj)); 78 79 dev_priv->mm.interruptible = was_interruptible; 80 } 81 82 end = obj->userptr.ptr + obj->base.size; 83 84 drm_gem_object_unreference(&obj->base); 85 mutex_unlock(&dev->struct_mutex); 86 87 return end; 88 } 89 90 static void *invalidate_range__linear(struct i915_mmu_notifier *mn, 91 struct mm_struct *mm, 92 unsigned long start, 93 unsigned long end) 94 { 95 struct i915_mmu_object *mo; 96 unsigned long serial; 97 98 restart: 99 serial = mn->serial; 100 list_for_each_entry(mo, &mn->linear, link) { 101 struct drm_i915_gem_object *obj; 102 103 if (mo->it.last < start || mo->it.start > end) 104 continue; 105 106 obj = mo->obj; 107 108 if (!kref_get_unless_zero(&obj->base.refcount)) 109 continue; 110 111 spin_unlock(&mn->lock); 112 113 cancel_userptr(obj); 114 115 spin_lock(&mn->lock); 116 if (serial != mn->serial) 117 goto restart; 118 } 119 120 return NULL; 121 } 122 123 static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, 124 struct mm_struct *mm, 125 unsigned long start, 126 unsigned long end) 127 { 128 struct i915_mmu_notifier *mn = container_of(_mn, struct i915_mmu_notifier, mn); 129 struct interval_tree_node *it = NULL; 130 unsigned long next = start; 131 unsigned long serial = 0; 132 133 end--; /* interval ranges are inclusive, but invalidate range is exclusive */ 134 while (next < end) { 135 struct drm_i915_gem_object *obj = NULL; 136 137 spin_lock(&mn->lock); 138 if (mn->has_linear) 139 it = invalidate_range__linear(mn, mm, start, end); 140 else if (serial == mn->serial) 141 it = interval_tree_iter_next(it, next, end); 142 else 143 it = interval_tree_iter_first(&mn->objects, start, end); 144 if (it != NULL) { 145 obj = container_of(it, struct i915_mmu_object, it)->obj; 146 147 /* The mmu_object is released late when destroying the 148 * GEM object so it is entirely possible to gain a 149 * reference on an object in the process of being freed 150 * since our serialisation is via the spinlock and not 151 * the struct_mutex - and consequently use it after it 152 * is freed and then double free it. 153 */ 154 if (!kref_get_unless_zero(&obj->base.refcount)) { 155 spin_unlock(&mn->lock); 156 serial = 0; 157 continue; 158 } 159 160 serial = mn->serial; 161 } 162 spin_unlock(&mn->lock); 163 if (obj == NULL) 164 return; 165 166 next = cancel_userptr(obj); 167 } 168 } 169 170 static const struct mmu_notifier_ops i915_gem_userptr_notifier = { 171 .invalidate_range_start = i915_gem_userptr_mn_invalidate_range_start, 172 }; 173 174 static struct i915_mmu_notifier * 175 __i915_mmu_notifier_lookup(struct drm_device *dev, struct mm_struct *mm) 176 { 177 struct drm_i915_private *dev_priv = to_i915(dev); 178 struct i915_mmu_notifier *mmu; 179 180 /* Protected by dev->struct_mutex */ 181 hash_for_each_possible(dev_priv->mmu_notifiers, mmu, node, (unsigned long)mm) 182 if (mmu->mm == mm) 183 return mmu; 184 185 return NULL; 186 } 187 188 static struct i915_mmu_notifier * 189 i915_mmu_notifier_get(struct drm_device *dev, struct mm_struct *mm) 190 { 191 struct drm_i915_private *dev_priv = to_i915(dev); 192 struct i915_mmu_notifier *mmu; 193 int ret; 194 195 lockdep_assert_held(&dev->struct_mutex); 196 197 mmu = __i915_mmu_notifier_lookup(dev, mm); 198 if (mmu) 199 return mmu; 200 201 mmu = kmalloc(sizeof(*mmu), GFP_KERNEL); 202 if (mmu == NULL) 203 return ERR_PTR(-ENOMEM); 204 205 spin_lock_init(&mmu->lock); 206 mmu->dev = dev; 207 mmu->mn.ops = &i915_gem_userptr_notifier; 208 mmu->mm = mm; 209 mmu->objects = RB_ROOT; 210 mmu->count = 0; 211 mmu->serial = 1; 212 INIT_LIST_HEAD(&mmu->linear); 213 mmu->has_linear = false; 214 215 /* Protected by mmap_sem (write-lock) */ 216 ret = __mmu_notifier_register(&mmu->mn, mm); 217 if (ret) { 218 kfree(mmu); 219 return ERR_PTR(ret); 220 } 221 222 /* Protected by dev->struct_mutex */ 223 hash_add(dev_priv->mmu_notifiers, &mmu->node, (unsigned long)mm); 224 return mmu; 225 } 226 227 static void 228 __i915_mmu_notifier_destroy_worker(struct work_struct *work) 229 { 230 struct i915_mmu_notifier *mmu = container_of(work, typeof(*mmu), work); 231 mmu_notifier_unregister(&mmu->mn, mmu->mm); 232 kfree(mmu); 233 } 234 235 static void 236 __i915_mmu_notifier_destroy(struct i915_mmu_notifier *mmu) 237 { 238 lockdep_assert_held(&mmu->dev->struct_mutex); 239 240 /* Protected by dev->struct_mutex */ 241 hash_del(&mmu->node); 242 243 /* Our lock ordering is: mmap_sem, mmu_notifier_scru, struct_mutex. 244 * We enter the function holding struct_mutex, therefore we need 245 * to drop our mutex prior to calling mmu_notifier_unregister in 246 * order to prevent lock inversion (and system-wide deadlock) 247 * between the mmap_sem and struct-mutex. Hence we defer the 248 * unregistration to a workqueue where we hold no locks. 249 */ 250 INIT_WORK(&mmu->work, __i915_mmu_notifier_destroy_worker); 251 schedule_work(&mmu->work); 252 } 253 254 static void __i915_mmu_notifier_update_serial(struct i915_mmu_notifier *mmu) 255 { 256 if (++mmu->serial == 0) 257 mmu->serial = 1; 258 } 259 260 static bool i915_mmu_notifier_has_linear(struct i915_mmu_notifier *mmu) 261 { 262 struct i915_mmu_object *mn; 263 264 list_for_each_entry(mn, &mmu->linear, link) 265 if (mn->is_linear) 266 return true; 267 268 return false; 269 } 270 271 static void 272 i915_mmu_notifier_del(struct i915_mmu_notifier *mmu, 273 struct i915_mmu_object *mn) 274 { 275 lockdep_assert_held(&mmu->dev->struct_mutex); 276 277 spin_lock(&mmu->lock); 278 list_del(&mn->link); 279 if (mn->is_linear) 280 mmu->has_linear = i915_mmu_notifier_has_linear(mmu); 281 else 282 interval_tree_remove(&mn->it, &mmu->objects); 283 __i915_mmu_notifier_update_serial(mmu); 284 spin_unlock(&mmu->lock); 285 286 /* Protected against _add() by dev->struct_mutex */ 287 if (--mmu->count == 0) 288 __i915_mmu_notifier_destroy(mmu); 289 } 290 291 static int 292 i915_mmu_notifier_add(struct i915_mmu_notifier *mmu, 293 struct i915_mmu_object *mn) 294 { 295 struct interval_tree_node *it; 296 int ret; 297 298 ret = i915_mutex_lock_interruptible(mmu->dev); 299 if (ret) 300 return ret; 301 302 /* Make sure we drop the final active reference (and thereby 303 * remove the objects from the interval tree) before we do 304 * the check for overlapping objects. 305 */ 306 i915_gem_retire_requests(mmu->dev); 307 308 spin_lock(&mmu->lock); 309 it = interval_tree_iter_first(&mmu->objects, 310 mn->it.start, mn->it.last); 311 if (it) { 312 struct drm_i915_gem_object *obj; 313 314 /* We only need to check the first object in the range as it 315 * either has cancelled gup work queued and we need to 316 * return back to the user to give time for the gup-workers 317 * to flush their object references upon which the object will 318 * be removed from the interval-tree, or the the range is 319 * still in use by another client and the overlap is invalid. 320 * 321 * If we do have an overlap, we cannot use the interval tree 322 * for fast range invalidation. 323 */ 324 325 obj = container_of(it, struct i915_mmu_object, it)->obj; 326 if (!obj->userptr.workers) 327 mmu->has_linear = mn->is_linear = true; 328 else 329 ret = -EAGAIN; 330 } else 331 interval_tree_insert(&mn->it, &mmu->objects); 332 333 if (ret == 0) { 334 list_add(&mn->link, &mmu->linear); 335 __i915_mmu_notifier_update_serial(mmu); 336 } 337 spin_unlock(&mmu->lock); 338 mutex_unlock(&mmu->dev->struct_mutex); 339 340 return ret; 341 } 342 343 static void 344 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 345 { 346 struct i915_mmu_object *mn; 347 348 mn = obj->userptr.mn; 349 if (mn == NULL) 350 return; 351 352 i915_mmu_notifier_del(mn->mmu, mn); 353 obj->userptr.mn = NULL; 354 } 355 356 static struct i915_mmu_notifier * 357 i915_mmu_notifier_find(struct i915_mm_struct *mm) 358 { 359 struct i915_mmu_notifier *mn = mm->mn; 360 361 mn = mm->mn; 362 if (mn) 363 return mn; 364 365 down_write(&mm->mm->mmap_sem); 366 mutex_lock(&to_i915(mm->dev)->mm_lock); 367 if ((mn = mm->mn) == NULL) { 368 mn = i915_mmu_notifier_create(mm->mm); 369 if (!IS_ERR(mn)) 370 mm->mn = mn; 371 } 372 mutex_unlock(&to_i915(mm->dev)->mm_lock); 373 up_write(&mm->mm->mmap_sem); 374 375 return mn; 376 } 377 378 static int 379 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 380 unsigned flags) 381 { 382 struct i915_mmu_notifier *mmu; 383 struct i915_mmu_object *mn; 384 int ret; 385 386 if (flags & I915_USERPTR_UNSYNCHRONIZED) 387 return capable(CAP_SYS_ADMIN) ? 0 : -EPERM; 388 389 down_write(&obj->userptr.mm->mmap_sem); 390 ret = i915_mutex_lock_interruptible(obj->base.dev); 391 if (ret == 0) { 392 mmu = i915_mmu_notifier_get(obj->base.dev, obj->userptr.mm); 393 if (!IS_ERR(mmu)) 394 mmu->count++; /* preemptive add to act as a refcount */ 395 else 396 ret = PTR_ERR(mmu); 397 mutex_unlock(&obj->base.dev->struct_mutex); 398 } 399 up_write(&obj->userptr.mm->mmap_sem); 400 if (ret) 401 return ret; 402 403 mn = kzalloc(sizeof(*mn), GFP_KERNEL); 404 if (mn == NULL) { 405 ret = -ENOMEM; 406 goto destroy_mmu; 407 } 408 409 mn->mmu = mmu; 410 mn->it.start = obj->userptr.ptr; 411 mn->it.last = mn->it.start + obj->base.size - 1; 412 mn->obj = obj; 413 414 ret = i915_mmu_notifier_add(mmu, mn); 415 if (ret) 416 goto free_mn; 417 418 obj->userptr.mn = mn; 419 return 0; 420 421 free_mn: 422 kfree(mn); 423 destroy_mmu: 424 mutex_lock(&obj->base.dev->struct_mutex); 425 if (--mmu->count == 0) 426 __i915_mmu_notifier_destroy(mmu); 427 mutex_unlock(&obj->base.dev->struct_mutex); 428 return ret; 429 } 430 431 #else 432 433 #if 0 434 static void 435 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 436 { 437 } 438 439 static int 440 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 441 unsigned flags) 442 { 443 if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0) 444 return -ENODEV; 445 446 447 return 0; 448 } 449 #endif 450 #endif 451 452 struct get_pages_work { 453 struct work_struct work; 454 struct drm_i915_gem_object *obj; 455 struct task_struct *task; 456 }; 457 458 459 #if IS_ENABLED(CONFIG_SWIOTLB) 460 #define swiotlb_active() swiotlb_nr_tbl() 461 #else 462 #define swiotlb_active() 0 463 #endif 464 465 #if 0 466 static int 467 st_set_pages(struct sg_table **st, struct vm_page **pvec, int num_pages) 468 { 469 struct scatterlist *sg; 470 int ret, n; 471 472 *st = kmalloc(sizeof(**st), M_DRM, M_WAITOK); 473 if (*st == NULL) 474 return -ENOMEM; 475 476 if (swiotlb_active()) { 477 ret = sg_alloc_table(*st, num_pages, GFP_KERNEL); 478 if (ret) 479 goto err; 480 481 for_each_sg((*st)->sgl, sg, num_pages, n) 482 sg_set_page(sg, pvec[n], PAGE_SIZE, 0); 483 } else { 484 ret = sg_alloc_table_from_pages(*st, pvec, num_pages, 485 0, num_pages << PAGE_SHIFT, 486 GFP_KERNEL); 487 if (ret) 488 goto err; 489 } 490 491 return 0; 492 493 err: 494 kfree(*st); 495 *st = NULL; 496 return ret; 497 } 498 499 static void 500 __i915_gem_userptr_get_pages_worker(struct work_struct *_work) 501 { 502 struct get_pages_work *work = container_of(_work, typeof(*work), work); 503 struct drm_i915_gem_object *obj = work->obj; 504 struct drm_device *dev = obj->base.dev; 505 const int num_pages = obj->base.size >> PAGE_SHIFT; 506 struct page **pvec; 507 int pinned, ret; 508 509 ret = -ENOMEM; 510 pinned = 0; 511 512 pvec = kmalloc(num_pages*sizeof(struct page *), 513 GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); 514 if (pvec == NULL) 515 pvec = drm_malloc_ab(num_pages, sizeof(struct page *)); 516 if (pvec != NULL) { 517 struct mm_struct *mm = obj->userptr.mm; 518 519 down_read(&mm->mmap_sem); 520 while (pinned < num_pages) { 521 ret = get_user_pages(work->task, mm, 522 obj->userptr.ptr + pinned * PAGE_SIZE, 523 num_pages - pinned, 524 !obj->userptr.read_only, 0, 525 pvec + pinned, NULL); 526 if (ret < 0) 527 break; 528 529 pinned += ret; 530 } 531 up_read(&mm->mmap_sem); 532 } 533 534 mutex_lock(&dev->struct_mutex); 535 if (obj->userptr.work != &work->work) { 536 ret = 0; 537 } else if (pinned == num_pages) { 538 ret = st_set_pages(&obj->pages, pvec, num_pages); 539 if (ret == 0) { 540 list_add_tail(&obj->global_list, &to_i915(dev)->mm.unbound_list); 541 pinned = 0; 542 } 543 } 544 545 obj->userptr.work = ERR_PTR(ret); 546 obj->userptr.workers--; 547 drm_gem_object_unreference(&obj->base); 548 mutex_unlock(&dev->struct_mutex); 549 550 release_pages(pvec, pinned, 0); 551 drm_free_large(pvec); 552 553 put_task_struct(work->task); 554 kfree(work); 555 } 556 557 static int 558 i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) 559 { 560 const int num_pages = obj->base.size >> PAGE_SHIFT; 561 struct page **pvec; 562 int pinned, ret; 563 564 /* If userspace should engineer that these pages are replaced in 565 * the vma between us binding this page into the GTT and completion 566 * of rendering... Their loss. If they change the mapping of their 567 * pages they need to create a new bo to point to the new vma. 568 * 569 * However, that still leaves open the possibility of the vma 570 * being copied upon fork. Which falls under the same userspace 571 * synchronisation issue as a regular bo, except that this time 572 * the process may not be expecting that a particular piece of 573 * memory is tied to the GPU. 574 * 575 * Fortunately, we can hook into the mmu_notifier in order to 576 * discard the page references prior to anything nasty happening 577 * to the vma (discard or cloning) which should prevent the more 578 * egregious cases from causing harm. 579 */ 580 581 pvec = NULL; 582 pinned = 0; 583 if (obj->userptr.mm == current->mm) { 584 pvec = kmalloc(num_pages*sizeof(struct page *), 585 GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); 586 if (pvec == NULL) { 587 pvec = drm_malloc_ab(num_pages, sizeof(struct page *)); 588 if (pvec == NULL) 589 return -ENOMEM; 590 } 591 592 pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages, 593 !obj->userptr.read_only, pvec); 594 } 595 if (pinned < num_pages) { 596 if (pinned < 0) { 597 ret = pinned; 598 pinned = 0; 599 } else { 600 /* Spawn a worker so that we can acquire the 601 * user pages without holding our mutex. Access 602 * to the user pages requires mmap_sem, and we have 603 * a strict lock ordering of mmap_sem, struct_mutex - 604 * we already hold struct_mutex here and so cannot 605 * call gup without encountering a lock inversion. 606 * 607 * Userspace will keep on repeating the operation 608 * (thanks to EAGAIN) until either we hit the fast 609 * path or the worker completes. If the worker is 610 * cancelled or superseded, the task is still run 611 * but the results ignored. (This leads to 612 * complications that we may have a stray object 613 * refcount that we need to be wary of when 614 * checking for existing objects during creation.) 615 * If the worker encounters an error, it reports 616 * that error back to this function through 617 * obj->userptr.work = ERR_PTR. 618 */ 619 ret = -EAGAIN; 620 if (obj->userptr.work == NULL && 621 obj->userptr.workers < I915_GEM_USERPTR_MAX_WORKERS) { 622 struct get_pages_work *work; 623 624 work = kmalloc(sizeof(*work), GFP_KERNEL); 625 if (work != NULL) { 626 obj->userptr.work = &work->work; 627 obj->userptr.workers++; 628 629 work->obj = obj; 630 drm_gem_object_reference(&obj->base); 631 632 work->task = current; 633 get_task_struct(work->task); 634 635 INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker); 636 schedule_work(&work->work); 637 } else 638 ret = -ENOMEM; 639 } else { 640 if (IS_ERR(obj->userptr.work)) { 641 ret = PTR_ERR(obj->userptr.work); 642 obj->userptr.work = NULL; 643 } 644 } 645 } 646 } else { 647 ret = st_set_pages(&obj->pages, pvec, num_pages); 648 if (ret == 0) { 649 obj->userptr.work = NULL; 650 pinned = 0; 651 } 652 } 653 654 release_pages(pvec, pinned, 0); 655 drm_free_large(pvec); 656 return ret; 657 } 658 659 static void 660 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) 661 { 662 struct sg_page_iter sg_iter; 663 664 BUG_ON(obj->userptr.work != NULL); 665 666 if (obj->madv != I915_MADV_WILLNEED) 667 obj->dirty = 0; 668 669 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { 670 struct page *page = sg_page_iter_page(&sg_iter); 671 672 if (obj->dirty) 673 set_page_dirty(page); 674 675 mark_page_accessed(page); 676 page_cache_release(page); 677 } 678 obj->dirty = 0; 679 680 sg_free_table(obj->pages); 681 kfree(obj->pages); 682 } 683 684 static void 685 i915_gem_userptr_release(struct drm_i915_gem_object *obj) 686 { 687 i915_gem_userptr_release__mmu_notifier(obj); 688 689 if (obj->userptr.mm) { 690 mmput(obj->userptr.mm); 691 obj->userptr.mm = NULL; 692 } 693 } 694 695 static int 696 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj) 697 { 698 if (obj->userptr.mn) 699 return 0; 700 701 return i915_gem_userptr_init__mmu_notifier(obj, 0); 702 } 703 704 static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { 705 .dmabuf_export = i915_gem_userptr_dmabuf_export, 706 .get_pages = i915_gem_userptr_get_pages, 707 .put_pages = i915_gem_userptr_put_pages, 708 .release = i915_gem_userptr_release, 709 }; 710 711 /** 712 * Creates a new mm object that wraps some normal memory from the process 713 * context - user memory. 714 * 715 * We impose several restrictions upon the memory being mapped 716 * into the GPU. 717 * 1. It must be page aligned (both start/end addresses, i.e ptr and size). 718 * 2. It must be normal system memory, not a pointer into another map of IO 719 * space (e.g. it must not be a GTT mmapping of another object). 720 * 3. We only allow a bo as large as we could in theory map into the GTT, 721 * that is we limit the size to the total size of the GTT. 722 * 4. The bo is marked as being snoopable. The backing pages are left 723 * accessible directly by the CPU, but reads and writes by the GPU may 724 * incur the cost of a snoop (unless you have an LLC architecture). 725 * 726 * Synchronisation between multiple users and the GPU is left to userspace 727 * through the normal set-domain-ioctl. The kernel will enforce that the 728 * GPU relinquishes the VMA before it is returned back to the system 729 * i.e. upon free(), munmap() or process termination. However, the userspace 730 * malloc() library may not immediately relinquish the VMA after free() and 731 * instead reuse it whilst the GPU is still reading and writing to the VMA. 732 * Caveat emptor. 733 * 734 * Also note, that the object created here is not currently a "first class" 735 * object, in that several ioctls are banned. These are the CPU access 736 * ioctls: mmap(), pwrite and pread. In practice, you are expected to use 737 * direct access via your pointer rather than use those ioctls. 738 * 739 * If you think this is a good interface to use to pass GPU memory between 740 * drivers, please use dma-buf instead. In fact, wherever possible use 741 * dma-buf instead. 742 */ 743 int 744 i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 745 { 746 struct drm_i915_private *dev_priv = dev->dev_private; 747 struct drm_i915_gem_userptr *args = data; 748 struct drm_i915_gem_object *obj; 749 int ret; 750 u32 handle; 751 752 if (args->flags & ~(I915_USERPTR_READ_ONLY | 753 I915_USERPTR_UNSYNCHRONIZED)) 754 return -EINVAL; 755 756 if (offset_in_page(args->user_ptr | args->user_size)) 757 return -EINVAL; 758 759 if (args->user_size > dev_priv->gtt.base.total) 760 return -E2BIG; 761 762 if (!access_ok(args->flags & I915_USERPTR_READ_ONLY ? VERIFY_READ : VERIFY_WRITE, 763 (char __user *)(unsigned long)args->user_ptr, args->user_size)) 764 return -EFAULT; 765 766 if (args->flags & I915_USERPTR_READ_ONLY) { 767 /* On almost all of the current hw, we cannot tell the GPU that a 768 * page is readonly, so this is just a placeholder in the uAPI. 769 */ 770 return -ENODEV; 771 } 772 773 /* Allocate the new object */ 774 obj = i915_gem_object_alloc(dev); 775 if (obj == NULL) 776 return -ENOMEM; 777 778 drm_gem_private_object_init(dev, &obj->base, args->user_size); 779 i915_gem_object_init(obj, &i915_gem_userptr_ops); 780 obj->cache_level = I915_CACHE_LLC; 781 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 782 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 783 784 obj->userptr.ptr = args->user_ptr; 785 obj->userptr.read_only = !!(args->flags & I915_USERPTR_READ_ONLY); 786 787 /* And keep a pointer to the current->mm for resolving the user pages 788 * at binding. This means that we need to hook into the mmu_notifier 789 * in order to detect if the mmu is destroyed. 790 */ 791 ret = -ENOMEM; 792 if ((obj->userptr.mm = get_task_mm(current))) 793 ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags); 794 if (ret == 0) 795 ret = drm_gem_handle_create(file, &obj->base, &handle); 796 797 /* drop reference from allocate - handle holds it now */ 798 drm_gem_object_unreference_unlocked(&obj->base); 799 if (ret) 800 return ret; 801 802 args->handle = handle; 803 return 0; 804 } 805 #endif 806 807 int 808 i915_gem_init_userptr(struct drm_device *dev) 809 { 810 #if defined(CONFIG_MMU_NOTIFIER) 811 struct drm_i915_private *dev_priv = to_i915(dev); 812 hash_init(dev_priv->mmu_notifiers); 813 #endif 814 return 0; 815 } 816