1 /* 2 * Copyright © 2008-2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 */ 27 28 #include <drm/drmP.h> 29 #include <drm/drm_vma_manager.h> 30 #include <drm/i915_drm.h> 31 #include "i915_drv.h" 32 #include "i915_gem_clflush.h" 33 #include "i915_vgpu.h" 34 #include "i915_trace.h" 35 #include "intel_drv.h" 36 #include "intel_frontbuffer.h" 37 #include "intel_mocs.h" 38 #include "i915_gemfs.h" 39 #include <linux/dma-fence-array.h> 40 #include <linux/kthread.h> 41 #include <linux/reservation.h> 42 #include <linux/shmem_fs.h> 43 #include <linux/slab.h> 44 #include <linux/stop_machine.h> 45 #include <linux/swap.h> 46 #include <linux/pci.h> 47 #include <linux/dma-buf.h> 48 #include <linux/swiotlb.h> 49 50 #include <sys/mman.h> 51 #include <vm/vm_map.h> 52 #include <vm/vm_param.h> 53 54 #undef USE_INSERT 55 56 static void i915_gem_flush_free_objects(struct drm_i915_private *i915); 57 58 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) 59 { 60 if (obj->cache_dirty) 61 return false; 62 63 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)) 64 return true; 65 66 return obj->pin_global; /* currently in use by HW, keep flushed */ 67 } 68 69 static int 70 insert_mappable_node(struct i915_ggtt *ggtt, 71 struct drm_mm_node *node, u32 size) 72 { 73 memset(node, 0, sizeof(*node)); 74 return drm_mm_insert_node_in_range(&ggtt->base.mm, node, 75 size, 0, I915_COLOR_UNEVICTABLE, 76 0, ggtt->mappable_end, 77 DRM_MM_INSERT_LOW); 78 } 79 80 static void 81 remove_mappable_node(struct drm_mm_node *node) 82 { 83 drm_mm_remove_node(node); 84 } 85 86 /* some bookkeeping */ 87 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, 88 u64 size) 89 { 90 lockmgr(&dev_priv->mm.object_stat_lock, LK_EXCLUSIVE); 91 dev_priv->mm.object_count++; 92 dev_priv->mm.object_memory += size; 93 lockmgr(&dev_priv->mm.object_stat_lock, LK_RELEASE); 94 } 95 96 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, 97 u64 size) 98 { 99 lockmgr(&dev_priv->mm.object_stat_lock, LK_EXCLUSIVE); 100 dev_priv->mm.object_count--; 101 dev_priv->mm.object_memory -= size; 102 lockmgr(&dev_priv->mm.object_stat_lock, LK_RELEASE); 103 } 104 105 static int 106 i915_gem_wait_for_error(struct i915_gpu_error *error) 107 { 108 int ret; 109 110 might_sleep(); 111 112 /* 113 * Only wait 10 seconds for the gpu reset to complete to avoid hanging 114 * userspace. If it takes that long something really bad is going on and 115 * we should simply try to bail out and fail as gracefully as possible. 116 */ 117 ret = wait_event_interruptible_timeout(error->reset_queue, 118 !i915_reset_backoff(error), 119 I915_RESET_TIMEOUT); 120 if (ret == 0) { 121 DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); 122 return -EIO; 123 } else if (ret < 0) { 124 return ret; 125 } else { 126 return 0; 127 } 128 } 129 130 int i915_mutex_lock_interruptible(struct drm_device *dev) 131 { 132 struct drm_i915_private *dev_priv = to_i915(dev); 133 int ret; 134 135 ret = i915_gem_wait_for_error(&dev_priv->gpu_error); 136 if (ret) 137 return ret; 138 139 ret = mutex_lock_interruptible(&dev->struct_mutex); 140 if (ret) 141 return ret; 142 143 return 0; 144 } 145 146 int 147 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 148 struct drm_file *file) 149 { 150 struct drm_i915_private *dev_priv = to_i915(dev); 151 struct i915_ggtt *ggtt = &dev_priv->ggtt; 152 struct drm_i915_gem_get_aperture *args = data; 153 struct i915_vma *vma; 154 u64 pinned; 155 156 pinned = ggtt->base.reserved; 157 mutex_lock(&dev->struct_mutex); 158 list_for_each_entry(vma, &ggtt->base.active_list, vm_link) 159 if (i915_vma_is_pinned(vma)) 160 pinned += vma->node.size; 161 list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link) 162 if (i915_vma_is_pinned(vma)) 163 pinned += vma->node.size; 164 mutex_unlock(&dev->struct_mutex); 165 166 args->aper_size = ggtt->base.total; 167 args->aper_available_size = args->aper_size - pinned; 168 169 return 0; 170 } 171 172 static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) 173 { 174 #if 0 175 struct address_space *mapping = obj->base.filp->f_mapping; 176 #else 177 vm_object_t vm_obj = obj->base.filp; 178 #endif 179 drm_dma_handle_t *phys; 180 struct sg_table *st; 181 struct scatterlist *sg; 182 char *vaddr; 183 int i; 184 int err; 185 186 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) 187 return -EINVAL; 188 189 /* Always aligning to the object size, allows a single allocation 190 * to handle all possible callers, and given typical object sizes, 191 * the alignment of the buddy allocation will naturally match. 192 */ 193 phys = drm_pci_alloc(obj->base.dev, 194 roundup_pow_of_two(obj->base.size), 195 roundup_pow_of_two(obj->base.size)); 196 if (!phys) 197 return -ENOMEM; 198 199 vaddr = phys->vaddr; 200 VM_OBJECT_LOCK(vm_obj); 201 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 202 struct page *page; 203 char *src; 204 205 #if 0 206 page = shmem_read_mapping_page(mapping, i); 207 #else 208 page = shmem_read_mapping_page(vm_obj, i); 209 #endif 210 if (IS_ERR(page)) { 211 err = PTR_ERR(page); 212 goto err_phys; 213 } 214 215 src = kmap_atomic(page); 216 memcpy(vaddr, src, PAGE_SIZE); 217 drm_clflush_virt_range(vaddr, PAGE_SIZE); 218 kunmap_atomic(src); 219 220 put_page(page); 221 vaddr += PAGE_SIZE; 222 } 223 VM_OBJECT_UNLOCK(vm_obj); 224 225 i915_gem_chipset_flush(to_i915(obj->base.dev)); 226 227 st = kmalloc(sizeof(*st), M_DRM, GFP_KERNEL); 228 if (!st) { 229 err = -ENOMEM; 230 goto err_phys; 231 } 232 233 if (sg_alloc_table(st, 1, GFP_KERNEL)) { 234 kfree(st); 235 err = -ENOMEM; 236 goto err_phys; 237 } 238 239 sg = st->sgl; 240 sg->offset = 0; 241 sg->length = obj->base.size; 242 243 sg_dma_address(sg) = phys->busaddr; 244 sg_dma_len(sg) = obj->base.size; 245 246 obj->phys_handle = phys; 247 248 __i915_gem_object_set_pages(obj, st, sg->length); 249 250 return 0; 251 252 err_phys: 253 drm_pci_free(obj->base.dev, phys); 254 255 return err; 256 } 257 258 static void __start_cpu_write(struct drm_i915_gem_object *obj) 259 { 260 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 261 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 262 if (cpu_write_needs_clflush(obj)) 263 obj->cache_dirty = true; 264 } 265 266 static void 267 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 268 struct sg_table *pages, 269 bool needs_clflush) 270 { 271 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); 272 273 if (obj->mm.madv == I915_MADV_DONTNEED) 274 obj->mm.dirty = false; 275 276 if (needs_clflush && 277 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0 && 278 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) 279 drm_clflush_sg(pages); 280 281 __start_cpu_write(obj); 282 } 283 284 static void 285 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 286 struct sg_table *pages) 287 { 288 __i915_gem_object_release_shmem(obj, pages, false); 289 290 if (obj->mm.dirty) { 291 #if 0 292 struct address_space *mapping = obj->base.filp->f_mapping; 293 #else 294 vm_object_t vm_obj = obj->base.filp; 295 #endif 296 char *vaddr = obj->phys_handle->vaddr; 297 int i; 298 299 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 300 struct page *page; 301 char *dst; 302 303 page = shmem_read_mapping_page(vm_obj, i); 304 if (IS_ERR(page)) 305 continue; 306 307 dst = kmap_atomic(page); 308 drm_clflush_virt_range(vaddr, PAGE_SIZE); 309 memcpy(dst, vaddr, PAGE_SIZE); 310 kunmap_atomic(dst); 311 312 set_page_dirty(page); 313 if (obj->mm.madv == I915_MADV_WILLNEED) 314 mark_page_accessed(page); 315 put_page(page); 316 vaddr += PAGE_SIZE; 317 } 318 obj->mm.dirty = false; 319 } 320 321 sg_free_table(pages); 322 kfree(pages); 323 324 drm_pci_free(obj->base.dev, obj->phys_handle); 325 } 326 327 static void 328 i915_gem_object_release_phys(struct drm_i915_gem_object *obj) 329 { 330 i915_gem_object_unpin_pages(obj); 331 } 332 333 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = { 334 .get_pages = i915_gem_object_get_pages_phys, 335 .put_pages = i915_gem_object_put_pages_phys, 336 .release = i915_gem_object_release_phys, 337 }; 338 339 static const struct drm_i915_gem_object_ops i915_gem_object_ops; 340 341 int i915_gem_object_unbind(struct drm_i915_gem_object *obj) 342 { 343 struct i915_vma *vma; 344 LINUX_LIST_HEAD(still_in_list); 345 int ret; 346 347 lockdep_assert_held(&obj->base.dev->struct_mutex); 348 349 /* Closed vma are removed from the obj->vma_list - but they may 350 * still have an active binding on the object. To remove those we 351 * must wait for all rendering to complete to the object (as unbinding 352 * must anyway), and retire the requests. 353 */ 354 ret = i915_gem_object_set_to_cpu_domain(obj, false); 355 if (ret) 356 return ret; 357 358 while ((vma = list_first_entry_or_null(&obj->vma_list, 359 struct i915_vma, 360 obj_link))) { 361 list_move_tail(&vma->obj_link, &still_in_list); 362 ret = i915_vma_unbind(vma); 363 if (ret) 364 break; 365 } 366 list_splice(&still_in_list, &obj->vma_list); 367 368 return ret; 369 } 370 371 static long 372 i915_gem_object_wait_fence(struct dma_fence *fence, 373 unsigned int flags, 374 long timeout, 375 struct intel_rps_client *rps_client) 376 { 377 struct drm_i915_gem_request *rq; 378 379 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1); 380 381 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 382 return timeout; 383 384 if (!dma_fence_is_i915(fence)) 385 return dma_fence_wait_timeout(fence, 386 flags & I915_WAIT_INTERRUPTIBLE, 387 timeout); 388 389 rq = to_request(fence); 390 if (i915_gem_request_completed(rq)) 391 goto out; 392 393 /* This client is about to stall waiting for the GPU. In many cases 394 * this is undesirable and limits the throughput of the system, as 395 * many clients cannot continue processing user input/output whilst 396 * blocked. RPS autotuning may take tens of milliseconds to respond 397 * to the GPU load and thus incurs additional latency for the client. 398 * We can circumvent that by promoting the GPU frequency to maximum 399 * before we wait. This makes the GPU throttle up much more quickly 400 * (good for benchmarks and user experience, e.g. window animations), 401 * but at a cost of spending more power processing the workload 402 * (bad for battery). Not all clients even want their results 403 * immediately and for them we should just let the GPU select its own 404 * frequency to maximise efficiency. To prevent a single client from 405 * forcing the clocks too high for the whole system, we only allow 406 * each client to waitboost once in a busy period. 407 */ 408 if (rps_client) { 409 if (INTEL_GEN(rq->i915) >= 6) 410 gen6_rps_boost(rq, rps_client); 411 else 412 rps_client = NULL; 413 } 414 415 timeout = i915_wait_request(rq, flags, timeout); 416 417 out: 418 if (flags & I915_WAIT_LOCKED && i915_gem_request_completed(rq)) 419 i915_gem_request_retire_upto(rq); 420 421 return timeout; 422 } 423 424 static long 425 i915_gem_object_wait_reservation(struct reservation_object *resv, 426 unsigned int flags, 427 long timeout, 428 struct intel_rps_client *rps_client) 429 { 430 unsigned int seq = __read_seqcount_begin(&resv->seq); 431 struct dma_fence *excl; 432 bool prune_fences = false; 433 434 if (flags & I915_WAIT_ALL) { 435 struct dma_fence **shared; 436 unsigned int count, i; 437 int ret; 438 439 ret = reservation_object_get_fences_rcu(resv, 440 &excl, &count, &shared); 441 if (ret) 442 return ret; 443 444 for (i = 0; i < count; i++) { 445 timeout = i915_gem_object_wait_fence(shared[i], 446 flags, timeout, 447 rps_client); 448 if (timeout < 0) 449 break; 450 451 dma_fence_put(shared[i]); 452 } 453 454 for (; i < count; i++) 455 dma_fence_put(shared[i]); 456 kfree(shared); 457 458 prune_fences = count && timeout >= 0; 459 } else { 460 excl = reservation_object_get_excl_rcu(resv); 461 } 462 463 if (excl && timeout >= 0) { 464 timeout = i915_gem_object_wait_fence(excl, flags, timeout, 465 rps_client); 466 prune_fences = timeout >= 0; 467 } 468 469 dma_fence_put(excl); 470 471 /* Oportunistically prune the fences iff we know they have *all* been 472 * signaled and that the reservation object has not been changed (i.e. 473 * no new fences have been added). 474 */ 475 if (prune_fences && !__read_seqcount_retry(&resv->seq, seq)) { 476 if (reservation_object_trylock(resv)) { 477 if (!__read_seqcount_retry(&resv->seq, seq)) 478 reservation_object_add_excl_fence(resv, NULL); 479 reservation_object_unlock(resv); 480 } 481 } 482 483 return timeout; 484 } 485 486 static void __fence_set_priority(struct dma_fence *fence, int prio) 487 { 488 struct drm_i915_gem_request *rq; 489 struct intel_engine_cs *engine; 490 491 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence)) 492 return; 493 494 rq = to_request(fence); 495 engine = rq->engine; 496 if (!engine->schedule) 497 return; 498 499 engine->schedule(rq, prio); 500 } 501 502 static void fence_set_priority(struct dma_fence *fence, int prio) 503 { 504 /* Recurse once into a fence-array */ 505 if (dma_fence_is_array(fence)) { 506 struct dma_fence_array *array = to_dma_fence_array(fence); 507 int i; 508 509 for (i = 0; i < array->num_fences; i++) 510 __fence_set_priority(array->fences[i], prio); 511 } else { 512 __fence_set_priority(fence, prio); 513 } 514 } 515 516 int 517 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 518 unsigned int flags, 519 int prio) 520 { 521 struct dma_fence *excl; 522 523 if (flags & I915_WAIT_ALL) { 524 struct dma_fence **shared; 525 unsigned int count, i; 526 int ret; 527 528 ret = reservation_object_get_fences_rcu(obj->resv, 529 &excl, &count, &shared); 530 if (ret) 531 return ret; 532 533 for (i = 0; i < count; i++) { 534 fence_set_priority(shared[i], prio); 535 dma_fence_put(shared[i]); 536 } 537 538 kfree(shared); 539 } else { 540 excl = reservation_object_get_excl_rcu(obj->resv); 541 } 542 543 if (excl) { 544 fence_set_priority(excl, prio); 545 dma_fence_put(excl); 546 } 547 return 0; 548 } 549 550 /** 551 * Waits for rendering to the object to be completed 552 * @obj: i915 gem object 553 * @flags: how to wait (under a lock, for all rendering or just for writes etc) 554 * @timeout: how long to wait 555 * @rps: client (user process) to charge for any waitboosting 556 */ 557 int 558 i915_gem_object_wait(struct drm_i915_gem_object *obj, 559 unsigned int flags, 560 long timeout, 561 struct intel_rps_client *rps_client) 562 { 563 might_sleep(); 564 #if IS_ENABLED(CONFIG_LOCKDEP) 565 GEM_BUG_ON(debug_locks && 566 !!lockdep_is_held(&obj->base.dev->struct_mutex) != 567 !!(flags & I915_WAIT_LOCKED)); 568 #endif 569 GEM_BUG_ON(timeout < 0); 570 571 timeout = i915_gem_object_wait_reservation(obj->resv, 572 flags, timeout, 573 rps_client); 574 return timeout < 0 ? timeout : 0; 575 } 576 577 static struct intel_rps_client *to_rps_client(struct drm_file *file) 578 { 579 struct drm_i915_file_private *fpriv = file->driver_priv; 580 581 return &fpriv->rps_client; 582 } 583 584 static int 585 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, 586 struct drm_i915_gem_pwrite *args, 587 struct drm_file *file) 588 { 589 void *vaddr = obj->phys_handle->vaddr + args->offset; 590 char __user *user_data = u64_to_user_ptr(args->data_ptr); 591 592 /* We manually control the domain here and pretend that it 593 * remains coherent i.e. in the GTT domain, like shmem_pwrite. 594 */ 595 intel_fb_obj_invalidate(obj, ORIGIN_CPU); 596 if (copy_from_user(vaddr, user_data, args->size)) 597 return -EFAULT; 598 599 drm_clflush_virt_range(vaddr, args->size); 600 i915_gem_chipset_flush(to_i915(obj->base.dev)); 601 602 intel_fb_obj_flush(obj, ORIGIN_CPU); 603 return 0; 604 } 605 606 void *i915_gem_object_alloc(struct drm_i915_private *dev_priv) 607 { 608 return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL); 609 } 610 611 void i915_gem_object_free(struct drm_i915_gem_object *obj) 612 { 613 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 614 kmem_cache_free(dev_priv->objects, obj); 615 } 616 617 static int 618 i915_gem_create(struct drm_file *file, 619 struct drm_i915_private *dev_priv, 620 uint64_t size, 621 uint32_t *handle_p) 622 { 623 struct drm_i915_gem_object *obj; 624 int ret; 625 u32 handle; 626 627 size = roundup(size, PAGE_SIZE); 628 if (size == 0) 629 return -EINVAL; 630 631 /* Allocate the new object */ 632 obj = i915_gem_object_create(dev_priv, size); 633 if (IS_ERR(obj)) 634 return PTR_ERR(obj); 635 636 ret = drm_gem_handle_create(file, &obj->base, &handle); 637 /* drop reference from allocate - handle holds it now */ 638 i915_gem_object_put(obj); 639 if (ret) 640 return ret; 641 642 *handle_p = handle; 643 return 0; 644 } 645 646 int 647 i915_gem_dumb_create(struct drm_file *file, 648 struct drm_device *dev, 649 struct drm_mode_create_dumb *args) 650 { 651 /* have to work out size/pitch and return them */ 652 args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64); 653 args->size = args->pitch * args->height; 654 return i915_gem_create(file, to_i915(dev), 655 args->size, &args->handle); 656 } 657 658 static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj) 659 { 660 return !(obj->cache_level == I915_CACHE_NONE || 661 obj->cache_level == I915_CACHE_WT); 662 } 663 664 /** 665 * Creates a new mm object and returns a handle to it. 666 * @dev: drm device pointer 667 * @data: ioctl data blob 668 * @file: drm file pointer 669 */ 670 int 671 i915_gem_create_ioctl(struct drm_device *dev, void *data, 672 struct drm_file *file) 673 { 674 struct drm_i915_private *dev_priv = to_i915(dev); 675 struct drm_i915_gem_create *args = data; 676 677 i915_gem_flush_free_objects(dev_priv); 678 679 return i915_gem_create(file, dev_priv, 680 args->size, &args->handle); 681 } 682 683 static inline enum fb_op_origin 684 fb_write_origin(struct drm_i915_gem_object *obj, unsigned int domain) 685 { 686 return (domain == I915_GEM_DOMAIN_GTT ? 687 obj->frontbuffer_ggtt_origin : ORIGIN_CPU); 688 } 689 690 static void 691 flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains) 692 { 693 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 694 695 if (!(obj->base.write_domain & flush_domains)) 696 return; 697 698 /* No actual flushing is required for the GTT write domain. Writes 699 * to it "immediately" go to main memory as far as we know, so there's 700 * no chipset flush. It also doesn't land in render cache. 701 * 702 * However, we do have to enforce the order so that all writes through 703 * the GTT land before any writes to the device, such as updates to 704 * the GATT itself. 705 * 706 * We also have to wait a bit for the writes to land from the GTT. 707 * An uncached read (i.e. mmio) seems to be ideal for the round-trip 708 * timing. This issue has only been observed when switching quickly 709 * between GTT writes and CPU reads from inside the kernel on recent hw, 710 * and it appears to only affect discrete GTT blocks (i.e. on LLC 711 * system agents we cannot reproduce this behaviour). 712 */ 713 wmb(); 714 715 switch (obj->base.write_domain) { 716 case I915_GEM_DOMAIN_GTT: 717 if (!HAS_LLC(dev_priv)) { 718 intel_runtime_pm_get(dev_priv); 719 spin_lock_irq(&dev_priv->uncore.lock); 720 POSTING_READ_FW(RING_HEAD(dev_priv->engine[RCS]->mmio_base)); 721 spin_unlock_irq(&dev_priv->uncore.lock); 722 intel_runtime_pm_put(dev_priv); 723 } 724 725 intel_fb_obj_flush(obj, 726 fb_write_origin(obj, I915_GEM_DOMAIN_GTT)); 727 break; 728 729 case I915_GEM_DOMAIN_CPU: 730 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC); 731 break; 732 733 case I915_GEM_DOMAIN_RENDER: 734 if (gpu_write_needs_clflush(obj)) 735 obj->cache_dirty = true; 736 break; 737 } 738 739 obj->base.write_domain = 0; 740 } 741 742 static inline int 743 __copy_to_user_swizzled(char __user *cpu_vaddr, 744 const char *gpu_vaddr, int gpu_offset, 745 int length) 746 { 747 int ret, cpu_offset = 0; 748 749 while (length > 0) { 750 int cacheline_end = ALIGN(gpu_offset + 1, 64); 751 int this_length = min(cacheline_end - gpu_offset, length); 752 int swizzled_gpu_offset = gpu_offset ^ 64; 753 754 ret = __copy_to_user(cpu_vaddr + cpu_offset, 755 gpu_vaddr + swizzled_gpu_offset, 756 this_length); 757 if (ret) 758 return ret + length; 759 760 cpu_offset += this_length; 761 gpu_offset += this_length; 762 length -= this_length; 763 } 764 765 return 0; 766 } 767 768 static inline int 769 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, 770 const char __user *cpu_vaddr, 771 int length) 772 { 773 int ret, cpu_offset = 0; 774 775 while (length > 0) { 776 int cacheline_end = ALIGN(gpu_offset + 1, 64); 777 int this_length = min(cacheline_end - gpu_offset, length); 778 int swizzled_gpu_offset = gpu_offset ^ 64; 779 780 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, 781 cpu_vaddr + cpu_offset, 782 this_length); 783 if (ret) 784 return ret + length; 785 786 cpu_offset += this_length; 787 gpu_offset += this_length; 788 length -= this_length; 789 } 790 791 return 0; 792 } 793 794 /* 795 * Pins the specified object's pages and synchronizes the object with 796 * GPU accesses. Sets needs_clflush to non-zero if the caller should 797 * flush the object from the CPU cache. 798 */ 799 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, 800 unsigned int *needs_clflush) 801 { 802 int ret; 803 804 lockdep_assert_held(&obj->base.dev->struct_mutex); 805 806 *needs_clflush = 0; 807 if (!i915_gem_object_has_struct_page(obj)) 808 return -ENODEV; 809 810 ret = i915_gem_object_wait(obj, 811 I915_WAIT_INTERRUPTIBLE | 812 I915_WAIT_LOCKED, 813 MAX_SCHEDULE_TIMEOUT, 814 NULL); 815 if (ret) 816 return ret; 817 818 ret = i915_gem_object_pin_pages(obj); 819 if (ret) 820 return ret; 821 822 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ || 823 !static_cpu_has(X86_FEATURE_CLFLUSH)) { 824 ret = i915_gem_object_set_to_cpu_domain(obj, false); 825 if (ret) 826 goto err_unpin; 827 else 828 goto out; 829 } 830 831 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 832 833 /* If we're not in the cpu read domain, set ourself into the gtt 834 * read domain and manually flush cachelines (if required). This 835 * optimizes for the case when the gpu will dirty the data 836 * anyway again before the next pread happens. 837 */ 838 if (!obj->cache_dirty && 839 !(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) 840 *needs_clflush = CLFLUSH_BEFORE; 841 842 out: 843 /* return with the pages pinned */ 844 return 0; 845 846 err_unpin: 847 i915_gem_object_unpin_pages(obj); 848 return ret; 849 } 850 851 int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, 852 unsigned int *needs_clflush) 853 { 854 int ret; 855 856 lockdep_assert_held(&obj->base.dev->struct_mutex); 857 858 *needs_clflush = 0; 859 if (!i915_gem_object_has_struct_page(obj)) 860 return -ENODEV; 861 862 ret = i915_gem_object_wait(obj, 863 I915_WAIT_INTERRUPTIBLE | 864 I915_WAIT_LOCKED | 865 I915_WAIT_ALL, 866 MAX_SCHEDULE_TIMEOUT, 867 NULL); 868 if (ret) 869 return ret; 870 871 ret = i915_gem_object_pin_pages(obj); 872 if (ret) 873 return ret; 874 875 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE || 876 !static_cpu_has(X86_FEATURE_CLFLUSH)) { 877 ret = i915_gem_object_set_to_cpu_domain(obj, true); 878 if (ret) 879 goto err_unpin; 880 else 881 goto out; 882 } 883 884 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 885 886 /* If we're not in the cpu write domain, set ourself into the 887 * gtt write domain and manually flush cachelines (as required). 888 * This optimizes for the case when the gpu will use the data 889 * right away and we therefore have to clflush anyway. 890 */ 891 if (!obj->cache_dirty) { 892 *needs_clflush |= CLFLUSH_AFTER; 893 894 /* 895 * Same trick applies to invalidate partially written 896 * cachelines read before writing. 897 */ 898 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) 899 *needs_clflush |= CLFLUSH_BEFORE; 900 } 901 902 out: 903 intel_fb_obj_invalidate(obj, ORIGIN_CPU); 904 obj->mm.dirty = true; 905 /* return with the pages pinned */ 906 return 0; 907 908 err_unpin: 909 i915_gem_object_unpin_pages(obj); 910 return ret; 911 } 912 913 static void 914 shmem_clflush_swizzled_range(char *addr, unsigned long length, 915 bool swizzled) 916 { 917 if (unlikely(swizzled)) { 918 unsigned long start = (unsigned long) addr; 919 unsigned long end = (unsigned long) addr + length; 920 921 /* For swizzling simply ensure that we always flush both 922 * channels. Lame, but simple and it works. Swizzled 923 * pwrite/pread is far from a hotpath - current userspace 924 * doesn't use it at all. */ 925 start = round_down(start, 128); 926 end = round_up(end, 128); 927 928 drm_clflush_virt_range((void *)start, end - start); 929 } else { 930 drm_clflush_virt_range(addr, length); 931 } 932 933 } 934 935 /* Only difference to the fast-path function is that this can handle bit17 936 * and uses non-atomic copy and kmap functions. */ 937 static int 938 shmem_pread_slow(struct page *page, int offset, int length, 939 char __user *user_data, 940 bool page_do_bit17_swizzling, bool needs_clflush) 941 { 942 char *vaddr; 943 int ret; 944 945 vaddr = kmap(page); 946 if (needs_clflush) 947 shmem_clflush_swizzled_range(vaddr + offset, length, 948 page_do_bit17_swizzling); 949 950 if (page_do_bit17_swizzling) 951 ret = __copy_to_user_swizzled(user_data, vaddr, offset, length); 952 else 953 ret = __copy_to_user(user_data, vaddr + offset, length); 954 kunmap(page); 955 956 return ret ? - EFAULT : 0; 957 } 958 959 static int 960 shmem_pread(struct page *page, int offset, int length, char __user *user_data, 961 bool page_do_bit17_swizzling, bool needs_clflush) 962 { 963 int ret; 964 965 ret = -ENODEV; 966 if (!page_do_bit17_swizzling) { 967 char *vaddr = kmap_atomic(page); 968 969 if (needs_clflush) 970 drm_clflush_virt_range(vaddr + offset, length); 971 ret = __copy_to_user_inatomic(user_data, vaddr + offset, length); 972 kunmap_atomic(vaddr); 973 } 974 if (ret == 0) 975 return 0; 976 977 return shmem_pread_slow(page, offset, length, user_data, 978 page_do_bit17_swizzling, needs_clflush); 979 } 980 981 static int 982 i915_gem_shmem_pread(struct drm_i915_gem_object *obj, 983 struct drm_i915_gem_pread *args) 984 { 985 char __user *user_data; 986 u64 remain; 987 unsigned int obj_do_bit17_swizzling; 988 unsigned int needs_clflush; 989 unsigned int idx, offset; 990 int ret; 991 992 obj_do_bit17_swizzling = 0; 993 if (i915_gem_object_needs_bit17_swizzle(obj)) 994 obj_do_bit17_swizzling = BIT(17); 995 996 ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex); 997 if (ret) 998 return ret; 999 1000 ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush); 1001 mutex_unlock(&obj->base.dev->struct_mutex); 1002 if (ret) 1003 return ret; 1004 1005 remain = args->size; 1006 user_data = u64_to_user_ptr(args->data_ptr); 1007 offset = offset_in_page(args->offset); 1008 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { 1009 struct page *page = i915_gem_object_get_page(obj, idx); 1010 int length; 1011 1012 length = remain; 1013 if (offset + length > PAGE_SIZE) 1014 length = PAGE_SIZE - offset; 1015 1016 ret = shmem_pread(page, offset, length, user_data, 1017 page_to_phys(page) & obj_do_bit17_swizzling, 1018 needs_clflush); 1019 if (ret) 1020 break; 1021 1022 remain -= length; 1023 user_data += length; 1024 offset = 0; 1025 } 1026 1027 i915_gem_obj_finish_shmem_access(obj); 1028 return ret; 1029 } 1030 1031 static inline bool 1032 gtt_user_read(struct io_mapping *mapping, 1033 loff_t base, int offset, 1034 char __user *user_data, int length) 1035 { 1036 void __iomem *vaddr; 1037 unsigned long unwritten; 1038 1039 /* We can use the cpu mem copy function because this is X86. */ 1040 vaddr = io_mapping_map_atomic_wc(mapping, base); 1041 unwritten = __copy_to_user_inatomic(user_data, 1042 (void __force *)vaddr + offset, 1043 length); 1044 io_mapping_unmap_atomic(vaddr); 1045 if (unwritten) { 1046 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE); 1047 unwritten = copy_to_user(user_data, 1048 (void __force *)vaddr + offset, 1049 length); 1050 io_mapping_unmap(vaddr); 1051 } 1052 return unwritten; 1053 } 1054 1055 static int 1056 i915_gem_gtt_pread(struct drm_i915_gem_object *obj, 1057 const struct drm_i915_gem_pread *args) 1058 { 1059 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1060 struct i915_ggtt *ggtt = &i915->ggtt; 1061 struct drm_mm_node node; 1062 struct i915_vma *vma; 1063 void __user *user_data; 1064 u64 remain, offset; 1065 int ret; 1066 1067 ret = mutex_lock_interruptible(&i915->drm.struct_mutex); 1068 if (ret) 1069 return ret; 1070 1071 intel_runtime_pm_get(i915); 1072 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 1073 PIN_MAPPABLE | 1074 PIN_NONFAULT | 1075 PIN_NONBLOCK); 1076 if (!IS_ERR(vma)) { 1077 node.start = i915_ggtt_offset(vma); 1078 node.allocated = false; 1079 ret = i915_vma_put_fence(vma); 1080 if (ret) { 1081 i915_vma_unpin(vma); 1082 vma = ERR_PTR(ret); 1083 } 1084 } 1085 if (IS_ERR(vma)) { 1086 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE); 1087 if (ret) 1088 goto out_unlock; 1089 GEM_BUG_ON(!node.allocated); 1090 } 1091 1092 ret = i915_gem_object_set_to_gtt_domain(obj, false); 1093 if (ret) 1094 goto out_unpin; 1095 1096 mutex_unlock(&i915->drm.struct_mutex); 1097 1098 user_data = u64_to_user_ptr(args->data_ptr); 1099 remain = args->size; 1100 offset = args->offset; 1101 1102 while (remain > 0) { 1103 /* Operation in this page 1104 * 1105 * page_base = page offset within aperture 1106 * page_offset = offset within page 1107 * page_length = bytes to copy for this page 1108 */ 1109 u32 page_base = node.start; 1110 unsigned page_offset = offset_in_page(offset); 1111 unsigned page_length = PAGE_SIZE - page_offset; 1112 page_length = remain < page_length ? remain : page_length; 1113 if (node.allocated) { 1114 wmb(); 1115 ggtt->base.insert_page(&ggtt->base, 1116 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), 1117 node.start, I915_CACHE_NONE, 0); 1118 wmb(); 1119 } else { 1120 page_base += offset & LINUX_PAGE_MASK; 1121 } 1122 1123 if (gtt_user_read(&ggtt->mappable, page_base, page_offset, 1124 user_data, page_length)) { 1125 ret = -EFAULT; 1126 break; 1127 } 1128 1129 remain -= page_length; 1130 user_data += page_length; 1131 offset += page_length; 1132 } 1133 1134 mutex_lock(&i915->drm.struct_mutex); 1135 out_unpin: 1136 if (node.allocated) { 1137 wmb(); 1138 ggtt->base.clear_range(&ggtt->base, 1139 node.start, node.size); 1140 remove_mappable_node(&node); 1141 } else { 1142 i915_vma_unpin(vma); 1143 } 1144 out_unlock: 1145 intel_runtime_pm_put(i915); 1146 mutex_unlock(&i915->drm.struct_mutex); 1147 1148 return ret; 1149 } 1150 1151 /** 1152 * Reads data from the object referenced by handle. 1153 * @dev: drm device pointer 1154 * @data: ioctl data blob 1155 * @file: drm file pointer 1156 * 1157 * On error, the contents of *data are undefined. 1158 */ 1159 int 1160 i915_gem_pread_ioctl(struct drm_device *dev, void *data, 1161 struct drm_file *file) 1162 { 1163 struct drm_i915_gem_pread *args = data; 1164 struct drm_i915_gem_object *obj; 1165 int ret; 1166 1167 if (args->size == 0) 1168 return 0; 1169 1170 #if 0 1171 if (!access_ok(VERIFY_WRITE, 1172 u64_to_user_ptr(args->data_ptr), 1173 args->size)) 1174 return -EFAULT; 1175 #endif 1176 1177 obj = i915_gem_object_lookup(file, args->handle); 1178 if (!obj) 1179 return -ENOENT; 1180 1181 /* Bounds check source. */ 1182 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) { 1183 ret = -EINVAL; 1184 goto out; 1185 } 1186 1187 trace_i915_gem_object_pread(obj, args->offset, args->size); 1188 1189 ret = i915_gem_object_wait(obj, 1190 I915_WAIT_INTERRUPTIBLE, 1191 MAX_SCHEDULE_TIMEOUT, 1192 to_rps_client(file)); 1193 if (ret) 1194 goto out; 1195 1196 ret = i915_gem_object_pin_pages(obj); 1197 if (ret) 1198 goto out; 1199 1200 ret = i915_gem_shmem_pread(obj, args); 1201 if (ret == -EFAULT || ret == -ENODEV) 1202 ret = i915_gem_gtt_pread(obj, args); 1203 1204 i915_gem_object_unpin_pages(obj); 1205 out: 1206 i915_gem_object_put(obj); 1207 return ret; 1208 } 1209 1210 /* This is the fast write path which cannot handle 1211 * page faults in the source data 1212 */ 1213 1214 static inline bool 1215 ggtt_write(struct io_mapping *mapping, 1216 loff_t base, int offset, 1217 char __user *user_data, int length) 1218 { 1219 void __iomem *vaddr; 1220 unsigned long unwritten; 1221 1222 /* We can use the cpu mem copy function because this is X86. */ 1223 vaddr = io_mapping_map_atomic_wc(mapping, base); 1224 unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset, 1225 user_data, length); 1226 io_mapping_unmap_atomic(vaddr); 1227 if (unwritten) { 1228 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE); 1229 unwritten = copy_from_user((void __force *)vaddr + offset, 1230 user_data, length); 1231 io_mapping_unmap(vaddr); 1232 } 1233 1234 return unwritten; 1235 } 1236 1237 /** 1238 * This is the fast pwrite path, where we copy the data directly from the 1239 * user into the GTT, uncached. 1240 * @obj: i915 GEM object 1241 * @args: pwrite arguments structure 1242 */ 1243 static int 1244 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, 1245 const struct drm_i915_gem_pwrite *args) 1246 { 1247 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1248 struct i915_ggtt *ggtt = &i915->ggtt; 1249 struct drm_mm_node node; 1250 struct i915_vma *vma; 1251 u64 remain, offset; 1252 void __user *user_data; 1253 int ret; 1254 1255 ret = mutex_lock_interruptible(&i915->drm.struct_mutex); 1256 if (ret) 1257 return ret; 1258 1259 if (i915_gem_object_has_struct_page(obj)) { 1260 /* 1261 * Avoid waking the device up if we can fallback, as 1262 * waking/resuming is very slow (worst-case 10-100 ms 1263 * depending on PCI sleeps and our own resume time). 1264 * This easily dwarfs any performance advantage from 1265 * using the cache bypass of indirect GGTT access. 1266 */ 1267 if (!intel_runtime_pm_get_if_in_use(i915)) { 1268 ret = -EFAULT; 1269 goto out_unlock; 1270 } 1271 } else { 1272 /* No backing pages, no fallback, we must force GGTT access */ 1273 intel_runtime_pm_get(i915); 1274 } 1275 1276 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 1277 PIN_MAPPABLE | 1278 PIN_NONFAULT | 1279 PIN_NONBLOCK); 1280 if (!IS_ERR(vma)) { 1281 node.start = i915_ggtt_offset(vma); 1282 node.allocated = false; 1283 ret = i915_vma_put_fence(vma); 1284 if (ret) { 1285 i915_vma_unpin(vma); 1286 vma = ERR_PTR(ret); 1287 } 1288 } 1289 if (IS_ERR(vma)) { 1290 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE); 1291 if (ret) 1292 goto out_rpm; 1293 GEM_BUG_ON(!node.allocated); 1294 } 1295 1296 ret = i915_gem_object_set_to_gtt_domain(obj, true); 1297 if (ret) 1298 goto out_unpin; 1299 1300 mutex_unlock(&i915->drm.struct_mutex); 1301 1302 intel_fb_obj_invalidate(obj, ORIGIN_CPU); 1303 1304 user_data = u64_to_user_ptr(args->data_ptr); 1305 offset = args->offset; 1306 remain = args->size; 1307 while (remain) { 1308 /* Operation in this page 1309 * 1310 * page_base = page offset within aperture 1311 * page_offset = offset within page 1312 * page_length = bytes to copy for this page 1313 */ 1314 u32 page_base = node.start; 1315 unsigned int page_offset = offset_in_page(offset); 1316 unsigned int page_length = PAGE_SIZE - page_offset; 1317 page_length = remain < page_length ? remain : page_length; 1318 if (node.allocated) { 1319 wmb(); /* flush the write before we modify the GGTT */ 1320 ggtt->base.insert_page(&ggtt->base, 1321 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), 1322 node.start, I915_CACHE_NONE, 0); 1323 wmb(); /* flush modifications to the GGTT (insert_page) */ 1324 } else { 1325 page_base += offset & LINUX_PAGE_MASK; 1326 } 1327 /* If we get a fault while copying data, then (presumably) our 1328 * source page isn't available. Return the error and we'll 1329 * retry in the slow path. 1330 * If the object is non-shmem backed, we retry again with the 1331 * path that handles page fault. 1332 */ 1333 if (ggtt_write(&ggtt->mappable, page_base, page_offset, 1334 user_data, page_length)) { 1335 ret = -EFAULT; 1336 break; 1337 } 1338 1339 remain -= page_length; 1340 user_data += page_length; 1341 offset += page_length; 1342 } 1343 intel_fb_obj_flush(obj, ORIGIN_CPU); 1344 1345 mutex_lock(&i915->drm.struct_mutex); 1346 out_unpin: 1347 if (node.allocated) { 1348 wmb(); 1349 ggtt->base.clear_range(&ggtt->base, 1350 node.start, node.size); 1351 remove_mappable_node(&node); 1352 } else { 1353 i915_vma_unpin(vma); 1354 } 1355 out_rpm: 1356 intel_runtime_pm_put(i915); 1357 out_unlock: 1358 mutex_unlock(&i915->drm.struct_mutex); 1359 return ret; 1360 } 1361 1362 static int 1363 shmem_pwrite_slow(struct page *page, int offset, int length, 1364 char __user *user_data, 1365 bool page_do_bit17_swizzling, 1366 bool needs_clflush_before, 1367 bool needs_clflush_after) 1368 { 1369 char *vaddr; 1370 int ret; 1371 1372 vaddr = kmap(page); 1373 if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) 1374 shmem_clflush_swizzled_range(vaddr + offset, length, 1375 page_do_bit17_swizzling); 1376 if (page_do_bit17_swizzling) 1377 ret = __copy_from_user_swizzled(vaddr, offset, user_data, 1378 length); 1379 else 1380 ret = __copy_from_user(vaddr + offset, user_data, length); 1381 if (needs_clflush_after) 1382 shmem_clflush_swizzled_range(vaddr + offset, length, 1383 page_do_bit17_swizzling); 1384 kunmap(page); 1385 1386 return ret ? -EFAULT : 0; 1387 } 1388 1389 /* Per-page copy function for the shmem pwrite fastpath. 1390 * Flushes invalid cachelines before writing to the target if 1391 * needs_clflush_before is set and flushes out any written cachelines after 1392 * writing if needs_clflush is set. 1393 */ 1394 static int 1395 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data, 1396 bool page_do_bit17_swizzling, 1397 bool needs_clflush_before, 1398 bool needs_clflush_after) 1399 { 1400 int ret; 1401 1402 ret = -ENODEV; 1403 if (!page_do_bit17_swizzling) { 1404 char *vaddr = kmap_atomic(page); 1405 1406 if (needs_clflush_before) 1407 drm_clflush_virt_range(vaddr + offset, len); 1408 ret = __copy_from_user_inatomic(vaddr + offset, user_data, len); 1409 if (needs_clflush_after) 1410 drm_clflush_virt_range(vaddr + offset, len); 1411 1412 kunmap_atomic(vaddr); 1413 } 1414 if (ret == 0) 1415 return ret; 1416 1417 return shmem_pwrite_slow(page, offset, len, user_data, 1418 page_do_bit17_swizzling, 1419 needs_clflush_before, 1420 needs_clflush_after); 1421 } 1422 1423 static int 1424 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, 1425 const struct drm_i915_gem_pwrite *args) 1426 { 1427 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1428 void __user *user_data; 1429 u64 remain; 1430 unsigned int obj_do_bit17_swizzling; 1431 unsigned int partial_cacheline_write; 1432 unsigned int needs_clflush; 1433 unsigned int offset, idx; 1434 int ret; 1435 #ifdef __DragonFly__ 1436 vm_object_t vm_obj; 1437 #endif 1438 1439 ret = mutex_lock_interruptible(&i915->drm.struct_mutex); 1440 if (ret) 1441 return ret; 1442 1443 ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush); 1444 mutex_unlock(&i915->drm.struct_mutex); 1445 if (ret) 1446 return ret; 1447 1448 obj_do_bit17_swizzling = 0; 1449 if (i915_gem_object_needs_bit17_swizzle(obj)) 1450 obj_do_bit17_swizzling = BIT(17); 1451 1452 /* If we don't overwrite a cacheline completely we need to be 1453 * careful to have up-to-date data by first clflushing. Don't 1454 * overcomplicate things and flush the entire patch. 1455 */ 1456 partial_cacheline_write = 0; 1457 if (needs_clflush & CLFLUSH_BEFORE) 1458 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1; 1459 1460 user_data = u64_to_user_ptr(args->data_ptr); 1461 remain = args->size; 1462 offset = offset_in_page(args->offset); 1463 #ifdef __DragonFly__ 1464 vm_obj = obj->base.filp; 1465 VM_OBJECT_LOCK(vm_obj); 1466 vm_object_pip_add(vm_obj, 1); 1467 #endif 1468 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { 1469 struct page *page = i915_gem_object_get_page(obj, idx); 1470 int length; 1471 1472 length = remain; 1473 if (offset + length > PAGE_SIZE) 1474 length = PAGE_SIZE - offset; 1475 1476 ret = shmem_pwrite(page, offset, length, user_data, 1477 page_to_phys(page) & obj_do_bit17_swizzling, 1478 (offset | length) & partial_cacheline_write, 1479 needs_clflush & CLFLUSH_AFTER); 1480 if (ret) 1481 break; 1482 1483 remain -= length; 1484 user_data += length; 1485 offset = 0; 1486 } 1487 #ifdef __DragonFly__ 1488 if (vm_obj != obj->base.filp) { 1489 kprintf("i915_gem_shmem_pwrite: VM_OBJECT CHANGED! %p %p\n", 1490 vm_obj, obj->base.filp); 1491 } 1492 vm_object_pip_wakeup(vm_obj); 1493 VM_OBJECT_UNLOCK(vm_obj); 1494 #endif 1495 1496 intel_fb_obj_flush(obj, ORIGIN_CPU); 1497 i915_gem_obj_finish_shmem_access(obj); 1498 return ret; 1499 } 1500 1501 /** 1502 * Writes data to the object referenced by handle. 1503 * @dev: drm device 1504 * @data: ioctl data blob 1505 * @file: drm file 1506 * 1507 * On error, the contents of the buffer that were to be modified are undefined. 1508 */ 1509 int 1510 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 1511 struct drm_file *file) 1512 { 1513 struct drm_i915_gem_pwrite *args = data; 1514 struct drm_i915_gem_object *obj; 1515 int ret; 1516 1517 if (args->size == 0) 1518 return 0; 1519 1520 #if 0 1521 if (!access_ok(VERIFY_READ, 1522 u64_to_user_ptr(args->data_ptr), 1523 args->size)) 1524 return -EFAULT; 1525 #endif 1526 1527 obj = i915_gem_object_lookup(file, args->handle); 1528 if (!obj) 1529 return -ENOENT; 1530 1531 /* Bounds check destination. */ 1532 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) { 1533 ret = -EINVAL; 1534 goto err; 1535 } 1536 1537 trace_i915_gem_object_pwrite(obj, args->offset, args->size); 1538 1539 ret = -ENODEV; 1540 if (obj->ops->pwrite) 1541 ret = obj->ops->pwrite(obj, args); 1542 if (ret != -ENODEV) 1543 goto err; 1544 1545 ret = i915_gem_object_wait(obj, 1546 I915_WAIT_INTERRUPTIBLE | 1547 I915_WAIT_ALL, 1548 MAX_SCHEDULE_TIMEOUT, 1549 to_rps_client(file)); 1550 if (ret) 1551 goto err; 1552 1553 ret = i915_gem_object_pin_pages(obj); 1554 if (ret) 1555 goto err; 1556 1557 ret = -EFAULT; 1558 /* We can only do the GTT pwrite on untiled buffers, as otherwise 1559 * it would end up going through the fenced access, and we'll get 1560 * different detiling behavior between reading and writing. 1561 * pread/pwrite currently are reading and writing from the CPU 1562 * perspective, requiring manual detiling by the client. 1563 */ 1564 if (!i915_gem_object_has_struct_page(obj) || 1565 cpu_write_needs_clflush(obj)) 1566 /* Note that the gtt paths might fail with non-page-backed user 1567 * pointers (e.g. gtt mappings when moving data between 1568 * textures). Fallback to the shmem path in that case. 1569 */ 1570 ret = i915_gem_gtt_pwrite_fast(obj, args); 1571 1572 if (ret == -EFAULT || ret == -ENOSPC) { 1573 if (obj->phys_handle) 1574 ret = i915_gem_phys_pwrite(obj, args, file); 1575 else 1576 ret = i915_gem_shmem_pwrite(obj, args); 1577 } 1578 1579 i915_gem_object_unpin_pages(obj); 1580 err: 1581 i915_gem_object_put(obj); 1582 return ret; 1583 } 1584 1585 static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) 1586 { 1587 struct drm_i915_private *i915; 1588 struct list_head *list; 1589 struct i915_vma *vma; 1590 1591 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 1592 1593 list_for_each_entry(vma, &obj->vma_list, obj_link) { 1594 if (!i915_vma_is_ggtt(vma)) 1595 break; 1596 1597 if (i915_vma_is_active(vma)) 1598 continue; 1599 1600 if (!drm_mm_node_allocated(&vma->node)) 1601 continue; 1602 1603 list_move_tail(&vma->vm_link, &vma->vm->inactive_list); 1604 } 1605 1606 i915 = to_i915(obj->base.dev); 1607 lockmgr(&i915->mm.obj_lock, LK_EXCLUSIVE); 1608 list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list; 1609 list_move_tail(&obj->mm.link, list); 1610 lockmgr(&i915->mm.obj_lock, LK_RELEASE); 1611 } 1612 1613 /** 1614 * Called when user space prepares to use an object with the CPU, either 1615 * through the mmap ioctl's mapping or a GTT mapping. 1616 * @dev: drm device 1617 * @data: ioctl data blob 1618 * @file: drm file 1619 */ 1620 int 1621 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1622 struct drm_file *file) 1623 { 1624 struct drm_i915_gem_set_domain *args = data; 1625 struct drm_i915_gem_object *obj; 1626 uint32_t read_domains = args->read_domains; 1627 uint32_t write_domain = args->write_domain; 1628 int err; 1629 1630 /* Only handle setting domains to types used by the CPU. */ 1631 if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS) 1632 return -EINVAL; 1633 1634 /* Having something in the write domain implies it's in the read 1635 * domain, and only that read domain. Enforce that in the request. 1636 */ 1637 if (write_domain != 0 && read_domains != write_domain) 1638 return -EINVAL; 1639 1640 obj = i915_gem_object_lookup(file, args->handle); 1641 if (!obj) 1642 return -ENOENT; 1643 1644 /* Try to flush the object off the GPU without holding the lock. 1645 * We will repeat the flush holding the lock in the normal manner 1646 * to catch cases where we are gazumped. 1647 */ 1648 err = i915_gem_object_wait(obj, 1649 I915_WAIT_INTERRUPTIBLE | 1650 (write_domain ? I915_WAIT_ALL : 0), 1651 MAX_SCHEDULE_TIMEOUT, 1652 to_rps_client(file)); 1653 if (err) 1654 goto out; 1655 1656 /* Flush and acquire obj->pages so that we are coherent through 1657 * direct access in memory with previous cached writes through 1658 * shmemfs and that our cache domain tracking remains valid. 1659 * For example, if the obj->filp was moved to swap without us 1660 * being notified and releasing the pages, we would mistakenly 1661 * continue to assume that the obj remained out of the CPU cached 1662 * domain. 1663 */ 1664 err = i915_gem_object_pin_pages(obj); 1665 if (err) 1666 goto out; 1667 1668 err = i915_mutex_lock_interruptible(dev); 1669 if (err) 1670 goto out_unpin; 1671 1672 if (read_domains & I915_GEM_DOMAIN_WC) 1673 err = i915_gem_object_set_to_wc_domain(obj, write_domain); 1674 else if (read_domains & I915_GEM_DOMAIN_GTT) 1675 err = i915_gem_object_set_to_gtt_domain(obj, write_domain); 1676 else 1677 err = i915_gem_object_set_to_cpu_domain(obj, write_domain); 1678 1679 /* And bump the LRU for this access */ 1680 i915_gem_object_bump_inactive_ggtt(obj); 1681 1682 mutex_unlock(&dev->struct_mutex); 1683 1684 if (write_domain != 0) 1685 intel_fb_obj_invalidate(obj, 1686 fb_write_origin(obj, write_domain)); 1687 1688 out_unpin: 1689 i915_gem_object_unpin_pages(obj); 1690 out: 1691 i915_gem_object_put(obj); 1692 return err; 1693 } 1694 1695 /** 1696 * Called when user space has done writes to this buffer 1697 * @dev: drm device 1698 * @data: ioctl data blob 1699 * @file: drm file 1700 */ 1701 int 1702 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1703 struct drm_file *file) 1704 { 1705 struct drm_i915_gem_sw_finish *args = data; 1706 struct drm_i915_gem_object *obj; 1707 1708 obj = i915_gem_object_lookup(file, args->handle); 1709 if (!obj) 1710 return -ENOENT; 1711 1712 /* Pinned buffers may be scanout, so flush the cache */ 1713 i915_gem_object_flush_if_display(obj); 1714 i915_gem_object_put(obj); 1715 1716 return 0; 1717 } 1718 1719 static int 1720 vm_object_map_wc_callback(vm_page_t p, void *data) 1721 { 1722 pmap_page_set_memattr(p, VM_MEMATTR_WRITE_COMBINING); 1723 1724 return 0; 1725 } 1726 1727 /** 1728 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address 1729 * it is mapped to. 1730 * @dev: drm device 1731 * @data: ioctl data blob 1732 * @file: drm file 1733 * 1734 * While the mapping holds a reference on the contents of the object, it doesn't 1735 * imply a ref on the object itself. 1736 * 1737 * IMPORTANT: 1738 * 1739 * DRM driver writers who look a this function as an example for how to do GEM 1740 * mmap support, please don't implement mmap support like here. The modern way 1741 * to implement DRM mmap support is with an mmap offset ioctl (like 1742 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly. 1743 * That way debug tooling like valgrind will understand what's going on, hiding 1744 * the mmap call in a driver private ioctl will break that. The i915 driver only 1745 * does cpu mmaps this way because we didn't know better. 1746 */ 1747 int 1748 i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1749 struct drm_file *file) 1750 { 1751 struct drm_i915_gem_mmap *args = data; 1752 struct drm_i915_gem_object *obj; 1753 unsigned long addr; 1754 #ifdef __DragonFly__ 1755 struct proc *p = curproc; 1756 vm_map_t map = &p->p_vmspace->vm_map; 1757 vm_size_t size; 1758 int error = 0, rv; 1759 struct vm_object *vm_obj; 1760 struct rb_vm_page_scan_info info; 1761 #endif 1762 1763 if (args->flags & ~(I915_MMAP_WC)) 1764 return -EINVAL; 1765 1766 if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT)) 1767 return -ENODEV; 1768 1769 obj = i915_gem_object_lookup(file, args->handle); 1770 if (!obj) 1771 return -ENOENT; 1772 1773 /* prime objects have no backing filp to GEM mmap 1774 * pages from. 1775 */ 1776 if (!obj->base.filp) { 1777 i915_gem_object_put(obj); 1778 return -EINVAL; 1779 } 1780 1781 #ifdef __DragonFly__ 1782 if (args->size == 0) 1783 goto out; 1784 1785 size = round_page(args->size); 1786 if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) { 1787 error = -ENOMEM; 1788 goto out; 1789 } 1790 1791 /* 1792 * Call hint to ensure that NULL is not returned as a valid address 1793 * and to reduce vm_map traversals. XXX causes instability, use a 1794 * fixed low address as the start point instead to avoid the NULL 1795 * return issue. 1796 */ 1797 addr = PAGE_SIZE; 1798 1799 /* 1800 * Use 256KB alignment. It is unclear why this matters for a 1801 * virtual address but it appears to fix a number of application/X 1802 * crashes and kms console switching is much faster. 1803 */ 1804 vm_object_hold(obj->base.filp); 1805 vm_object_reference_locked(obj->base.filp); 1806 vm_object_drop(obj->base.filp); 1807 1808 /* Something gets wrong here: fails to mmap 4096 */ 1809 rv = vm_map_find(map, obj->base.filp, NULL, 1810 args->offset, &addr, args->size, 1811 256 * 1024, /* align */ 1812 TRUE, /* fitit */ 1813 VM_MAPTYPE_NORMAL, VM_SUBSYS_DRM_GEM, 1814 VM_PROT_READ | VM_PROT_WRITE, /* prot */ 1815 VM_PROT_READ | VM_PROT_WRITE, /* max */ 1816 MAP_SHARED /* cow */); 1817 if (rv != KERN_SUCCESS) { 1818 vm_object_deallocate(obj->base.filp); 1819 error = -vm_mmap_to_errno(rv); 1820 } else { 1821 args->addr_ptr = (uint64_t)addr; 1822 } 1823 #else 1824 addr = vm_mmap(obj->base.filp, 0, args->size, 1825 PROT_READ | PROT_WRITE, MAP_SHARED, 1826 args->offset); 1827 #endif /* __DragonFly__ */ 1828 if (args->flags & I915_MMAP_WC) { /* I915_PARAM_MMAP_VERSION */ 1829 struct mm_struct *mm = current->mm; 1830 #if 0 1831 struct vm_area_struct *vma; 1832 #endif 1833 1834 if (down_write_killable(&mm->mmap_sem)) { 1835 i915_gem_object_put(obj); 1836 return -EINTR; 1837 } 1838 #ifdef __DragonFly__ 1839 vm_obj = obj->base.filp; 1840 vm_object_hold(vm_obj); 1841 vm_obj->memattr = pgprot_writecombine(vm_obj->memattr); 1842 /* Change attributes of all pages in the mapping here */ 1843 info.error = 0; 1844 info.count = 0; 1845 vm_page_rb_tree_RB_SCAN( 1846 &vm_obj->rb_memq, /* *head */ 1847 NULL, /* *scancmp */ 1848 vm_object_map_wc_callback, /* *callback */ 1849 &info /* *data */ 1850 ); 1851 vm_object_drop(vm_obj); 1852 #else 1853 vma = find_vma(mm, addr); 1854 if (vma) 1855 vma->vm_page_prot = 1856 pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); 1857 else 1858 addr = -ENOMEM; 1859 #endif /* __DragonFly__ */ 1860 up_write(&mm->mmap_sem); 1861 1862 /* This may race, but that's ok, it only gets set */ 1863 WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU); 1864 } 1865 1866 out: 1867 i915_gem_object_put(obj); 1868 if (IS_ERR((void *)addr)) 1869 return addr; 1870 1871 args->addr_ptr = (uint64_t) addr; 1872 1873 return 0; 1874 } 1875 1876 static unsigned int tile_row_pages(struct drm_i915_gem_object *obj) 1877 { 1878 return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT; 1879 } 1880 1881 /** 1882 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps 1883 * 1884 * A history of the GTT mmap interface: 1885 * 1886 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to 1887 * aligned and suitable for fencing, and still fit into the available 1888 * mappable space left by the pinned display objects. A classic problem 1889 * we called the page-fault-of-doom where we would ping-pong between 1890 * two objects that could not fit inside the GTT and so the memcpy 1891 * would page one object in at the expense of the other between every 1892 * single byte. 1893 * 1894 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none 1895 * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the 1896 * object is too large for the available space (or simply too large 1897 * for the mappable aperture!), a view is created instead and faulted 1898 * into userspace. (This view is aligned and sized appropriately for 1899 * fenced access.) 1900 * 1901 * 2 - Recognise WC as a separate cache domain so that we can flush the 1902 * delayed writes via GTT before performing direct access via WC. 1903 * 1904 * Restrictions: 1905 * 1906 * * snoopable objects cannot be accessed via the GTT. It can cause machine 1907 * hangs on some architectures, corruption on others. An attempt to service 1908 * a GTT page fault from a snoopable object will generate a SIGBUS. 1909 * 1910 * * the object must be able to fit into RAM (physical memory, though no 1911 * limited to the mappable aperture). 1912 * 1913 * 1914 * Caveats: 1915 * 1916 * * a new GTT page fault will synchronize rendering from the GPU and flush 1917 * all data to system memory. Subsequent access will not be synchronized. 1918 * 1919 * * all mappings are revoked on runtime device suspend. 1920 * 1921 * * there are only 8, 16 or 32 fence registers to share between all users 1922 * (older machines require fence register for display and blitter access 1923 * as well). Contention of the fence registers will cause the previous users 1924 * to be unmapped and any new access will generate new page faults. 1925 * 1926 * * running out of memory while servicing a fault may generate a SIGBUS, 1927 * rather than the expected SIGSEGV. 1928 */ 1929 int i915_gem_mmap_gtt_version(void) 1930 { 1931 return 2; 1932 } 1933 1934 static inline struct i915_ggtt_view 1935 compute_partial_view(struct drm_i915_gem_object *obj, 1936 pgoff_t page_offset, 1937 unsigned int chunk) 1938 { 1939 struct i915_ggtt_view view; 1940 1941 if (i915_gem_object_is_tiled(obj)) 1942 chunk = roundup(chunk, tile_row_pages(obj)); 1943 1944 view.type = I915_GGTT_VIEW_PARTIAL; 1945 view.partial.offset = rounddown(page_offset, chunk); 1946 view.partial.size = 1947 min_t(unsigned int, chunk, 1948 (obj->base.size >> PAGE_SHIFT) - view.partial.offset); 1949 1950 /* If the partial covers the entire object, just create a normal VMA. */ 1951 if (chunk >= obj->base.size >> PAGE_SHIFT) 1952 view.type = I915_GGTT_VIEW_NORMAL; 1953 1954 return view; 1955 } 1956 1957 #ifdef __DragonFly__ 1958 static inline void drm_vma_node_unmap(struct drm_vma_offset_node *node, 1959 struct address_space *file_mapping) 1960 { 1961 struct drm_i915_gem_object *obj = container_of( 1962 node,struct drm_i915_gem_object, base.vma_node); 1963 vm_object_t devobj; 1964 1965 devobj = cdev_pager_lookup(obj); 1966 if (devobj != NULL) { 1967 VM_OBJECT_LOCK(devobj); 1968 vm_object_page_remove(devobj, 0, 0, false); 1969 VM_OBJECT_UNLOCK(devobj); 1970 vm_object_deallocate(devobj); 1971 } 1972 } 1973 #endif 1974 1975 static void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj) 1976 { 1977 struct i915_vma *vma; 1978 1979 GEM_BUG_ON(!obj->userfault_count); 1980 1981 obj->userfault_count = 0; 1982 list_del(&obj->userfault_link); 1983 #ifdef __DragonFly__ 1984 drm_vma_node_unmap(&obj->base.vma_node, NULL); 1985 #else 1986 drm_vma_node_unmap(&obj->base.vma_node, 1987 obj->base.dev->anon_inode->i_mapping); 1988 #endif 1989 1990 list_for_each_entry(vma, &obj->vma_list, obj_link) { 1991 if (!i915_vma_is_ggtt(vma)) 1992 break; 1993 1994 i915_vma_unset_userfault(vma); 1995 } 1996 } 1997 1998 /** 1999 * i915_gem_fault - fault a page into the GTT 2000 * 2001 * vm_obj is locked on entry and expected to be locked on return. 2002 * 2003 * This is a OBJT_MGTDEVICE object, *mres will be NULL and should be set 2004 * to the desired vm_page. The page is not indexed into the vm_obj. 2005 * 2006 * XXX Most GEM calls appear to be interruptable, but we can't hard loop 2007 * in that case. Release all resources and wait 1 tick before retrying. 2008 * This is a huge problem which needs to be fixed by getting rid of most 2009 * of the interruptability. The linux code does not retry but does appear 2010 * to have some sort of mechanism (VM_FAULT_NOPAGE ?) for the higher level 2011 * to be able to retry. 2012 * 2013 * -- 2014 * @vma: VMA in question 2015 * @vmf: fault info 2016 * 2017 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped 2018 * from userspace. The fault handler takes care of binding the object to 2019 * the GTT (if needed), allocating and programming a fence register (again, 2020 * only if needed based on whether the old reg is still valid or the object 2021 * is tiled) and inserting a new PTE into the faulting process. 2022 * 2023 * Note that the faulting process may involve evicting existing objects 2024 * from the GTT and/or fence registers to make room. So performance may 2025 * suffer if the GTT working set is large or there are few fence registers 2026 * left. 2027 * 2028 * The current feature set supported by i915_gem_fault() and thus GTT mmaps 2029 * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version). 2030 * vm_obj is locked on entry and expected to be locked on return. The VM 2031 * pager has placed an anonymous memory page at (obj,offset) which we have 2032 * to replace. 2033 */ 2034 // int i915_gem_fault(struct vm_fault *vmf) 2035 int i915_gem_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, vm_page_t *mres) 2036 { 2037 #define MIN_CHUNK_PAGES ((1 << 20) >> PAGE_SHIFT) /* 1 MiB */ 2038 struct vm_area_struct *area; 2039 struct drm_i915_gem_object *obj = to_intel_bo(vm_obj->handle); 2040 struct drm_device *dev = obj->base.dev; 2041 struct drm_i915_private *dev_priv = to_i915(dev); 2042 struct i915_ggtt *ggtt = &dev_priv->ggtt; 2043 bool write = !!(prot & VM_PROT_WRITE); 2044 struct i915_vma *vma; 2045 pgoff_t page_offset; 2046 vm_page_t m; 2047 unsigned int flags; 2048 int ret; 2049 #ifdef __DragonFly__ 2050 int didref = 0; 2051 struct vm_area_struct vmas; 2052 2053 /* Fill-in vm_area_struct */ 2054 area = &vmas; 2055 area->vm_private_data = vm_obj->handle; 2056 area->vm_start = 0; 2057 area->vm_end = obj->base.size; 2058 area->vm_mm = current->mm; 2059 // XXX: in Linux, mmap_sem is held on entry of this function 2060 // XXX: should that be an exclusive lock ? 2061 down_read(&area->vm_mm->mmap_sem); 2062 #endif 2063 2064 /* We don't use vmf->pgoff since that has the fake offset */ 2065 page_offset = (unsigned long)offset >> PAGE_SHIFT; 2066 2067 /* 2068 * vm_fault() has supplied us with a busied page placeholding 2069 * the operation. This presents a lock order reversal issue 2070 * again i915_gem_release_mmap() for our device mutex. 2071 * 2072 * Deal with the problem by getting rid of the placeholder now, 2073 * and then dealing with the potential for a new placeholder when 2074 * we try to insert later. 2075 */ 2076 KKASSERT(*mres == NULL); 2077 m = NULL; 2078 2079 retry: 2080 trace_i915_gem_object_fault(obj, page_offset, true, write); 2081 2082 /* Try to flush the object off the GPU first without holding the lock. 2083 * Upon acquiring the lock, we will perform our sanity checks and then 2084 * repeat the flush holding the lock in the normal manner to catch cases 2085 * where we are gazumped. 2086 */ 2087 ret = i915_gem_object_wait(obj, 2088 I915_WAIT_INTERRUPTIBLE, 2089 MAX_SCHEDULE_TIMEOUT, 2090 NULL); 2091 if (ret) 2092 goto err; 2093 2094 ret = i915_gem_object_pin_pages(obj); 2095 if (ret) 2096 goto err; 2097 2098 intel_runtime_pm_get(dev_priv); 2099 2100 ret = i915_mutex_lock_interruptible(dev); 2101 if (ret) { 2102 if (ret != -EINTR) 2103 kprintf("i915: caught bug(%d) (mutex_lock_inter)\n", ret); 2104 goto err_rpm; 2105 } 2106 2107 /* Access to snoopable pages through the GTT is incoherent. */ 2108 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) { 2109 kprintf("i915: caught bug() (cache_level %d %d)\n", 2110 (obj->cache_level), !HAS_LLC(dev_priv)); 2111 ret = -EFAULT; 2112 goto err_unlock; 2113 } 2114 2115 /* If the object is smaller than a couple of partial vma, it is 2116 * not worth only creating a single partial vma - we may as well 2117 * clear enough space for the full object. 2118 */ 2119 flags = PIN_MAPPABLE; 2120 if (obj->base.size > 2 * MIN_CHUNK_PAGES << PAGE_SHIFT) 2121 flags |= PIN_NONBLOCK | PIN_NONFAULT; 2122 2123 /* Now pin it into the GTT as needed */ 2124 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, flags); 2125 if (IS_ERR(vma)) { 2126 /* Use a partial view if it is bigger than available space */ 2127 struct i915_ggtt_view view = 2128 compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES); 2129 2130 kprintf("i915_gem_fault: CHUNKING PASS\n"); 2131 2132 /* Userspace is now writing through an untracked VMA, abandon 2133 * all hope that the hardware is able to track future writes. 2134 */ 2135 obj->frontbuffer_ggtt_origin = ORIGIN_CPU; 2136 2137 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE); 2138 } 2139 if (IS_ERR(vma)) { 2140 kprintf("i915: caught bug() (VMA error %ld objsize %ld)\n", 2141 PTR_ERR(vma), obj->base.size); 2142 ret = PTR_ERR(vma); 2143 goto err_unlock; 2144 } 2145 2146 ret = i915_gem_object_set_to_gtt_domain(obj, write); 2147 if (ret) { 2148 kprintf("i915: caught bug(%d) (set_to_gtt_dom)\n", ret); 2149 goto err_unpin; 2150 } 2151 2152 ret = i915_vma_pin_fence(vma); 2153 if (ret) { 2154 kprintf("i915: caught bug(%d) (vma_pin_fence)\n", ret); 2155 goto err_unpin; 2156 } 2157 2158 #ifdef __DragonFly__ 2159 /* 2160 * Add a pip count to avoid destruction and certain other 2161 * complex operations (such as collapses?) while unlocked. 2162 */ 2163 vm_object_pip_add(vm_obj, 1); 2164 didref = 1; 2165 2166 ret = 0; 2167 2168 #if 0 2169 /* NO LONGER USED */ 2170 { 2171 vm_page_t m; 2172 2173 m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 2174 if (m) { 2175 if (vm_page_busy_try(m, false)) { 2176 kprintf("i915_gem_fault: BUSY\n"); 2177 ret = -EINTR; 2178 goto err_unpin; 2179 } 2180 } 2181 goto have_page; 2182 } 2183 #endif 2184 2185 /* Finally, remap it using the new GTT offset */ 2186 m = vm_phys_fictitious_to_vm_page(ggtt->mappable_base + 2187 vma->node.start + offset); 2188 2189 if (m == NULL) { 2190 kprintf("i915: caught bug() (phys_fict_to_vm)\n"); 2191 ret = -EFAULT; 2192 goto err_fence; 2193 } 2194 KASSERT((m->flags & PG_FICTITIOUS) != 0, ("not fictitious %p", m)); 2195 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 2196 2197 /* 2198 * Try to busy the page. Fails on non-zero return. 2199 */ 2200 if (vm_page_busy_try(m, false)) { 2201 kprintf("i915_gem_fault: BUSY(2)\n"); 2202 ret = -EINTR; 2203 goto err_fence; 2204 } 2205 m->valid = VM_PAGE_BITS_ALL; 2206 2207 #ifdef USE_INSERT 2208 /* NO LONGER USED */ 2209 /* 2210 * This should always work since we already checked via a lookup 2211 * above. 2212 */ 2213 if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset)) == FALSE) { 2214 kprintf("i915:gem_fault: page %p,%jd already in object\n", 2215 vm_obj, 2216 OFF_TO_IDX(offset)); 2217 vm_page_wakeup(m); 2218 ret = -EINTR; 2219 goto err_unpin; 2220 } 2221 have_page: 2222 #endif 2223 *mres = m; 2224 ret = VM_PAGER_OK; 2225 2226 /* 2227 * ALTERNATIVE ERROR RETURN. 2228 * 2229 * OBJECT EXPECTED TO BE LOCKED. 2230 */ 2231 #endif 2232 2233 /* Mark as being mmapped into userspace for later revocation */ 2234 assert_rpm_wakelock_held(dev_priv); 2235 if (!i915_vma_set_userfault(vma) && !obj->userfault_count++) 2236 list_add(&obj->userfault_link, &dev_priv->mm.userfault_list); 2237 GEM_BUG_ON(!obj->userfault_count); 2238 2239 err_fence: 2240 i915_vma_unpin_fence(vma); 2241 err_unpin: 2242 __i915_vma_unpin(vma); 2243 err_unlock: 2244 mutex_unlock(&dev->struct_mutex); 2245 err_rpm: 2246 intel_runtime_pm_put(dev_priv); 2247 i915_gem_object_unpin_pages(obj); 2248 err: 2249 switch (ret) { 2250 case -EIO: 2251 /* 2252 * We eat errors when the gpu is terminally wedged to avoid 2253 * userspace unduly crashing (gl has no provisions for mmaps to 2254 * fail). But any other -EIO isn't ours (e.g. swap in failure) 2255 * and so needs to be reported. 2256 */ 2257 if (!i915_terminally_wedged(&dev_priv->gpu_error)) { 2258 // ret = VM_FAULT_SIGBUS; 2259 break; 2260 } 2261 case -EAGAIN: 2262 /* 2263 * EAGAIN means the gpu is hung and we'll wait for the error 2264 * handler to reset everything when re-faulting in 2265 * i915_mutex_lock_interruptible. 2266 */ 2267 case -ERESTARTSYS: 2268 case -EINTR: 2269 #ifdef __DragonFly__ 2270 if (didref) { 2271 kprintf("i915: caught bug(%d) (retry)\n", ret); 2272 vm_object_pip_wakeup(vm_obj); 2273 didref = 0; 2274 } 2275 VM_OBJECT_UNLOCK(vm_obj); 2276 int dummy; 2277 tsleep(&dummy, 0, "delay", 1); /* XXX */ 2278 VM_OBJECT_LOCK(vm_obj); 2279 goto retry; 2280 case VM_PAGER_OK: 2281 break; 2282 #endif 2283 default: 2284 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret); 2285 ret = VM_PAGER_ERROR; 2286 break; 2287 } 2288 2289 #ifdef __DragonFly__ 2290 if (didref) 2291 vm_object_pip_wakeup(vm_obj); 2292 else 2293 kprintf("i915: caught bug(%d)\n", ret); 2294 2295 up_read(&area->vm_mm->mmap_sem); 2296 #endif 2297 2298 return ret; 2299 } 2300 2301 /** 2302 * i915_gem_release_mmap - remove physical page mappings 2303 * @obj: obj in question 2304 * 2305 * Preserve the reservation of the mmapping with the DRM core code, but 2306 * relinquish ownership of the pages back to the system. 2307 * 2308 * It is vital that we remove the page mapping if we have mapped a tiled 2309 * object through the GTT and then lose the fence register due to 2310 * resource pressure. Similarly if the object has been moved out of the 2311 * aperture, than pages mapped into userspace must be revoked. Removing the 2312 * mapping will then trigger a page fault on the next user access, allowing 2313 * fixup by i915_gem_fault(). 2314 */ 2315 void 2316 i915_gem_release_mmap(struct drm_i915_gem_object *obj) 2317 { 2318 struct drm_i915_private *i915 = to_i915(obj->base.dev); 2319 2320 /* Serialisation between user GTT access and our code depends upon 2321 * revoking the CPU's PTE whilst the mutex is held. The next user 2322 * pagefault then has to wait until we release the mutex. 2323 * 2324 * Note that RPM complicates somewhat by adding an additional 2325 * requirement that operations to the GGTT be made holding the RPM 2326 * wakeref. 2327 */ 2328 lockdep_assert_held(&i915->drm.struct_mutex); 2329 intel_runtime_pm_get(i915); 2330 2331 if (!obj->userfault_count) 2332 goto out; 2333 2334 __i915_gem_object_release_mmap(obj); 2335 2336 /* Ensure that the CPU's PTE are revoked and there are not outstanding 2337 * memory transactions from userspace before we return. The TLB 2338 * flushing implied above by changing the PTE above *should* be 2339 * sufficient, an extra barrier here just provides us with a bit 2340 * of paranoid documentation about our requirement to serialise 2341 * memory writes before touching registers / GSM. 2342 */ 2343 wmb(); 2344 2345 out: 2346 intel_runtime_pm_put(i915); 2347 } 2348 2349 void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv) 2350 { 2351 struct drm_i915_gem_object *obj, *on; 2352 int i; 2353 2354 /* 2355 * Only called during RPM suspend. All users of the userfault_list 2356 * must be holding an RPM wakeref to ensure that this can not 2357 * run concurrently with themselves (and use the struct_mutex for 2358 * protection between themselves). 2359 */ 2360 2361 list_for_each_entry_safe(obj, on, 2362 &dev_priv->mm.userfault_list, userfault_link) 2363 __i915_gem_object_release_mmap(obj); 2364 2365 /* The fence will be lost when the device powers down. If any were 2366 * in use by hardware (i.e. they are pinned), we should not be powering 2367 * down! All other fences will be reacquired by the user upon waking. 2368 */ 2369 for (i = 0; i < dev_priv->num_fence_regs; i++) { 2370 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2371 2372 /* Ideally we want to assert that the fence register is not 2373 * live at this point (i.e. that no piece of code will be 2374 * trying to write through fence + GTT, as that both violates 2375 * our tracking of activity and associated locking/barriers, 2376 * but also is illegal given that the hw is powered down). 2377 * 2378 * Previously we used reg->pin_count as a "liveness" indicator. 2379 * That is not sufficient, and we need a more fine-grained 2380 * tool if we want to have a sanity check here. 2381 */ 2382 2383 if (!reg->vma) 2384 continue; 2385 2386 GEM_BUG_ON(i915_vma_has_userfault(reg->vma)); 2387 reg->dirty = true; 2388 } 2389 } 2390 2391 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj) 2392 { 2393 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 2394 int err; 2395 2396 err = drm_gem_create_mmap_offset(&obj->base); 2397 if (likely(!err)) 2398 return 0; 2399 2400 /* Attempt to reap some mmap space from dead objects */ 2401 do { 2402 err = i915_gem_wait_for_idle(dev_priv, I915_WAIT_INTERRUPTIBLE); 2403 if (err) 2404 break; 2405 2406 i915_gem_drain_freed_objects(dev_priv); 2407 err = drm_gem_create_mmap_offset(&obj->base); 2408 if (!err) 2409 break; 2410 2411 } while (flush_delayed_work(&dev_priv->gt.retire_work)); 2412 2413 return err; 2414 } 2415 2416 #if 0 2417 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj) 2418 { 2419 drm_gem_free_mmap_offset(&obj->base); 2420 } 2421 #endif 2422 2423 int 2424 i915_gem_mmap_gtt(struct drm_file *file, 2425 struct drm_device *dev, 2426 uint32_t handle, 2427 uint64_t *offset) 2428 { 2429 struct drm_i915_gem_object *obj; 2430 int ret; 2431 2432 obj = i915_gem_object_lookup(file, handle); 2433 if (!obj) 2434 return -ENOENT; 2435 2436 ret = i915_gem_object_create_mmap_offset(obj); 2437 if (ret == 0) 2438 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 2439 DRM_GEM_MAPPING_KEY; 2440 2441 i915_gem_object_put(obj); 2442 return ret; 2443 } 2444 2445 /** 2446 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 2447 * @dev: DRM device 2448 * @data: GTT mapping ioctl data 2449 * @file: GEM object info 2450 * 2451 * Simply returns the fake offset to userspace so it can mmap it. 2452 * The mmap call will end up in drm_gem_mmap(), which will set things 2453 * up so we can get faults in the handler above. 2454 * 2455 * The fault handler will take care of binding the object into the GTT 2456 * (since it may have been evicted to make room for something), allocating 2457 * a fence register, and mapping the appropriate aperture address into 2458 * userspace. 2459 */ 2460 int 2461 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 2462 struct drm_file *file) 2463 { 2464 struct drm_i915_gem_mmap_gtt *args = data; 2465 2466 return i915_gem_mmap_gtt(file, dev, args->handle, (uint64_t *)&args->offset); 2467 } 2468 2469 /* Immediately discard the backing storage */ 2470 static void 2471 i915_gem_object_truncate(struct drm_i915_gem_object *obj) 2472 { 2473 vm_object_t vm_obj = obj->base.filp; 2474 2475 if (obj->base.filp == NULL) 2476 return; 2477 2478 VM_OBJECT_LOCK(vm_obj); 2479 vm_object_page_remove(vm_obj, 0, 0, false); 2480 VM_OBJECT_UNLOCK(vm_obj); 2481 2482 /* Our goal here is to return as much of the memory as 2483 * is possible back to the system as we are called from OOM. 2484 * To do this we must instruct the shmfs to drop all of its 2485 * backing pages, *now*. 2486 */ 2487 #if 0 2488 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); 2489 #endif 2490 obj->mm.madv = __I915_MADV_PURGED; 2491 obj->mm.pages = ERR_PTR(-EFAULT); 2492 } 2493 2494 /* Try to discard unwanted pages */ 2495 void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj) 2496 { 2497 #if 0 2498 struct address_space *mapping; 2499 #endif 2500 2501 lockdep_assert_held(&obj->mm.lock); 2502 GEM_BUG_ON(i915_gem_object_has_pages(obj)); 2503 2504 switch (obj->mm.madv) { 2505 case I915_MADV_DONTNEED: 2506 i915_gem_object_truncate(obj); 2507 case __I915_MADV_PURGED: 2508 return; 2509 } 2510 2511 if (obj->base.filp == NULL) 2512 return; 2513 2514 #if 0 2515 mapping = obj->base.filp->f_mapping, 2516 invalidate_mapping_pages(mapping, 0, (loff_t)-1); 2517 #endif 2518 invalidate_mapping_pages(obj->base.filp, 0, (loff_t)-1); 2519 } 2520 2521 static void 2522 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj, 2523 struct sg_table *pages) 2524 { 2525 struct sgt_iter sgt_iter; 2526 struct page *page; 2527 2528 __i915_gem_object_release_shmem(obj, pages, true); 2529 2530 i915_gem_gtt_finish_pages(obj, pages); 2531 2532 if (i915_gem_object_needs_bit17_swizzle(obj)) 2533 i915_gem_object_save_bit_17_swizzle(obj, pages); 2534 2535 for_each_sgt_page(page, sgt_iter, pages) { 2536 if (obj->mm.dirty) 2537 set_page_dirty(page); 2538 2539 if (obj->mm.madv == I915_MADV_WILLNEED) 2540 mark_page_accessed(page); 2541 2542 put_page(page); 2543 } 2544 obj->mm.dirty = false; 2545 2546 sg_free_table(pages); 2547 kfree(pages); 2548 } 2549 2550 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) 2551 { 2552 struct radix_tree_iter iter; 2553 void __rcu **slot; 2554 2555 rcu_read_lock(); 2556 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0) 2557 radix_tree_delete(&obj->mm.get_page.radix, iter.index); 2558 rcu_read_unlock(); 2559 } 2560 2561 void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, 2562 enum i915_mm_subclass subclass) 2563 { 2564 struct drm_i915_private *i915 = to_i915(obj->base.dev); 2565 struct sg_table *pages; 2566 2567 if (i915_gem_object_has_pinned_pages(obj)) 2568 return; 2569 2570 GEM_BUG_ON(obj->bind_count); 2571 if (!i915_gem_object_has_pages(obj)) 2572 return; 2573 2574 /* May be called by shrinker from within get_pages() (on another bo) */ 2575 mutex_lock_nested(&obj->mm.lock, subclass); 2576 if (unlikely(atomic_read(&obj->mm.pages_pin_count))) 2577 goto unlock; 2578 2579 /* ->put_pages might need to allocate memory for the bit17 swizzle 2580 * array, hence protect them from being reaped by removing them from gtt 2581 * lists early. */ 2582 pages = fetch_and_zero(&obj->mm.pages); 2583 GEM_BUG_ON(!pages); 2584 2585 lockmgr(&i915->mm.obj_lock, LK_EXCLUSIVE); 2586 list_del(&obj->mm.link); 2587 lockmgr(&i915->mm.obj_lock, LK_RELEASE); 2588 2589 if (obj->mm.mapping) { 2590 void *ptr; 2591 2592 ptr = page_mask_bits(obj->mm.mapping); 2593 if (is_vmalloc_addr(ptr)) 2594 vunmap(ptr); 2595 else 2596 kunmap(kmap_to_page(ptr)); 2597 2598 obj->mm.mapping = NULL; 2599 } 2600 2601 __i915_gem_object_reset_page_iter(obj); 2602 2603 if (!IS_ERR(pages)) 2604 obj->ops->put_pages(obj, pages); 2605 2606 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0; 2607 2608 unlock: 2609 mutex_unlock(&obj->mm.lock); 2610 } 2611 2612 static bool i915_sg_trim(struct sg_table *orig_st) 2613 { 2614 struct sg_table new_st; 2615 struct scatterlist *sg, *new_sg; 2616 unsigned int i; 2617 2618 if (orig_st->nents == orig_st->orig_nents) 2619 return false; 2620 2621 if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL | __GFP_NOWARN)) 2622 return false; 2623 2624 new_sg = new_st.sgl; 2625 for_each_sg(orig_st->sgl, sg, orig_st->nents, i) { 2626 sg_set_page(new_sg, sg_page(sg), sg->length, 0); 2627 /* called before being DMA mapped, no need to copy sg->dma_* */ 2628 new_sg = sg_next(new_sg); 2629 } 2630 GEM_BUG_ON(new_sg); /* Should walk exactly nents and hit the end */ 2631 2632 sg_free_table(orig_st); 2633 2634 *orig_st = new_st; 2635 return true; 2636 } 2637 2638 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) 2639 { 2640 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 2641 const unsigned long page_count = obj->base.size / PAGE_SIZE; 2642 unsigned long i; 2643 struct vm_object *mapping; 2644 struct sg_table *st; 2645 struct scatterlist *sg; 2646 struct sgt_iter sgt_iter; 2647 struct page *page; 2648 unsigned long last_pfn = 0; /* suppress gcc warning */ 2649 unsigned int max_segment = i915_sg_segment_size(); 2650 unsigned int sg_page_sizes; 2651 gfp_t noreclaim; 2652 int ret; 2653 2654 /* Assert that the object is not currently in any GPU domain. As it 2655 * wasn't in the GTT, there shouldn't be any way it could have been in 2656 * a GPU cache 2657 */ 2658 GEM_BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); 2659 GEM_BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); 2660 2661 st = kmalloc(sizeof(*st), M_DRM, GFP_KERNEL); 2662 if (st == NULL) 2663 return -ENOMEM; 2664 2665 rebuild_st: 2666 if (sg_alloc_table(st, page_count, GFP_KERNEL)) { 2667 kfree(st); 2668 return -ENOMEM; 2669 } 2670 2671 /* Get the list of pages out of our struct file. They'll be pinned 2672 * at this point until we release them. 2673 * 2674 * Fail silently without starting the shrinker 2675 */ 2676 #ifdef __DragonFly__ 2677 mapping = obj->base.filp; 2678 VM_OBJECT_LOCK(mapping); 2679 #endif 2680 noreclaim = mapping_gfp_constraint(mapping, 2681 ~(__GFP_IO | __GFP_RECLAIM)); 2682 noreclaim |= __GFP_NORETRY | __GFP_NOWARN; 2683 2684 sg = st->sgl; 2685 st->nents = 0; 2686 sg_page_sizes = 0; 2687 for (i = 0; i < page_count; i++) { 2688 const unsigned int shrink[] = { 2689 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND | I915_SHRINK_PURGEABLE, 2690 0, 2691 }, *s = shrink; 2692 gfp_t gfp = noreclaim; 2693 2694 do { 2695 page = shmem_read_mapping_page_gfp(mapping, i, gfp); 2696 if (likely(!IS_ERR(page))) 2697 break; 2698 2699 if (!*s) { 2700 ret = PTR_ERR(page); 2701 goto err_sg; 2702 } 2703 2704 i915_gem_shrink(dev_priv, 2 * page_count, NULL, *s++); 2705 cond_resched(); 2706 2707 /* We've tried hard to allocate the memory by reaping 2708 * our own buffer, now let the real VM do its job and 2709 * go down in flames if truly OOM. 2710 * 2711 * However, since graphics tend to be disposable, 2712 * defer the oom here by reporting the ENOMEM back 2713 * to userspace. 2714 */ 2715 if (!*s) { 2716 /* reclaim and warn, but no oom */ 2717 gfp = mapping_gfp_mask(mapping); 2718 2719 /* Our bo are always dirty and so we require 2720 * kswapd to reclaim our pages (direct reclaim 2721 * does not effectively begin pageout of our 2722 * buffers on its own). However, direct reclaim 2723 * only waits for kswapd when under allocation 2724 * congestion. So as a result __GFP_RECLAIM is 2725 * unreliable and fails to actually reclaim our 2726 * dirty pages -- unless you try over and over 2727 * again with !__GFP_NORETRY. However, we still 2728 * want to fail this allocation rather than 2729 * trigger the out-of-memory killer and for 2730 * this we want __GFP_RETRY_MAYFAIL. 2731 */ 2732 gfp |= __GFP_RETRY_MAYFAIL; 2733 } 2734 } while (1); 2735 2736 if (!i || 2737 sg->length >= max_segment || 2738 page_to_pfn(page) != last_pfn + 1) { 2739 if (i) { 2740 sg_page_sizes |= sg->length; 2741 sg = sg_next(sg); 2742 } 2743 st->nents++; 2744 sg_set_page(sg, page, PAGE_SIZE, 0); 2745 } else { 2746 sg->length += PAGE_SIZE; 2747 } 2748 last_pfn = page_to_pfn(page); 2749 2750 /* Check that the i965g/gm workaround works. */ 2751 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL)); 2752 } 2753 if (sg) { /* loop terminated early; short sg table */ 2754 sg_page_sizes |= sg->length; 2755 sg_mark_end(sg); 2756 } 2757 #ifdef __DragonFly__ 2758 VM_OBJECT_UNLOCK(mapping); 2759 #endif 2760 2761 /* Trim unused sg entries to avoid wasting memory. */ 2762 i915_sg_trim(st); 2763 2764 ret = i915_gem_gtt_prepare_pages(obj, st); 2765 if (ret) { 2766 /* DMA remapping failed? One possible cause is that 2767 * it could not reserve enough large entries, asking 2768 * for PAGE_SIZE chunks instead may be helpful. 2769 */ 2770 if (max_segment > PAGE_SIZE) { 2771 for_each_sgt_page(page, sgt_iter, st) 2772 put_page(page); 2773 sg_free_table(st); 2774 2775 max_segment = PAGE_SIZE; 2776 goto rebuild_st; 2777 } else { 2778 dev_warn(&dev_priv->drm.pdev->dev, 2779 "Failed to DMA remap %lu pages\n", 2780 page_count); 2781 goto err_pages; 2782 } 2783 } 2784 2785 if (i915_gem_object_needs_bit17_swizzle(obj)) 2786 i915_gem_object_do_bit_17_swizzle(obj, st); 2787 2788 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 2789 2790 return 0; 2791 2792 err_sg: 2793 sg_mark_end(sg); 2794 err_pages: 2795 for_each_sgt_page(page, sgt_iter, st) 2796 put_page(page); 2797 #ifdef __DragonFly__ 2798 VM_OBJECT_UNLOCK(mapping); 2799 #endif 2800 sg_free_table(st); 2801 kfree(st); 2802 2803 /* shmemfs first checks if there is enough memory to allocate the page 2804 * and reports ENOSPC should there be insufficient, along with the usual 2805 * ENOMEM for a genuine allocation failure. 2806 * 2807 * We use ENOSPC in our driver to mean that we have run out of aperture 2808 * space and so want to translate the error from shmemfs back to our 2809 * usual understanding of ENOMEM. 2810 */ 2811 if (ret == -ENOSPC) 2812 ret = -ENOMEM; 2813 2814 return ret; 2815 } 2816 2817 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, 2818 struct sg_table *pages, 2819 unsigned int sg_page_sizes) 2820 { 2821 struct drm_i915_private *i915 = to_i915(obj->base.dev); 2822 unsigned long supported = INTEL_INFO(i915)->page_sizes; 2823 int i; 2824 2825 lockdep_assert_held(&obj->mm.lock); 2826 2827 obj->mm.get_page.sg_pos = pages->sgl; 2828 obj->mm.get_page.sg_idx = 0; 2829 2830 obj->mm.pages = pages; 2831 2832 if (i915_gem_object_is_tiled(obj) && 2833 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { 2834 GEM_BUG_ON(obj->mm.quirked); 2835 __i915_gem_object_pin_pages(obj); 2836 obj->mm.quirked = true; 2837 } 2838 2839 GEM_BUG_ON(!sg_page_sizes); 2840 obj->mm.page_sizes.phys = sg_page_sizes; 2841 2842 /* 2843 * Calculate the supported page-sizes which fit into the given 2844 * sg_page_sizes. This will give us the page-sizes which we may be able 2845 * to use opportunistically when later inserting into the GTT. For 2846 * example if phys=2G, then in theory we should be able to use 1G, 2M, 2847 * 64K or 4K pages, although in practice this will depend on a number of 2848 * other factors. 2849 */ 2850 obj->mm.page_sizes.sg = 0; 2851 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { 2852 if (obj->mm.page_sizes.phys & ~0u << i) 2853 obj->mm.page_sizes.sg |= BIT(i); 2854 } 2855 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg)); 2856 2857 lockmgr(&i915->mm.obj_lock, LK_EXCLUSIVE); 2858 list_add(&obj->mm.link, &i915->mm.unbound_list); 2859 lockmgr(&i915->mm.obj_lock, LK_RELEASE); 2860 } 2861 2862 static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj) 2863 { 2864 int err; 2865 2866 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { 2867 DRM_DEBUG("Attempting to obtain a purgeable object\n"); 2868 return -EFAULT; 2869 } 2870 2871 err = obj->ops->get_pages(obj); 2872 GEM_BUG_ON(!err && IS_ERR_OR_NULL(obj->mm.pages)); 2873 2874 return err; 2875 } 2876 2877 /* Ensure that the associated pages are gathered from the backing storage 2878 * and pinned into our object. i915_gem_object_pin_pages() may be called 2879 * multiple times before they are released by a single call to 2880 * i915_gem_object_unpin_pages() - once the pages are no longer referenced 2881 * either as a result of memory pressure (reaping pages under the shrinker) 2882 * or as the object is itself released. 2883 */ 2884 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) 2885 { 2886 int err; 2887 2888 err = mutex_lock_interruptible(&obj->mm.lock); 2889 if (err) 2890 return err; 2891 2892 if (unlikely(!i915_gem_object_has_pages(obj))) { 2893 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); 2894 2895 err = ____i915_gem_object_get_pages(obj); 2896 if (err) 2897 goto unlock; 2898 2899 smp_mb__before_atomic(); 2900 } 2901 atomic_inc(&obj->mm.pages_pin_count); 2902 2903 unlock: 2904 mutex_unlock(&obj->mm.lock); 2905 return err; 2906 } 2907 2908 /* The 'mapping' part of i915_gem_object_pin_map() below */ 2909 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj, 2910 enum i915_map_type type) 2911 { 2912 unsigned long n_pages = obj->base.size >> PAGE_SHIFT; 2913 struct sg_table *sgt = obj->mm.pages; 2914 struct sgt_iter sgt_iter; 2915 struct page *page; 2916 struct page *stack_pages[32]; 2917 struct page **pages = stack_pages; 2918 unsigned long i = 0; 2919 pgprot_t pgprot; 2920 void *addr; 2921 2922 /* A single page can always be kmapped */ 2923 if (n_pages == 1 && type == I915_MAP_WB) 2924 return kmap(sg_page(sgt->sgl)); 2925 2926 if (n_pages > ARRAY_SIZE(stack_pages)) { 2927 /* Too big for stack -- allocate temporary array instead */ 2928 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL); 2929 if (!pages) 2930 return NULL; 2931 } 2932 2933 for_each_sgt_page(page, sgt_iter, sgt) 2934 pages[i++] = page; 2935 2936 /* Check that we have the expected number of pages */ 2937 GEM_BUG_ON(i != n_pages); 2938 2939 switch (type) { 2940 default: 2941 MISSING_CASE(type); 2942 /* fallthrough to use PAGE_KERNEL anyway */ 2943 case I915_MAP_WB: 2944 pgprot = PAGE_KERNEL; 2945 break; 2946 case I915_MAP_WC: 2947 pgprot = pgprot_writecombine(PAGE_KERNEL_IO); 2948 break; 2949 } 2950 addr = vmap(pages, n_pages, 0, pgprot); 2951 2952 if (pages != stack_pages) 2953 kvfree(pages); 2954 2955 return addr; 2956 } 2957 2958 /* get, pin, and map the pages of the object into kernel space */ 2959 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, 2960 enum i915_map_type type) 2961 { 2962 enum i915_map_type has_type; 2963 bool pinned; 2964 void *ptr; 2965 int ret; 2966 2967 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); 2968 2969 ret = mutex_lock_interruptible(&obj->mm.lock); 2970 if (ret) 2971 return ERR_PTR(ret); 2972 2973 pinned = !(type & I915_MAP_OVERRIDE); 2974 type &= ~I915_MAP_OVERRIDE; 2975 2976 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) { 2977 if (unlikely(!i915_gem_object_has_pages(obj))) { 2978 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); 2979 2980 ret = ____i915_gem_object_get_pages(obj); 2981 if (ret) 2982 goto err_unlock; 2983 2984 smp_mb__before_atomic(); 2985 } 2986 atomic_inc(&obj->mm.pages_pin_count); 2987 pinned = false; 2988 } 2989 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 2990 2991 ptr = page_unpack_bits(obj->mm.mapping, &has_type); 2992 if (ptr && has_type != type) { 2993 if (pinned) { 2994 ret = -EBUSY; 2995 goto err_unpin; 2996 } 2997 2998 if (is_vmalloc_addr(ptr)) 2999 vunmap(ptr); 3000 else 3001 kunmap(kmap_to_page(ptr)); 3002 3003 ptr = obj->mm.mapping = NULL; 3004 } 3005 3006 if (!ptr) { 3007 ptr = i915_gem_object_map(obj, type); 3008 if (!ptr) { 3009 ret = -ENOMEM; 3010 goto err_unpin; 3011 } 3012 3013 obj->mm.mapping = page_pack_bits(ptr, type); 3014 } 3015 3016 out_unlock: 3017 mutex_unlock(&obj->mm.lock); 3018 return ptr; 3019 3020 err_unpin: 3021 atomic_dec(&obj->mm.pages_pin_count); 3022 err_unlock: 3023 ptr = ERR_PTR(ret); 3024 goto out_unlock; 3025 } 3026 3027 static int 3028 i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj, 3029 const struct drm_i915_gem_pwrite *arg) 3030 { 3031 #ifndef __DragonFly__ 3032 struct address_space *mapping = obj->base.filp->f_mapping; 3033 #endif 3034 char __user *user_data = u64_to_user_ptr(arg->data_ptr); 3035 u64 remain, offset; 3036 unsigned int pg; 3037 3038 /* Before we instantiate/pin the backing store for our use, we 3039 * can prepopulate the shmemfs filp efficiently using a write into 3040 * the pagecache. We avoid the penalty of instantiating all the 3041 * pages, important if the user is just writing to a few and never 3042 * uses the object on the GPU, and using a direct write into shmemfs 3043 * allows it to avoid the cost of retrieving a page (either swapin 3044 * or clearing-before-use) before it is overwritten. 3045 */ 3046 if (i915_gem_object_has_pages(obj)) 3047 return -ENODEV; 3048 3049 if (obj->mm.madv != I915_MADV_WILLNEED) 3050 return -EFAULT; 3051 3052 /* Before the pages are instantiated the object is treated as being 3053 * in the CPU domain. The pages will be clflushed as required before 3054 * use, and we can freely write into the pages directly. If userspace 3055 * races pwrite with any other operation; corruption will ensue - 3056 * that is userspace's prerogative! 3057 */ 3058 3059 remain = arg->size; 3060 offset = arg->offset; 3061 pg = offset_in_page(offset); 3062 3063 do { 3064 unsigned int len, unwritten; 3065 struct page *page; 3066 void *data, *vaddr; 3067 int err; 3068 3069 len = PAGE_SIZE - pg; 3070 if (len > remain) 3071 len = remain; 3072 3073 #ifndef __DragonFly__ 3074 err = pagecache_write_begin(obj->base.filp, mapping, 3075 #else 3076 err = pagecache_write_begin(obj->base.filp, NULL, 3077 #endif 3078 offset, len, 0, 3079 &page, &data); 3080 if (err < 0) 3081 return err; 3082 3083 vaddr = kmap(page); 3084 unwritten = copy_from_user(vaddr + pg, user_data, len); 3085 kunmap(page); 3086 3087 #ifndef __DragonFly__ 3088 err = pagecache_write_end(obj->base.filp, mapping, 3089 #else 3090 err = pagecache_write_end(obj->base.filp, NULL, 3091 #endif 3092 offset, len, len - unwritten, 3093 page, data); 3094 if (err < 0) 3095 return err; 3096 3097 if (unwritten) 3098 return -EFAULT; 3099 3100 remain -= len; 3101 user_data += len; 3102 offset += len; 3103 pg = 0; 3104 } while (remain); 3105 3106 return 0; 3107 } 3108 3109 static bool ban_context(const struct i915_gem_context *ctx, 3110 unsigned int score) 3111 { 3112 return (i915_gem_context_is_bannable(ctx) && 3113 score >= CONTEXT_SCORE_BAN_THRESHOLD); 3114 } 3115 3116 static void i915_gem_context_mark_guilty(struct i915_gem_context *ctx) 3117 { 3118 unsigned int score; 3119 bool banned; 3120 3121 atomic_inc(&ctx->guilty_count); 3122 3123 score = atomic_add_return(CONTEXT_SCORE_GUILTY, &ctx->ban_score); 3124 banned = ban_context(ctx, score); 3125 DRM_DEBUG_DRIVER("context %s marked guilty (score %d) banned? %s\n", 3126 ctx->name, score, yesno(banned)); 3127 if (!banned) 3128 return; 3129 3130 i915_gem_context_set_banned(ctx); 3131 if (!IS_ERR_OR_NULL(ctx->file_priv)) { 3132 atomic_inc(&ctx->file_priv->context_bans); 3133 DRM_DEBUG_DRIVER("client %s has had %d context banned\n", 3134 ctx->name, atomic_read(&ctx->file_priv->context_bans)); 3135 } 3136 } 3137 3138 static void i915_gem_context_mark_innocent(struct i915_gem_context *ctx) 3139 { 3140 atomic_inc(&ctx->active_count); 3141 } 3142 3143 struct drm_i915_gem_request * 3144 i915_gem_find_active_request(struct intel_engine_cs *engine) 3145 { 3146 struct drm_i915_gem_request *request, *active = NULL; 3147 unsigned long flags; 3148 3149 /* We are called by the error capture and reset at a random 3150 * point in time. In particular, note that neither is crucially 3151 * ordered with an interrupt. After a hang, the GPU is dead and we 3152 * assume that no more writes can happen (we waited long enough for 3153 * all writes that were in transaction to be flushed) - adding an 3154 * extra delay for a recent interrupt is pointless. Hence, we do 3155 * not need an engine->irq_seqno_barrier() before the seqno reads. 3156 */ 3157 spin_lock_irqsave(&engine->timeline->lock, flags); 3158 list_for_each_entry(request, &engine->timeline->requests, link) { 3159 if (__i915_gem_request_completed(request, 3160 request->global_seqno)) 3161 continue; 3162 3163 GEM_BUG_ON(request->engine != engine); 3164 GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 3165 &request->fence.flags)); 3166 3167 active = request; 3168 break; 3169 } 3170 spin_unlock_irqrestore(&engine->timeline->lock, flags); 3171 3172 return active; 3173 } 3174 3175 static bool engine_stalled(struct intel_engine_cs *engine) 3176 { 3177 if (!engine->hangcheck.stalled) 3178 return false; 3179 3180 /* Check for possible seqno movement after hang declaration */ 3181 if (engine->hangcheck.seqno != intel_engine_get_seqno(engine)) { 3182 DRM_DEBUG_DRIVER("%s pardoned\n", engine->name); 3183 return false; 3184 } 3185 3186 return true; 3187 } 3188 3189 /* 3190 * Ensure irq handler finishes, and not run again. 3191 * Also return the active request so that we only search for it once. 3192 */ 3193 struct drm_i915_gem_request * 3194 i915_gem_reset_prepare_engine(struct intel_engine_cs *engine) 3195 { 3196 struct drm_i915_gem_request *request = NULL; 3197 3198 /* 3199 * During the reset sequence, we must prevent the engine from 3200 * entering RC6. As the context state is undefined until we restart 3201 * the engine, if it does enter RC6 during the reset, the state 3202 * written to the powercontext is undefined and so we may lose 3203 * GPU state upon resume, i.e. fail to restart after a reset. 3204 */ 3205 intel_uncore_forcewake_get(engine->i915, FORCEWAKE_ALL); 3206 3207 /* 3208 * Prevent the signaler thread from updating the request 3209 * state (by calling dma_fence_signal) as we are processing 3210 * the reset. The write from the GPU of the seqno is 3211 * asynchronous and the signaler thread may see a different 3212 * value to us and declare the request complete, even though 3213 * the reset routine have picked that request as the active 3214 * (incomplete) request. This conflict is not handled 3215 * gracefully! 3216 */ 3217 kthread_park(engine->breadcrumbs.signaler); 3218 3219 /* 3220 * Prevent request submission to the hardware until we have 3221 * completed the reset in i915_gem_reset_finish(). If a request 3222 * is completed by one engine, it may then queue a request 3223 * to a second via its engine->irq_tasklet *just* as we are 3224 * calling engine->init_hw() and also writing the ELSP. 3225 * Turning off the engine->irq_tasklet until the reset is over 3226 * prevents the race. 3227 */ 3228 tasklet_kill(&engine->execlists.irq_tasklet); 3229 tasklet_disable(&engine->execlists.irq_tasklet); 3230 3231 if (engine->irq_seqno_barrier) 3232 engine->irq_seqno_barrier(engine); 3233 3234 request = i915_gem_find_active_request(engine); 3235 if (request && request->fence.error == -EIO) 3236 request = ERR_PTR(-EIO); /* Previous reset failed! */ 3237 3238 return request; 3239 } 3240 3241 int i915_gem_reset_prepare(struct drm_i915_private *dev_priv) 3242 { 3243 struct intel_engine_cs *engine; 3244 struct drm_i915_gem_request *request; 3245 enum intel_engine_id id; 3246 int err = 0; 3247 3248 for_each_engine(engine, dev_priv, id) { 3249 request = i915_gem_reset_prepare_engine(engine); 3250 if (IS_ERR(request)) { 3251 err = PTR_ERR(request); 3252 continue; 3253 } 3254 3255 engine->hangcheck.active_request = request; 3256 } 3257 3258 i915_gem_revoke_fences(dev_priv); 3259 3260 return err; 3261 } 3262 3263 static void skip_request(struct drm_i915_gem_request *request) 3264 { 3265 void *vaddr = request->ring->vaddr; 3266 u32 head; 3267 3268 /* As this request likely depends on state from the lost 3269 * context, clear out all the user operations leaving the 3270 * breadcrumb at the end (so we get the fence notifications). 3271 */ 3272 head = request->head; 3273 if (request->postfix < head) { 3274 memset(vaddr + head, 0, request->ring->size - head); 3275 head = 0; 3276 } 3277 memset(vaddr + head, 0, request->postfix - head); 3278 3279 dma_fence_set_error(&request->fence, -EIO); 3280 } 3281 3282 static void engine_skip_context(struct drm_i915_gem_request *request) 3283 { 3284 struct intel_engine_cs *engine = request->engine; 3285 struct i915_gem_context *hung_ctx = request->ctx; 3286 struct intel_timeline *timeline; 3287 unsigned long flags; 3288 3289 timeline = i915_gem_context_lookup_timeline(hung_ctx, engine); 3290 3291 spin_lock_irqsave(&engine->timeline->lock, flags); 3292 lockmgr(&timeline->lock, LK_EXCLUSIVE); 3293 3294 list_for_each_entry_continue(request, &engine->timeline->requests, link) 3295 if (request->ctx == hung_ctx) 3296 skip_request(request); 3297 3298 list_for_each_entry(request, &timeline->requests, link) 3299 skip_request(request); 3300 3301 lockmgr(&timeline->lock, LK_RELEASE); 3302 spin_unlock_irqrestore(&engine->timeline->lock, flags); 3303 } 3304 3305 /* Returns the request if it was guilty of the hang */ 3306 static struct drm_i915_gem_request * 3307 i915_gem_reset_request(struct intel_engine_cs *engine, 3308 struct drm_i915_gem_request *request) 3309 { 3310 /* The guilty request will get skipped on a hung engine. 3311 * 3312 * Users of client default contexts do not rely on logical 3313 * state preserved between batches so it is safe to execute 3314 * queued requests following the hang. Non default contexts 3315 * rely on preserved state, so skipping a batch loses the 3316 * evolution of the state and it needs to be considered corrupted. 3317 * Executing more queued batches on top of corrupted state is 3318 * risky. But we take the risk by trying to advance through 3319 * the queued requests in order to make the client behaviour 3320 * more predictable around resets, by not throwing away random 3321 * amount of batches it has prepared for execution. Sophisticated 3322 * clients can use gem_reset_stats_ioctl and dma fence status 3323 * (exported via sync_file info ioctl on explicit fences) to observe 3324 * when it loses the context state and should rebuild accordingly. 3325 * 3326 * The context ban, and ultimately the client ban, mechanism are safety 3327 * valves if client submission ends up resulting in nothing more than 3328 * subsequent hangs. 3329 */ 3330 3331 if (engine_stalled(engine)) { 3332 i915_gem_context_mark_guilty(request->ctx); 3333 skip_request(request); 3334 3335 /* If this context is now banned, skip all pending requests. */ 3336 if (i915_gem_context_is_banned(request->ctx)) 3337 engine_skip_context(request); 3338 } else { 3339 /* 3340 * Since this is not the hung engine, it may have advanced 3341 * since the hang declaration. Double check by refinding 3342 * the active request at the time of the reset. 3343 */ 3344 request = i915_gem_find_active_request(engine); 3345 if (request) { 3346 i915_gem_context_mark_innocent(request->ctx); 3347 dma_fence_set_error(&request->fence, -EAGAIN); 3348 3349 /* Rewind the engine to replay the incomplete rq */ 3350 spin_lock_irq(&engine->timeline->lock); 3351 request = list_prev_entry(request, link); 3352 if (&request->link == &engine->timeline->requests) 3353 request = NULL; 3354 spin_unlock_irq(&engine->timeline->lock); 3355 } 3356 } 3357 3358 return request; 3359 } 3360 3361 void i915_gem_reset_engine(struct intel_engine_cs *engine, 3362 struct drm_i915_gem_request *request) 3363 { 3364 engine->irq_posted = 0; 3365 3366 if (request) 3367 request = i915_gem_reset_request(engine, request); 3368 3369 if (request) { 3370 DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n", 3371 engine->name, request->global_seqno); 3372 } 3373 3374 /* Setup the CS to resume from the breadcrumb of the hung request */ 3375 engine->reset_hw(engine, request); 3376 } 3377 3378 void i915_gem_reset(struct drm_i915_private *dev_priv) 3379 { 3380 struct intel_engine_cs *engine; 3381 enum intel_engine_id id; 3382 3383 lockdep_assert_held(&dev_priv->drm.struct_mutex); 3384 3385 i915_gem_retire_requests(dev_priv); 3386 3387 for_each_engine(engine, dev_priv, id) { 3388 struct i915_gem_context *ctx; 3389 3390 i915_gem_reset_engine(engine, engine->hangcheck.active_request); 3391 ctx = fetch_and_zero(&engine->last_retired_context); 3392 if (ctx) 3393 engine->context_unpin(engine, ctx); 3394 } 3395 3396 i915_gem_restore_fences(dev_priv); 3397 3398 if (dev_priv->gt.awake) { 3399 intel_sanitize_gt_powersave(dev_priv); 3400 intel_enable_gt_powersave(dev_priv); 3401 if (INTEL_GEN(dev_priv) >= 6) 3402 gen6_rps_busy(dev_priv); 3403 } 3404 } 3405 3406 void i915_gem_reset_finish_engine(struct intel_engine_cs *engine) 3407 { 3408 tasklet_enable(&engine->execlists.irq_tasklet); 3409 kthread_unpark(engine->breadcrumbs.signaler); 3410 3411 intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL); 3412 } 3413 3414 void i915_gem_reset_finish(struct drm_i915_private *dev_priv) 3415 { 3416 struct intel_engine_cs *engine; 3417 enum intel_engine_id id; 3418 3419 lockdep_assert_held(&dev_priv->drm.struct_mutex); 3420 3421 for_each_engine(engine, dev_priv, id) { 3422 engine->hangcheck.active_request = NULL; 3423 i915_gem_reset_finish_engine(engine); 3424 } 3425 } 3426 3427 static void nop_submit_request(struct drm_i915_gem_request *request) 3428 { 3429 dma_fence_set_error(&request->fence, -EIO); 3430 3431 i915_gem_request_submit(request); 3432 } 3433 3434 static void nop_complete_submit_request(struct drm_i915_gem_request *request) 3435 { 3436 unsigned long flags; 3437 3438 dma_fence_set_error(&request->fence, -EIO); 3439 3440 spin_lock_irqsave(&request->engine->timeline->lock, flags); 3441 __i915_gem_request_submit(request); 3442 intel_engine_init_global_seqno(request->engine, request->global_seqno); 3443 spin_unlock_irqrestore(&request->engine->timeline->lock, flags); 3444 } 3445 3446 void i915_gem_set_wedged(struct drm_i915_private *i915) 3447 { 3448 struct intel_engine_cs *engine; 3449 enum intel_engine_id id; 3450 3451 /* 3452 * First, stop submission to hw, but do not yet complete requests by 3453 * rolling the global seqno forward (since this would complete requests 3454 * for which we haven't set the fence error to EIO yet). 3455 */ 3456 for_each_engine(engine, i915, id) 3457 engine->submit_request = nop_submit_request; 3458 3459 /* 3460 * Make sure no one is running the old callback before we proceed with 3461 * cancelling requests and resetting the completion tracking. Otherwise 3462 * we might submit a request to the hardware which never completes. 3463 */ 3464 synchronize_rcu(); 3465 3466 for_each_engine(engine, i915, id) { 3467 /* Mark all executing requests as skipped */ 3468 engine->cancel_requests(engine); 3469 3470 /* 3471 * Only once we've force-cancelled all in-flight requests can we 3472 * start to complete all requests. 3473 */ 3474 engine->submit_request = nop_complete_submit_request; 3475 } 3476 3477 /* 3478 * Make sure no request can slip through without getting completed by 3479 * either this call here to intel_engine_init_global_seqno, or the one 3480 * in nop_complete_submit_request. 3481 */ 3482 synchronize_rcu(); 3483 3484 for_each_engine(engine, i915, id) { 3485 unsigned long flags; 3486 3487 /* Mark all pending requests as complete so that any concurrent 3488 * (lockless) lookup doesn't try and wait upon the request as we 3489 * reset it. 3490 */ 3491 spin_lock_irqsave(&engine->timeline->lock, flags); 3492 intel_engine_init_global_seqno(engine, 3493 intel_engine_last_submit(engine)); 3494 spin_unlock_irqrestore(&engine->timeline->lock, flags); 3495 } 3496 3497 set_bit(I915_WEDGED, &i915->gpu_error.flags); 3498 wake_up_all(&i915->gpu_error.reset_queue); 3499 } 3500 3501 bool i915_gem_unset_wedged(struct drm_i915_private *i915) 3502 { 3503 struct i915_gem_timeline *tl; 3504 int i; 3505 3506 lockdep_assert_held(&i915->drm.struct_mutex); 3507 if (!test_bit(I915_WEDGED, &i915->gpu_error.flags)) 3508 return true; 3509 3510 /* Before unwedging, make sure that all pending operations 3511 * are flushed and errored out - we may have requests waiting upon 3512 * third party fences. We marked all inflight requests as EIO, and 3513 * every execbuf since returned EIO, for consistency we want all 3514 * the currently pending requests to also be marked as EIO, which 3515 * is done inside our nop_submit_request - and so we must wait. 3516 * 3517 * No more can be submitted until we reset the wedged bit. 3518 */ 3519 list_for_each_entry(tl, &i915->gt.timelines, link) { 3520 for (i = 0; i < ARRAY_SIZE(tl->engine); i++) { 3521 struct drm_i915_gem_request *rq; 3522 3523 rq = i915_gem_active_peek(&tl->engine[i].last_request, 3524 &i915->drm.struct_mutex); 3525 if (!rq) 3526 continue; 3527 3528 /* We can't use our normal waiter as we want to 3529 * avoid recursively trying to handle the current 3530 * reset. The basic dma_fence_default_wait() installs 3531 * a callback for dma_fence_signal(), which is 3532 * triggered by our nop handler (indirectly, the 3533 * callback enables the signaler thread which is 3534 * woken by the nop_submit_request() advancing the seqno 3535 * and when the seqno passes the fence, the signaler 3536 * then signals the fence waking us up). 3537 */ 3538 if (dma_fence_default_wait(&rq->fence, true, 3539 MAX_SCHEDULE_TIMEOUT) < 0) 3540 return false; 3541 } 3542 } 3543 3544 /* Undo nop_submit_request. We prevent all new i915 requests from 3545 * being queued (by disallowing execbuf whilst wedged) so having 3546 * waited for all active requests above, we know the system is idle 3547 * and do not have to worry about a thread being inside 3548 * engine->submit_request() as we swap over. So unlike installing 3549 * the nop_submit_request on reset, we can do this from normal 3550 * context and do not require stop_machine(). 3551 */ 3552 intel_engines_reset_default_submission(i915); 3553 i915_gem_contexts_lost(i915); 3554 3555 smp_mb__before_atomic(); /* complete takeover before enabling execbuf */ 3556 clear_bit(I915_WEDGED, &i915->gpu_error.flags); 3557 3558 return true; 3559 } 3560 3561 static void 3562 i915_gem_retire_work_handler(struct work_struct *work) 3563 { 3564 struct drm_i915_private *dev_priv = 3565 container_of(work, typeof(*dev_priv), gt.retire_work.work); 3566 struct drm_device *dev = &dev_priv->drm; 3567 3568 /* Come back later if the device is busy... */ 3569 if (mutex_trylock(&dev->struct_mutex)) { 3570 i915_gem_retire_requests(dev_priv); 3571 mutex_unlock(&dev->struct_mutex); 3572 } 3573 3574 /* Keep the retire handler running until we are finally idle. 3575 * We do not need to do this test under locking as in the worst-case 3576 * we queue the retire worker once too often. 3577 */ 3578 if (READ_ONCE(dev_priv->gt.awake)) { 3579 i915_queue_hangcheck(dev_priv); 3580 queue_delayed_work(dev_priv->wq, 3581 &dev_priv->gt.retire_work, 3582 round_jiffies_up_relative(HZ)); 3583 } 3584 } 3585 3586 static void 3587 i915_gem_idle_work_handler(struct work_struct *work) 3588 { 3589 struct drm_i915_private *dev_priv = 3590 container_of(work, typeof(*dev_priv), gt.idle_work.work); 3591 struct drm_device *dev = &dev_priv->drm; 3592 bool rearm_hangcheck; 3593 3594 if (!READ_ONCE(dev_priv->gt.awake)) 3595 return; 3596 3597 /* 3598 * Wait for last execlists context complete, but bail out in case a 3599 * new request is submitted. 3600 */ 3601 wait_for(intel_engines_are_idle(dev_priv), 10); 3602 if (READ_ONCE(dev_priv->gt.active_requests)) 3603 return; 3604 3605 rearm_hangcheck = 3606 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); 3607 3608 if (!mutex_trylock(&dev->struct_mutex)) { 3609 /* Currently busy, come back later */ 3610 mod_delayed_work(dev_priv->wq, 3611 &dev_priv->gt.idle_work, 3612 msecs_to_jiffies(50)); 3613 goto out_rearm; 3614 } 3615 3616 /* 3617 * New request retired after this work handler started, extend active 3618 * period until next instance of the work. 3619 */ 3620 if (work_pending(work)) 3621 goto out_unlock; 3622 3623 if (dev_priv->gt.active_requests) 3624 goto out_unlock; 3625 3626 if (wait_for(intel_engines_are_idle(dev_priv), 10)) 3627 DRM_ERROR("Timeout waiting for engines to idle\n"); 3628 3629 intel_engines_mark_idle(dev_priv); 3630 i915_gem_timelines_mark_idle(dev_priv); 3631 3632 GEM_BUG_ON(!dev_priv->gt.awake); 3633 dev_priv->gt.awake = false; 3634 rearm_hangcheck = false; 3635 3636 if (INTEL_GEN(dev_priv) >= 6) 3637 gen6_rps_idle(dev_priv); 3638 intel_runtime_pm_put(dev_priv); 3639 out_unlock: 3640 mutex_unlock(&dev->struct_mutex); 3641 3642 out_rearm: 3643 if (rearm_hangcheck) { 3644 GEM_BUG_ON(!dev_priv->gt.awake); 3645 i915_queue_hangcheck(dev_priv); 3646 } 3647 } 3648 3649 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) 3650 { 3651 struct drm_i915_private *i915 = to_i915(gem->dev); 3652 struct drm_i915_gem_object *obj = to_intel_bo(gem); 3653 struct drm_i915_file_private *fpriv = file->driver_priv; 3654 struct i915_lut_handle *lut, *ln; 3655 3656 mutex_lock(&i915->drm.struct_mutex); 3657 3658 list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) { 3659 struct i915_gem_context *ctx = lut->ctx; 3660 struct i915_vma *vma; 3661 3662 GEM_BUG_ON(ctx->file_priv == ERR_PTR(-EBADF)); 3663 if (ctx->file_priv != fpriv) 3664 continue; 3665 3666 vma = radix_tree_delete(&ctx->handles_vma, lut->handle); 3667 GEM_BUG_ON(vma->obj != obj); 3668 3669 /* We allow the process to have multiple handles to the same 3670 * vma, in the same fd namespace, by virtue of flink/open. 3671 */ 3672 GEM_BUG_ON(!vma->open_count); 3673 if (!--vma->open_count && !i915_vma_is_ggtt(vma)) 3674 i915_vma_close(vma); 3675 3676 list_del(&lut->obj_link); 3677 list_del(&lut->ctx_link); 3678 3679 kmem_cache_free(i915->luts, lut); 3680 __i915_gem_object_release_unless_active(obj); 3681 } 3682 3683 mutex_unlock(&i915->drm.struct_mutex); 3684 } 3685 3686 static unsigned long to_wait_timeout(s64 timeout_ns) 3687 { 3688 if (timeout_ns < 0) 3689 return MAX_SCHEDULE_TIMEOUT; 3690 3691 if (timeout_ns == 0) 3692 return 0; 3693 3694 return nsecs_to_jiffies_timeout(timeout_ns); 3695 } 3696 3697 /** 3698 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT 3699 * @dev: drm device pointer 3700 * @data: ioctl data blob 3701 * @file: drm file pointer 3702 * 3703 * Returns 0 if successful, else an error is returned with the remaining time in 3704 * the timeout parameter. 3705 * -ETIME: object is still busy after timeout 3706 * -ERESTARTSYS: signal interrupted the wait 3707 * -ENONENT: object doesn't exist 3708 * Also possible, but rare: 3709 * -EAGAIN: incomplete, restart syscall 3710 * -ENOMEM: damn 3711 * -ENODEV: Internal IRQ fail 3712 * -E?: The add request failed 3713 * 3714 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any 3715 * non-zero timeout parameter the wait ioctl will wait for the given number of 3716 * nanoseconds on an object becoming unbusy. Since the wait itself does so 3717 * without holding struct_mutex the object may become re-busied before this 3718 * function completes. A similar but shorter * race condition exists in the busy 3719 * ioctl 3720 */ 3721 int 3722 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 3723 { 3724 struct drm_i915_gem_wait *args = data; 3725 struct drm_i915_gem_object *obj; 3726 ktime_t start; 3727 long ret; 3728 3729 if (args->flags != 0) 3730 return -EINVAL; 3731 3732 obj = i915_gem_object_lookup(file, args->bo_handle); 3733 if (!obj) 3734 return -ENOENT; 3735 3736 start = ktime_get(); 3737 3738 ret = i915_gem_object_wait(obj, 3739 I915_WAIT_INTERRUPTIBLE | I915_WAIT_ALL, 3740 to_wait_timeout(args->timeout_ns), 3741 to_rps_client(file)); 3742 3743 if (args->timeout_ns > 0) { 3744 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start)); 3745 if (args->timeout_ns < 0) 3746 args->timeout_ns = 0; 3747 3748 /* 3749 * Apparently ktime isn't accurate enough and occasionally has a 3750 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch 3751 * things up to make the test happy. We allow up to 1 jiffy. 3752 * 3753 * This is a regression from the timespec->ktime conversion. 3754 */ 3755 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns)) 3756 args->timeout_ns = 0; 3757 3758 /* 3759 * Apparently ktime isn't accurate enough and occasionally has a 3760 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch 3761 * things up to make the test happy. We allow up to 1 jiffy. 3762 * 3763 * This is a regression from the timespec->ktime conversion. 3764 */ 3765 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns)) 3766 args->timeout_ns = 0; 3767 3768 /* Asked to wait beyond the jiffie/scheduler precision? */ 3769 if (ret == -ETIME && args->timeout_ns) 3770 ret = -EAGAIN; 3771 } 3772 3773 i915_gem_object_put(obj); 3774 return ret; 3775 } 3776 3777 static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags) 3778 { 3779 int ret, i; 3780 3781 for (i = 0; i < ARRAY_SIZE(tl->engine); i++) { 3782 ret = i915_gem_active_wait(&tl->engine[i].last_request, flags); 3783 if (ret) 3784 return ret; 3785 } 3786 3787 return 0; 3788 } 3789 3790 static int wait_for_engines(struct drm_i915_private *i915) 3791 { 3792 if (wait_for(intel_engines_are_idle(i915), 50)) { 3793 DRM_ERROR("Failed to idle engines, declaring wedged!\n"); 3794 i915_gem_set_wedged(i915); 3795 return -EIO; 3796 } 3797 3798 return 0; 3799 } 3800 3801 int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags) 3802 { 3803 int ret; 3804 3805 /* If the device is asleep, we have no requests outstanding */ 3806 if (!READ_ONCE(i915->gt.awake)) 3807 return 0; 3808 3809 if (flags & I915_WAIT_LOCKED) { 3810 struct i915_gem_timeline *tl; 3811 3812 lockdep_assert_held(&i915->drm.struct_mutex); 3813 3814 list_for_each_entry(tl, &i915->gt.timelines, link) { 3815 ret = wait_for_timeline(tl, flags); 3816 if (ret) 3817 return ret; 3818 } 3819 3820 i915_gem_retire_requests(i915); 3821 GEM_BUG_ON(i915->gt.active_requests); 3822 3823 ret = wait_for_engines(i915); 3824 } else { 3825 ret = wait_for_timeline(&i915->gt.global_timeline, flags); 3826 } 3827 3828 return ret; 3829 } 3830 3831 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj) 3832 { 3833 /* 3834 * We manually flush the CPU domain so that we can override and 3835 * force the flush for the display, and perform it asyncrhonously. 3836 */ 3837 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 3838 if (obj->cache_dirty) 3839 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE); 3840 obj->base.write_domain = 0; 3841 } 3842 3843 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj) 3844 { 3845 if (!READ_ONCE(obj->pin_global)) 3846 return; 3847 3848 mutex_lock(&obj->base.dev->struct_mutex); 3849 __i915_gem_object_flush_for_display(obj); 3850 mutex_unlock(&obj->base.dev->struct_mutex); 3851 } 3852 3853 /** 3854 * Moves a single object to the WC read, and possibly write domain. 3855 * @obj: object to act on 3856 * @write: ask for write access or read only 3857 * 3858 * This function returns when the move is complete, including waiting on 3859 * flushes to occur. 3860 */ 3861 int 3862 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write) 3863 { 3864 int ret; 3865 3866 lockdep_assert_held(&obj->base.dev->struct_mutex); 3867 3868 ret = i915_gem_object_wait(obj, 3869 I915_WAIT_INTERRUPTIBLE | 3870 I915_WAIT_LOCKED | 3871 (write ? I915_WAIT_ALL : 0), 3872 MAX_SCHEDULE_TIMEOUT, 3873 NULL); 3874 if (ret) 3875 return ret; 3876 3877 if (obj->base.write_domain == I915_GEM_DOMAIN_WC) 3878 return 0; 3879 3880 /* Flush and acquire obj->pages so that we are coherent through 3881 * direct access in memory with previous cached writes through 3882 * shmemfs and that our cache domain tracking remains valid. 3883 * For example, if the obj->filp was moved to swap without us 3884 * being notified and releasing the pages, we would mistakenly 3885 * continue to assume that the obj remained out of the CPU cached 3886 * domain. 3887 */ 3888 ret = i915_gem_object_pin_pages(obj); 3889 if (ret) 3890 return ret; 3891 3892 flush_write_domain(obj, ~I915_GEM_DOMAIN_WC); 3893 3894 /* Serialise direct access to this object with the barriers for 3895 * coherent writes from the GPU, by effectively invalidating the 3896 * WC domain upon first access. 3897 */ 3898 if ((obj->base.read_domains & I915_GEM_DOMAIN_WC) == 0) 3899 mb(); 3900 3901 /* It should now be out of any other write domains, and we can update 3902 * the domain values for our changes. 3903 */ 3904 GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_WC) != 0); 3905 obj->base.read_domains |= I915_GEM_DOMAIN_WC; 3906 if (write) { 3907 obj->base.read_domains = I915_GEM_DOMAIN_WC; 3908 obj->base.write_domain = I915_GEM_DOMAIN_WC; 3909 obj->mm.dirty = true; 3910 } 3911 3912 i915_gem_object_unpin_pages(obj); 3913 return 0; 3914 } 3915 3916 /** 3917 * Moves a single object to the GTT read, and possibly write domain. 3918 * @obj: object to act on 3919 * @write: ask for write access or read only 3920 * 3921 * This function returns when the move is complete, including waiting on 3922 * flushes to occur. 3923 */ 3924 int 3925 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 3926 { 3927 int ret; 3928 3929 lockdep_assert_held(&obj->base.dev->struct_mutex); 3930 3931 ret = i915_gem_object_wait(obj, 3932 I915_WAIT_INTERRUPTIBLE | 3933 I915_WAIT_LOCKED | 3934 (write ? I915_WAIT_ALL : 0), 3935 MAX_SCHEDULE_TIMEOUT, 3936 NULL); 3937 if (ret) 3938 return ret; 3939 3940 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 3941 return 0; 3942 3943 /* Flush and acquire obj->pages so that we are coherent through 3944 * direct access in memory with previous cached writes through 3945 * shmemfs and that our cache domain tracking remains valid. 3946 * For example, if the obj->filp was moved to swap without us 3947 * being notified and releasing the pages, we would mistakenly 3948 * continue to assume that the obj remained out of the CPU cached 3949 * domain. 3950 */ 3951 ret = i915_gem_object_pin_pages(obj); 3952 if (ret) 3953 return ret; 3954 3955 flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT); 3956 3957 /* Serialise direct access to this object with the barriers for 3958 * coherent writes from the GPU, by effectively invalidating the 3959 * GTT domain upon first access. 3960 */ 3961 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 3962 mb(); 3963 3964 /* It should now be out of any other write domains, and we can update 3965 * the domain values for our changes. 3966 */ 3967 GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 3968 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 3969 if (write) { 3970 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 3971 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 3972 obj->mm.dirty = true; 3973 } 3974 3975 i915_gem_object_unpin_pages(obj); 3976 return 0; 3977 } 3978 3979 /** 3980 * Changes the cache-level of an object across all VMA. 3981 * @obj: object to act on 3982 * @cache_level: new cache level to set for the object 3983 * 3984 * After this function returns, the object will be in the new cache-level 3985 * across all GTT and the contents of the backing storage will be coherent, 3986 * with respect to the new cache-level. In order to keep the backing storage 3987 * coherent for all users, we only allow a single cache level to be set 3988 * globally on the object and prevent it from being changed whilst the 3989 * hardware is reading from the object. That is if the object is currently 3990 * on the scanout it will be set to uncached (or equivalent display 3991 * cache coherency) and all non-MOCS GPU access will also be uncached so 3992 * that all direct access to the scanout remains coherent. 3993 */ 3994 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 3995 enum i915_cache_level cache_level) 3996 { 3997 struct i915_vma *vma; 3998 int ret; 3999 4000 lockdep_assert_held(&obj->base.dev->struct_mutex); 4001 4002 if (obj->cache_level == cache_level) 4003 return 0; 4004 4005 /* Inspect the list of currently bound VMA and unbind any that would 4006 * be invalid given the new cache-level. This is principally to 4007 * catch the issue of the CS prefetch crossing page boundaries and 4008 * reading an invalid PTE on older architectures. 4009 */ 4010 restart: 4011 list_for_each_entry(vma, &obj->vma_list, obj_link) { 4012 if (!drm_mm_node_allocated(&vma->node)) 4013 continue; 4014 4015 if (i915_vma_is_pinned(vma)) { 4016 DRM_DEBUG("can not change the cache level of pinned objects\n"); 4017 return -EBUSY; 4018 } 4019 4020 if (i915_gem_valid_gtt_space(vma, cache_level)) 4021 continue; 4022 4023 ret = i915_vma_unbind(vma); 4024 if (ret) 4025 return ret; 4026 4027 /* As unbinding may affect other elements in the 4028 * obj->vma_list (due to side-effects from retiring 4029 * an active vma), play safe and restart the iterator. 4030 */ 4031 goto restart; 4032 } 4033 4034 /* We can reuse the existing drm_mm nodes but need to change the 4035 * cache-level on the PTE. We could simply unbind them all and 4036 * rebind with the correct cache-level on next use. However since 4037 * we already have a valid slot, dma mapping, pages etc, we may as 4038 * rewrite the PTE in the belief that doing so tramples upon less 4039 * state and so involves less work. 4040 */ 4041 if (obj->bind_count) { 4042 /* Before we change the PTE, the GPU must not be accessing it. 4043 * If we wait upon the object, we know that all the bound 4044 * VMA are no longer active. 4045 */ 4046 ret = i915_gem_object_wait(obj, 4047 I915_WAIT_INTERRUPTIBLE | 4048 I915_WAIT_LOCKED | 4049 I915_WAIT_ALL, 4050 MAX_SCHEDULE_TIMEOUT, 4051 NULL); 4052 if (ret) 4053 return ret; 4054 4055 if (!HAS_LLC(to_i915(obj->base.dev)) && 4056 cache_level != I915_CACHE_NONE) { 4057 /* Access to snoopable pages through the GTT is 4058 * incoherent and on some machines causes a hard 4059 * lockup. Relinquish the CPU mmaping to force 4060 * userspace to refault in the pages and we can 4061 * then double check if the GTT mapping is still 4062 * valid for that pointer access. 4063 */ 4064 i915_gem_release_mmap(obj); 4065 4066 /* As we no longer need a fence for GTT access, 4067 * we can relinquish it now (and so prevent having 4068 * to steal a fence from someone else on the next 4069 * fence request). Note GPU activity would have 4070 * dropped the fence as all snoopable access is 4071 * supposed to be linear. 4072 */ 4073 list_for_each_entry(vma, &obj->vma_list, obj_link) { 4074 ret = i915_vma_put_fence(vma); 4075 if (ret) 4076 return ret; 4077 } 4078 } else { 4079 /* We either have incoherent backing store and 4080 * so no GTT access or the architecture is fully 4081 * coherent. In such cases, existing GTT mmaps 4082 * ignore the cache bit in the PTE and we can 4083 * rewrite it without confusing the GPU or having 4084 * to force userspace to fault back in its mmaps. 4085 */ 4086 } 4087 4088 list_for_each_entry(vma, &obj->vma_list, obj_link) { 4089 if (!drm_mm_node_allocated(&vma->node)) 4090 continue; 4091 4092 ret = i915_vma_bind(vma, cache_level, PIN_UPDATE); 4093 if (ret) 4094 return ret; 4095 } 4096 } 4097 4098 list_for_each_entry(vma, &obj->vma_list, obj_link) 4099 vma->node.color = cache_level; 4100 i915_gem_object_set_cache_coherency(obj, cache_level); 4101 obj->cache_dirty = true; /* Always invalidate stale cachelines */ 4102 4103 return 0; 4104 } 4105 4106 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, 4107 struct drm_file *file) 4108 { 4109 struct drm_i915_gem_caching *args = data; 4110 struct drm_i915_gem_object *obj; 4111 int err = 0; 4112 4113 rcu_read_lock(); 4114 obj = i915_gem_object_lookup_rcu(file, args->handle); 4115 if (!obj) { 4116 err = -ENOENT; 4117 goto out; 4118 } 4119 4120 switch (obj->cache_level) { 4121 case I915_CACHE_LLC: 4122 case I915_CACHE_L3_LLC: 4123 args->caching = I915_CACHING_CACHED; 4124 break; 4125 4126 case I915_CACHE_WT: 4127 args->caching = I915_CACHING_DISPLAY; 4128 break; 4129 4130 default: 4131 args->caching = I915_CACHING_NONE; 4132 break; 4133 } 4134 out: 4135 rcu_read_unlock(); 4136 return err; 4137 } 4138 4139 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, 4140 struct drm_file *file) 4141 { 4142 struct drm_i915_private *i915 = to_i915(dev); 4143 struct drm_i915_gem_caching *args = data; 4144 struct drm_i915_gem_object *obj; 4145 enum i915_cache_level level; 4146 int ret = 0; 4147 4148 switch (args->caching) { 4149 case I915_CACHING_NONE: 4150 level = I915_CACHE_NONE; 4151 break; 4152 case I915_CACHING_CACHED: 4153 /* 4154 * Due to a HW issue on BXT A stepping, GPU stores via a 4155 * snooped mapping may leave stale data in a corresponding CPU 4156 * cacheline, whereas normally such cachelines would get 4157 * invalidated. 4158 */ 4159 if (!HAS_LLC(i915) && !HAS_SNOOP(i915)) 4160 return -ENODEV; 4161 4162 level = I915_CACHE_LLC; 4163 break; 4164 case I915_CACHING_DISPLAY: 4165 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE; 4166 break; 4167 default: 4168 return -EINVAL; 4169 } 4170 4171 obj = i915_gem_object_lookup(file, args->handle); 4172 if (!obj) 4173 return -ENOENT; 4174 4175 if (obj->cache_level == level) 4176 goto out; 4177 4178 ret = i915_gem_object_wait(obj, 4179 I915_WAIT_INTERRUPTIBLE, 4180 MAX_SCHEDULE_TIMEOUT, 4181 to_rps_client(file)); 4182 if (ret) 4183 goto out; 4184 4185 ret = i915_mutex_lock_interruptible(dev); 4186 if (ret) 4187 goto out; 4188 4189 ret = i915_gem_object_set_cache_level(obj, level); 4190 mutex_unlock(&dev->struct_mutex); 4191 4192 out: 4193 i915_gem_object_put(obj); 4194 return ret; 4195 } 4196 4197 /* 4198 * Prepare buffer for display plane (scanout, cursors, etc). 4199 * Can be called from an uninterruptible phase (modesetting) and allows 4200 * any flushes to be pipelined (for pageflips). 4201 */ 4202 struct i915_vma * 4203 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 4204 u32 alignment, 4205 const struct i915_ggtt_view *view) 4206 { 4207 struct i915_vma *vma; 4208 int ret; 4209 4210 lockdep_assert_held(&obj->base.dev->struct_mutex); 4211 4212 /* Mark the global pin early so that we account for the 4213 * display coherency whilst setting up the cache domains. 4214 */ 4215 obj->pin_global++; 4216 4217 /* The display engine is not coherent with the LLC cache on gen6. As 4218 * a result, we make sure that the pinning that is about to occur is 4219 * done with uncached PTEs. This is lowest common denominator for all 4220 * chipsets. 4221 * 4222 * However for gen6+, we could do better by using the GFDT bit instead 4223 * of uncaching, which would allow us to flush all the LLC-cached data 4224 * with that bit in the PTE to main memory with just one PIPE_CONTROL. 4225 */ 4226 ret = i915_gem_object_set_cache_level(obj, 4227 HAS_WT(to_i915(obj->base.dev)) ? 4228 I915_CACHE_WT : I915_CACHE_NONE); 4229 if (ret) { 4230 vma = ERR_PTR(ret); 4231 goto err_unpin_global; 4232 } 4233 4234 /* As the user may map the buffer once pinned in the display plane 4235 * (e.g. libkms for the bootup splash), we have to ensure that we 4236 * always use map_and_fenceable for all scanout buffers. However, 4237 * it may simply be too big to fit into mappable, in which case 4238 * put it anyway and hope that userspace can cope (but always first 4239 * try to preserve the existing ABI). 4240 */ 4241 vma = ERR_PTR(-ENOSPC); 4242 if (!view || view->type == I915_GGTT_VIEW_NORMAL) 4243 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, 4244 PIN_MAPPABLE | PIN_NONBLOCK); 4245 if (IS_ERR(vma)) { 4246 struct drm_i915_private *i915 = to_i915(obj->base.dev); 4247 unsigned int flags; 4248 4249 /* Valleyview is definitely limited to scanning out the first 4250 * 512MiB. Lets presume this behaviour was inherited from the 4251 * g4x display engine and that all earlier gen are similarly 4252 * limited. Testing suggests that it is a little more 4253 * complicated than this. For example, Cherryview appears quite 4254 * happy to scanout from anywhere within its global aperture. 4255 */ 4256 flags = 0; 4257 if (HAS_GMCH_DISPLAY(i915)) 4258 flags = PIN_MAPPABLE; 4259 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags); 4260 } 4261 if (IS_ERR(vma)) 4262 goto err_unpin_global; 4263 4264 vma->display_alignment = max_t(u64, vma->display_alignment, alignment); 4265 4266 /* Treat this as an end-of-frame, like intel_user_framebuffer_dirty() */ 4267 __i915_gem_object_flush_for_display(obj); 4268 intel_fb_obj_flush(obj, ORIGIN_DIRTYFB); 4269 4270 /* It should now be out of any other write domains, and we can update 4271 * the domain values for our changes. 4272 */ 4273 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 4274 4275 return vma; 4276 4277 err_unpin_global: 4278 obj->pin_global--; 4279 return vma; 4280 } 4281 4282 void 4283 i915_gem_object_unpin_from_display_plane(struct i915_vma *vma) 4284 { 4285 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); 4286 4287 if (WARN_ON(vma->obj->pin_global == 0)) 4288 return; 4289 4290 if (--vma->obj->pin_global == 0) 4291 vma->display_alignment = I915_GTT_MIN_ALIGNMENT; 4292 4293 /* Bump the LRU to try and avoid premature eviction whilst flipping */ 4294 i915_gem_object_bump_inactive_ggtt(vma->obj); 4295 4296 i915_vma_unpin(vma); 4297 } 4298 4299 /** 4300 * Moves a single object to the CPU read, and possibly write domain. 4301 * @obj: object to act on 4302 * @write: requesting write or read-only access 4303 * 4304 * This function returns when the move is complete, including waiting on 4305 * flushes to occur. 4306 */ 4307 int 4308 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 4309 { 4310 int ret; 4311 4312 lockdep_assert_held(&obj->base.dev->struct_mutex); 4313 4314 ret = i915_gem_object_wait(obj, 4315 I915_WAIT_INTERRUPTIBLE | 4316 I915_WAIT_LOCKED | 4317 (write ? I915_WAIT_ALL : 0), 4318 MAX_SCHEDULE_TIMEOUT, 4319 NULL); 4320 if (ret) 4321 return ret; 4322 4323 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 4324 4325 /* Flush the CPU cache if it's still invalid. */ 4326 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 4327 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC); 4328 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 4329 } 4330 4331 /* It should now be out of any other write domains, and we can update 4332 * the domain values for our changes. 4333 */ 4334 GEM_BUG_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU); 4335 4336 /* If we're writing through the CPU, then the GPU read domains will 4337 * need to be invalidated at next use. 4338 */ 4339 if (write) 4340 __start_cpu_write(obj); 4341 4342 return 0; 4343 } 4344 4345 /* Throttle our rendering by waiting until the ring has completed our requests 4346 * emitted over 20 msec ago. 4347 * 4348 * Note that if we were to use the current jiffies each time around the loop, 4349 * we wouldn't escape the function with any frames outstanding if the time to 4350 * render a frame was over 20ms. 4351 * 4352 * This should get us reasonable parallelism between CPU and GPU but also 4353 * relatively low latency when blocking on a particular request to finish. 4354 */ 4355 static int 4356 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 4357 { 4358 struct drm_i915_private *dev_priv = to_i915(dev); 4359 struct drm_i915_file_private *file_priv = file->driver_priv; 4360 unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES; 4361 struct drm_i915_gem_request *request, *target = NULL; 4362 long ret; 4363 4364 /* ABI: return -EIO if already wedged */ 4365 if (i915_terminally_wedged(&dev_priv->gpu_error)) 4366 return -EIO; 4367 4368 lockmgr(&file_priv->mm.lock, LK_EXCLUSIVE); 4369 list_for_each_entry(request, &file_priv->mm.request_list, client_link) { 4370 if (time_after_eq(request->emitted_jiffies, recent_enough)) 4371 break; 4372 4373 if (target) { 4374 list_del(&target->client_link); 4375 target->file_priv = NULL; 4376 } 4377 4378 target = request; 4379 } 4380 if (target) 4381 i915_gem_request_get(target); 4382 lockmgr(&file_priv->mm.lock, LK_RELEASE); 4383 4384 if (target == NULL) 4385 return 0; 4386 4387 ret = i915_wait_request(target, 4388 I915_WAIT_INTERRUPTIBLE, 4389 MAX_SCHEDULE_TIMEOUT); 4390 i915_gem_request_put(target); 4391 4392 return ret < 0 ? ret : 0; 4393 } 4394 4395 struct i915_vma * 4396 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, 4397 const struct i915_ggtt_view *view, 4398 u64 size, 4399 u64 alignment, 4400 u64 flags) 4401 { 4402 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 4403 struct i915_address_space *vm = &dev_priv->ggtt.base; 4404 struct i915_vma *vma; 4405 int ret; 4406 4407 lockdep_assert_held(&obj->base.dev->struct_mutex); 4408 4409 if (!view && flags & PIN_MAPPABLE) { 4410 /* If the required space is larger than the available 4411 * aperture, we will not able to find a slot for the 4412 * object and unbinding the object now will be in 4413 * vain. Worse, doing so may cause us to ping-pong 4414 * the object in and out of the Global GTT and 4415 * waste a lot of cycles under the mutex. 4416 */ 4417 if (obj->base.size > dev_priv->ggtt.mappable_end) 4418 return ERR_PTR(-E2BIG); 4419 4420 /* If NONBLOCK is set the caller is optimistically 4421 * trying to cache the full object within the mappable 4422 * aperture, and *must* have a fallback in place for 4423 * situations where we cannot bind the object. We 4424 * can be a little more lax here and use the fallback 4425 * more often to avoid costly migrations of ourselves 4426 * and other objects within the aperture. 4427 * 4428 * Half-the-aperture is used as a simple heuristic. 4429 * More interesting would to do search for a free 4430 * block prior to making the commitment to unbind. 4431 * That caters for the self-harm case, and with a 4432 * little more heuristics (e.g. NOFAULT, NOEVICT) 4433 * we could try to minimise harm to others. 4434 */ 4435 if (flags & PIN_NONBLOCK && 4436 obj->base.size > dev_priv->ggtt.mappable_end / 2) 4437 return ERR_PTR(-ENOSPC); 4438 } 4439 4440 vma = i915_vma_instance(obj, vm, view); 4441 if (unlikely(IS_ERR(vma))) 4442 return vma; 4443 4444 if (i915_vma_misplaced(vma, size, alignment, flags)) { 4445 if (flags & PIN_NONBLOCK) { 4446 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) 4447 return ERR_PTR(-ENOSPC); 4448 4449 if (flags & PIN_MAPPABLE && 4450 vma->fence_size > dev_priv->ggtt.mappable_end / 2) 4451 return ERR_PTR(-ENOSPC); 4452 } 4453 4454 WARN(i915_vma_is_pinned(vma), 4455 "bo is already pinned in ggtt with incorrect alignment:" 4456 " offset=%08x, req.alignment=%llx," 4457 " req.map_and_fenceable=%d, vma->map_and_fenceable=%d\n", 4458 i915_ggtt_offset(vma), alignment, 4459 !!(flags & PIN_MAPPABLE), 4460 i915_vma_is_map_and_fenceable(vma)); 4461 ret = i915_vma_unbind(vma); 4462 if (ret) 4463 return ERR_PTR(ret); 4464 } 4465 4466 ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL); 4467 if (ret) 4468 return ERR_PTR(ret); 4469 4470 return vma; 4471 } 4472 4473 static __always_inline unsigned int __busy_read_flag(unsigned int id) 4474 { 4475 /* Note that we could alias engines in the execbuf API, but 4476 * that would be very unwise as it prevents userspace from 4477 * fine control over engine selection. Ahem. 4478 * 4479 * This should be something like EXEC_MAX_ENGINE instead of 4480 * I915_NUM_ENGINES. 4481 */ 4482 BUILD_BUG_ON(I915_NUM_ENGINES > 16); 4483 return 0x10000 << id; 4484 } 4485 4486 static __always_inline unsigned int __busy_write_id(unsigned int id) 4487 { 4488 /* The uABI guarantees an active writer is also amongst the read 4489 * engines. This would be true if we accessed the activity tracking 4490 * under the lock, but as we perform the lookup of the object and 4491 * its activity locklessly we can not guarantee that the last_write 4492 * being active implies that we have set the same engine flag from 4493 * last_read - hence we always set both read and write busy for 4494 * last_write. 4495 */ 4496 return id | __busy_read_flag(id); 4497 } 4498 4499 #pragma GCC diagnostic push 4500 #pragma GCC diagnostic ignored "-Wdiscarded-qualifiers" 4501 4502 static __always_inline unsigned int 4503 __busy_set_if_active(const struct dma_fence *fence, 4504 unsigned int (*flag)(unsigned int id)) 4505 { 4506 struct drm_i915_gem_request *rq; 4507 4508 /* We have to check the current hw status of the fence as the uABI 4509 * guarantees forward progress. We could rely on the idle worker 4510 * to eventually flush us, but to minimise latency just ask the 4511 * hardware. 4512 * 4513 * Note we only report on the status of native fences. 4514 */ 4515 if (!dma_fence_is_i915(fence)) 4516 return 0; 4517 4518 /* opencode to_request() in order to avoid const warnings */ 4519 rq = container_of(fence, struct drm_i915_gem_request, fence); 4520 if (i915_gem_request_completed(rq)) 4521 return 0; 4522 4523 return flag(rq->engine->uabi_id); 4524 } 4525 #pragma GCC diagnostic pop 4526 4527 static __always_inline unsigned int 4528 busy_check_reader(const struct dma_fence *fence) 4529 { 4530 return __busy_set_if_active(fence, __busy_read_flag); 4531 } 4532 4533 static __always_inline unsigned int 4534 busy_check_writer(const struct dma_fence *fence) 4535 { 4536 if (!fence) 4537 return 0; 4538 4539 return __busy_set_if_active(fence, __busy_write_id); 4540 } 4541 4542 int 4543 i915_gem_busy_ioctl(struct drm_device *dev, void *data, 4544 struct drm_file *file) 4545 { 4546 struct drm_i915_gem_busy *args = data; 4547 struct drm_i915_gem_object *obj; 4548 struct reservation_object_list *list; 4549 unsigned int seq; 4550 int err; 4551 4552 err = -ENOENT; 4553 rcu_read_lock(); 4554 obj = i915_gem_object_lookup_rcu(file, args->handle); 4555 if (!obj) 4556 goto out; 4557 4558 /* A discrepancy here is that we do not report the status of 4559 * non-i915 fences, i.e. even though we may report the object as idle, 4560 * a call to set-domain may still stall waiting for foreign rendering. 4561 * This also means that wait-ioctl may report an object as busy, 4562 * where busy-ioctl considers it idle. 4563 * 4564 * We trade the ability to warn of foreign fences to report on which 4565 * i915 engines are active for the object. 4566 * 4567 * Alternatively, we can trade that extra information on read/write 4568 * activity with 4569 * args->busy = 4570 * !reservation_object_test_signaled_rcu(obj->resv, true); 4571 * to report the overall busyness. This is what the wait-ioctl does. 4572 * 4573 */ 4574 retry: 4575 seq = raw_read_seqcount(&obj->resv->seq); 4576 4577 /* Translate the exclusive fence to the READ *and* WRITE engine */ 4578 args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl)); 4579 4580 /* Translate shared fences to READ set of engines */ 4581 list = rcu_dereference(obj->resv->fence); 4582 if (list) { 4583 unsigned int shared_count = list->shared_count, i; 4584 4585 for (i = 0; i < shared_count; ++i) { 4586 struct dma_fence *fence = 4587 rcu_dereference(list->shared[i]); 4588 4589 args->busy |= busy_check_reader(fence); 4590 } 4591 } 4592 4593 if (args->busy && read_seqcount_retry(&obj->resv->seq, seq)) 4594 goto retry; 4595 4596 err = 0; 4597 out: 4598 rcu_read_unlock(); 4599 return err; 4600 } 4601 4602 int 4603 i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 4604 struct drm_file *file_priv) 4605 { 4606 return i915_gem_ring_throttle(dev, file_priv); 4607 } 4608 4609 int 4610 i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 4611 struct drm_file *file_priv) 4612 { 4613 struct drm_i915_private *dev_priv = to_i915(dev); 4614 struct drm_i915_gem_madvise *args = data; 4615 struct drm_i915_gem_object *obj; 4616 int err; 4617 4618 switch (args->madv) { 4619 case I915_MADV_DONTNEED: 4620 case I915_MADV_WILLNEED: 4621 break; 4622 default: 4623 return -EINVAL; 4624 } 4625 4626 obj = i915_gem_object_lookup(file_priv, args->handle); 4627 if (!obj) 4628 return -ENOENT; 4629 4630 err = mutex_lock_interruptible(&obj->mm.lock); 4631 if (err) 4632 goto out; 4633 4634 if (i915_gem_object_has_pages(obj) && 4635 i915_gem_object_is_tiled(obj) && 4636 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) { 4637 if (obj->mm.madv == I915_MADV_WILLNEED) { 4638 GEM_BUG_ON(!obj->mm.quirked); 4639 __i915_gem_object_unpin_pages(obj); 4640 obj->mm.quirked = false; 4641 } 4642 if (args->madv == I915_MADV_WILLNEED) { 4643 GEM_BUG_ON(obj->mm.quirked); 4644 __i915_gem_object_pin_pages(obj); 4645 obj->mm.quirked = true; 4646 } 4647 } 4648 4649 if (obj->mm.madv != __I915_MADV_PURGED) 4650 obj->mm.madv = args->madv; 4651 4652 /* if the object is no longer attached, discard its backing storage */ 4653 if (obj->mm.madv == I915_MADV_DONTNEED && 4654 !i915_gem_object_has_pages(obj)) 4655 i915_gem_object_truncate(obj); 4656 4657 args->retained = obj->mm.madv != __I915_MADV_PURGED; 4658 mutex_unlock(&obj->mm.lock); 4659 4660 out: 4661 i915_gem_object_put(obj); 4662 return err; 4663 } 4664 4665 static void 4666 frontbuffer_retire(struct i915_gem_active *active, 4667 struct drm_i915_gem_request *request) 4668 { 4669 struct drm_i915_gem_object *obj = 4670 container_of(active, typeof(*obj), frontbuffer_write); 4671 4672 intel_fb_obj_flush(obj, ORIGIN_CS); 4673 } 4674 4675 void i915_gem_object_init(struct drm_i915_gem_object *obj, 4676 const struct drm_i915_gem_object_ops *ops) 4677 { 4678 lockinit(&obj->mm.lock, "i9goml", 0, LK_CANRECURSE); 4679 4680 INIT_LIST_HEAD(&obj->vma_list); 4681 INIT_LIST_HEAD(&obj->lut_list); 4682 INIT_LIST_HEAD(&obj->batch_pool_link); 4683 4684 obj->ops = ops; 4685 4686 reservation_object_init(&obj->__builtin_resv); 4687 obj->resv = &obj->__builtin_resv; 4688 4689 obj->frontbuffer_ggtt_origin = ORIGIN_GTT; 4690 init_request_active(&obj->frontbuffer_write, frontbuffer_retire); 4691 4692 obj->mm.madv = I915_MADV_WILLNEED; 4693 INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN); 4694 lockinit(&obj->mm.get_page.lock, "i915ogpl", 0, LK_CANRECURSE); 4695 4696 i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size); 4697 } 4698 4699 static const struct drm_i915_gem_object_ops i915_gem_object_ops = { 4700 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | 4701 I915_GEM_OBJECT_IS_SHRINKABLE, 4702 4703 .get_pages = i915_gem_object_get_pages_gtt, 4704 .put_pages = i915_gem_object_put_pages_gtt, 4705 4706 .pwrite = i915_gem_object_pwrite_gtt, 4707 }; 4708 4709 static int i915_gem_object_create_shmem(struct drm_device *dev, 4710 struct drm_gem_object *obj, 4711 size_t size) 4712 { 4713 #ifndef __DragonFly__ 4714 struct drm_i915_private *i915 = to_i915(dev); 4715 unsigned long flags = VM_NORESERVE; 4716 struct file *filp; 4717 #endif 4718 4719 #ifndef __DragonFly__ 4720 drm_gem_private_object_init(dev, obj, size); 4721 #else 4722 drm_gem_object_init(dev, obj, size); 4723 #endif 4724 4725 #ifndef __DragonFly__ 4726 if (i915->mm.gemfs) 4727 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size, 4728 flags); 4729 else 4730 filp = shmem_file_setup("i915", size, flags); 4731 4732 if (IS_ERR(filp)) 4733 return PTR_ERR(filp); 4734 4735 obj->filp = filp; 4736 #endif 4737 4738 return 0; 4739 } 4740 4741 struct drm_i915_gem_object * 4742 i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size) 4743 { 4744 struct drm_i915_gem_object *obj; 4745 #if 0 4746 struct address_space *mapping; 4747 #endif 4748 unsigned int cache_level; 4749 gfp_t mask; 4750 int ret; 4751 4752 /* There is a prevalence of the assumption that we fit the object's 4753 * page count inside a 32bit _signed_ variable. Let's document this and 4754 * catch if we ever need to fix it. In the meantime, if you do spot 4755 * such a local variable, please consider fixing! 4756 */ 4757 if (size >> PAGE_SHIFT > INT_MAX) 4758 return ERR_PTR(-E2BIG); 4759 4760 if (overflows_type(size, obj->base.size)) 4761 return ERR_PTR(-E2BIG); 4762 4763 obj = i915_gem_object_alloc(dev_priv); 4764 if (obj == NULL) 4765 return ERR_PTR(-ENOMEM); 4766 4767 ret = i915_gem_object_create_shmem(&dev_priv->drm, &obj->base, size); 4768 if (ret) 4769 goto fail; 4770 4771 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 4772 if (IS_I965GM(dev_priv) || IS_I965G(dev_priv)) { 4773 /* 965gm cannot relocate objects above 4GiB. */ 4774 mask &= ~__GFP_HIGHMEM; 4775 mask |= __GFP_DMA32; 4776 } 4777 4778 #if 0 4779 mapping = obj->base.filp->f_mapping; 4780 mapping_set_gfp_mask(mapping, mask); 4781 #endif 4782 4783 i915_gem_object_init(obj, &i915_gem_object_ops); 4784 4785 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 4786 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 4787 4788 if (HAS_LLC(dev_priv)) 4789 /* On some devices, we can have the GPU use the LLC (the CPU 4790 * cache) for about a 10% performance improvement 4791 * compared to uncached. Graphics requests other than 4792 * display scanout are coherent with the CPU in 4793 * accessing this cache. This means in this mode we 4794 * don't need to clflush on the CPU side, and on the 4795 * GPU side we only need to flush internal caches to 4796 * get data visible to the CPU. 4797 * 4798 * However, we maintain the display planes as UC, and so 4799 * need to rebind when first used as such. 4800 */ 4801 cache_level = I915_CACHE_LLC; 4802 else 4803 cache_level = I915_CACHE_NONE; 4804 4805 i915_gem_object_set_cache_coherency(obj, cache_level); 4806 4807 trace_i915_gem_object_create(obj); 4808 4809 return obj; 4810 4811 fail: 4812 i915_gem_object_free(obj); 4813 return ERR_PTR(ret); 4814 } 4815 4816 static bool discard_backing_storage(struct drm_i915_gem_object *obj) 4817 { 4818 /* If we are the last user of the backing storage (be it shmemfs 4819 * pages or stolen etc), we know that the pages are going to be 4820 * immediately released. In this case, we can then skip copying 4821 * back the contents from the GPU. 4822 */ 4823 4824 if (obj->mm.madv != I915_MADV_WILLNEED) 4825 return false; 4826 4827 if (obj->base.filp == NULL) 4828 return true; 4829 4830 /* At first glance, this looks racy, but then again so would be 4831 * userspace racing mmap against close. However, the first external 4832 * reference to the filp can only be obtained through the 4833 * i915_gem_mmap_ioctl() which safeguards us against the user 4834 * acquiring such a reference whilst we are in the middle of 4835 * freeing the object. 4836 */ 4837 #if 0 4838 return atomic_long_read(&obj->base.filp->f_count) == 1; 4839 #else 4840 return false; 4841 #endif 4842 } 4843 4844 static void __i915_gem_free_objects(struct drm_i915_private *i915, 4845 struct llist_node *freed) 4846 { 4847 struct drm_i915_gem_object *obj, *on; 4848 4849 intel_runtime_pm_get(i915); 4850 llist_for_each_entry_safe(obj, on, freed, freed) { 4851 struct i915_vma *vma, *vn; 4852 4853 trace_i915_gem_object_destroy(obj); 4854 4855 mutex_lock(&i915->drm.struct_mutex); 4856 4857 GEM_BUG_ON(i915_gem_object_is_active(obj)); 4858 list_for_each_entry_safe(vma, vn, 4859 &obj->vma_list, obj_link) { 4860 GEM_BUG_ON(i915_vma_is_active(vma)); 4861 vma->flags &= ~I915_VMA_PIN_MASK; 4862 i915_vma_close(vma); 4863 } 4864 GEM_BUG_ON(!list_empty(&obj->vma_list)); 4865 GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma_tree)); 4866 4867 /* This serializes freeing with the shrinker. Since the free 4868 * is delayed, first by RCU then by the workqueue, we want the 4869 * shrinker to be able to free pages of unreferenced objects, 4870 * or else we may oom whilst there are plenty of deferred 4871 * freed objects. 4872 */ 4873 if (i915_gem_object_has_pages(obj)) { 4874 lockmgr(&i915->mm.obj_lock, LK_EXCLUSIVE); 4875 list_del_init(&obj->mm.link); 4876 lockmgr(&i915->mm.obj_lock, LK_RELEASE); 4877 } 4878 4879 mutex_unlock(&i915->drm.struct_mutex); 4880 4881 GEM_BUG_ON(obj->bind_count); 4882 GEM_BUG_ON(obj->userfault_count); 4883 GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits)); 4884 GEM_BUG_ON(!list_empty(&obj->lut_list)); 4885 4886 if (obj->ops->release) 4887 obj->ops->release(obj); 4888 4889 #if 0 4890 if (WARN_ON(i915_gem_object_has_pinned_pages(obj))) 4891 #else 4892 if (i915_gem_object_has_pinned_pages(obj)) 4893 #endif 4894 atomic_set(&obj->mm.pages_pin_count, 0); 4895 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 4896 GEM_BUG_ON(i915_gem_object_has_pages(obj)); 4897 4898 if (obj->base.import_attach) 4899 drm_prime_gem_destroy(&obj->base, NULL); 4900 4901 reservation_object_fini(&obj->__builtin_resv); 4902 drm_gem_object_release(&obj->base); 4903 i915_gem_info_remove_obj(i915, obj->base.size); 4904 4905 kfree(obj->bit_17); 4906 i915_gem_object_free(obj); 4907 4908 if (on) 4909 cond_resched(); 4910 } 4911 intel_runtime_pm_put(i915); 4912 } 4913 4914 static void i915_gem_flush_free_objects(struct drm_i915_private *i915) 4915 { 4916 struct llist_node *freed; 4917 4918 /* Free the oldest, most stale object to keep the free_list short */ 4919 freed = NULL; 4920 if (!llist_empty(&i915->mm.free_list)) { /* quick test for hotpath */ 4921 /* Only one consumer of llist_del_first() allowed */ 4922 lockmgr(&i915->mm.free_lock, LK_EXCLUSIVE); 4923 freed = llist_del_first(&i915->mm.free_list); 4924 lockmgr(&i915->mm.free_lock, LK_RELEASE); 4925 } 4926 if (unlikely(freed)) { 4927 freed->next = NULL; 4928 __i915_gem_free_objects(i915, freed); 4929 } 4930 } 4931 4932 static void __i915_gem_free_work(struct work_struct *work) 4933 { 4934 struct drm_i915_private *i915 = 4935 container_of(work, struct drm_i915_private, mm.free_work); 4936 struct llist_node *freed; 4937 4938 /* All file-owned VMA should have been released by this point through 4939 * i915_gem_close_object(), or earlier by i915_gem_context_close(). 4940 * However, the object may also be bound into the global GTT (e.g. 4941 * older GPUs without per-process support, or for direct access through 4942 * the GTT either for the user or for scanout). Those VMA still need to 4943 * unbound now. 4944 */ 4945 4946 lockmgr(&i915->mm.free_lock, LK_EXCLUSIVE); 4947 while ((freed = llist_del_all(&i915->mm.free_list))) { 4948 lockmgr(&i915->mm.free_lock, LK_RELEASE); 4949 4950 __i915_gem_free_objects(i915, freed); 4951 if (need_resched()) 4952 return; 4953 4954 lockmgr(&i915->mm.free_lock, LK_EXCLUSIVE); 4955 } 4956 lockmgr(&i915->mm.free_lock, LK_RELEASE); 4957 } 4958 4959 static void __i915_gem_free_object_rcu(struct rcu_head *head) 4960 { 4961 struct drm_i915_gem_object *obj = 4962 container_of(head, typeof(*obj), rcu); 4963 struct drm_i915_private *i915 = to_i915(obj->base.dev); 4964 4965 /* We can't simply use call_rcu() from i915_gem_free_object() 4966 * as we need to block whilst unbinding, and the call_rcu 4967 * task may be called from softirq context. So we take a 4968 * detour through a worker. 4969 */ 4970 if (llist_add(&obj->freed, &i915->mm.free_list)) 4971 schedule_work(&i915->mm.free_work); 4972 } 4973 4974 void i915_gem_free_object(struct drm_gem_object *gem_obj) 4975 { 4976 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 4977 4978 if (obj->mm.quirked) 4979 __i915_gem_object_unpin_pages(obj); 4980 4981 if (discard_backing_storage(obj)) 4982 obj->mm.madv = I915_MADV_DONTNEED; 4983 4984 /* Before we free the object, make sure any pure RCU-only 4985 * read-side critical sections are complete, e.g. 4986 * i915_gem_busy_ioctl(). For the corresponding synchronized 4987 * lookup see i915_gem_object_lookup_rcu(). 4988 */ 4989 call_rcu(&obj->rcu, __i915_gem_free_object_rcu); 4990 } 4991 4992 void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj) 4993 { 4994 lockdep_assert_held(&obj->base.dev->struct_mutex); 4995 4996 if (!i915_gem_object_has_active_reference(obj) && 4997 i915_gem_object_is_active(obj)) 4998 i915_gem_object_set_active_reference(obj); 4999 else 5000 i915_gem_object_put(obj); 5001 } 5002 5003 static void assert_kernel_context_is_current(struct drm_i915_private *dev_priv) 5004 { 5005 struct intel_engine_cs *engine; 5006 enum intel_engine_id id; 5007 5008 for_each_engine(engine, dev_priv, id) 5009 GEM_BUG_ON(engine->last_retired_context && 5010 !i915_gem_context_is_kernel(engine->last_retired_context)); 5011 } 5012 5013 void i915_gem_sanitize(struct drm_i915_private *i915) 5014 { 5015 if (i915_terminally_wedged(&i915->gpu_error)) { 5016 mutex_lock(&i915->drm.struct_mutex); 5017 i915_gem_unset_wedged(i915); 5018 mutex_unlock(&i915->drm.struct_mutex); 5019 } 5020 5021 /* 5022 * If we inherit context state from the BIOS or earlier occupants 5023 * of the GPU, the GPU may be in an inconsistent state when we 5024 * try to take over. The only way to remove the earlier state 5025 * is by resetting. However, resetting on earlier gen is tricky as 5026 * it may impact the display and we are uncertain about the stability 5027 * of the reset, so this could be applied to even earlier gen. 5028 */ 5029 if (INTEL_GEN(i915) >= 5) { 5030 int reset = intel_gpu_reset(i915, ALL_ENGINES); 5031 WARN_ON(reset && reset != -ENODEV); 5032 } 5033 } 5034 5035 int i915_gem_suspend(struct drm_i915_private *dev_priv) 5036 { 5037 struct drm_device *dev = &dev_priv->drm; 5038 int ret; 5039 5040 intel_runtime_pm_get(dev_priv); 5041 intel_suspend_gt_powersave(dev_priv); 5042 5043 mutex_lock(&dev->struct_mutex); 5044 5045 /* We have to flush all the executing contexts to main memory so 5046 * that they can saved in the hibernation image. To ensure the last 5047 * context image is coherent, we have to switch away from it. That 5048 * leaves the dev_priv->kernel_context still active when 5049 * we actually suspend, and its image in memory may not match the GPU 5050 * state. Fortunately, the kernel_context is disposable and we do 5051 * not rely on its state. 5052 */ 5053 if (!i915_terminally_wedged(&dev_priv->gpu_error)) { 5054 ret = i915_gem_switch_to_kernel_context(dev_priv); 5055 if (ret) 5056 goto err_unlock; 5057 5058 ret = i915_gem_wait_for_idle(dev_priv, 5059 I915_WAIT_INTERRUPTIBLE | 5060 I915_WAIT_LOCKED); 5061 if (ret && ret != -EIO) 5062 goto err_unlock; 5063 5064 assert_kernel_context_is_current(dev_priv); 5065 } 5066 i915_gem_contexts_lost(dev_priv); 5067 mutex_unlock(&dev->struct_mutex); 5068 5069 intel_guc_suspend(dev_priv); 5070 5071 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); 5072 cancel_delayed_work_sync(&dev_priv->gt.retire_work); 5073 5074 /* As the idle_work is rearming if it detects a race, play safe and 5075 * repeat the flush until it is definitely idle. 5076 */ 5077 drain_delayed_work(&dev_priv->gt.idle_work); 5078 5079 /* Assert that we sucessfully flushed all the work and 5080 * reset the GPU back to its idle, low power state. 5081 */ 5082 WARN_ON(dev_priv->gt.awake); 5083 if (WARN_ON(!intel_engines_are_idle(dev_priv))) 5084 i915_gem_set_wedged(dev_priv); /* no hope, discard everything */ 5085 5086 /* 5087 * Neither the BIOS, ourselves or any other kernel 5088 * expects the system to be in execlists mode on startup, 5089 * so we need to reset the GPU back to legacy mode. And the only 5090 * known way to disable logical contexts is through a GPU reset. 5091 * 5092 * So in order to leave the system in a known default configuration, 5093 * always reset the GPU upon unload and suspend. Afterwards we then 5094 * clean up the GEM state tracking, flushing off the requests and 5095 * leaving the system in a known idle state. 5096 * 5097 * Note that is of the upmost importance that the GPU is idle and 5098 * all stray writes are flushed *before* we dismantle the backing 5099 * storage for the pinned objects. 5100 * 5101 * However, since we are uncertain that resetting the GPU on older 5102 * machines is a good idea, we don't - just in case it leaves the 5103 * machine in an unusable condition. 5104 */ 5105 i915_gem_sanitize(dev_priv); 5106 5107 intel_runtime_pm_put(dev_priv); 5108 return 0; 5109 5110 err_unlock: 5111 mutex_unlock(&dev->struct_mutex); 5112 intel_runtime_pm_put(dev_priv); 5113 return ret; 5114 } 5115 5116 void i915_gem_resume(struct drm_i915_private *dev_priv) 5117 { 5118 struct drm_device *dev = &dev_priv->drm; 5119 5120 WARN_ON(dev_priv->gt.awake); 5121 5122 mutex_lock(&dev->struct_mutex); 5123 i915_gem_restore_gtt_mappings(dev_priv); 5124 i915_gem_restore_fences(dev_priv); 5125 5126 /* As we didn't flush the kernel context before suspend, we cannot 5127 * guarantee that the context image is complete. So let's just reset 5128 * it and start again. 5129 */ 5130 dev_priv->gt.resume(dev_priv); 5131 5132 mutex_unlock(&dev->struct_mutex); 5133 } 5134 5135 void i915_gem_init_swizzling(struct drm_i915_private *dev_priv) 5136 { 5137 if (INTEL_GEN(dev_priv) < 5 || 5138 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 5139 return; 5140 5141 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 5142 DISP_TILE_SURFACE_SWIZZLING); 5143 5144 if (IS_GEN5(dev_priv)) 5145 return; 5146 5147 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 5148 if (IS_GEN6(dev_priv)) 5149 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); 5150 else if (IS_GEN7(dev_priv)) 5151 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); 5152 else if (IS_GEN8(dev_priv)) 5153 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW)); 5154 else 5155 BUG(); 5156 } 5157 5158 static void init_unused_ring(struct drm_i915_private *dev_priv, u32 base) 5159 { 5160 I915_WRITE(RING_CTL(base), 0); 5161 I915_WRITE(RING_HEAD(base), 0); 5162 I915_WRITE(RING_TAIL(base), 0); 5163 I915_WRITE(RING_START(base), 0); 5164 } 5165 5166 static void init_unused_rings(struct drm_i915_private *dev_priv) 5167 { 5168 if (IS_I830(dev_priv)) { 5169 init_unused_ring(dev_priv, PRB1_BASE); 5170 init_unused_ring(dev_priv, SRB0_BASE); 5171 init_unused_ring(dev_priv, SRB1_BASE); 5172 init_unused_ring(dev_priv, SRB2_BASE); 5173 init_unused_ring(dev_priv, SRB3_BASE); 5174 } else if (IS_GEN2(dev_priv)) { 5175 init_unused_ring(dev_priv, SRB0_BASE); 5176 init_unused_ring(dev_priv, SRB1_BASE); 5177 } else if (IS_GEN3(dev_priv)) { 5178 init_unused_ring(dev_priv, PRB1_BASE); 5179 init_unused_ring(dev_priv, PRB2_BASE); 5180 } 5181 } 5182 5183 static int __i915_gem_restart_engines(void *data) 5184 { 5185 struct drm_i915_private *i915 = data; 5186 struct intel_engine_cs *engine; 5187 enum intel_engine_id id; 5188 int err; 5189 5190 for_each_engine(engine, i915, id) { 5191 err = engine->init_hw(engine); 5192 if (err) 5193 return err; 5194 } 5195 5196 return 0; 5197 } 5198 5199 int i915_gem_init_hw(struct drm_i915_private *dev_priv) 5200 { 5201 int ret; 5202 5203 dev_priv->gt.last_init_time = ktime_get(); 5204 5205 /* Double layer security blanket, see i915_gem_init() */ 5206 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 5207 5208 if (HAS_EDRAM(dev_priv) && INTEL_GEN(dev_priv) < 9) 5209 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf)); 5210 5211 if (IS_HASWELL(dev_priv)) 5212 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev_priv) ? 5213 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED); 5214 5215 if (HAS_PCH_NOP(dev_priv)) { 5216 if (IS_IVYBRIDGE(dev_priv)) { 5217 u32 temp = I915_READ(GEN7_MSG_CTL); 5218 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK); 5219 I915_WRITE(GEN7_MSG_CTL, temp); 5220 } else if (INTEL_GEN(dev_priv) >= 7) { 5221 u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT); 5222 temp &= ~RESET_PCH_HANDSHAKE_ENABLE; 5223 I915_WRITE(HSW_NDE_RSTWRN_OPT, temp); 5224 } 5225 } 5226 5227 i915_gem_init_swizzling(dev_priv); 5228 5229 /* 5230 * At least 830 can leave some of the unused rings 5231 * "active" (ie. head != tail) after resume which 5232 * will prevent c3 entry. Makes sure all unused rings 5233 * are totally idle. 5234 */ 5235 init_unused_rings(dev_priv); 5236 5237 BUG_ON(!dev_priv->kernel_context); 5238 if (i915_terminally_wedged(&dev_priv->gpu_error)) { 5239 ret = -EIO; 5240 goto out; 5241 } 5242 5243 ret = i915_ppgtt_init_hw(dev_priv); 5244 if (ret) { 5245 DRM_ERROR("PPGTT enable HW failed %d\n", ret); 5246 goto out; 5247 } 5248 5249 /* Need to do basic initialisation of all rings first: */ 5250 ret = __i915_gem_restart_engines(dev_priv); 5251 if (ret) 5252 goto out; 5253 5254 intel_mocs_init_l3cc_table(dev_priv); 5255 5256 /* We can't enable contexts until all firmware is loaded */ 5257 ret = intel_uc_init_hw(dev_priv); 5258 if (ret) 5259 goto out; 5260 5261 out: 5262 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5263 return ret; 5264 } 5265 5266 bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value) 5267 { 5268 if (INTEL_INFO(dev_priv)->gen < 6) 5269 return false; 5270 5271 /* TODO: make semaphores and Execlists play nicely together */ 5272 if (i915_modparams.enable_execlists) 5273 return false; 5274 5275 if (value >= 0) 5276 return value; 5277 5278 /* Enable semaphores on SNB when IO remapping is off */ 5279 if (IS_GEN6(dev_priv) && intel_vtd_active()) 5280 return false; 5281 5282 return true; 5283 } 5284 5285 int i915_gem_init(struct drm_i915_private *dev_priv) 5286 { 5287 int ret; 5288 5289 /* 5290 * We need to fallback to 4K pages since gvt gtt handling doesn't 5291 * support huge page entries - we will need to check either hypervisor 5292 * mm can support huge guest page or just do emulation in gvt. 5293 */ 5294 if (intel_vgpu_active(dev_priv)) 5295 mkwrite_device_info(dev_priv)->page_sizes = 5296 I915_GTT_PAGE_SIZE_4K; 5297 5298 dev_priv->mm.unordered_timeline = dma_fence_context_alloc(1); 5299 5300 if (!i915_modparams.enable_execlists) { 5301 dev_priv->gt.resume = intel_legacy_submission_resume; 5302 dev_priv->gt.cleanup_engine = intel_engine_cleanup; 5303 } else { 5304 dev_priv->gt.resume = intel_lr_context_resume; 5305 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup; 5306 } 5307 5308 ret = i915_gem_init_userptr(dev_priv); 5309 if (ret) 5310 return ret; 5311 5312 /* This is just a security blanket to placate dragons. 5313 * On some systems, we very sporadically observe that the first TLBs 5314 * used by the CS may be stale, despite us poking the TLB reset. If 5315 * we hold the forcewake during initialisation these problems 5316 * just magically go away. 5317 */ 5318 mutex_lock(&dev_priv->drm.struct_mutex); 5319 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 5320 5321 ret = i915_gem_init_ggtt(dev_priv); 5322 if (ret) 5323 goto out_unlock; 5324 5325 ret = i915_gem_contexts_init(dev_priv); 5326 if (ret) 5327 goto out_unlock; 5328 5329 ret = intel_engines_init(dev_priv); 5330 if (ret) 5331 goto out_unlock; 5332 5333 ret = i915_gem_init_hw(dev_priv); 5334 if (ret == -EIO) { 5335 /* Allow engine initialisation to fail by marking the GPU as 5336 * wedged. But we only want to do this where the GPU is angry, 5337 * for all other failure, such as an allocation failure, bail. 5338 */ 5339 if (!i915_terminally_wedged(&dev_priv->gpu_error)) { 5340 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n"); 5341 i915_gem_set_wedged(dev_priv); 5342 } 5343 ret = 0; 5344 } 5345 5346 out_unlock: 5347 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5348 mutex_unlock(&dev_priv->drm.struct_mutex); 5349 5350 return ret; 5351 } 5352 5353 void i915_gem_init_mmio(struct drm_i915_private *i915) 5354 { 5355 i915_gem_sanitize(i915); 5356 } 5357 5358 void 5359 i915_gem_cleanup_engines(struct drm_i915_private *dev_priv) 5360 { 5361 struct intel_engine_cs *engine; 5362 enum intel_engine_id id; 5363 5364 for_each_engine(engine, dev_priv, id) 5365 dev_priv->gt.cleanup_engine(engine); 5366 } 5367 5368 void 5369 i915_gem_load_init_fences(struct drm_i915_private *dev_priv) 5370 { 5371 int i; 5372 5373 if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) && 5374 !IS_CHERRYVIEW(dev_priv)) 5375 dev_priv->num_fence_regs = 32; 5376 else if (INTEL_INFO(dev_priv)->gen >= 4 || 5377 IS_I945G(dev_priv) || IS_I945GM(dev_priv) || 5378 IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) 5379 dev_priv->num_fence_regs = 16; 5380 else 5381 dev_priv->num_fence_regs = 8; 5382 5383 if (intel_vgpu_active(dev_priv)) 5384 dev_priv->num_fence_regs = 5385 I915_READ(vgtif_reg(avail_rs.fence_num)); 5386 5387 /* Initialize fence registers to zero */ 5388 for (i = 0; i < dev_priv->num_fence_regs; i++) { 5389 struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i]; 5390 5391 fence->i915 = dev_priv; 5392 fence->id = i; 5393 list_add_tail(&fence->link, &dev_priv->mm.fence_list); 5394 } 5395 i915_gem_restore_fences(dev_priv); 5396 5397 i915_gem_detect_bit_6_swizzle(dev_priv); 5398 } 5399 5400 int 5401 i915_gem_load_init(struct drm_i915_private *dev_priv) 5402 { 5403 int err = -ENOMEM; 5404 5405 dev_priv->objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN); 5406 if (!dev_priv->objects) 5407 goto err_out; 5408 5409 dev_priv->vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN); 5410 if (!dev_priv->vmas) 5411 goto err_objects; 5412 5413 dev_priv->luts = KMEM_CACHE(i915_lut_handle, 0); 5414 if (!dev_priv->luts) 5415 goto err_vmas; 5416 5417 dev_priv->requests = KMEM_CACHE(drm_i915_gem_request, 5418 SLAB_HWCACHE_ALIGN | 5419 SLAB_RECLAIM_ACCOUNT | 5420 SLAB_TYPESAFE_BY_RCU); 5421 if (!dev_priv->requests) 5422 goto err_luts; 5423 5424 dev_priv->dependencies = KMEM_CACHE(i915_dependency, 5425 SLAB_HWCACHE_ALIGN | 5426 SLAB_RECLAIM_ACCOUNT); 5427 if (!dev_priv->dependencies) 5428 goto err_requests; 5429 5430 dev_priv->priorities = KMEM_CACHE(i915_priolist, SLAB_HWCACHE_ALIGN); 5431 if (!dev_priv->priorities) 5432 goto err_dependencies; 5433 5434 mutex_lock(&dev_priv->drm.struct_mutex); 5435 INIT_LIST_HEAD(&dev_priv->gt.timelines); 5436 err = i915_gem_timeline_init__global(dev_priv); 5437 mutex_unlock(&dev_priv->drm.struct_mutex); 5438 if (err) 5439 goto err_priorities; 5440 5441 INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work); 5442 5443 lockinit(&dev_priv->mm.obj_lock, "i9dpmmo", 0, 0); 5444 lockinit(&dev_priv->mm.free_lock, "i9dpmmf", 0, 0); 5445 init_llist_head(&dev_priv->mm.free_list); 5446 INIT_LIST_HEAD(&dev_priv->mm.unbound_list); 5447 INIT_LIST_HEAD(&dev_priv->mm.bound_list); 5448 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 5449 INIT_LIST_HEAD(&dev_priv->mm.userfault_list); 5450 5451 INIT_DELAYED_WORK(&dev_priv->gt.retire_work, 5452 i915_gem_retire_work_handler); 5453 INIT_DELAYED_WORK(&dev_priv->gt.idle_work, 5454 i915_gem_idle_work_handler); 5455 init_waitqueue_head(&dev_priv->gpu_error.wait_queue); 5456 init_waitqueue_head(&dev_priv->gpu_error.reset_queue); 5457 5458 atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0); 5459 5460 lockinit(&dev_priv->fb_tracking.lock, "drmftl", 0, 0); 5461 5462 return 0; 5463 5464 err_priorities: 5465 kmem_cache_destroy(dev_priv->priorities); 5466 err_dependencies: 5467 kmem_cache_destroy(dev_priv->dependencies); 5468 err_requests: 5469 kmem_cache_destroy(dev_priv->requests); 5470 err_luts: 5471 kmem_cache_destroy(dev_priv->luts); 5472 err_vmas: 5473 kmem_cache_destroy(dev_priv->vmas); 5474 err_objects: 5475 kmem_cache_destroy(dev_priv->objects); 5476 err_out: 5477 return err; 5478 } 5479 5480 void i915_gem_load_cleanup(struct drm_i915_private *dev_priv) 5481 { 5482 i915_gem_drain_freed_objects(dev_priv); 5483 WARN_ON(!llist_empty(&dev_priv->mm.free_list)); 5484 WARN_ON(dev_priv->mm.object_count); 5485 5486 mutex_lock(&dev_priv->drm.struct_mutex); 5487 i915_gem_timeline_fini(&dev_priv->gt.global_timeline); 5488 WARN_ON(!list_empty(&dev_priv->gt.timelines)); 5489 mutex_unlock(&dev_priv->drm.struct_mutex); 5490 5491 kmem_cache_destroy(dev_priv->priorities); 5492 kmem_cache_destroy(dev_priv->dependencies); 5493 kmem_cache_destroy(dev_priv->requests); 5494 kmem_cache_destroy(dev_priv->luts); 5495 kmem_cache_destroy(dev_priv->vmas); 5496 kmem_cache_destroy(dev_priv->objects); 5497 5498 /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */ 5499 rcu_barrier(); 5500 5501 #if 0 5502 i915_gemfs_fini(dev_priv); 5503 #endif 5504 } 5505 5506 int i915_gem_freeze(struct drm_i915_private *dev_priv) 5507 { 5508 /* Discard all purgeable objects, let userspace recover those as 5509 * required after resuming. 5510 */ 5511 i915_gem_shrink_all(dev_priv); 5512 5513 return 0; 5514 } 5515 5516 int i915_gem_freeze_late(struct drm_i915_private *dev_priv) 5517 { 5518 struct drm_i915_gem_object *obj; 5519 struct list_head *phases[] = { 5520 &dev_priv->mm.unbound_list, 5521 &dev_priv->mm.bound_list, 5522 NULL 5523 }, **p; 5524 5525 /* Called just before we write the hibernation image. 5526 * 5527 * We need to update the domain tracking to reflect that the CPU 5528 * will be accessing all the pages to create and restore from the 5529 * hibernation, and so upon restoration those pages will be in the 5530 * CPU domain. 5531 * 5532 * To make sure the hibernation image contains the latest state, 5533 * we update that state just before writing out the image. 5534 * 5535 * To try and reduce the hibernation image, we manually shrink 5536 * the objects as well, see i915_gem_freeze() 5537 */ 5538 5539 i915_gem_shrink(dev_priv, -1UL, NULL, I915_SHRINK_UNBOUND); 5540 i915_gem_drain_freed_objects(dev_priv); 5541 5542 lockmgr(&dev_priv->mm.obj_lock, LK_EXCLUSIVE); 5543 for (p = phases; *p; p++) { 5544 list_for_each_entry(obj, *p, mm.link) 5545 __start_cpu_write(obj); 5546 } 5547 lockmgr(&dev_priv->mm.obj_lock, LK_RELEASE); 5548 5549 return 0; 5550 } 5551 5552 void i915_gem_release(struct drm_device *dev, struct drm_file *file) 5553 { 5554 struct drm_i915_file_private *file_priv = file->driver_priv; 5555 struct drm_i915_gem_request *request; 5556 5557 /* Clean up our request list when the client is going away, so that 5558 * later retire_requests won't dereference our soon-to-be-gone 5559 * file_priv. 5560 */ 5561 lockmgr(&file_priv->mm.lock, LK_EXCLUSIVE); 5562 list_for_each_entry(request, &file_priv->mm.request_list, client_link) 5563 request->file_priv = NULL; 5564 lockmgr(&file_priv->mm.lock, LK_RELEASE); 5565 } 5566 5567 #ifdef __DragonFly__ 5568 int 5569 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 5570 vm_ooffset_t foff, struct ucred *cred, u_short *color) 5571 { 5572 *color = 0; /* XXXKIB */ 5573 return (0); 5574 } 5575 5576 void 5577 i915_gem_pager_dtor(void *handle) 5578 { 5579 struct drm_gem_object *obj = handle; 5580 struct drm_device *dev = obj->dev; 5581 5582 drm_gem_free_mmap_offset(obj); 5583 mutex_lock(&dev->struct_mutex); 5584 i915_gem_release_mmap(to_intel_bo(obj)); 5585 drm_gem_object_unreference(obj); 5586 mutex_unlock(&dev->struct_mutex); 5587 } 5588 #endif 5589 5590 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file) 5591 { 5592 struct drm_i915_file_private *file_priv; 5593 int ret; 5594 5595 DRM_DEBUG("\n"); 5596 5597 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL); 5598 if (!file_priv) 5599 return -ENOMEM; 5600 5601 file->driver_priv = file_priv; 5602 file_priv->dev_priv = i915; 5603 file_priv->file = file; 5604 5605 lockinit(&file_priv->mm.lock, "i915_priv", 0, 0); 5606 INIT_LIST_HEAD(&file_priv->mm.request_list); 5607 5608 file_priv->bsd_engine = -1; 5609 5610 ret = i915_gem_context_open(i915, file); 5611 if (ret) 5612 kfree(file_priv); 5613 5614 return ret; 5615 } 5616 5617 /** 5618 * i915_gem_track_fb - update frontbuffer tracking 5619 * @old: current GEM buffer for the frontbuffer slots 5620 * @new: new GEM buffer for the frontbuffer slots 5621 * @frontbuffer_bits: bitmask of frontbuffer slots 5622 * 5623 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them 5624 * from @old and setting them in @new. Both @old and @new can be NULL. 5625 */ 5626 void i915_gem_track_fb(struct drm_i915_gem_object *old, 5627 struct drm_i915_gem_object *new, 5628 unsigned frontbuffer_bits) 5629 { 5630 /* Control of individual bits within the mask are guarded by 5631 * the owning plane->mutex, i.e. we can never see concurrent 5632 * manipulation of individual bits. But since the bitfield as a whole 5633 * is updated using RMW, we need to use atomics in order to update 5634 * the bits. 5635 */ 5636 BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES > 5637 sizeof(atomic_t) * BITS_PER_BYTE); 5638 5639 if (old) { 5640 WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits)); 5641 atomic_andnot(frontbuffer_bits, &old->frontbuffer_bits); 5642 } 5643 5644 if (new) { 5645 WARN_ON(atomic_read(&new->frontbuffer_bits) & frontbuffer_bits); 5646 atomic_or(frontbuffer_bits, &new->frontbuffer_bits); 5647 } 5648 } 5649 5650 /* Allocate a new GEM object and fill it with the supplied data */ 5651 struct drm_i915_gem_object * 5652 i915_gem_object_create_from_data(struct drm_i915_private *dev_priv, 5653 const void *data, size_t size) 5654 { 5655 struct drm_i915_gem_object *obj; 5656 struct vm_object *file; 5657 size_t offset; 5658 int err; 5659 5660 obj = i915_gem_object_create(dev_priv, round_up(size, PAGE_SIZE)); 5661 if (IS_ERR(obj)) 5662 return obj; 5663 5664 GEM_BUG_ON(obj->base.write_domain != I915_GEM_DOMAIN_CPU); 5665 5666 file = obj->base.filp; 5667 offset = 0; 5668 do { 5669 unsigned int len = min_t(typeof(size), size, PAGE_SIZE); 5670 struct page *page; 5671 void *pgdata, *vaddr; 5672 5673 err = pagecache_write_begin(file, NULL, 5674 offset, len, 0, 5675 &page, &pgdata); 5676 if (err < 0) 5677 goto fail; 5678 5679 vaddr = kmap(page); 5680 memcpy(vaddr, data, len); 5681 kunmap(page); 5682 5683 #ifndef __DragonFly__ 5684 err = pagecache_write_end(file, file->f_mapping, 5685 #else 5686 err = pagecache_write_end(file, NULL, 5687 #endif 5688 offset, len, len, 5689 page, pgdata); 5690 if (err < 0) 5691 goto fail; 5692 5693 size -= len; 5694 data += len; 5695 offset += len; 5696 } while (size); 5697 5698 return obj; 5699 5700 fail: 5701 i915_gem_object_put(obj); 5702 return ERR_PTR(err); 5703 } 5704 5705 struct scatterlist * 5706 i915_gem_object_get_sg(struct drm_i915_gem_object *obj, 5707 unsigned int n, 5708 unsigned int *offset) 5709 { 5710 struct i915_gem_object_page_iter *iter = &obj->mm.get_page; 5711 struct scatterlist *sg; 5712 unsigned int idx, count; 5713 5714 might_sleep(); 5715 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT); 5716 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 5717 5718 /* As we iterate forward through the sg, we record each entry in a 5719 * radixtree for quick repeated (backwards) lookups. If we have seen 5720 * this index previously, we will have an entry for it. 5721 * 5722 * Initial lookup is O(N), but this is amortized to O(1) for 5723 * sequential page access (where each new request is consecutive 5724 * to the previous one). Repeated lookups are O(lg(obj->base.size)), 5725 * i.e. O(1) with a large constant! 5726 */ 5727 if (n < READ_ONCE(iter->sg_idx)) 5728 goto lookup; 5729 5730 mutex_lock(&iter->lock); 5731 5732 /* We prefer to reuse the last sg so that repeated lookup of this 5733 * (or the subsequent) sg are fast - comparing against the last 5734 * sg is faster than going through the radixtree. 5735 */ 5736 5737 sg = iter->sg_pos; 5738 idx = iter->sg_idx; 5739 count = __sg_page_count(sg); 5740 5741 while (idx + count <= n) { 5742 unsigned long exception, i; 5743 int ret; 5744 5745 /* If we cannot allocate and insert this entry, or the 5746 * individual pages from this range, cancel updating the 5747 * sg_idx so that on this lookup we are forced to linearly 5748 * scan onwards, but on future lookups we will try the 5749 * insertion again (in which case we need to be careful of 5750 * the error return reporting that we have already inserted 5751 * this index). 5752 */ 5753 ret = radix_tree_insert(&iter->radix, idx, sg); 5754 if (ret && ret != -EEXIST) 5755 goto scan; 5756 5757 exception = 5758 RADIX_TREE_EXCEPTIONAL_ENTRY | 5759 idx << RADIX_TREE_EXCEPTIONAL_SHIFT; 5760 for (i = 1; i < count; i++) { 5761 ret = radix_tree_insert(&iter->radix, idx + i, 5762 (void *)exception); 5763 if (ret && ret != -EEXIST) 5764 goto scan; 5765 } 5766 5767 idx += count; 5768 sg = ____sg_next(sg); 5769 count = __sg_page_count(sg); 5770 } 5771 5772 scan: 5773 iter->sg_pos = sg; 5774 iter->sg_idx = idx; 5775 5776 mutex_unlock(&iter->lock); 5777 5778 if (unlikely(n < idx)) /* insertion completed by another thread */ 5779 goto lookup; 5780 5781 /* In case we failed to insert the entry into the radixtree, we need 5782 * to look beyond the current sg. 5783 */ 5784 while (idx + count <= n) { 5785 idx += count; 5786 sg = ____sg_next(sg); 5787 count = __sg_page_count(sg); 5788 } 5789 5790 *offset = n - idx; 5791 return sg; 5792 5793 lookup: 5794 rcu_read_lock(); 5795 5796 sg = radix_tree_lookup(&iter->radix, n); 5797 GEM_BUG_ON(!sg); 5798 5799 /* If this index is in the middle of multi-page sg entry, 5800 * the radixtree will contain an exceptional entry that points 5801 * to the start of that range. We will return the pointer to 5802 * the base page and the offset of this page within the 5803 * sg entry's range. 5804 */ 5805 *offset = 0; 5806 if (unlikely(radix_tree_exception(sg))) { 5807 unsigned long base = 5808 (unsigned long)sg >> RADIX_TREE_EXCEPTIONAL_SHIFT; 5809 5810 sg = radix_tree_lookup(&iter->radix, base); 5811 GEM_BUG_ON(!sg); 5812 5813 *offset = n - base; 5814 } 5815 5816 rcu_read_unlock(); 5817 5818 return sg; 5819 } 5820 5821 struct page * 5822 i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) 5823 { 5824 struct scatterlist *sg; 5825 unsigned int offset; 5826 5827 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); 5828 5829 sg = i915_gem_object_get_sg(obj, n, &offset); 5830 return nth_page(sg_page(sg), offset); 5831 } 5832 5833 /* Like i915_gem_object_get_page(), but mark the returned page dirty */ 5834 struct page * 5835 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, 5836 unsigned int n) 5837 { 5838 struct page *page; 5839 5840 page = i915_gem_object_get_page(obj, n); 5841 if (!obj->mm.dirty) 5842 set_page_dirty(page); 5843 5844 return page; 5845 } 5846 5847 dma_addr_t 5848 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, 5849 unsigned long n) 5850 { 5851 struct scatterlist *sg; 5852 unsigned int offset; 5853 5854 sg = i915_gem_object_get_sg(obj, n, &offset); 5855 return sg_dma_address(sg) + (offset << PAGE_SHIFT); 5856 } 5857 5858 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align) 5859 { 5860 struct sg_table *pages; 5861 int err; 5862 5863 if (align > obj->base.size) 5864 return -EINVAL; 5865 5866 if (obj->ops == &i915_gem_phys_ops) 5867 return 0; 5868 5869 if (obj->ops != &i915_gem_object_ops) 5870 return -EINVAL; 5871 5872 err = i915_gem_object_unbind(obj); 5873 if (err) 5874 return err; 5875 5876 mutex_lock(&obj->mm.lock); 5877 5878 if (obj->mm.madv != I915_MADV_WILLNEED) { 5879 err = -EFAULT; 5880 goto err_unlock; 5881 } 5882 5883 if (obj->mm.quirked) { 5884 err = -EFAULT; 5885 goto err_unlock; 5886 } 5887 5888 if (obj->mm.mapping) { 5889 err = -EBUSY; 5890 goto err_unlock; 5891 } 5892 5893 pages = fetch_and_zero(&obj->mm.pages); 5894 if (pages) { 5895 struct drm_i915_private *i915 = to_i915(obj->base.dev); 5896 5897 __i915_gem_object_reset_page_iter(obj); 5898 5899 lockmgr(&i915->mm.obj_lock, LK_EXCLUSIVE); 5900 list_del(&obj->mm.link); 5901 lockmgr(&i915->mm.obj_lock, LK_RELEASE); 5902 } 5903 5904 obj->ops = &i915_gem_phys_ops; 5905 5906 err = ____i915_gem_object_get_pages(obj); 5907 if (err) 5908 goto err_xfer; 5909 5910 /* Perma-pin (until release) the physical set of pages */ 5911 __i915_gem_object_pin_pages(obj); 5912 5913 if (!IS_ERR_OR_NULL(pages)) 5914 i915_gem_object_ops.put_pages(obj, pages); 5915 mutex_unlock(&obj->mm.lock); 5916 return 0; 5917 5918 err_xfer: 5919 obj->ops = &i915_gem_object_ops; 5920 obj->mm.pages = pages; 5921 err_unlock: 5922 mutex_unlock(&obj->mm.lock); 5923 return err; 5924 } 5925 5926 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 5927 #include "selftests/scatterlist.c" 5928 #include "selftests/mock_gem_device.c" 5929 #include "selftests/huge_gem_object.c" 5930 #include "selftests/huge_pages.c" 5931 #include "selftests/i915_gem_object.c" 5932 #include "selftests/i915_gem_coherency.c" 5933 #endif 5934