1 /* 2 * Copyright © 2008-2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 */ 27 28 #include <drm/drmP.h> 29 #include <drm/drm_vma_manager.h> 30 #include <drm/i915_drm.h> 31 #include "i915_drv.h" 32 #include "i915_vgpu.h" 33 #include "i915_trace.h" 34 #include "intel_drv.h" 35 #include "intel_mocs.h" 36 #include <linux/shmem_fs.h> 37 #include <linux/slab.h> 38 #include <linux/swap.h> 39 #include <linux/pci.h> 40 #include <linux/dma-buf.h> 41 42 #include <sys/mman.h> 43 #include <vm/vm_map.h> 44 #include <vm/vm_param.h> 45 46 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); 47 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); 48 static void 49 i915_gem_object_retire__write(struct drm_i915_gem_object *obj); 50 static void 51 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring); 52 53 static bool cpu_cache_is_coherent(struct drm_device *dev, 54 enum i915_cache_level level) 55 { 56 return HAS_LLC(dev) || level != I915_CACHE_NONE; 57 } 58 59 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) 60 { 61 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 62 return false; 63 64 if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) 65 return true; 66 67 return obj->pin_display; 68 } 69 70 static int 71 insert_mappable_node(struct drm_i915_private *i915, 72 struct drm_mm_node *node, u32 size) 73 { 74 memset(node, 0, sizeof(*node)); 75 return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node, 76 size, 0, 0, 0, 77 i915->ggtt.mappable_end, 78 DRM_MM_SEARCH_DEFAULT, 79 DRM_MM_CREATE_DEFAULT); 80 } 81 82 static void 83 remove_mappable_node(struct drm_mm_node *node) 84 { 85 drm_mm_remove_node(node); 86 } 87 88 /* some bookkeeping */ 89 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, 90 size_t size) 91 { 92 spin_lock(&dev_priv->mm.object_stat_lock); 93 dev_priv->mm.object_count++; 94 dev_priv->mm.object_memory += size; 95 spin_unlock(&dev_priv->mm.object_stat_lock); 96 } 97 98 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, 99 size_t size) 100 { 101 spin_lock(&dev_priv->mm.object_stat_lock); 102 dev_priv->mm.object_count--; 103 dev_priv->mm.object_memory -= size; 104 spin_unlock(&dev_priv->mm.object_stat_lock); 105 } 106 107 static int 108 i915_gem_wait_for_error(struct i915_gpu_error *error) 109 { 110 int ret; 111 112 if (!i915_reset_in_progress(error)) 113 return 0; 114 115 /* 116 * Only wait 10 seconds for the gpu reset to complete to avoid hanging 117 * userspace. If it takes that long something really bad is going on and 118 * we should simply try to bail out and fail as gracefully as possible. 119 */ 120 ret = wait_event_interruptible_timeout(error->reset_queue, 121 !i915_reset_in_progress(error), 122 10*HZ); 123 if (ret == 0) { 124 DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); 125 return -EIO; 126 } else if (ret < 0) { 127 return ret; 128 } else { 129 return 0; 130 } 131 } 132 133 int i915_mutex_lock_interruptible(struct drm_device *dev) 134 { 135 struct drm_i915_private *dev_priv = dev->dev_private; 136 int ret; 137 138 ret = i915_gem_wait_for_error(&dev_priv->gpu_error); 139 if (ret) 140 return ret; 141 142 ret = mutex_lock_interruptible(&dev->struct_mutex); 143 if (ret) 144 return ret; 145 146 WARN_ON(i915_verify_lists(dev)); 147 return 0; 148 } 149 150 int 151 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 152 struct drm_file *file) 153 { 154 struct drm_i915_private *dev_priv = to_i915(dev); 155 struct i915_ggtt *ggtt = &dev_priv->ggtt; 156 struct drm_i915_gem_get_aperture *args = data; 157 struct i915_vma *vma; 158 size_t pinned; 159 160 pinned = 0; 161 mutex_lock(&dev->struct_mutex); 162 list_for_each_entry(vma, &ggtt->base.active_list, vm_link) 163 if (vma->pin_count) 164 pinned += vma->node.size; 165 list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link) 166 if (vma->pin_count) 167 pinned += vma->node.size; 168 mutex_unlock(&dev->struct_mutex); 169 170 args->aper_size = ggtt->base.total; 171 args->aper_available_size = args->aper_size - pinned; 172 173 return 0; 174 } 175 176 #if 0 177 static int 178 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) 179 { 180 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; 181 char *vaddr = obj->phys_handle->vaddr; 182 struct sg_table *st; 183 struct scatterlist *sg; 184 int i; 185 186 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) 187 return -EINVAL; 188 189 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 190 struct page *page; 191 char *src; 192 193 page = shmem_read_mapping_page(mapping, i); 194 if (IS_ERR(page)) 195 return PTR_ERR(page); 196 197 src = kmap_atomic(page); 198 memcpy(vaddr, src, PAGE_SIZE); 199 drm_clflush_virt_range(vaddr, PAGE_SIZE); 200 kunmap_atomic(src); 201 202 put_page(page); 203 vaddr += PAGE_SIZE; 204 } 205 206 i915_gem_chipset_flush(to_i915(obj->base.dev)); 207 208 st = kmalloc(sizeof(*st), GFP_KERNEL); 209 if (st == NULL) 210 return -ENOMEM; 211 212 if (sg_alloc_table(st, 1, GFP_KERNEL)) { 213 kfree(st); 214 return -ENOMEM; 215 } 216 217 sg = st->sgl; 218 sg->offset = 0; 219 sg->length = obj->base.size; 220 221 sg_dma_address(sg) = obj->phys_handle->busaddr; 222 sg_dma_len(sg) = obj->base.size; 223 224 obj->pages = st; 225 return 0; 226 } 227 228 static void 229 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) 230 { 231 int ret; 232 233 BUG_ON(obj->madv == __I915_MADV_PURGED); 234 235 ret = i915_gem_object_set_to_cpu_domain(obj, true); 236 if (WARN_ON(ret)) { 237 /* In the event of a disaster, abandon all caches and 238 * hope for the best. 239 */ 240 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; 241 } 242 243 if (obj->madv == I915_MADV_DONTNEED) 244 obj->dirty = 0; 245 246 if (obj->dirty) { 247 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; 248 char *vaddr = obj->phys_handle->vaddr; 249 int i; 250 251 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 252 struct page *page; 253 char *dst; 254 255 page = shmem_read_mapping_page(mapping, i); 256 if (IS_ERR(page)) 257 continue; 258 259 dst = kmap_atomic(page); 260 drm_clflush_virt_range(vaddr, PAGE_SIZE); 261 memcpy(dst, vaddr, PAGE_SIZE); 262 kunmap_atomic(dst); 263 264 set_page_dirty(page); 265 if (obj->madv == I915_MADV_WILLNEED) 266 mark_page_accessed(page); 267 put_page(page); 268 vaddr += PAGE_SIZE; 269 } 270 obj->dirty = 0; 271 } 272 273 sg_free_table(obj->pages); 274 kfree(obj->pages); 275 } 276 277 static void 278 i915_gem_object_release_phys(struct drm_i915_gem_object *obj) 279 { 280 drm_pci_free(obj->base.dev, obj->phys_handle); 281 } 282 283 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = { 284 .get_pages = i915_gem_object_get_pages_phys, 285 .put_pages = i915_gem_object_put_pages_phys, 286 .release = i915_gem_object_release_phys, 287 }; 288 #endif 289 290 static int 291 drop_pages(struct drm_i915_gem_object *obj) 292 { 293 struct i915_vma *vma, *next; 294 int ret; 295 296 drm_gem_object_reference(&obj->base); 297 list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) 298 if (i915_vma_unbind(vma)) 299 break; 300 301 ret = i915_gem_object_put_pages(obj); 302 drm_gem_object_unreference(&obj->base); 303 304 return ret; 305 } 306 307 int 308 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, 309 int align) 310 { 311 drm_dma_handle_t *phys; 312 int ret; 313 314 if (obj->phys_handle) { 315 if ((unsigned long)obj->phys_handle->vaddr & (align -1)) 316 return -EBUSY; 317 318 return 0; 319 } 320 321 if (obj->madv != I915_MADV_WILLNEED) 322 return -EFAULT; 323 324 if (obj->base.filp == NULL) 325 return -EINVAL; 326 327 ret = drop_pages(obj); 328 if (ret) 329 return ret; 330 331 /* create a new object */ 332 phys = drm_pci_alloc(obj->base.dev, obj->base.size, align); 333 if (!phys) 334 return -ENOMEM; 335 336 obj->phys_handle = phys; 337 #if 0 338 obj->ops = &i915_gem_phys_ops; 339 #endif 340 341 return i915_gem_object_get_pages(obj); 342 } 343 344 static int 345 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, 346 struct drm_i915_gem_pwrite *args, 347 struct drm_file *file_priv) 348 { 349 struct drm_device *dev = obj->base.dev; 350 void *vaddr = obj->phys_handle->vaddr + args->offset; 351 char __user *user_data = u64_to_user_ptr(args->data_ptr); 352 int ret = 0; 353 354 /* We manually control the domain here and pretend that it 355 * remains coherent i.e. in the GTT domain, like shmem_pwrite. 356 */ 357 ret = i915_gem_object_wait_rendering(obj, false); 358 if (ret) 359 return ret; 360 361 intel_fb_obj_invalidate(obj, ORIGIN_CPU); 362 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { 363 unsigned long unwritten; 364 365 /* The physical object once assigned is fixed for the lifetime 366 * of the obj, so we can safely drop the lock and continue 367 * to access vaddr. 368 */ 369 mutex_unlock(&dev->struct_mutex); 370 unwritten = copy_from_user(vaddr, user_data, args->size); 371 mutex_lock(&dev->struct_mutex); 372 if (unwritten) { 373 ret = -EFAULT; 374 goto out; 375 } 376 } 377 378 drm_clflush_virt_range(vaddr, args->size); 379 i915_gem_chipset_flush(to_i915(dev)); 380 381 out: 382 intel_fb_obj_flush(obj, false, ORIGIN_CPU); 383 return ret; 384 } 385 386 void *i915_gem_object_alloc(struct drm_device *dev) 387 { 388 return kzalloc(sizeof(struct drm_i915_gem_object), GFP_KERNEL); 389 } 390 391 void i915_gem_object_free(struct drm_i915_gem_object *obj) 392 { 393 kfree(obj); 394 } 395 396 static int 397 i915_gem_create(struct drm_file *file, 398 struct drm_device *dev, 399 uint64_t size, 400 uint32_t *handle_p) 401 { 402 struct drm_i915_gem_object *obj; 403 int ret; 404 u32 handle; 405 406 size = roundup(size, PAGE_SIZE); 407 if (size == 0) 408 return -EINVAL; 409 410 /* Allocate the new object */ 411 obj = i915_gem_object_create(dev, size); 412 if (IS_ERR(obj)) 413 return PTR_ERR(obj); 414 415 ret = drm_gem_handle_create(file, &obj->base, &handle); 416 /* drop reference from allocate - handle holds it now */ 417 drm_gem_object_unreference_unlocked(&obj->base); 418 if (ret) 419 return ret; 420 421 *handle_p = handle; 422 return 0; 423 } 424 425 int 426 i915_gem_dumb_create(struct drm_file *file, 427 struct drm_device *dev, 428 struct drm_mode_create_dumb *args) 429 { 430 /* have to work out size/pitch and return them */ 431 args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64); 432 args->size = args->pitch * args->height; 433 return i915_gem_create(file, dev, 434 args->size, &args->handle); 435 } 436 437 /** 438 * Creates a new mm object and returns a handle to it. 439 * @dev: drm device pointer 440 * @data: ioctl data blob 441 * @file: drm file pointer 442 */ 443 int 444 i915_gem_create_ioctl(struct drm_device *dev, void *data, 445 struct drm_file *file) 446 { 447 struct drm_i915_gem_create *args = data; 448 449 return i915_gem_create(file, dev, 450 args->size, &args->handle); 451 } 452 453 static inline int 454 __copy_to_user_swizzled(char __user *cpu_vaddr, 455 const char *gpu_vaddr, int gpu_offset, 456 int length) 457 { 458 int ret, cpu_offset = 0; 459 460 while (length > 0) { 461 int cacheline_end = ALIGN(gpu_offset + 1, 64); 462 int this_length = min(cacheline_end - gpu_offset, length); 463 int swizzled_gpu_offset = gpu_offset ^ 64; 464 465 ret = __copy_to_user(cpu_vaddr + cpu_offset, 466 gpu_vaddr + swizzled_gpu_offset, 467 this_length); 468 if (ret) 469 return ret + length; 470 471 cpu_offset += this_length; 472 gpu_offset += this_length; 473 length -= this_length; 474 } 475 476 return 0; 477 } 478 479 static inline int 480 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, 481 const char __user *cpu_vaddr, 482 int length) 483 { 484 int ret, cpu_offset = 0; 485 486 while (length > 0) { 487 int cacheline_end = ALIGN(gpu_offset + 1, 64); 488 int this_length = min(cacheline_end - gpu_offset, length); 489 int swizzled_gpu_offset = gpu_offset ^ 64; 490 491 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, 492 cpu_vaddr + cpu_offset, 493 this_length); 494 if (ret) 495 return ret + length; 496 497 cpu_offset += this_length; 498 gpu_offset += this_length; 499 length -= this_length; 500 } 501 502 return 0; 503 } 504 505 /* 506 * Pins the specified object's pages and synchronizes the object with 507 * GPU accesses. Sets needs_clflush to non-zero if the caller should 508 * flush the object from the CPU cache. 509 */ 510 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, 511 int *needs_clflush) 512 { 513 int ret; 514 515 *needs_clflush = 0; 516 517 if (WARN_ON(!i915_gem_object_has_struct_page(obj))) 518 return -EINVAL; 519 520 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { 521 /* If we're not in the cpu read domain, set ourself into the gtt 522 * read domain and manually flush cachelines (if required). This 523 * optimizes for the case when the gpu will dirty the data 524 * anyway again before the next pread happens. */ 525 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev, 526 obj->cache_level); 527 ret = i915_gem_object_wait_rendering(obj, true); 528 if (ret) 529 return ret; 530 } 531 532 ret = i915_gem_object_get_pages(obj); 533 if (ret) 534 return ret; 535 536 i915_gem_object_pin_pages(obj); 537 538 return ret; 539 } 540 541 /* Per-page copy function for the shmem pread fastpath. 542 * Flushes invalid cachelines before reading the target if 543 * needs_clflush is set. */ 544 static int 545 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length, 546 char __user *user_data, 547 bool page_do_bit17_swizzling, bool needs_clflush) 548 { 549 char *vaddr; 550 int ret; 551 552 if (unlikely(page_do_bit17_swizzling)) 553 return -EINVAL; 554 555 vaddr = kmap_atomic(page); 556 if (needs_clflush) 557 drm_clflush_virt_range(vaddr + shmem_page_offset, 558 page_length); 559 ret = __copy_to_user_inatomic(user_data, 560 vaddr + shmem_page_offset, 561 page_length); 562 kunmap_atomic(vaddr); 563 564 return ret ? -EFAULT : 0; 565 } 566 567 static void 568 shmem_clflush_swizzled_range(char *addr, unsigned long length, 569 bool swizzled) 570 { 571 if (unlikely(swizzled)) { 572 unsigned long start = (unsigned long) addr; 573 unsigned long end = (unsigned long) addr + length; 574 575 /* For swizzling simply ensure that we always flush both 576 * channels. Lame, but simple and it works. Swizzled 577 * pwrite/pread is far from a hotpath - current userspace 578 * doesn't use it at all. */ 579 start = round_down(start, 128); 580 end = round_up(end, 128); 581 582 drm_clflush_virt_range((void *)start, end - start); 583 } else { 584 drm_clflush_virt_range(addr, length); 585 } 586 587 } 588 589 /* Only difference to the fast-path function is that this can handle bit17 590 * and uses non-atomic copy and kmap functions. */ 591 static int 592 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, 593 char __user *user_data, 594 bool page_do_bit17_swizzling, bool needs_clflush) 595 { 596 char *vaddr; 597 int ret; 598 599 vaddr = kmap(page); 600 if (needs_clflush) 601 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 602 page_length, 603 page_do_bit17_swizzling); 604 605 if (page_do_bit17_swizzling) 606 ret = __copy_to_user_swizzled(user_data, 607 vaddr, shmem_page_offset, 608 page_length); 609 else 610 ret = __copy_to_user(user_data, 611 vaddr + shmem_page_offset, 612 page_length); 613 kunmap(page); 614 615 return ret ? - EFAULT : 0; 616 } 617 618 static inline unsigned long 619 slow_user_access(struct io_mapping *mapping, 620 uint64_t page_base, int page_offset, 621 char __user *user_data, 622 unsigned long length, bool pwrite) 623 { 624 void __iomem *ioaddr; 625 void *vaddr; 626 uint64_t unwritten; 627 628 ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE); 629 /* We can use the cpu mem copy function because this is X86. */ 630 vaddr = (void __force *)ioaddr + page_offset; 631 if (pwrite) 632 unwritten = __copy_from_user(vaddr, user_data, length); 633 else 634 unwritten = __copy_to_user(user_data, vaddr, length); 635 636 io_mapping_unmap(ioaddr); 637 return unwritten; 638 } 639 640 static int 641 i915_gem_gtt_pread(struct drm_device *dev, 642 struct drm_i915_gem_object *obj, uint64_t size, 643 uint64_t data_offset, uint64_t data_ptr) 644 { 645 struct drm_i915_private *dev_priv = dev->dev_private; 646 struct i915_ggtt *ggtt = &dev_priv->ggtt; 647 struct drm_mm_node node; 648 char __user *user_data; 649 uint64_t remain; 650 uint64_t offset; 651 int ret; 652 653 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE); 654 if (ret) { 655 ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE); 656 if (ret) 657 goto out; 658 659 ret = i915_gem_object_get_pages(obj); 660 if (ret) { 661 remove_mappable_node(&node); 662 goto out; 663 } 664 665 i915_gem_object_pin_pages(obj); 666 } else { 667 node.start = i915_gem_obj_ggtt_offset(obj); 668 node.allocated = false; 669 ret = i915_gem_object_put_fence(obj); 670 if (ret) 671 goto out_unpin; 672 } 673 674 ret = i915_gem_object_set_to_gtt_domain(obj, false); 675 if (ret) 676 goto out_unpin; 677 678 user_data = u64_to_user_ptr(data_ptr); 679 remain = size; 680 offset = data_offset; 681 682 mutex_unlock(&dev->struct_mutex); 683 if (likely(!i915.prefault_disable)) { 684 ret = fault_in_multipages_writeable(user_data, remain); 685 if (ret) { 686 mutex_lock(&dev->struct_mutex); 687 goto out_unpin; 688 } 689 } 690 691 while (remain > 0) { 692 /* Operation in this page 693 * 694 * page_base = page offset within aperture 695 * page_offset = offset within page 696 * page_length = bytes to copy for this page 697 */ 698 u32 page_base = node.start; 699 unsigned page_offset = offset_in_page(offset); 700 unsigned page_length = PAGE_SIZE - page_offset; 701 page_length = remain < page_length ? remain : page_length; 702 if (node.allocated) { 703 wmb(); 704 ggtt->base.insert_page(&ggtt->base, 705 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), 706 node.start, 707 I915_CACHE_NONE, 0); 708 wmb(); 709 } else { 710 page_base += offset & PAGE_MASK; 711 } 712 /* This is a slow read/write as it tries to read from 713 * and write to user memory which may result into page 714 * faults, and so we cannot perform this under struct_mutex. 715 */ 716 if (slow_user_access(ggtt->mappable, page_base, 717 page_offset, user_data, 718 page_length, false)) { 719 ret = -EFAULT; 720 break; 721 } 722 723 remain -= page_length; 724 user_data += page_length; 725 offset += page_length; 726 } 727 728 mutex_lock(&dev->struct_mutex); 729 if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) { 730 /* The user has modified the object whilst we tried 731 * reading from it, and we now have no idea what domain 732 * the pages should be in. As we have just been touching 733 * them directly, flush everything back to the GTT 734 * domain. 735 */ 736 ret = i915_gem_object_set_to_gtt_domain(obj, false); 737 } 738 739 out_unpin: 740 if (node.allocated) { 741 wmb(); 742 ggtt->base.clear_range(&ggtt->base, 743 node.start, node.size, 744 true); 745 i915_gem_object_unpin_pages(obj); 746 remove_mappable_node(&node); 747 } else { 748 i915_gem_object_ggtt_unpin(obj); 749 } 750 out: 751 return ret; 752 } 753 754 static int 755 i915_gem_shmem_pread(struct drm_device *dev, 756 struct drm_i915_gem_object *obj, 757 struct drm_i915_gem_pread *args, 758 struct drm_file *file) 759 { 760 char __user *user_data; 761 ssize_t remain; 762 loff_t offset; 763 int shmem_page_offset, page_length, ret = 0; 764 int obj_do_bit17_swizzling, page_do_bit17_swizzling; 765 int prefaulted = 0; 766 int needs_clflush = 0; 767 struct sg_page_iter sg_iter; 768 769 if (!i915_gem_object_has_struct_page(obj)) 770 return -ENODEV; 771 772 user_data = u64_to_user_ptr(args->data_ptr); 773 remain = args->size; 774 775 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 776 777 ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush); 778 if (ret) 779 return ret; 780 781 offset = args->offset; 782 783 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 784 offset >> PAGE_SHIFT) { 785 struct page *page = sg_page_iter_page(&sg_iter); 786 787 if (remain <= 0) 788 break; 789 790 /* Operation in this page 791 * 792 * shmem_page_offset = offset within page in shmem file 793 * page_length = bytes to copy for this page 794 */ 795 shmem_page_offset = offset_in_page(offset); 796 page_length = remain; 797 if ((shmem_page_offset + page_length) > PAGE_SIZE) 798 page_length = PAGE_SIZE - shmem_page_offset; 799 800 page_do_bit17_swizzling = obj_do_bit17_swizzling && 801 (page_to_phys(page) & (1 << 17)) != 0; 802 803 ret = shmem_pread_fast(page, shmem_page_offset, page_length, 804 user_data, page_do_bit17_swizzling, 805 needs_clflush); 806 if (ret == 0) 807 goto next_page; 808 809 mutex_unlock(&dev->struct_mutex); 810 811 if (likely(!i915.prefault_disable) && !prefaulted) { 812 ret = fault_in_multipages_writeable(user_data, remain); 813 /* Userspace is tricking us, but we've already clobbered 814 * its pages with the prefault and promised to write the 815 * data up to the first fault. Hence ignore any errors 816 * and just continue. */ 817 (void)ret; 818 prefaulted = 1; 819 } 820 821 ret = shmem_pread_slow(page, shmem_page_offset, page_length, 822 user_data, page_do_bit17_swizzling, 823 needs_clflush); 824 825 mutex_lock(&dev->struct_mutex); 826 827 if (ret) 828 goto out; 829 830 next_page: 831 remain -= page_length; 832 user_data += page_length; 833 offset += page_length; 834 } 835 836 out: 837 i915_gem_object_unpin_pages(obj); 838 839 return ret; 840 } 841 842 /** 843 * Reads data from the object referenced by handle. 844 * @dev: drm device pointer 845 * @data: ioctl data blob 846 * @file: drm file pointer 847 * 848 * On error, the contents of *data are undefined. 849 */ 850 int 851 i915_gem_pread_ioctl(struct drm_device *dev, void *data, 852 struct drm_file *file) 853 { 854 struct drm_i915_gem_pread *args = data; 855 struct drm_i915_gem_object *obj; 856 int ret = 0; 857 858 if (args->size == 0) 859 return 0; 860 861 #if 0 862 if (!access_ok(VERIFY_WRITE, 863 u64_to_user_ptr(args->data_ptr), 864 args->size)) 865 return -EFAULT; 866 #endif 867 868 ret = i915_mutex_lock_interruptible(dev); 869 if (ret) 870 return ret; 871 872 obj = to_intel_bo(drm_gem_object_lookup(file, args->handle)); 873 if (&obj->base == NULL) { 874 ret = -ENOENT; 875 goto unlock; 876 } 877 878 /* Bounds check source. */ 879 if (args->offset > obj->base.size || 880 args->size > obj->base.size - args->offset) { 881 ret = -EINVAL; 882 goto out; 883 } 884 885 trace_i915_gem_object_pread(obj, args->offset, args->size); 886 887 ret = i915_gem_shmem_pread(dev, obj, args, file); 888 889 /* pread for non shmem backed objects */ 890 if (ret == -EFAULT || ret == -ENODEV) 891 ret = i915_gem_gtt_pread(dev, obj, args->size, 892 args->offset, args->data_ptr); 893 894 out: 895 drm_gem_object_unreference(&obj->base); 896 unlock: 897 mutex_unlock(&dev->struct_mutex); 898 return ret; 899 } 900 901 /* This is the fast write path which cannot handle 902 * page faults in the source data 903 */ 904 905 static inline int 906 fast_user_write(struct io_mapping *mapping, 907 loff_t page_base, int page_offset, 908 char __user *user_data, 909 int length) 910 { 911 void __iomem *vaddr_atomic; 912 void *vaddr; 913 unsigned long unwritten; 914 915 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base); 916 /* We can use the cpu mem copy function because this is X86. */ 917 vaddr = (void __force*)vaddr_atomic + page_offset; 918 unwritten = __copy_from_user_inatomic_nocache(vaddr, 919 user_data, length); 920 io_mapping_unmap_atomic(vaddr_atomic); 921 return unwritten; 922 } 923 924 /** 925 * This is the fast pwrite path, where we copy the data directly from the 926 * user into the GTT, uncached. 927 * @dev: drm device pointer 928 * @obj: i915 gem object 929 * @args: pwrite arguments structure 930 * @file: drm file pointer 931 */ 932 static int 933 i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, 934 struct drm_i915_gem_object *obj, 935 struct drm_i915_gem_pwrite *args, 936 struct drm_file *file) 937 { 938 struct i915_ggtt *ggtt = &i915->ggtt; 939 struct drm_device *dev = obj->base.dev; 940 struct drm_mm_node node; 941 uint64_t remain, offset; 942 char __user *user_data; 943 int ret; 944 bool hit_slow_path = false; 945 946 if (obj->tiling_mode != I915_TILING_NONE) 947 return -EFAULT; 948 949 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK); 950 if (ret) { 951 ret = insert_mappable_node(i915, &node, PAGE_SIZE); 952 if (ret) 953 goto out; 954 955 ret = i915_gem_object_get_pages(obj); 956 if (ret) { 957 remove_mappable_node(&node); 958 goto out; 959 } 960 961 i915_gem_object_pin_pages(obj); 962 } else { 963 node.start = i915_gem_obj_ggtt_offset(obj); 964 node.allocated = false; 965 ret = i915_gem_object_put_fence(obj); 966 if (ret) 967 goto out_unpin; 968 } 969 970 ret = i915_gem_object_set_to_gtt_domain(obj, true); 971 if (ret) 972 goto out_unpin; 973 974 intel_fb_obj_invalidate(obj, ORIGIN_GTT); 975 obj->dirty = true; 976 977 user_data = u64_to_user_ptr(args->data_ptr); 978 offset = args->offset; 979 remain = args->size; 980 while (remain) { 981 /* Operation in this page 982 * 983 * page_base = page offset within aperture 984 * page_offset = offset within page 985 * page_length = bytes to copy for this page 986 */ 987 u32 page_base = node.start; 988 unsigned page_offset = offset_in_page(offset); 989 unsigned page_length = PAGE_SIZE - page_offset; 990 page_length = remain < page_length ? remain : page_length; 991 if (node.allocated) { 992 wmb(); /* flush the write before we modify the GGTT */ 993 ggtt->base.insert_page(&ggtt->base, 994 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), 995 node.start, I915_CACHE_NONE, 0); 996 wmb(); /* flush modifications to the GGTT (insert_page) */ 997 } else { 998 page_base += offset & LINUX_PAGE_MASK; 999 } 1000 /* If we get a fault while copying data, then (presumably) our 1001 * source page isn't available. Return the error and we'll 1002 * retry in the slow path. 1003 * If the object is non-shmem backed, we retry again with the 1004 * path that handles page fault. 1005 */ 1006 if (fast_user_write(ggtt->mappable, page_base, 1007 page_offset, user_data, page_length)) { 1008 hit_slow_path = true; 1009 mutex_unlock(&dev->struct_mutex); 1010 if (slow_user_access(ggtt->mappable, 1011 page_base, 1012 page_offset, user_data, 1013 page_length, true)) { 1014 ret = -EFAULT; 1015 mutex_lock(&dev->struct_mutex); 1016 goto out_flush; 1017 } 1018 1019 mutex_lock(&dev->struct_mutex); 1020 } 1021 1022 remain -= page_length; 1023 user_data += page_length; 1024 offset += page_length; 1025 } 1026 1027 out_flush: 1028 if (hit_slow_path) { 1029 if (ret == 0 && 1030 (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) { 1031 /* The user has modified the object whilst we tried 1032 * reading from it, and we now have no idea what domain 1033 * the pages should be in. As we have just been touching 1034 * them directly, flush everything back to the GTT 1035 * domain. 1036 */ 1037 ret = i915_gem_object_set_to_gtt_domain(obj, false); 1038 } 1039 } 1040 1041 intel_fb_obj_flush(obj, false, ORIGIN_GTT); 1042 out_unpin: 1043 if (node.allocated) { 1044 wmb(); 1045 ggtt->base.clear_range(&ggtt->base, 1046 node.start, node.size, 1047 true); 1048 i915_gem_object_unpin_pages(obj); 1049 remove_mappable_node(&node); 1050 } else { 1051 i915_gem_object_ggtt_unpin(obj); 1052 } 1053 out: 1054 return ret; 1055 } 1056 1057 /* Per-page copy function for the shmem pwrite fastpath. 1058 * Flushes invalid cachelines before writing to the target if 1059 * needs_clflush_before is set and flushes out any written cachelines after 1060 * writing if needs_clflush is set. */ 1061 static int 1062 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length, 1063 char __user *user_data, 1064 bool page_do_bit17_swizzling, 1065 bool needs_clflush_before, 1066 bool needs_clflush_after) 1067 { 1068 char *vaddr; 1069 int ret; 1070 1071 if (unlikely(page_do_bit17_swizzling)) 1072 return -EINVAL; 1073 1074 vaddr = kmap_atomic(page); 1075 if (needs_clflush_before) 1076 drm_clflush_virt_range(vaddr + shmem_page_offset, 1077 page_length); 1078 ret = __copy_from_user_inatomic(vaddr + shmem_page_offset, 1079 user_data, page_length); 1080 if (needs_clflush_after) 1081 drm_clflush_virt_range(vaddr + shmem_page_offset, 1082 page_length); 1083 kunmap_atomic(vaddr); 1084 1085 return ret ? -EFAULT : 0; 1086 } 1087 1088 /* Only difference to the fast-path function is that this can handle bit17 1089 * and uses non-atomic copy and kmap functions. */ 1090 static int 1091 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, 1092 char __user *user_data, 1093 bool page_do_bit17_swizzling, 1094 bool needs_clflush_before, 1095 bool needs_clflush_after) 1096 { 1097 char *vaddr; 1098 int ret; 1099 1100 vaddr = kmap(page); 1101 if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) 1102 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 1103 page_length, 1104 page_do_bit17_swizzling); 1105 if (page_do_bit17_swizzling) 1106 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, 1107 user_data, 1108 page_length); 1109 else 1110 ret = __copy_from_user(vaddr + shmem_page_offset, 1111 user_data, 1112 page_length); 1113 if (needs_clflush_after) 1114 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 1115 page_length, 1116 page_do_bit17_swizzling); 1117 kunmap(page); 1118 1119 return ret ? -EFAULT : 0; 1120 } 1121 1122 static int 1123 i915_gem_shmem_pwrite(struct drm_device *dev, 1124 struct drm_i915_gem_object *obj, 1125 struct drm_i915_gem_pwrite *args, 1126 struct drm_file *file) 1127 { 1128 ssize_t remain; 1129 loff_t offset; 1130 char __user *user_data; 1131 int shmem_page_offset, page_length, ret = 0; 1132 int obj_do_bit17_swizzling, page_do_bit17_swizzling; 1133 int hit_slowpath = 0; 1134 int needs_clflush_after = 0; 1135 int needs_clflush_before = 0; 1136 struct sg_page_iter sg_iter; 1137 1138 user_data = u64_to_user_ptr(args->data_ptr); 1139 remain = args->size; 1140 1141 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 1142 1143 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1144 /* If we're not in the cpu write domain, set ourself into the gtt 1145 * write domain and manually flush cachelines (if required). This 1146 * optimizes for the case when the gpu will use the data 1147 * right away and we therefore have to clflush anyway. */ 1148 needs_clflush_after = cpu_write_needs_clflush(obj); 1149 ret = i915_gem_object_wait_rendering(obj, false); 1150 if (ret) 1151 return ret; 1152 } 1153 /* Same trick applies to invalidate partially written cachelines read 1154 * before writing. */ 1155 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 1156 needs_clflush_before = 1157 !cpu_cache_is_coherent(dev, obj->cache_level); 1158 1159 ret = i915_gem_object_get_pages(obj); 1160 if (ret) 1161 return ret; 1162 1163 intel_fb_obj_invalidate(obj, ORIGIN_CPU); 1164 1165 i915_gem_object_pin_pages(obj); 1166 1167 offset = args->offset; 1168 obj->dirty = 1; 1169 1170 VM_OBJECT_LOCK(obj->base.filp); 1171 vm_object_pip_add(obj->base.filp, 1); 1172 1173 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 1174 offset >> PAGE_SHIFT) { 1175 struct page *page = sg_page_iter_page(&sg_iter); 1176 int partial_cacheline_write; 1177 1178 if (remain <= 0) 1179 break; 1180 1181 /* Operation in this page 1182 * 1183 * shmem_page_offset = offset within page in shmem file 1184 * page_length = bytes to copy for this page 1185 */ 1186 shmem_page_offset = offset_in_page(offset); 1187 1188 page_length = remain; 1189 if ((shmem_page_offset + page_length) > PAGE_SIZE) 1190 page_length = PAGE_SIZE - shmem_page_offset; 1191 1192 /* If we don't overwrite a cacheline completely we need to be 1193 * careful to have up-to-date data by first clflushing. Don't 1194 * overcomplicate things and flush the entire patch. */ 1195 partial_cacheline_write = needs_clflush_before && 1196 ((shmem_page_offset | page_length) 1197 & (cpu_clflush_line_size - 1)); 1198 1199 page_do_bit17_swizzling = obj_do_bit17_swizzling && 1200 (page_to_phys(page) & (1 << 17)) != 0; 1201 1202 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, 1203 user_data, page_do_bit17_swizzling, 1204 partial_cacheline_write, 1205 needs_clflush_after); 1206 if (ret == 0) 1207 goto next_page; 1208 1209 hit_slowpath = 1; 1210 mutex_unlock(&dev->struct_mutex); 1211 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, 1212 user_data, page_do_bit17_swizzling, 1213 partial_cacheline_write, 1214 needs_clflush_after); 1215 1216 mutex_lock(&dev->struct_mutex); 1217 1218 if (ret) 1219 goto out; 1220 1221 next_page: 1222 remain -= page_length; 1223 user_data += page_length; 1224 offset += page_length; 1225 } 1226 vm_object_pip_wakeup(obj->base.filp); 1227 VM_OBJECT_UNLOCK(obj->base.filp); 1228 1229 out: 1230 i915_gem_object_unpin_pages(obj); 1231 1232 if (hit_slowpath) { 1233 /* 1234 * Fixup: Flush cpu caches in case we didn't flush the dirty 1235 * cachelines in-line while writing and the object moved 1236 * out of the cpu write domain while we've dropped the lock. 1237 */ 1238 if (!needs_clflush_after && 1239 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1240 if (i915_gem_clflush_object(obj, obj->pin_display)) 1241 needs_clflush_after = true; 1242 } 1243 } 1244 1245 if (needs_clflush_after) 1246 i915_gem_chipset_flush(to_i915(dev)); 1247 else 1248 obj->cache_dirty = true; 1249 1250 intel_fb_obj_flush(obj, false, ORIGIN_CPU); 1251 return ret; 1252 } 1253 1254 /** 1255 * Writes data to the object referenced by handle. 1256 * @dev: drm device 1257 * @data: ioctl data blob 1258 * @file: drm file 1259 * 1260 * On error, the contents of the buffer that were to be modified are undefined. 1261 */ 1262 int 1263 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 1264 struct drm_file *file) 1265 { 1266 struct drm_i915_private *dev_priv = dev->dev_private; 1267 struct drm_i915_gem_pwrite *args = data; 1268 struct drm_i915_gem_object *obj; 1269 int ret; 1270 1271 if (args->size == 0) 1272 return 0; 1273 1274 #if 0 1275 if (!access_ok(VERIFY_READ, 1276 u64_to_user_ptr(args->data_ptr), 1277 args->size)) 1278 return -EFAULT; 1279 #endif 1280 1281 if (likely(!i915.prefault_disable)) { 1282 ret = fault_in_multipages_readable(u64_to_user_ptr(args->data_ptr), 1283 args->size); 1284 if (ret) 1285 return -EFAULT; 1286 } 1287 1288 intel_runtime_pm_get(dev_priv); 1289 1290 ret = i915_mutex_lock_interruptible(dev); 1291 if (ret) 1292 goto put_rpm; 1293 1294 obj = to_intel_bo(drm_gem_object_lookup(file, args->handle)); 1295 if (&obj->base == NULL) { 1296 ret = -ENOENT; 1297 goto unlock; 1298 } 1299 1300 /* Bounds check destination. */ 1301 if (args->offset > obj->base.size || 1302 args->size > obj->base.size - args->offset) { 1303 ret = -EINVAL; 1304 goto out; 1305 } 1306 1307 trace_i915_gem_object_pwrite(obj, args->offset, args->size); 1308 1309 ret = -EFAULT; 1310 /* We can only do the GTT pwrite on untiled buffers, as otherwise 1311 * it would end up going through the fenced access, and we'll get 1312 * different detiling behavior between reading and writing. 1313 * pread/pwrite currently are reading and writing from the CPU 1314 * perspective, requiring manual detiling by the client. 1315 */ 1316 if (!i915_gem_object_has_struct_page(obj) || 1317 cpu_write_needs_clflush(obj)) { 1318 ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file); 1319 /* Note that the gtt paths might fail with non-page-backed user 1320 * pointers (e.g. gtt mappings when moving data between 1321 * textures). Fallback to the shmem path in that case. */ 1322 } 1323 1324 if (ret == -EFAULT) { 1325 if (obj->phys_handle) 1326 ret = i915_gem_phys_pwrite(obj, args, file); 1327 else if (i915_gem_object_has_struct_page(obj)) 1328 ret = i915_gem_shmem_pwrite(dev, obj, args, file); 1329 else 1330 ret = -ENODEV; 1331 } 1332 1333 out: 1334 drm_gem_object_unreference(&obj->base); 1335 unlock: 1336 mutex_unlock(&dev->struct_mutex); 1337 put_rpm: 1338 intel_runtime_pm_put(dev_priv); 1339 1340 return ret; 1341 } 1342 1343 static int 1344 i915_gem_check_wedge(unsigned reset_counter, bool interruptible) 1345 { 1346 if (__i915_terminally_wedged(reset_counter)) 1347 return -EIO; 1348 1349 if (__i915_reset_in_progress(reset_counter)) { 1350 /* Non-interruptible callers can't handle -EAGAIN, hence return 1351 * -EIO unconditionally for these. */ 1352 if (!interruptible) 1353 return -EIO; 1354 1355 return -EAGAIN; 1356 } 1357 1358 return 0; 1359 } 1360 1361 static void fake_irq(unsigned long data) 1362 { 1363 wake_up_process((struct task_struct *)data); 1364 } 1365 1366 static bool missed_irq(struct drm_i915_private *dev_priv, 1367 struct intel_engine_cs *engine) 1368 { 1369 return test_bit(engine->id, &dev_priv->gpu_error.missed_irq_rings); 1370 } 1371 1372 #if 0 1373 static unsigned long local_clock_us(unsigned *cpu) 1374 { 1375 unsigned long t; 1376 1377 /* Cheaply and approximately convert from nanoseconds to microseconds. 1378 * The result and subsequent calculations are also defined in the same 1379 * approximate microseconds units. The principal source of timing 1380 * error here is from the simple truncation. 1381 * 1382 * Note that local_clock() is only defined wrt to the current CPU; 1383 * the comparisons are no longer valid if we switch CPUs. Instead of 1384 * blocking preemption for the entire busywait, we can detect the CPU 1385 * switch and use that as indicator of system load and a reason to 1386 * stop busywaiting, see busywait_stop(). 1387 */ 1388 *cpu = get_cpu(); 1389 t = local_clock() >> 10; 1390 put_cpu(); 1391 1392 return t; 1393 } 1394 1395 static bool busywait_stop(unsigned long timeout, unsigned cpu) 1396 { 1397 unsigned this_cpu; 1398 1399 if (time_after(local_clock_us(&this_cpu), timeout)) 1400 return true; 1401 1402 return this_cpu != cpu; 1403 } 1404 1405 static int __i915_spin_request(struct drm_i915_gem_request *req, int state) 1406 { 1407 unsigned long timeout; 1408 unsigned cpu; 1409 1410 /* When waiting for high frequency requests, e.g. during synchronous 1411 * rendering split between the CPU and GPU, the finite amount of time 1412 * required to set up the irq and wait upon it limits the response 1413 * rate. By busywaiting on the request completion for a short while we 1414 * can service the high frequency waits as quick as possible. However, 1415 * if it is a slow request, we want to sleep as quickly as possible. 1416 * The tradeoff between waiting and sleeping is roughly the time it 1417 * takes to sleep on a request, on the order of a microsecond. 1418 */ 1419 1420 if (req->engine->irq_refcount) 1421 return -EBUSY; 1422 1423 /* Only spin if we know the GPU is processing this request */ 1424 if (!i915_gem_request_started(req, true)) 1425 return -EAGAIN; 1426 1427 timeout = local_clock_us(&cpu) + 5; 1428 while (!need_resched()) { 1429 if (i915_gem_request_completed(req, true)) 1430 return 0; 1431 1432 if (signal_pending_state(state, current)) 1433 break; 1434 1435 if (busywait_stop(timeout, cpu)) 1436 break; 1437 1438 cpu_relax_lowlatency(); 1439 } 1440 1441 if (i915_gem_request_completed(req, false)) 1442 return 0; 1443 1444 return -EAGAIN; 1445 } 1446 #endif 1447 1448 /** 1449 * __i915_wait_request - wait until execution of request has finished 1450 * @req: duh! 1451 * @interruptible: do an interruptible wait (normally yes) 1452 * @timeout: in - how long to wait (NULL forever); out - how much time remaining 1453 * @rps: RPS client 1454 * 1455 * Note: It is of utmost importance that the passed in seqno and reset_counter 1456 * values have been read by the caller in an smp safe manner. Where read-side 1457 * locks are involved, it is sufficient to read the reset_counter before 1458 * unlocking the lock that protects the seqno. For lockless tricks, the 1459 * reset_counter _must_ be read before, and an appropriate smp_rmb must be 1460 * inserted. 1461 * 1462 * Returns 0 if the request was found within the alloted time. Else returns the 1463 * errno with remaining time filled in timeout argument. 1464 */ 1465 int __i915_wait_request(struct drm_i915_gem_request *req, 1466 bool interruptible, 1467 s64 *timeout, 1468 struct intel_rps_client *rps) 1469 { 1470 struct intel_engine_cs *engine = i915_gem_request_get_engine(req); 1471 struct drm_i915_private *dev_priv = req->i915; 1472 const bool irq_test_in_progress = 1473 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_engine_flag(engine); 1474 int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; 1475 DEFINE_WAIT(reset); 1476 DEFINE_WAIT(wait); 1477 unsigned long timeout_expire; 1478 s64 before = 0; /* Only to silence a compiler warning. */ 1479 int ret; 1480 int sl_timeout = 1; 1481 1482 WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled"); 1483 1484 if (list_empty(&req->list)) 1485 return 0; 1486 1487 if (i915_gem_request_completed(req, true)) 1488 return 0; 1489 1490 timeout_expire = 0; 1491 if (timeout) { 1492 if (WARN_ON(*timeout < 0)) 1493 return -EINVAL; 1494 1495 if (*timeout == 0) 1496 return -ETIME; 1497 1498 timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout); 1499 1500 /* 1501 * Record current time in case interrupted by signal, or wedged. 1502 */ 1503 before = ktime_get_raw_ns(); 1504 } 1505 1506 if (INTEL_INFO(dev_priv)->gen >= 6) 1507 gen6_rps_boost(dev_priv, rps, req->emitted_jiffies); 1508 1509 trace_i915_gem_request_wait_begin(req); 1510 1511 /* Optimistic spin for the next jiffie before touching IRQs */ 1512 #if 0 1513 ret = __i915_spin_request(req, state); 1514 if (ret == 0) 1515 goto out; 1516 #endif 1517 1518 if (!irq_test_in_progress && WARN_ON(!engine->irq_get(engine))) { 1519 ret = -ENODEV; 1520 goto out; 1521 } 1522 1523 add_wait_queue(&dev_priv->gpu_error.wait_queue, &reset); 1524 lockmgr(&engine->irq_queue.lock, LK_EXCLUSIVE); 1525 for (;;) { 1526 struct timer_list timer; 1527 1528 prepare_to_wait(&engine->irq_queue, &wait, state); 1529 1530 /* We need to check whether any gpu reset happened in between 1531 * the request being submitted and now. If a reset has occurred, 1532 * the seqno will have been advance past ours and our request 1533 * is complete. If we are in the process of handling a reset, 1534 * the request is effectively complete as the rendering will 1535 * be discarded, but we need to return in order to drop the 1536 * struct_mutex. 1537 */ 1538 if (i915_reset_in_progress(&dev_priv->gpu_error)) { 1539 ret = 0; 1540 break; 1541 } 1542 1543 if (i915_gem_request_completed(req, false)) { 1544 ret = 0; 1545 break; 1546 } 1547 1548 if (signal_pending_state(state, current)) { 1549 ret = -ERESTARTSYS; 1550 break; 1551 } 1552 1553 if (timeout && time_after_eq(jiffies, timeout_expire)) { 1554 ret = -ETIME; 1555 break; 1556 } 1557 1558 /* Ensure that even if the GPU hangs, we get woken up. 1559 * 1560 * However, note that if no one is waiting, we never notice 1561 * a gpu hang. Eventually, we will have to wait for a resource 1562 * held by the GPU and so trigger a hangcheck. In the most 1563 * pathological case, this will be upon memory starvation! 1564 */ 1565 i915_queue_hangcheck(dev_priv); 1566 1567 timer.function = NULL; 1568 if (timeout || missed_irq(dev_priv, engine)) { 1569 unsigned long expire; 1570 1571 setup_timer_on_stack(&timer, fake_irq, (unsigned long)current); 1572 expire = missed_irq(dev_priv, engine) ? jiffies + 1 : timeout_expire; 1573 sl_timeout = expire - jiffies; 1574 if (sl_timeout < 1) 1575 sl_timeout = 1; 1576 mod_timer(&timer, expire); 1577 } 1578 1579 #if 0 1580 io_schedule(); 1581 #endif 1582 1583 if (timer.function) { 1584 del_singleshot_timer_sync(&timer); 1585 destroy_timer_on_stack(&timer); 1586 } 1587 1588 lksleep(&engine->irq_queue, &engine->irq_queue.lock, 1589 interruptible ? PCATCH : 0, "lwe", sl_timeout); 1590 } 1591 lockmgr(&engine->irq_queue.lock, LK_RELEASE); 1592 #if 0 1593 remove_wait_queue(&dev_priv->gpu_error.wait_queue, &reset); 1594 #endif 1595 1596 if (!irq_test_in_progress) 1597 engine->irq_put(engine); 1598 1599 finish_wait(&engine->irq_queue, &wait); 1600 1601 out: 1602 trace_i915_gem_request_wait_end(req); 1603 1604 if (timeout) { 1605 s64 tres = *timeout - (ktime_get_raw_ns() - before); 1606 1607 *timeout = tres < 0 ? 0 : tres; 1608 1609 /* 1610 * Apparently ktime isn't accurate enough and occasionally has a 1611 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch 1612 * things up to make the test happy. We allow up to 1 jiffy. 1613 * 1614 * This is a regrssion from the timespec->ktime conversion. 1615 */ 1616 if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000) 1617 *timeout = 0; 1618 } 1619 1620 return ret; 1621 } 1622 1623 int i915_gem_request_add_to_client(struct drm_i915_gem_request *req, 1624 struct drm_file *file) 1625 { 1626 struct drm_i915_file_private *file_priv; 1627 1628 WARN_ON(!req || !file || req->file_priv); 1629 1630 if (!req || !file) 1631 return -EINVAL; 1632 1633 if (req->file_priv) 1634 return -EINVAL; 1635 1636 file_priv = file->driver_priv; 1637 1638 spin_lock(&file_priv->mm.lock); 1639 req->file_priv = file_priv; 1640 list_add_tail(&req->client_list, &file_priv->mm.request_list); 1641 spin_unlock(&file_priv->mm.lock); 1642 1643 req->pid = curproc->p_pid; 1644 1645 return 0; 1646 } 1647 1648 static inline void 1649 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 1650 { 1651 struct drm_i915_file_private *file_priv = request->file_priv; 1652 1653 if (!file_priv) 1654 return; 1655 1656 spin_lock(&file_priv->mm.lock); 1657 list_del(&request->client_list); 1658 request->file_priv = NULL; 1659 spin_unlock(&file_priv->mm.lock); 1660 1661 #if 0 1662 put_pid(request->pid); 1663 request->pid = NULL; 1664 #endif 1665 } 1666 1667 static void i915_gem_request_retire(struct drm_i915_gem_request *request) 1668 { 1669 trace_i915_gem_request_retire(request); 1670 1671 /* We know the GPU must have read the request to have 1672 * sent us the seqno + interrupt, so use the position 1673 * of tail of the request to update the last known position 1674 * of the GPU head. 1675 * 1676 * Note this requires that we are always called in request 1677 * completion order. 1678 */ 1679 request->ringbuf->last_retired_head = request->postfix; 1680 1681 list_del_init(&request->list); 1682 i915_gem_request_remove_from_client(request); 1683 1684 if (request->previous_context) { 1685 if (i915.enable_execlists) 1686 intel_lr_context_unpin(request->previous_context, 1687 request->engine); 1688 } 1689 1690 i915_gem_context_unreference(request->ctx); 1691 i915_gem_request_unreference(request); 1692 } 1693 1694 static void 1695 __i915_gem_request_retire__upto(struct drm_i915_gem_request *req) 1696 { 1697 struct intel_engine_cs *engine = req->engine; 1698 struct drm_i915_gem_request *tmp; 1699 1700 lockdep_assert_held(&engine->i915->dev->struct_mutex); 1701 1702 if (list_empty(&req->list)) 1703 return; 1704 1705 do { 1706 tmp = list_first_entry(&engine->request_list, 1707 typeof(*tmp), list); 1708 1709 i915_gem_request_retire(tmp); 1710 } while (tmp != req); 1711 1712 WARN_ON(i915_verify_lists(engine->dev)); 1713 } 1714 1715 /** 1716 * Waits for a request to be signaled, and cleans up the 1717 * request and object lists appropriately for that event. 1718 * @req: request to wait on 1719 */ 1720 int 1721 i915_wait_request(struct drm_i915_gem_request *req) 1722 { 1723 struct drm_i915_private *dev_priv = req->i915; 1724 bool interruptible; 1725 int ret; 1726 1727 interruptible = dev_priv->mm.interruptible; 1728 1729 BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex)); 1730 1731 ret = __i915_wait_request(req, interruptible, NULL, NULL); 1732 if (ret) 1733 return ret; 1734 1735 /* If the GPU hung, we want to keep the requests to find the guilty. */ 1736 if (!i915_reset_in_progress(&dev_priv->gpu_error)) 1737 __i915_gem_request_retire__upto(req); 1738 1739 return 0; 1740 } 1741 1742 /** 1743 * Ensures that all rendering to the object has completed and the object is 1744 * safe to unbind from the GTT or access from the CPU. 1745 * @obj: i915 gem object 1746 * @readonly: waiting for read access or write 1747 */ 1748 int 1749 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, 1750 bool readonly) 1751 { 1752 int ret, i; 1753 1754 if (!obj->active) 1755 return 0; 1756 1757 if (readonly) { 1758 if (obj->last_write_req != NULL) { 1759 ret = i915_wait_request(obj->last_write_req); 1760 if (ret) 1761 return ret; 1762 1763 i = obj->last_write_req->engine->id; 1764 if (obj->last_read_req[i] == obj->last_write_req) 1765 i915_gem_object_retire__read(obj, i); 1766 else 1767 i915_gem_object_retire__write(obj); 1768 } 1769 } else { 1770 for (i = 0; i < I915_NUM_ENGINES; i++) { 1771 if (obj->last_read_req[i] == NULL) 1772 continue; 1773 1774 ret = i915_wait_request(obj->last_read_req[i]); 1775 if (ret) 1776 return ret; 1777 1778 i915_gem_object_retire__read(obj, i); 1779 } 1780 GEM_BUG_ON(obj->active); 1781 } 1782 1783 return 0; 1784 } 1785 1786 static void 1787 i915_gem_object_retire_request(struct drm_i915_gem_object *obj, 1788 struct drm_i915_gem_request *req) 1789 { 1790 int ring = req->engine->id; 1791 1792 if (obj->last_read_req[ring] == req) 1793 i915_gem_object_retire__read(obj, ring); 1794 else if (obj->last_write_req == req) 1795 i915_gem_object_retire__write(obj); 1796 1797 if (!i915_reset_in_progress(&req->i915->gpu_error)) 1798 __i915_gem_request_retire__upto(req); 1799 } 1800 1801 /* A nonblocking variant of the above wait. This is a highly dangerous routine 1802 * as the object state may change during this call. 1803 */ 1804 static __must_check int 1805 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, 1806 struct intel_rps_client *rps, 1807 bool readonly) 1808 { 1809 struct drm_device *dev = obj->base.dev; 1810 struct drm_i915_private *dev_priv = dev->dev_private; 1811 struct drm_i915_gem_request *requests[I915_NUM_ENGINES]; 1812 int ret, i, n = 0; 1813 1814 BUG_ON(!mutex_is_locked(&dev->struct_mutex)); 1815 BUG_ON(!dev_priv->mm.interruptible); 1816 1817 if (!obj->active) 1818 return 0; 1819 1820 if (readonly) { 1821 struct drm_i915_gem_request *req; 1822 1823 req = obj->last_write_req; 1824 if (req == NULL) 1825 return 0; 1826 1827 requests[n++] = i915_gem_request_reference(req); 1828 } else { 1829 for (i = 0; i < I915_NUM_ENGINES; i++) { 1830 struct drm_i915_gem_request *req; 1831 1832 req = obj->last_read_req[i]; 1833 if (req == NULL) 1834 continue; 1835 1836 requests[n++] = i915_gem_request_reference(req); 1837 } 1838 } 1839 1840 mutex_unlock(&dev->struct_mutex); 1841 ret = 0; 1842 for (i = 0; ret == 0 && i < n; i++) 1843 ret = __i915_wait_request(requests[i], true, NULL, rps); 1844 mutex_lock(&dev->struct_mutex); 1845 1846 for (i = 0; i < n; i++) { 1847 if (ret == 0) 1848 i915_gem_object_retire_request(obj, requests[i]); 1849 i915_gem_request_unreference(requests[i]); 1850 } 1851 1852 return ret; 1853 } 1854 1855 static struct intel_rps_client *to_rps_client(struct drm_file *file) 1856 { 1857 struct drm_i915_file_private *fpriv = file->driver_priv; 1858 return &fpriv->rps; 1859 } 1860 1861 static enum fb_op_origin 1862 write_origin(struct drm_i915_gem_object *obj, unsigned domain) 1863 { 1864 return domain == I915_GEM_DOMAIN_GTT && !obj->has_wc_mmap ? 1865 ORIGIN_GTT : ORIGIN_CPU; 1866 } 1867 1868 /** 1869 * Called when user space prepares to use an object with the CPU, either 1870 * through the mmap ioctl's mapping or a GTT mapping. 1871 * @dev: drm device 1872 * @data: ioctl data blob 1873 * @file: drm file 1874 */ 1875 int 1876 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1877 struct drm_file *file) 1878 { 1879 struct drm_i915_gem_set_domain *args = data; 1880 struct drm_i915_gem_object *obj; 1881 uint32_t read_domains = args->read_domains; 1882 uint32_t write_domain = args->write_domain; 1883 int ret; 1884 1885 /* Only handle setting domains to types used by the CPU. */ 1886 if (write_domain & I915_GEM_GPU_DOMAINS) 1887 return -EINVAL; 1888 1889 if (read_domains & I915_GEM_GPU_DOMAINS) 1890 return -EINVAL; 1891 1892 /* Having something in the write domain implies it's in the read 1893 * domain, and only that read domain. Enforce that in the request. 1894 */ 1895 if (write_domain != 0 && read_domains != write_domain) 1896 return -EINVAL; 1897 1898 ret = i915_mutex_lock_interruptible(dev); 1899 if (ret) 1900 return ret; 1901 1902 obj = to_intel_bo(drm_gem_object_lookup(file, args->handle)); 1903 if (&obj->base == NULL) { 1904 ret = -ENOENT; 1905 goto unlock; 1906 } 1907 1908 /* Try to flush the object off the GPU without holding the lock. 1909 * We will repeat the flush holding the lock in the normal manner 1910 * to catch cases where we are gazumped. 1911 */ 1912 ret = i915_gem_object_wait_rendering__nonblocking(obj, 1913 to_rps_client(file), 1914 !write_domain); 1915 if (ret) 1916 goto unref; 1917 1918 if (read_domains & I915_GEM_DOMAIN_GTT) 1919 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1920 else 1921 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1922 1923 if (write_domain != 0) 1924 intel_fb_obj_invalidate(obj, write_origin(obj, write_domain)); 1925 1926 unref: 1927 drm_gem_object_unreference(&obj->base); 1928 unlock: 1929 mutex_unlock(&dev->struct_mutex); 1930 return ret; 1931 } 1932 1933 /** 1934 * Called when user space has done writes to this buffer 1935 * @dev: drm device 1936 * @data: ioctl data blob 1937 * @file: drm file 1938 */ 1939 int 1940 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1941 struct drm_file *file) 1942 { 1943 struct drm_i915_gem_sw_finish *args = data; 1944 struct drm_i915_gem_object *obj; 1945 int ret = 0; 1946 1947 ret = i915_mutex_lock_interruptible(dev); 1948 if (ret) 1949 return ret; 1950 1951 obj = to_intel_bo(drm_gem_object_lookup(file, args->handle)); 1952 if (&obj->base == NULL) { 1953 ret = -ENOENT; 1954 goto unlock; 1955 } 1956 1957 /* Pinned buffers may be scanout, so flush the cache */ 1958 if (obj->pin_display) 1959 i915_gem_object_flush_cpu_write_domain(obj); 1960 1961 drm_gem_object_unreference(&obj->base); 1962 unlock: 1963 mutex_unlock(&dev->struct_mutex); 1964 return ret; 1965 } 1966 1967 /** 1968 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address 1969 * it is mapped to. 1970 * @dev: drm device 1971 * @data: ioctl data blob 1972 * @file: drm file 1973 * 1974 * While the mapping holds a reference on the contents of the object, it doesn't 1975 * imply a ref on the object itself. 1976 * 1977 * IMPORTANT: 1978 * 1979 * DRM driver writers who look a this function as an example for how to do GEM 1980 * mmap support, please don't implement mmap support like here. The modern way 1981 * to implement DRM mmap support is with an mmap offset ioctl (like 1982 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly. 1983 * That way debug tooling like valgrind will understand what's going on, hiding 1984 * the mmap call in a driver private ioctl will break that. The i915 driver only 1985 * does cpu mmaps this way because we didn't know better. 1986 */ 1987 int 1988 i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1989 struct drm_file *file) 1990 { 1991 struct drm_i915_gem_mmap *args = data; 1992 struct drm_gem_object *obj; 1993 unsigned long addr; 1994 1995 struct proc *p = curproc; 1996 vm_map_t map = &p->p_vmspace->vm_map; 1997 vm_size_t size; 1998 int error = 0, rv; 1999 2000 if (args->flags & ~(I915_MMAP_WC)) 2001 return -EINVAL; 2002 2003 #if 0 2004 if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT)) 2005 return -ENODEV; 2006 #endif 2007 2008 obj = drm_gem_object_lookup(file, args->handle); 2009 if (obj == NULL) 2010 return -ENOENT; 2011 2012 if (args->size == 0) 2013 goto out; 2014 2015 size = round_page(args->size); 2016 if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) { 2017 error = -ENOMEM; 2018 goto out; 2019 } 2020 2021 /* prime objects have no backing filp to GEM mmap 2022 * pages from. 2023 */ 2024 if (!obj->filp) { 2025 drm_gem_object_unreference_unlocked(obj); 2026 return -EINVAL; 2027 } 2028 2029 /* 2030 * Call hint to ensure that NULL is not returned as a valid address 2031 * and to reduce vm_map traversals. XXX causes instability, use a 2032 * fixed low address as the start point instead to avoid the NULL 2033 * return issue. 2034 */ 2035 addr = PAGE_SIZE; 2036 2037 /* 2038 * Use 256KB alignment. It is unclear why this matters for a 2039 * virtual address but it appears to fix a number of application/X 2040 * crashes and kms console switching is much faster. 2041 */ 2042 vm_object_hold(obj->filp); 2043 vm_object_reference_locked(obj->filp); 2044 vm_object_drop(obj->filp); 2045 2046 /* Something gets wrong here: fails to mmap 4096 */ 2047 rv = vm_map_find(map, obj->filp, NULL, 2048 args->offset, &addr, args->size, 2049 256 * 1024, /* align */ 2050 TRUE, /* fitit */ 2051 VM_MAPTYPE_NORMAL, VM_SUBSYS_DRM_GEM, 2052 VM_PROT_READ | VM_PROT_WRITE, /* prot */ 2053 VM_PROT_READ | VM_PROT_WRITE, /* max */ 2054 MAP_SHARED /* cow */); 2055 if (rv != KERN_SUCCESS) { 2056 vm_object_deallocate(obj->filp); 2057 error = -vm_mmap_to_errno(rv); 2058 } else { 2059 args->addr_ptr = (uint64_t)addr; 2060 } 2061 out: 2062 drm_gem_object_unreference_unlocked(obj); 2063 return (error); 2064 } 2065 2066 /** 2067 * i915_gem_fault - fault a page into the GTT 2068 * 2069 * vm_obj is locked on entry and expected to be locked on return. 2070 * 2071 * The vm_pager has placemarked the object with an anonymous memory page 2072 * which we must replace atomically to avoid races against concurrent faults 2073 * on the same page. XXX we currently are unable to do this atomically. 2074 * 2075 * If we are to return an error we should not touch the anonymous page, 2076 * the caller will deallocate it. 2077 * 2078 * XXX Most GEM calls appear to be interruptable, but we can't hard loop 2079 * in that case. Release all resources and wait 1 tick before retrying. 2080 * This is a huge problem which needs to be fixed by getting rid of most 2081 * of the interruptability. The linux code does not retry but does appear 2082 * to have some sort of mechanism (VM_FAULT_NOPAGE ?) for the higher level 2083 * to be able to retry. 2084 * 2085 * -- 2086 * @vma: VMA in question 2087 * @vmf: fault info 2088 * 2089 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped 2090 * from userspace. The fault handler takes care of binding the object to 2091 * the GTT (if needed), allocating and programming a fence register (again, 2092 * only if needed based on whether the old reg is still valid or the object 2093 * is tiled) and inserting a new PTE into the faulting process. 2094 * 2095 * Note that the faulting process may involve evicting existing objects 2096 * from the GTT and/or fence registers to make room. So performance may 2097 * suffer if the GTT working set is large or there are few fence registers 2098 * left. 2099 * 2100 * vm_obj is locked on entry and expected to be locked on return. The VM 2101 * pager has placed an anonymous memory page at (obj,offset) which we have 2102 * to replace. 2103 */ 2104 int i915_gem_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, vm_page_t *mres) 2105 { 2106 struct drm_i915_gem_object *obj = to_intel_bo(vm_obj->handle); 2107 struct drm_device *dev = obj->base.dev; 2108 struct drm_i915_private *dev_priv = to_i915(dev); 2109 struct i915_ggtt *ggtt = &dev_priv->ggtt; 2110 struct i915_ggtt_view view = i915_ggtt_view_normal; 2111 unsigned long page_offset; 2112 vm_page_t m; 2113 int ret = 0; 2114 bool write = !!(prot & VM_PROT_WRITE); 2115 2116 intel_runtime_pm_get(dev_priv); 2117 2118 /* We don't use vmf->pgoff since that has the fake offset */ 2119 page_offset = (unsigned long)offset; 2120 2121 /* 2122 * vm_fault() has supplied us with a busied page placeholding 2123 * the operation. This presents a lock order reversal issue 2124 * again i915_gem_release_mmap() for our device mutex. 2125 * 2126 * Deal with the problem by getting rid of the placeholder now, 2127 * and then dealing with the potential for a new placeholder when 2128 * we try to insert later. 2129 */ 2130 if (*mres != NULL) { 2131 m = *mres; 2132 *mres = NULL; 2133 if ((m->busy_count & PBUSY_LOCKED) == 0) 2134 kprintf("i915_gem_fault: Page was not busy\n"); 2135 else 2136 vm_page_remove(m); 2137 vm_page_free(m); 2138 } 2139 2140 m = NULL; 2141 2142 retry: 2143 ret = i915_mutex_lock_interruptible(dev); 2144 if (ret) 2145 goto out; 2146 2147 trace_i915_gem_object_fault(obj, page_offset, true, write); 2148 2149 /* Try to flush the object off the GPU first without holding the lock. 2150 * Upon reacquiring the lock, we will perform our sanity checks and then 2151 * repeat the flush holding the lock in the normal manner to catch cases 2152 * where we are gazumped. 2153 */ 2154 ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write); 2155 if (ret) 2156 goto unlock; 2157 2158 /* Access to snoopable pages through the GTT is incoherent. */ 2159 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) { 2160 ret = -EFAULT; 2161 goto unlock; 2162 } 2163 2164 /* Use a partial view if the object is bigger than the aperture. */ 2165 if (obj->base.size >= ggtt->mappable_end && 2166 obj->tiling_mode == I915_TILING_NONE) { 2167 #if 0 2168 static const unsigned int chunk_size = 256; // 1 MiB 2169 2170 memset(&view, 0, sizeof(view)); 2171 view.type = I915_GGTT_VIEW_PARTIAL; 2172 view.params.partial.offset = rounddown(page_offset, chunk_size); 2173 view.params.partial.size = 2174 min_t(unsigned int, 2175 chunk_size, 2176 (vma->vm_end - vma->vm_start)/PAGE_SIZE - 2177 view.params.partial.offset); 2178 #endif 2179 } 2180 2181 /* Now pin it into the GTT if needed */ 2182 ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE); 2183 if (ret) 2184 goto unlock; 2185 2186 ret = i915_gem_object_set_to_gtt_domain(obj, write); 2187 if (ret) 2188 goto unpin; 2189 2190 ret = i915_gem_object_get_fence(obj); 2191 if (ret) 2192 goto unpin; 2193 2194 /* 2195 * START FREEBSD MAGIC 2196 * 2197 * Add a pip count to avoid destruction and certain other 2198 * complex operations (such as collapses?) while unlocked. 2199 */ 2200 vm_object_pip_add(vm_obj, 1); 2201 2202 ret = 0; 2203 m = NULL; 2204 2205 /* 2206 * Since the object lock was dropped, another thread might have 2207 * faulted on the same GTT address and instantiated the mapping. 2208 * Recheck. 2209 */ 2210 m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 2211 if (m != NULL) { 2212 /* 2213 * Try to busy the page, retry on failure (non-zero ret). 2214 */ 2215 if (vm_page_busy_try(m, false)) { 2216 kprintf("i915_gem_fault: BUSY\n"); 2217 ret = -EINTR; 2218 goto unlock; 2219 } 2220 goto have_page; 2221 } 2222 /* 2223 * END FREEBSD MAGIC 2224 */ 2225 2226 obj->fault_mappable = true; 2227 2228 /* Finally, remap it using the new GTT offset */ 2229 m = vm_phys_fictitious_to_vm_page(ggtt->mappable_base + 2230 i915_gem_obj_ggtt_offset_view(obj, &view) + offset); 2231 if (m == NULL) { 2232 ret = -EFAULT; 2233 goto unpin; 2234 } 2235 KASSERT((m->flags & PG_FICTITIOUS) != 0, ("not fictitious %p", m)); 2236 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 2237 2238 /* 2239 * Try to busy the page. Fails on non-zero return. 2240 */ 2241 if (vm_page_busy_try(m, false)) { 2242 kprintf("i915_gem_fault: BUSY(2)\n"); 2243 ret = -EINTR; 2244 goto unpin; 2245 } 2246 m->valid = VM_PAGE_BITS_ALL; 2247 2248 #if 1 2249 /* 2250 * This should always work since we already checked via a lookup 2251 * above. 2252 */ 2253 if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset)) == FALSE) { 2254 kprintf("i915:gem_fault: page %p,%jd already in object\n", 2255 vm_obj, 2256 OFF_TO_IDX(offset)); 2257 vm_page_wakeup(m); 2258 ret = -EINTR; 2259 goto unpin; 2260 } 2261 #else 2262 /* NOT COMPILED ATM */ 2263 if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) { 2264 /* Overriding existing pages in partial view does not cause 2265 * us any trouble as TLBs are still valid because the fault 2266 * is due to userspace losing part of the mapping or never 2267 * having accessed it before (at this partials' range). 2268 */ 2269 unsigned long base = vma->vm_start + 2270 (view.params.partial.offset << PAGE_SHIFT); 2271 unsigned int i; 2272 2273 for (i = 0; i < view.params.partial.size; i++) { 2274 ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i); 2275 if (ret) 2276 break; 2277 } 2278 2279 obj->fault_mappable = true; 2280 } else { 2281 if (!obj->fault_mappable) { 2282 unsigned long size = min_t(unsigned long, 2283 vma->vm_end - vma->vm_start, 2284 obj->base.size); 2285 int i; 2286 2287 for (i = 0; i < size >> PAGE_SHIFT; i++) { 2288 ret = vm_insert_pfn(vma, 2289 (unsigned long)vma->vm_start + i * PAGE_SIZE, 2290 pfn + i); 2291 if (ret) 2292 break; 2293 } 2294 2295 obj->fault_mappable = true; 2296 } else 2297 ret = vm_insert_pfn(vma, 2298 (unsigned long)vmf->virtual_address, 2299 pfn + page_offset); 2300 } 2301 #endif 2302 2303 have_page: 2304 *mres = m; 2305 2306 i915_gem_object_ggtt_unpin_view(obj, &view); 2307 mutex_unlock(&dev->struct_mutex); 2308 ret = VM_PAGER_OK; 2309 goto done; 2310 2311 /* 2312 * ALTERNATIVE ERROR RETURN. 2313 * 2314 * OBJECT EXPECTED TO BE LOCKED. 2315 */ 2316 unpin: 2317 i915_gem_object_ggtt_unpin_view(obj, &view); 2318 unlock: 2319 mutex_unlock(&dev->struct_mutex); 2320 out: 2321 switch (ret) { 2322 case -EIO: 2323 /* 2324 * We eat errors when the gpu is terminally wedged to avoid 2325 * userspace unduly crashing (gl has no provisions for mmaps to 2326 * fail). But any other -EIO isn't ours (e.g. swap in failure) 2327 * and so needs to be reported. 2328 */ 2329 if (!i915_terminally_wedged(&dev_priv->gpu_error)) { 2330 // ret = VM_FAULT_SIGBUS; 2331 break; 2332 } 2333 case -EAGAIN: 2334 /* 2335 * EAGAIN means the gpu is hung and we'll wait for the error 2336 * handler to reset everything when re-faulting in 2337 * i915_mutex_lock_interruptible. 2338 */ 2339 case -ERESTARTSYS: 2340 case -EINTR: 2341 VM_OBJECT_UNLOCK(vm_obj); 2342 int dummy; 2343 tsleep(&dummy, 0, "delay", 1); /* XXX */ 2344 VM_OBJECT_LOCK(vm_obj); 2345 goto retry; 2346 default: 2347 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret); 2348 ret = VM_PAGER_ERROR; 2349 break; 2350 } 2351 2352 done: 2353 vm_object_pip_wakeup(vm_obj); 2354 2355 intel_runtime_pm_put(dev_priv); 2356 return ret; 2357 } 2358 2359 /** 2360 * i915_gem_release_mmap - remove physical page mappings 2361 * @obj: obj in question 2362 * 2363 * Preserve the reservation of the mmapping with the DRM core code, but 2364 * relinquish ownership of the pages back to the system. 2365 * 2366 * It is vital that we remove the page mapping if we have mapped a tiled 2367 * object through the GTT and then lose the fence register due to 2368 * resource pressure. Similarly if the object has been moved out of the 2369 * aperture, than pages mapped into userspace must be revoked. Removing the 2370 * mapping will then trigger a page fault on the next user access, allowing 2371 * fixup by i915_gem_fault(). 2372 */ 2373 void 2374 i915_gem_release_mmap(struct drm_i915_gem_object *obj) 2375 { 2376 vm_object_t devobj; 2377 vm_page_t m; 2378 int i, page_count; 2379 2380 /* Serialisation between user GTT access and our code depends upon 2381 * revoking the CPU's PTE whilst the mutex is held. The next user 2382 * pagefault then has to wait until we release the mutex. 2383 */ 2384 lockdep_assert_held(&obj->base.dev->struct_mutex); 2385 2386 if (!obj->fault_mappable) 2387 return; 2388 2389 devobj = cdev_pager_lookup(obj); 2390 if (devobj != NULL) { 2391 page_count = OFF_TO_IDX(obj->base.size); 2392 2393 VM_OBJECT_LOCK(devobj); 2394 for (i = 0; i < page_count; i++) { 2395 m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm"); 2396 if (m == NULL) 2397 continue; 2398 cdev_pager_free_page(devobj, m); 2399 } 2400 VM_OBJECT_UNLOCK(devobj); 2401 vm_object_deallocate(devobj); 2402 } 2403 2404 /* Ensure that the CPU's PTE are revoked and there are not outstanding 2405 * memory transactions from userspace before we return. The TLB 2406 * flushing implied above by changing the PTE above *should* be 2407 * sufficient, an extra barrier here just provides us with a bit 2408 * of paranoid documentation about our requirement to serialise 2409 * memory writes before touching registers / GSM. 2410 */ 2411 wmb(); 2412 2413 obj->fault_mappable = false; 2414 } 2415 2416 void 2417 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv) 2418 { 2419 struct drm_i915_gem_object *obj; 2420 2421 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) 2422 i915_gem_release_mmap(obj); 2423 } 2424 2425 uint32_t 2426 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 2427 { 2428 uint32_t gtt_size; 2429 2430 if (INTEL_INFO(dev)->gen >= 4 || 2431 tiling_mode == I915_TILING_NONE) 2432 return size; 2433 2434 /* Previous chips need a power-of-two fence region when tiling */ 2435 if (IS_GEN3(dev)) 2436 gtt_size = 1024*1024; 2437 else 2438 gtt_size = 512*1024; 2439 2440 while (gtt_size < size) 2441 gtt_size <<= 1; 2442 2443 return gtt_size; 2444 } 2445 2446 /** 2447 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 2448 * @dev: drm device 2449 * @size: object size 2450 * @tiling_mode: tiling mode 2451 * @fenced: is fenced alignemned required or not 2452 * 2453 * Return the required GTT alignment for an object, taking into account 2454 * potential fence register mapping. 2455 */ 2456 uint32_t 2457 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, 2458 int tiling_mode, bool fenced) 2459 { 2460 /* 2461 * Minimum alignment is 4k (GTT page size), but might be greater 2462 * if a fence register is needed for the object. 2463 */ 2464 if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) || 2465 tiling_mode == I915_TILING_NONE) 2466 return 4096; 2467 2468 /* 2469 * Previous chips need to be aligned to the size of the smallest 2470 * fence register that can contain the object. 2471 */ 2472 return i915_gem_get_gtt_size(dev, size, tiling_mode); 2473 } 2474 2475 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj) 2476 { 2477 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 2478 int ret; 2479 2480 dev_priv->mm.shrinker_no_lock_stealing = true; 2481 2482 ret = drm_gem_create_mmap_offset(&obj->base); 2483 if (ret != -ENOSPC) 2484 goto out; 2485 2486 /* Badly fragmented mmap space? The only way we can recover 2487 * space is by destroying unwanted objects. We can't randomly release 2488 * mmap_offsets as userspace expects them to be persistent for the 2489 * lifetime of the objects. The closest we can is to release the 2490 * offsets on purgeable objects by truncating it and marking it purged, 2491 * which prevents userspace from ever using that object again. 2492 */ 2493 i915_gem_shrink(dev_priv, 2494 obj->base.size >> PAGE_SHIFT, 2495 I915_SHRINK_BOUND | 2496 I915_SHRINK_UNBOUND | 2497 I915_SHRINK_PURGEABLE); 2498 ret = drm_gem_create_mmap_offset(&obj->base); 2499 if (ret != -ENOSPC) 2500 goto out; 2501 2502 i915_gem_shrink_all(dev_priv); 2503 ret = drm_gem_create_mmap_offset(&obj->base); 2504 out: 2505 dev_priv->mm.shrinker_no_lock_stealing = false; 2506 2507 return ret; 2508 } 2509 2510 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj) 2511 { 2512 drm_gem_free_mmap_offset(&obj->base); 2513 } 2514 2515 int 2516 i915_gem_mmap_gtt(struct drm_file *file, 2517 struct drm_device *dev, 2518 uint32_t handle, 2519 uint64_t *offset) 2520 { 2521 struct drm_i915_gem_object *obj; 2522 int ret; 2523 2524 ret = i915_mutex_lock_interruptible(dev); 2525 if (ret) 2526 return ret; 2527 2528 obj = to_intel_bo(drm_gem_object_lookup(file, handle)); 2529 if (&obj->base == NULL) { 2530 ret = -ENOENT; 2531 goto unlock; 2532 } 2533 2534 if (obj->madv != I915_MADV_WILLNEED) { 2535 DRM_DEBUG("Attempting to mmap a purgeable buffer\n"); 2536 ret = -EFAULT; 2537 goto out; 2538 } 2539 2540 ret = i915_gem_object_create_mmap_offset(obj); 2541 if (ret) 2542 goto out; 2543 2544 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 2545 DRM_GEM_MAPPING_KEY; 2546 2547 out: 2548 drm_gem_object_unreference(&obj->base); 2549 unlock: 2550 mutex_unlock(&dev->struct_mutex); 2551 return ret; 2552 } 2553 2554 /** 2555 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 2556 * @dev: DRM device 2557 * @data: GTT mapping ioctl data 2558 * @file: GEM object info 2559 * 2560 * Simply returns the fake offset to userspace so it can mmap it. 2561 * The mmap call will end up in drm_gem_mmap(), which will set things 2562 * up so we can get faults in the handler above. 2563 * 2564 * The fault handler will take care of binding the object into the GTT 2565 * (since it may have been evicted to make room for something), allocating 2566 * a fence register, and mapping the appropriate aperture address into 2567 * userspace. 2568 */ 2569 int 2570 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 2571 struct drm_file *file) 2572 { 2573 struct drm_i915_gem_mmap_gtt *args = data; 2574 2575 return i915_gem_mmap_gtt(file, dev, args->handle, (uint64_t *)&args->offset); 2576 } 2577 2578 /* Immediately discard the backing storage */ 2579 static void 2580 i915_gem_object_truncate(struct drm_i915_gem_object *obj) 2581 { 2582 vm_object_t vm_obj; 2583 2584 vm_obj = obj->base.filp; 2585 VM_OBJECT_LOCK(vm_obj); 2586 vm_object_page_remove(vm_obj, 0, 0, false); 2587 VM_OBJECT_UNLOCK(vm_obj); 2588 2589 obj->madv = __I915_MADV_PURGED; 2590 } 2591 2592 /* Try to discard unwanted pages */ 2593 static void 2594 i915_gem_object_invalidate(struct drm_i915_gem_object *obj) 2595 { 2596 #if 0 2597 struct address_space *mapping; 2598 #endif 2599 2600 switch (obj->madv) { 2601 case I915_MADV_DONTNEED: 2602 i915_gem_object_truncate(obj); 2603 case __I915_MADV_PURGED: 2604 return; 2605 } 2606 2607 if (obj->base.filp == NULL) 2608 return; 2609 2610 #if 0 2611 mapping = file_inode(obj->base.filp)->i_mapping, 2612 #endif 2613 invalidate_mapping_pages(obj->base.filp, 0, (loff_t)-1); 2614 } 2615 2616 static void 2617 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 2618 { 2619 struct sgt_iter sgt_iter; 2620 struct page *page; 2621 int ret; 2622 2623 BUG_ON(obj->madv == __I915_MADV_PURGED); 2624 2625 ret = i915_gem_object_set_to_cpu_domain(obj, true); 2626 if (WARN_ON(ret)) { 2627 /* In the event of a disaster, abandon all caches and 2628 * hope for the best. 2629 */ 2630 i915_gem_clflush_object(obj, true); 2631 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; 2632 } 2633 2634 i915_gem_gtt_finish_object(obj); 2635 2636 if (i915_gem_object_needs_bit17_swizzle(obj)) 2637 i915_gem_object_save_bit_17_swizzle(obj); 2638 2639 if (obj->madv == I915_MADV_DONTNEED) 2640 obj->dirty = 0; 2641 2642 for_each_sgt_page(page, sgt_iter, obj->pages) { 2643 if (obj->dirty) 2644 set_page_dirty(page); 2645 2646 if (obj->madv == I915_MADV_WILLNEED) 2647 mark_page_accessed(page); 2648 2649 vm_page_busy_wait((struct vm_page *)page, FALSE, "i915gem"); 2650 vm_page_unwire((struct vm_page *)page, 1); 2651 vm_page_wakeup((struct vm_page *)page); 2652 } 2653 obj->dirty = 0; 2654 2655 sg_free_table(obj->pages); 2656 kfree(obj->pages); 2657 } 2658 2659 int 2660 i915_gem_object_put_pages(struct drm_i915_gem_object *obj) 2661 { 2662 const struct drm_i915_gem_object_ops *ops = obj->ops; 2663 2664 if (obj->pages == NULL) 2665 return 0; 2666 2667 if (obj->pages_pin_count) 2668 return -EBUSY; 2669 2670 BUG_ON(i915_gem_obj_bound_any(obj)); 2671 2672 /* ->put_pages might need to allocate memory for the bit17 swizzle 2673 * array, hence protect them from being reaped by removing them from gtt 2674 * lists early. */ 2675 list_del(&obj->global_list); 2676 2677 if (obj->mapping) { 2678 if (is_vmalloc_addr(obj->mapping)) 2679 vunmap(obj->mapping); 2680 else 2681 kunmap(kmap_to_page(obj->mapping)); 2682 obj->mapping = NULL; 2683 } 2684 2685 ops->put_pages(obj); 2686 obj->pages = NULL; 2687 2688 i915_gem_object_invalidate(obj); 2689 2690 return 0; 2691 } 2692 2693 static int 2694 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) 2695 { 2696 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 2697 int page_count, i; 2698 vm_object_t vm_obj; 2699 struct sg_table *st; 2700 struct scatterlist *sg; 2701 struct sgt_iter sgt_iter; 2702 struct page *page; 2703 unsigned long last_pfn = 0; /* suppress gcc warning */ 2704 int ret; 2705 2706 /* Assert that the object is not currently in any GPU domain. As it 2707 * wasn't in the GTT, there shouldn't be any way it could have been in 2708 * a GPU cache 2709 */ 2710 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); 2711 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); 2712 2713 st = kmalloc(sizeof(*st), M_DRM, GFP_KERNEL); 2714 if (st == NULL) 2715 return -ENOMEM; 2716 2717 page_count = obj->base.size / PAGE_SIZE; 2718 if (sg_alloc_table(st, page_count, GFP_KERNEL)) { 2719 kfree(st); 2720 return -ENOMEM; 2721 } 2722 2723 /* Get the list of pages out of our struct file. They'll be pinned 2724 * at this point until we release them. 2725 * 2726 * Fail silently without starting the shrinker 2727 */ 2728 vm_obj = obj->base.filp; 2729 VM_OBJECT_LOCK(vm_obj); 2730 sg = st->sgl; 2731 st->nents = 0; 2732 for (i = 0; i < page_count; i++) { 2733 page = shmem_read_mapping_page(vm_obj, i); 2734 if (IS_ERR(page)) { 2735 i915_gem_shrink(dev_priv, 2736 page_count, 2737 I915_SHRINK_BOUND | 2738 I915_SHRINK_UNBOUND | 2739 I915_SHRINK_PURGEABLE); 2740 page = shmem_read_mapping_page(vm_obj, i); 2741 } 2742 if (IS_ERR(page)) { 2743 /* We've tried hard to allocate the memory by reaping 2744 * our own buffer, now let the real VM do its job and 2745 * go down in flames if truly OOM. 2746 */ 2747 i915_gem_shrink_all(dev_priv); 2748 page = shmem_read_mapping_page(vm_obj, i); 2749 if (IS_ERR(page)) { 2750 ret = PTR_ERR(page); 2751 goto err_pages; 2752 } 2753 } 2754 #ifdef CONFIG_SWIOTLB 2755 if (swiotlb_nr_tbl()) { 2756 st->nents++; 2757 sg_set_page(sg, page, PAGE_SIZE, 0); 2758 sg = sg_next(sg); 2759 continue; 2760 } 2761 #endif 2762 if (!i || page_to_pfn(page) != last_pfn + 1) { 2763 if (i) 2764 sg = sg_next(sg); 2765 st->nents++; 2766 sg_set_page(sg, page, PAGE_SIZE, 0); 2767 } else { 2768 sg->length += PAGE_SIZE; 2769 } 2770 last_pfn = page_to_pfn(page); 2771 2772 /* Check that the i965g/gm workaround works. */ 2773 } 2774 #ifdef CONFIG_SWIOTLB 2775 if (!swiotlb_nr_tbl()) 2776 #endif 2777 sg_mark_end(sg); 2778 obj->pages = st; 2779 VM_OBJECT_UNLOCK(vm_obj); 2780 2781 ret = i915_gem_gtt_prepare_object(obj); 2782 if (ret) 2783 goto err_pages; 2784 2785 if (i915_gem_object_needs_bit17_swizzle(obj)) 2786 i915_gem_object_do_bit_17_swizzle(obj); 2787 2788 if (obj->tiling_mode != I915_TILING_NONE && 2789 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) 2790 i915_gem_object_pin_pages(obj); 2791 2792 return 0; 2793 2794 err_pages: 2795 sg_mark_end(sg); 2796 for_each_sgt_page(page, sgt_iter, st) 2797 { 2798 struct vm_page *vmp = (struct vm_page *)page; 2799 vm_page_busy_wait(vmp, FALSE, "i915gem"); 2800 vm_page_unwire(vmp, 0); 2801 vm_page_wakeup(vmp); 2802 } 2803 VM_OBJECT_UNLOCK(vm_obj); 2804 sg_free_table(st); 2805 kfree(st); 2806 2807 /* shmemfs first checks if there is enough memory to allocate the page 2808 * and reports ENOSPC should there be insufficient, along with the usual 2809 * ENOMEM for a genuine allocation failure. 2810 * 2811 * We use ENOSPC in our driver to mean that we have run out of aperture 2812 * space and so want to translate the error from shmemfs back to our 2813 * usual understanding of ENOMEM. 2814 */ 2815 if (ret == -ENOSPC) 2816 ret = -ENOMEM; 2817 2818 return ret; 2819 } 2820 2821 /* Ensure that the associated pages are gathered from the backing storage 2822 * and pinned into our object. i915_gem_object_get_pages() may be called 2823 * multiple times before they are released by a single call to 2824 * i915_gem_object_put_pages() - once the pages are no longer referenced 2825 * either as a result of memory pressure (reaping pages under the shrinker) 2826 * or as the object is itself released. 2827 */ 2828 int 2829 i915_gem_object_get_pages(struct drm_i915_gem_object *obj) 2830 { 2831 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 2832 const struct drm_i915_gem_object_ops *ops = obj->ops; 2833 int ret; 2834 2835 if (obj->pages) 2836 return 0; 2837 2838 if (obj->madv != I915_MADV_WILLNEED) { 2839 DRM_DEBUG("Attempting to obtain a purgeable object\n"); 2840 return -EFAULT; 2841 } 2842 2843 BUG_ON(obj->pages_pin_count); 2844 2845 ret = ops->get_pages(obj); 2846 if (ret) 2847 return ret; 2848 2849 list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list); 2850 2851 obj->get_page.sg = obj->pages->sgl; 2852 obj->get_page.last = 0; 2853 2854 return 0; 2855 } 2856 2857 /* The 'mapping' part of i915_gem_object_pin_map() below */ 2858 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj) 2859 { 2860 unsigned long n_pages = obj->base.size >> PAGE_SHIFT; 2861 struct sg_table *sgt = obj->pages; 2862 struct sgt_iter sgt_iter; 2863 struct page *page; 2864 struct page *stack_pages[32]; 2865 struct page **pages = stack_pages; 2866 unsigned long i = 0; 2867 void *addr; 2868 2869 /* A single page can always be kmapped */ 2870 if (n_pages == 1) 2871 return kmap(sg_page(sgt->sgl)); 2872 2873 if (n_pages > ARRAY_SIZE(stack_pages)) { 2874 /* Too big for stack -- allocate temporary array instead */ 2875 pages = drm_malloc_gfp(n_pages, sizeof(*pages), GFP_TEMPORARY); 2876 if (!pages) 2877 return NULL; 2878 } 2879 2880 for_each_sgt_page(page, sgt_iter, sgt) 2881 pages[i++] = page; 2882 2883 /* Check that we have the expected number of pages */ 2884 GEM_BUG_ON(i != n_pages); 2885 2886 addr = vmap(pages, n_pages, 0, PAGE_KERNEL); 2887 2888 if (pages != stack_pages) 2889 drm_free_large(pages); 2890 2891 return addr; 2892 } 2893 2894 /* get, pin, and map the pages of the object into kernel space */ 2895 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj) 2896 { 2897 int ret; 2898 2899 lockdep_assert_held(&obj->base.dev->struct_mutex); 2900 2901 ret = i915_gem_object_get_pages(obj); 2902 if (ret) 2903 return ERR_PTR(ret); 2904 2905 i915_gem_object_pin_pages(obj); 2906 2907 if (!obj->mapping) { 2908 obj->mapping = i915_gem_object_map(obj); 2909 if (!obj->mapping) { 2910 i915_gem_object_unpin_pages(obj); 2911 return ERR_PTR(-ENOMEM); 2912 } 2913 } 2914 2915 return obj->mapping; 2916 } 2917 2918 void i915_vma_move_to_active(struct i915_vma *vma, 2919 struct drm_i915_gem_request *req) 2920 { 2921 struct drm_i915_gem_object *obj = vma->obj; 2922 struct intel_engine_cs *engine; 2923 2924 engine = i915_gem_request_get_engine(req); 2925 2926 /* Add a reference if we're newly entering the active list. */ 2927 if (obj->active == 0) 2928 drm_gem_object_reference(&obj->base); 2929 obj->active |= intel_engine_flag(engine); 2930 2931 list_move_tail(&obj->engine_list[engine->id], &engine->active_list); 2932 i915_gem_request_assign(&obj->last_read_req[engine->id], req); 2933 2934 list_move_tail(&vma->vm_link, &vma->vm->active_list); 2935 } 2936 2937 static void 2938 i915_gem_object_retire__write(struct drm_i915_gem_object *obj) 2939 { 2940 GEM_BUG_ON(obj->last_write_req == NULL); 2941 GEM_BUG_ON(!(obj->active & intel_engine_flag(obj->last_write_req->engine))); 2942 2943 i915_gem_request_assign(&obj->last_write_req, NULL); 2944 intel_fb_obj_flush(obj, true, ORIGIN_CS); 2945 } 2946 2947 static void 2948 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring) 2949 { 2950 struct i915_vma *vma; 2951 2952 GEM_BUG_ON(obj->last_read_req[ring] == NULL); 2953 GEM_BUG_ON(!(obj->active & (1 << ring))); 2954 2955 list_del_init(&obj->engine_list[ring]); 2956 i915_gem_request_assign(&obj->last_read_req[ring], NULL); 2957 2958 if (obj->last_write_req && obj->last_write_req->engine->id == ring) 2959 i915_gem_object_retire__write(obj); 2960 2961 obj->active &= ~(1 << ring); 2962 if (obj->active) 2963 return; 2964 2965 /* Bump our place on the bound list to keep it roughly in LRU order 2966 * so that we don't steal from recently used but inactive objects 2967 * (unless we are forced to ofc!) 2968 */ 2969 list_move_tail(&obj->global_list, 2970 &to_i915(obj->base.dev)->mm.bound_list); 2971 2972 list_for_each_entry(vma, &obj->vma_list, obj_link) { 2973 if (!list_empty(&vma->vm_link)) 2974 list_move_tail(&vma->vm_link, &vma->vm->inactive_list); 2975 } 2976 2977 i915_gem_request_assign(&obj->last_fenced_req, NULL); 2978 drm_gem_object_unreference(&obj->base); 2979 } 2980 2981 static int 2982 i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno) 2983 { 2984 struct intel_engine_cs *engine; 2985 int ret; 2986 2987 /* Carefully retire all requests without writing to the rings */ 2988 for_each_engine(engine, dev_priv) { 2989 ret = intel_engine_idle(engine); 2990 if (ret) 2991 return ret; 2992 } 2993 i915_gem_retire_requests(dev_priv); 2994 2995 /* Finally reset hw state */ 2996 for_each_engine(engine, dev_priv) 2997 intel_ring_init_seqno(engine, seqno); 2998 2999 return 0; 3000 } 3001 3002 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) 3003 { 3004 struct drm_i915_private *dev_priv = dev->dev_private; 3005 int ret; 3006 3007 if (seqno == 0) 3008 return -EINVAL; 3009 3010 /* HWS page needs to be set less than what we 3011 * will inject to ring 3012 */ 3013 ret = i915_gem_init_seqno(dev_priv, seqno - 1); 3014 if (ret) 3015 return ret; 3016 3017 /* Carefully set the last_seqno value so that wrap 3018 * detection still works 3019 */ 3020 dev_priv->next_seqno = seqno; 3021 dev_priv->last_seqno = seqno - 1; 3022 if (dev_priv->last_seqno == 0) 3023 dev_priv->last_seqno--; 3024 3025 return 0; 3026 } 3027 3028 int 3029 i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno) 3030 { 3031 /* reserve 0 for non-seqno */ 3032 if (dev_priv->next_seqno == 0) { 3033 int ret = i915_gem_init_seqno(dev_priv, 0); 3034 if (ret) 3035 return ret; 3036 3037 dev_priv->next_seqno = 1; 3038 } 3039 3040 *seqno = dev_priv->last_seqno = dev_priv->next_seqno++; 3041 return 0; 3042 } 3043 3044 /* 3045 * NB: This function is not allowed to fail. Doing so would mean the the 3046 * request is not being tracked for completion but the work itself is 3047 * going to happen on the hardware. This would be a Bad Thing(tm). 3048 */ 3049 void __i915_add_request(struct drm_i915_gem_request *request, 3050 struct drm_i915_gem_object *obj, 3051 bool flush_caches) 3052 { 3053 struct intel_engine_cs *engine; 3054 struct drm_i915_private *dev_priv; 3055 struct intel_ringbuffer *ringbuf; 3056 u32 request_start; 3057 u32 reserved_tail; 3058 int ret; 3059 3060 if (WARN_ON(request == NULL)) 3061 return; 3062 3063 engine = request->engine; 3064 dev_priv = request->i915; 3065 ringbuf = request->ringbuf; 3066 3067 /* 3068 * To ensure that this call will not fail, space for its emissions 3069 * should already have been reserved in the ring buffer. Let the ring 3070 * know that it is time to use that space up. 3071 */ 3072 request_start = intel_ring_get_tail(ringbuf); 3073 reserved_tail = request->reserved_space; 3074 request->reserved_space = 0; 3075 3076 /* 3077 * Emit any outstanding flushes - execbuf can fail to emit the flush 3078 * after having emitted the batchbuffer command. Hence we need to fix 3079 * things up similar to emitting the lazy request. The difference here 3080 * is that the flush _must_ happen before the next request, no matter 3081 * what. 3082 */ 3083 if (flush_caches) { 3084 if (i915.enable_execlists) 3085 ret = logical_ring_flush_all_caches(request); 3086 else 3087 ret = intel_ring_flush_all_caches(request); 3088 /* Not allowed to fail! */ 3089 WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret); 3090 } 3091 3092 trace_i915_gem_request_add(request); 3093 3094 request->head = request_start; 3095 3096 /* Whilst this request exists, batch_obj will be on the 3097 * active_list, and so will hold the active reference. Only when this 3098 * request is retired will the the batch_obj be moved onto the 3099 * inactive_list and lose its active reference. Hence we do not need 3100 * to explicitly hold another reference here. 3101 */ 3102 request->batch_obj = obj; 3103 3104 /* Seal the request and mark it as pending execution. Note that 3105 * we may inspect this state, without holding any locks, during 3106 * hangcheck. Hence we apply the barrier to ensure that we do not 3107 * see a more recent value in the hws than we are tracking. 3108 */ 3109 request->emitted_jiffies = jiffies; 3110 request->previous_seqno = engine->last_submitted_seqno; 3111 smp_store_mb(engine->last_submitted_seqno, request->seqno); 3112 list_add_tail(&request->list, &engine->request_list); 3113 3114 /* Record the position of the start of the request so that 3115 * should we detect the updated seqno part-way through the 3116 * GPU processing the request, we never over-estimate the 3117 * position of the head. 3118 */ 3119 request->postfix = intel_ring_get_tail(ringbuf); 3120 3121 if (i915.enable_execlists) 3122 ret = engine->emit_request(request); 3123 else { 3124 ret = engine->add_request(request); 3125 3126 request->tail = intel_ring_get_tail(ringbuf); 3127 } 3128 /* Not allowed to fail! */ 3129 WARN(ret, "emit|add_request failed: %d!\n", ret); 3130 3131 queue_delayed_work(dev_priv->wq, 3132 &dev_priv->mm.retire_work, 3133 round_jiffies_up_relative(HZ)); 3134 intel_mark_busy(dev_priv); 3135 3136 /* Sanity check that the reserved size was large enough. */ 3137 ret = intel_ring_get_tail(ringbuf) - request_start; 3138 if (ret < 0) 3139 ret += ringbuf->size; 3140 WARN_ONCE(ret > reserved_tail, 3141 "Not enough space reserved (%d bytes) " 3142 "for adding the request (%d bytes)\n", 3143 reserved_tail, ret); 3144 } 3145 3146 static bool i915_context_is_banned(struct drm_i915_private *dev_priv, 3147 const struct i915_gem_context *ctx) 3148 { 3149 unsigned long elapsed; 3150 3151 elapsed = get_seconds() - ctx->hang_stats.guilty_ts; 3152 3153 if (ctx->hang_stats.banned) 3154 return true; 3155 3156 if (ctx->hang_stats.ban_period_seconds && 3157 elapsed <= ctx->hang_stats.ban_period_seconds) { 3158 if (!i915_gem_context_is_default(ctx)) { 3159 DRM_DEBUG("context hanging too fast, banning!\n"); 3160 return true; 3161 } else if (i915_stop_ring_allow_ban(dev_priv)) { 3162 if (i915_stop_ring_allow_warn(dev_priv)) 3163 DRM_ERROR("gpu hanging too fast, banning!\n"); 3164 return true; 3165 } 3166 } 3167 3168 return false; 3169 } 3170 3171 static void i915_set_reset_status(struct drm_i915_private *dev_priv, 3172 struct i915_gem_context *ctx, 3173 const bool guilty) 3174 { 3175 struct i915_ctx_hang_stats *hs; 3176 3177 if (WARN_ON(!ctx)) 3178 return; 3179 3180 hs = &ctx->hang_stats; 3181 3182 if (guilty) { 3183 hs->banned = i915_context_is_banned(dev_priv, ctx); 3184 hs->batch_active++; 3185 hs->guilty_ts = get_seconds(); 3186 } else { 3187 hs->batch_pending++; 3188 } 3189 } 3190 3191 void i915_gem_request_free(struct kref *req_ref) 3192 { 3193 struct drm_i915_gem_request *req = container_of(req_ref, 3194 typeof(*req), ref); 3195 kmem_cache_free(req->i915->requests, req); 3196 } 3197 3198 static inline int 3199 __i915_gem_request_alloc(struct intel_engine_cs *engine, 3200 struct i915_gem_context *ctx, 3201 struct drm_i915_gem_request **req_out) 3202 { 3203 struct drm_i915_private *dev_priv = engine->i915; 3204 unsigned reset_counter = i915_reset_counter(&dev_priv->gpu_error); 3205 struct drm_i915_gem_request *req; 3206 int ret; 3207 3208 if (!req_out) 3209 return -EINVAL; 3210 3211 *req_out = NULL; 3212 3213 /* ABI: Before userspace accesses the GPU (e.g. execbuffer), report 3214 * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex 3215 * and restart. 3216 */ 3217 ret = i915_gem_check_wedge(reset_counter, dev_priv->mm.interruptible); 3218 if (ret) 3219 return ret; 3220 3221 req = kzalloc(sizeof(*req), GFP_KERNEL); 3222 if (req == NULL) 3223 return -ENOMEM; 3224 3225 ret = i915_gem_get_seqno(engine->i915, &req->seqno); 3226 if (ret) 3227 goto err; 3228 3229 kref_init(&req->ref); 3230 req->i915 = dev_priv; 3231 req->engine = engine; 3232 req->ctx = ctx; 3233 i915_gem_context_reference(req->ctx); 3234 3235 /* 3236 * Reserve space in the ring buffer for all the commands required to 3237 * eventually emit this request. This is to guarantee that the 3238 * i915_add_request() call can't fail. Note that the reserve may need 3239 * to be redone if the request is not actually submitted straight 3240 * away, e.g. because a GPU scheduler has deferred it. 3241 */ 3242 req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST; 3243 3244 if (i915.enable_execlists) 3245 ret = intel_logical_ring_alloc_request_extras(req); 3246 else 3247 ret = intel_ring_alloc_request_extras(req); 3248 if (ret) 3249 goto err_ctx; 3250 3251 *req_out = req; 3252 return 0; 3253 3254 err_ctx: 3255 i915_gem_context_unreference(ctx); 3256 err: 3257 kfree(req); 3258 return ret; 3259 } 3260 3261 /** 3262 * i915_gem_request_alloc - allocate a request structure 3263 * 3264 * @engine: engine that we wish to issue the request on. 3265 * @ctx: context that the request will be associated with. 3266 * This can be NULL if the request is not directly related to 3267 * any specific user context, in which case this function will 3268 * choose an appropriate context to use. 3269 * 3270 * Returns a pointer to the allocated request if successful, 3271 * or an error code if not. 3272 */ 3273 struct drm_i915_gem_request * 3274 i915_gem_request_alloc(struct intel_engine_cs *engine, 3275 struct i915_gem_context *ctx) 3276 { 3277 struct drm_i915_gem_request *req; 3278 int err; 3279 3280 if (ctx == NULL) 3281 ctx = engine->i915->kernel_context; 3282 err = __i915_gem_request_alloc(engine, ctx, &req); 3283 return err ? ERR_PTR(err) : req; 3284 } 3285 3286 struct drm_i915_gem_request * 3287 i915_gem_find_active_request(struct intel_engine_cs *engine) 3288 { 3289 struct drm_i915_gem_request *request; 3290 3291 list_for_each_entry(request, &engine->request_list, list) { 3292 if (i915_gem_request_completed(request, false)) 3293 continue; 3294 3295 return request; 3296 } 3297 3298 return NULL; 3299 } 3300 3301 static void i915_gem_reset_engine_status(struct drm_i915_private *dev_priv, 3302 struct intel_engine_cs *engine) 3303 { 3304 struct drm_i915_gem_request *request; 3305 bool ring_hung; 3306 3307 request = i915_gem_find_active_request(engine); 3308 3309 if (request == NULL) 3310 return; 3311 3312 ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG; 3313 3314 i915_set_reset_status(dev_priv, request->ctx, ring_hung); 3315 3316 list_for_each_entry_continue(request, &engine->request_list, list) 3317 i915_set_reset_status(dev_priv, request->ctx, false); 3318 } 3319 3320 static void i915_gem_reset_engine_cleanup(struct drm_i915_private *dev_priv, 3321 struct intel_engine_cs *engine) 3322 { 3323 struct intel_ringbuffer *buffer; 3324 3325 while (!list_empty(&engine->active_list)) { 3326 struct drm_i915_gem_object *obj; 3327 3328 obj = list_first_entry(&engine->active_list, 3329 struct drm_i915_gem_object, 3330 engine_list[engine->id]); 3331 3332 i915_gem_object_retire__read(obj, engine->id); 3333 } 3334 3335 /* 3336 * Clear the execlists queue up before freeing the requests, as those 3337 * are the ones that keep the context and ringbuffer backing objects 3338 * pinned in place. 3339 */ 3340 3341 if (i915.enable_execlists) { 3342 /* Ensure irq handler finishes or is cancelled. */ 3343 tasklet_kill(&engine->irq_tasklet); 3344 3345 intel_execlists_cancel_requests(engine); 3346 } 3347 3348 /* 3349 * We must free the requests after all the corresponding objects have 3350 * been moved off active lists. Which is the same order as the normal 3351 * retire_requests function does. This is important if object hold 3352 * implicit references on things like e.g. ppgtt address spaces through 3353 * the request. 3354 */ 3355 while (!list_empty(&engine->request_list)) { 3356 struct drm_i915_gem_request *request; 3357 3358 request = list_first_entry(&engine->request_list, 3359 struct drm_i915_gem_request, 3360 list); 3361 3362 i915_gem_request_retire(request); 3363 } 3364 3365 /* Having flushed all requests from all queues, we know that all 3366 * ringbuffers must now be empty. However, since we do not reclaim 3367 * all space when retiring the request (to prevent HEADs colliding 3368 * with rapid ringbuffer wraparound) the amount of available space 3369 * upon reset is less than when we start. Do one more pass over 3370 * all the ringbuffers to reset last_retired_head. 3371 */ 3372 list_for_each_entry(buffer, &engine->buffers, link) { 3373 buffer->last_retired_head = buffer->tail; 3374 intel_ring_update_space(buffer); 3375 } 3376 3377 intel_ring_init_seqno(engine, engine->last_submitted_seqno); 3378 } 3379 3380 void i915_gem_reset(struct drm_device *dev) 3381 { 3382 struct drm_i915_private *dev_priv = dev->dev_private; 3383 struct intel_engine_cs *engine; 3384 3385 /* 3386 * Before we free the objects from the requests, we need to inspect 3387 * them for finding the guilty party. As the requests only borrow 3388 * their reference to the objects, the inspection must be done first. 3389 */ 3390 for_each_engine(engine, dev_priv) 3391 i915_gem_reset_engine_status(dev_priv, engine); 3392 3393 for_each_engine(engine, dev_priv) 3394 i915_gem_reset_engine_cleanup(dev_priv, engine); 3395 3396 i915_gem_context_reset(dev); 3397 3398 i915_gem_restore_fences(dev); 3399 3400 WARN_ON(i915_verify_lists(dev)); 3401 } 3402 3403 /** 3404 * This function clears the request list as sequence numbers are passed. 3405 * @engine: engine to retire requests on 3406 */ 3407 void 3408 i915_gem_retire_requests_ring(struct intel_engine_cs *engine) 3409 { 3410 WARN_ON(i915_verify_lists(engine->dev)); 3411 3412 /* Retire requests first as we use it above for the early return. 3413 * If we retire requests last, we may use a later seqno and so clear 3414 * the requests lists without clearing the active list, leading to 3415 * confusion. 3416 */ 3417 while (!list_empty(&engine->request_list)) { 3418 struct drm_i915_gem_request *request; 3419 3420 request = list_first_entry(&engine->request_list, 3421 struct drm_i915_gem_request, 3422 list); 3423 3424 if (!i915_gem_request_completed(request, true)) 3425 break; 3426 3427 i915_gem_request_retire(request); 3428 } 3429 3430 /* Move any buffers on the active list that are no longer referenced 3431 * by the ringbuffer to the flushing/inactive lists as appropriate, 3432 * before we free the context associated with the requests. 3433 */ 3434 while (!list_empty(&engine->active_list)) { 3435 struct drm_i915_gem_object *obj; 3436 3437 obj = list_first_entry(&engine->active_list, 3438 struct drm_i915_gem_object, 3439 engine_list[engine->id]); 3440 3441 if (!list_empty(&obj->last_read_req[engine->id]->list)) 3442 break; 3443 3444 i915_gem_object_retire__read(obj, engine->id); 3445 } 3446 3447 if (unlikely(engine->trace_irq_req && 3448 i915_gem_request_completed(engine->trace_irq_req, true))) { 3449 engine->irq_put(engine); 3450 i915_gem_request_assign(&engine->trace_irq_req, NULL); 3451 } 3452 3453 WARN_ON(i915_verify_lists(engine->dev)); 3454 } 3455 3456 bool 3457 i915_gem_retire_requests(struct drm_i915_private *dev_priv) 3458 { 3459 struct intel_engine_cs *engine; 3460 bool idle = true; 3461 3462 for_each_engine(engine, dev_priv) { 3463 i915_gem_retire_requests_ring(engine); 3464 idle &= list_empty(&engine->request_list); 3465 if (i915.enable_execlists) { 3466 spin_lock_bh(&engine->execlist_lock); 3467 idle &= list_empty(&engine->execlist_queue); 3468 spin_unlock_bh(&engine->execlist_lock); 3469 } 3470 } 3471 3472 if (idle) 3473 mod_delayed_work(dev_priv->wq, 3474 &dev_priv->mm.idle_work, 3475 msecs_to_jiffies(100)); 3476 3477 return idle; 3478 } 3479 3480 static void 3481 i915_gem_retire_work_handler(struct work_struct *work) 3482 { 3483 struct drm_i915_private *dev_priv = 3484 container_of(work, typeof(*dev_priv), mm.retire_work.work); 3485 struct drm_device *dev = dev_priv->dev; 3486 bool idle; 3487 3488 /* Come back later if the device is busy... */ 3489 idle = false; 3490 if (mutex_trylock(&dev->struct_mutex)) { 3491 idle = i915_gem_retire_requests(dev_priv); 3492 mutex_unlock(&dev->struct_mutex); 3493 } 3494 if (!idle) 3495 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 3496 round_jiffies_up_relative(HZ)); 3497 } 3498 3499 static void 3500 i915_gem_idle_work_handler(struct work_struct *work) 3501 { 3502 struct drm_i915_private *dev_priv = 3503 container_of(work, typeof(*dev_priv), mm.idle_work.work); 3504 struct drm_device *dev = dev_priv->dev; 3505 struct intel_engine_cs *engine; 3506 3507 for_each_engine(engine, dev_priv) 3508 if (!list_empty(&engine->request_list)) 3509 return; 3510 3511 /* we probably should sync with hangcheck here, using cancel_work_sync. 3512 * Also locking seems to be fubar here, engine->request_list is protected 3513 * by dev->struct_mutex. */ 3514 3515 intel_mark_idle(dev_priv); 3516 3517 if (mutex_trylock(&dev->struct_mutex)) { 3518 for_each_engine(engine, dev_priv) 3519 i915_gem_batch_pool_fini(&engine->batch_pool); 3520 3521 mutex_unlock(&dev->struct_mutex); 3522 } 3523 } 3524 3525 /** 3526 * Ensures that an object will eventually get non-busy by flushing any required 3527 * write domains, emitting any outstanding lazy request and retiring and 3528 * completed requests. 3529 * @obj: object to flush 3530 */ 3531 static int 3532 i915_gem_object_flush_active(struct drm_i915_gem_object *obj) 3533 { 3534 int i; 3535 3536 if (!obj->active) 3537 return 0; 3538 3539 for (i = 0; i < I915_NUM_ENGINES; i++) { 3540 struct drm_i915_gem_request *req; 3541 3542 req = obj->last_read_req[i]; 3543 if (req == NULL) 3544 continue; 3545 3546 if (i915_gem_request_completed(req, true)) 3547 i915_gem_object_retire__read(obj, i); 3548 } 3549 3550 return 0; 3551 } 3552 3553 /** 3554 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT 3555 * @dev: drm device pointer 3556 * @data: ioctl data blob 3557 * @file: drm file pointer 3558 * 3559 * Returns 0 if successful, else an error is returned with the remaining time in 3560 * the timeout parameter. 3561 * -ETIME: object is still busy after timeout 3562 * -ERESTARTSYS: signal interrupted the wait 3563 * -ENONENT: object doesn't exist 3564 * Also possible, but rare: 3565 * -EAGAIN: GPU wedged 3566 * -ENOMEM: damn 3567 * -ENODEV: Internal IRQ fail 3568 * -E?: The add request failed 3569 * 3570 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any 3571 * non-zero timeout parameter the wait ioctl will wait for the given number of 3572 * nanoseconds on an object becoming unbusy. Since the wait itself does so 3573 * without holding struct_mutex the object may become re-busied before this 3574 * function completes. A similar but shorter * race condition exists in the busy 3575 * ioctl 3576 */ 3577 int 3578 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 3579 { 3580 struct drm_i915_gem_wait *args = data; 3581 struct drm_i915_gem_object *obj; 3582 struct drm_i915_gem_request *req[I915_NUM_ENGINES]; 3583 int i, n = 0; 3584 int ret; 3585 3586 if (args->flags != 0) 3587 return -EINVAL; 3588 3589 ret = i915_mutex_lock_interruptible(dev); 3590 if (ret) 3591 return ret; 3592 3593 obj = to_intel_bo(drm_gem_object_lookup(file, args->bo_handle)); 3594 if (&obj->base == NULL) { 3595 mutex_unlock(&dev->struct_mutex); 3596 return -ENOENT; 3597 } 3598 3599 /* Need to make sure the object gets inactive eventually. */ 3600 ret = i915_gem_object_flush_active(obj); 3601 if (ret) 3602 goto out; 3603 3604 if (!obj->active) 3605 goto out; 3606 3607 /* Do this after OLR check to make sure we make forward progress polling 3608 * on this IOCTL with a timeout == 0 (like busy ioctl) 3609 */ 3610 if (args->timeout_ns == 0) { 3611 ret = -ETIME; 3612 goto out; 3613 } 3614 3615 drm_gem_object_unreference(&obj->base); 3616 3617 for (i = 0; i < I915_NUM_ENGINES; i++) { 3618 if (obj->last_read_req[i] == NULL) 3619 continue; 3620 3621 req[n++] = i915_gem_request_reference(obj->last_read_req[i]); 3622 } 3623 3624 mutex_unlock(&dev->struct_mutex); 3625 3626 for (i = 0; i < n; i++) { 3627 if (ret == 0) 3628 ret = __i915_wait_request(req[i], true, 3629 args->timeout_ns > 0 ? &args->timeout_ns : NULL, 3630 to_rps_client(file)); 3631 i915_gem_request_unreference(req[i]); 3632 } 3633 return ret; 3634 3635 out: 3636 drm_gem_object_unreference(&obj->base); 3637 mutex_unlock(&dev->struct_mutex); 3638 return ret; 3639 } 3640 3641 static int 3642 __i915_gem_object_sync(struct drm_i915_gem_object *obj, 3643 struct intel_engine_cs *to, 3644 struct drm_i915_gem_request *from_req, 3645 struct drm_i915_gem_request **to_req) 3646 { 3647 struct intel_engine_cs *from; 3648 int ret; 3649 3650 from = i915_gem_request_get_engine(from_req); 3651 if (to == from) 3652 return 0; 3653 3654 if (i915_gem_request_completed(from_req, true)) 3655 return 0; 3656 3657 if (!i915_semaphore_is_enabled(to_i915(obj->base.dev))) { 3658 struct drm_i915_private *i915 = to_i915(obj->base.dev); 3659 ret = __i915_wait_request(from_req, 3660 i915->mm.interruptible, 3661 NULL, 3662 &i915->rps.semaphores); 3663 if (ret) 3664 return ret; 3665 3666 i915_gem_object_retire_request(obj, from_req); 3667 } else { 3668 int idx = intel_ring_sync_index(from, to); 3669 u32 seqno = i915_gem_request_get_seqno(from_req); 3670 3671 WARN_ON(!to_req); 3672 3673 if (seqno <= from->semaphore.sync_seqno[idx]) 3674 return 0; 3675 3676 if (*to_req == NULL) { 3677 struct drm_i915_gem_request *req; 3678 3679 req = i915_gem_request_alloc(to, NULL); 3680 if (IS_ERR(req)) 3681 return PTR_ERR(req); 3682 3683 *to_req = req; 3684 } 3685 3686 trace_i915_gem_ring_sync_to(*to_req, from, from_req); 3687 ret = to->semaphore.sync_to(*to_req, from, seqno); 3688 if (ret) 3689 return ret; 3690 3691 /* We use last_read_req because sync_to() 3692 * might have just caused seqno wrap under 3693 * the radar. 3694 */ 3695 from->semaphore.sync_seqno[idx] = 3696 i915_gem_request_get_seqno(obj->last_read_req[from->id]); 3697 } 3698 3699 return 0; 3700 } 3701 3702 /** 3703 * i915_gem_object_sync - sync an object to a ring. 3704 * 3705 * @obj: object which may be in use on another ring. 3706 * @to: ring we wish to use the object on. May be NULL. 3707 * @to_req: request we wish to use the object for. See below. 3708 * This will be allocated and returned if a request is 3709 * required but not passed in. 3710 * 3711 * This code is meant to abstract object synchronization with the GPU. 3712 * Calling with NULL implies synchronizing the object with the CPU 3713 * rather than a particular GPU ring. Conceptually we serialise writes 3714 * between engines inside the GPU. We only allow one engine to write 3715 * into a buffer at any time, but multiple readers. To ensure each has 3716 * a coherent view of memory, we must: 3717 * 3718 * - If there is an outstanding write request to the object, the new 3719 * request must wait for it to complete (either CPU or in hw, requests 3720 * on the same ring will be naturally ordered). 3721 * 3722 * - If we are a write request (pending_write_domain is set), the new 3723 * request must wait for outstanding read requests to complete. 3724 * 3725 * For CPU synchronisation (NULL to) no request is required. For syncing with 3726 * rings to_req must be non-NULL. However, a request does not have to be 3727 * pre-allocated. If *to_req is NULL and sync commands will be emitted then a 3728 * request will be allocated automatically and returned through *to_req. Note 3729 * that it is not guaranteed that commands will be emitted (because the system 3730 * might already be idle). Hence there is no need to create a request that 3731 * might never have any work submitted. Note further that if a request is 3732 * returned in *to_req, it is the responsibility of the caller to submit 3733 * that request (after potentially adding more work to it). 3734 * 3735 * Returns 0 if successful, else propagates up the lower layer error. 3736 */ 3737 int 3738 i915_gem_object_sync(struct drm_i915_gem_object *obj, 3739 struct intel_engine_cs *to, 3740 struct drm_i915_gem_request **to_req) 3741 { 3742 const bool readonly = obj->base.pending_write_domain == 0; 3743 struct drm_i915_gem_request *req[I915_NUM_ENGINES]; 3744 int ret, i, n; 3745 3746 if (!obj->active) 3747 return 0; 3748 3749 if (to == NULL) 3750 return i915_gem_object_wait_rendering(obj, readonly); 3751 3752 n = 0; 3753 if (readonly) { 3754 if (obj->last_write_req) 3755 req[n++] = obj->last_write_req; 3756 } else { 3757 for (i = 0; i < I915_NUM_ENGINES; i++) 3758 if (obj->last_read_req[i]) 3759 req[n++] = obj->last_read_req[i]; 3760 } 3761 for (i = 0; i < n; i++) { 3762 ret = __i915_gem_object_sync(obj, to, req[i], to_req); 3763 if (ret) 3764 return ret; 3765 } 3766 3767 return 0; 3768 } 3769 3770 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 3771 { 3772 u32 old_write_domain, old_read_domains; 3773 3774 /* Force a pagefault for domain tracking on next user access */ 3775 i915_gem_release_mmap(obj); 3776 3777 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 3778 return; 3779 3780 old_read_domains = obj->base.read_domains; 3781 old_write_domain = obj->base.write_domain; 3782 3783 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 3784 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 3785 3786 trace_i915_gem_object_change_domain(obj, 3787 old_read_domains, 3788 old_write_domain); 3789 } 3790 3791 static void __i915_vma_iounmap(struct i915_vma *vma) 3792 { 3793 GEM_BUG_ON(vma->pin_count); 3794 3795 if (vma->iomap == NULL) 3796 return; 3797 3798 io_mapping_unmap(vma->iomap); 3799 vma->iomap = NULL; 3800 } 3801 3802 static int __i915_vma_unbind(struct i915_vma *vma, bool wait) 3803 { 3804 struct drm_i915_gem_object *obj = vma->obj; 3805 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3806 int ret; 3807 3808 if (list_empty(&vma->obj_link)) 3809 return 0; 3810 3811 if (!drm_mm_node_allocated(&vma->node)) { 3812 i915_gem_vma_destroy(vma); 3813 return 0; 3814 } 3815 3816 if (vma->pin_count) 3817 return -EBUSY; 3818 3819 BUG_ON(obj->pages == NULL); 3820 3821 if (wait) { 3822 ret = i915_gem_object_wait_rendering(obj, false); 3823 if (ret) 3824 return ret; 3825 } 3826 3827 if (vma->is_ggtt && vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) { 3828 i915_gem_object_finish_gtt(obj); 3829 3830 /* release the fence reg _after_ flushing */ 3831 ret = i915_gem_object_put_fence(obj); 3832 if (ret) 3833 return ret; 3834 3835 __i915_vma_iounmap(vma); 3836 } 3837 3838 trace_i915_vma_unbind(vma); 3839 3840 vma->vm->unbind_vma(vma); 3841 vma->bound = 0; 3842 3843 list_del_init(&vma->vm_link); 3844 if (vma->is_ggtt) { 3845 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) { 3846 obj->map_and_fenceable = false; 3847 } else if (vma->ggtt_view.pages) { 3848 sg_free_table(vma->ggtt_view.pages); 3849 kfree(vma->ggtt_view.pages); 3850 } 3851 vma->ggtt_view.pages = NULL; 3852 } 3853 3854 drm_mm_remove_node(&vma->node); 3855 i915_gem_vma_destroy(vma); 3856 3857 /* Since the unbound list is global, only move to that list if 3858 * no more VMAs exist. */ 3859 if (list_empty(&obj->vma_list)) 3860 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list); 3861 3862 /* And finally now the object is completely decoupled from this vma, 3863 * we can drop its hold on the backing storage and allow it to be 3864 * reaped by the shrinker. 3865 */ 3866 i915_gem_object_unpin_pages(obj); 3867 3868 return 0; 3869 } 3870 3871 int i915_vma_unbind(struct i915_vma *vma) 3872 { 3873 return __i915_vma_unbind(vma, true); 3874 } 3875 3876 int __i915_vma_unbind_no_wait(struct i915_vma *vma) 3877 { 3878 return __i915_vma_unbind(vma, false); 3879 } 3880 3881 int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv) 3882 { 3883 struct intel_engine_cs *engine; 3884 int ret; 3885 3886 lockdep_assert_held(&dev_priv->dev->struct_mutex); 3887 3888 for_each_engine(engine, dev_priv) { 3889 if (engine->last_context == NULL) 3890 continue; 3891 3892 ret = intel_engine_idle(engine); 3893 if (ret) 3894 return ret; 3895 } 3896 3897 WARN_ON(i915_verify_lists(dev)); 3898 return 0; 3899 } 3900 3901 static bool i915_gem_valid_gtt_space(struct i915_vma *vma, 3902 unsigned long cache_level) 3903 { 3904 struct drm_mm_node *gtt_space = &vma->node; 3905 struct drm_mm_node *other; 3906 3907 /* 3908 * On some machines we have to be careful when putting differing types 3909 * of snoopable memory together to avoid the prefetcher crossing memory 3910 * domains and dying. During vm initialisation, we decide whether or not 3911 * these constraints apply and set the drm_mm.color_adjust 3912 * appropriately. 3913 */ 3914 if (vma->vm->mm.color_adjust == NULL) 3915 return true; 3916 3917 if (!drm_mm_node_allocated(gtt_space)) 3918 return true; 3919 3920 if (list_empty(>t_space->node_list)) 3921 return true; 3922 3923 other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list); 3924 if (other->allocated && !other->hole_follows && other->color != cache_level) 3925 return false; 3926 3927 other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list); 3928 if (other->allocated && !gtt_space->hole_follows && other->color != cache_level) 3929 return false; 3930 3931 return true; 3932 } 3933 3934 /** 3935 * Finds free space in the GTT aperture and binds the object or a view of it 3936 * there. 3937 * @obj: object to bind 3938 * @vm: address space to bind into 3939 * @ggtt_view: global gtt view if applicable 3940 * @alignment: requested alignment 3941 * @flags: mask of PIN_* flags to use 3942 */ 3943 static struct i915_vma * 3944 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, 3945 struct i915_address_space *vm, 3946 const struct i915_ggtt_view *ggtt_view, 3947 unsigned alignment, 3948 uint64_t flags) 3949 { 3950 struct drm_device *dev = obj->base.dev; 3951 struct drm_i915_private *dev_priv = to_i915(dev); 3952 struct i915_ggtt *ggtt = &dev_priv->ggtt; 3953 u32 fence_alignment, unfenced_alignment; 3954 u32 search_flag, alloc_flag; 3955 u64 start, end; 3956 u64 size, fence_size; 3957 struct i915_vma *vma; 3958 int ret; 3959 3960 if (i915_is_ggtt(vm)) { 3961 u32 view_size; 3962 3963 if (WARN_ON(!ggtt_view)) 3964 return ERR_PTR(-EINVAL); 3965 3966 view_size = i915_ggtt_view_size(obj, ggtt_view); 3967 3968 fence_size = i915_gem_get_gtt_size(dev, 3969 view_size, 3970 obj->tiling_mode); 3971 fence_alignment = i915_gem_get_gtt_alignment(dev, 3972 view_size, 3973 obj->tiling_mode, 3974 true); 3975 unfenced_alignment = i915_gem_get_gtt_alignment(dev, 3976 view_size, 3977 obj->tiling_mode, 3978 false); 3979 size = flags & PIN_MAPPABLE ? fence_size : view_size; 3980 } else { 3981 fence_size = i915_gem_get_gtt_size(dev, 3982 obj->base.size, 3983 obj->tiling_mode); 3984 fence_alignment = i915_gem_get_gtt_alignment(dev, 3985 obj->base.size, 3986 obj->tiling_mode, 3987 true); 3988 unfenced_alignment = 3989 i915_gem_get_gtt_alignment(dev, 3990 obj->base.size, 3991 obj->tiling_mode, 3992 false); 3993 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size; 3994 } 3995 3996 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; 3997 end = vm->total; 3998 if (flags & PIN_MAPPABLE) 3999 end = min_t(u64, end, ggtt->mappable_end); 4000 if (flags & PIN_ZONE_4G) 4001 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE); 4002 4003 if (alignment == 0) 4004 alignment = flags & PIN_MAPPABLE ? fence_alignment : 4005 unfenced_alignment; 4006 if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) { 4007 DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n", 4008 ggtt_view ? ggtt_view->type : 0, 4009 alignment); 4010 return ERR_PTR(-EINVAL); 4011 } 4012 4013 /* If binding the object/GGTT view requires more space than the entire 4014 * aperture has, reject it early before evicting everything in a vain 4015 * attempt to find space. 4016 */ 4017 if (size > end) { 4018 DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n", 4019 ggtt_view ? ggtt_view->type : 0, 4020 size, 4021 flags & PIN_MAPPABLE ? "mappable" : "total", 4022 end); 4023 return ERR_PTR(-E2BIG); 4024 } 4025 4026 ret = i915_gem_object_get_pages(obj); 4027 if (ret) 4028 return ERR_PTR(ret); 4029 4030 i915_gem_object_pin_pages(obj); 4031 4032 vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) : 4033 i915_gem_obj_lookup_or_create_vma(obj, vm); 4034 4035 if (IS_ERR(vma)) 4036 goto err_unpin; 4037 4038 if (flags & PIN_OFFSET_FIXED) { 4039 uint64_t offset = flags & PIN_OFFSET_MASK; 4040 4041 if (offset & (alignment - 1) || offset + size > end) { 4042 ret = -EINVAL; 4043 goto err_free_vma; 4044 } 4045 vma->node.start = offset; 4046 vma->node.size = size; 4047 vma->node.color = obj->cache_level; 4048 ret = drm_mm_reserve_node(&vm->mm, &vma->node); 4049 if (ret) { 4050 ret = i915_gem_evict_for_vma(vma); 4051 if (ret == 0) 4052 ret = drm_mm_reserve_node(&vm->mm, &vma->node); 4053 } 4054 if (ret) 4055 goto err_free_vma; 4056 } else { 4057 if (flags & PIN_HIGH) { 4058 search_flag = DRM_MM_SEARCH_BELOW; 4059 alloc_flag = DRM_MM_CREATE_TOP; 4060 } else { 4061 search_flag = DRM_MM_SEARCH_DEFAULT; 4062 alloc_flag = DRM_MM_CREATE_DEFAULT; 4063 } 4064 4065 search_free: 4066 ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node, 4067 size, alignment, 4068 obj->cache_level, 4069 start, end, 4070 search_flag, 4071 alloc_flag); 4072 if (ret) { 4073 ret = i915_gem_evict_something(dev, vm, size, alignment, 4074 obj->cache_level, 4075 start, end, 4076 flags); 4077 if (ret == 0) 4078 goto search_free; 4079 4080 goto err_free_vma; 4081 } 4082 } 4083 if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) { 4084 ret = -EINVAL; 4085 goto err_remove_node; 4086 } 4087 4088 trace_i915_vma_bind(vma, flags); 4089 ret = i915_vma_bind(vma, obj->cache_level, flags); 4090 if (ret) 4091 goto err_remove_node; 4092 4093 list_move_tail(&obj->global_list, &dev_priv->mm.bound_list); 4094 list_add_tail(&vma->vm_link, &vm->inactive_list); 4095 4096 return vma; 4097 4098 err_remove_node: 4099 drm_mm_remove_node(&vma->node); 4100 err_free_vma: 4101 i915_gem_vma_destroy(vma); 4102 vma = ERR_PTR(ret); 4103 err_unpin: 4104 i915_gem_object_unpin_pages(obj); 4105 return vma; 4106 } 4107 4108 bool 4109 i915_gem_clflush_object(struct drm_i915_gem_object *obj, 4110 bool force) 4111 { 4112 /* If we don't have a page list set up, then we're not pinned 4113 * to GPU, and we can ignore the cache flush because it'll happen 4114 * again at bind time. 4115 */ 4116 if (obj->pages == NULL) 4117 return false; 4118 4119 /* 4120 * Stolen memory is always coherent with the GPU as it is explicitly 4121 * marked as wc by the system, or the system is cache-coherent. 4122 */ 4123 if (obj->stolen || obj->phys_handle) 4124 return false; 4125 4126 /* If the GPU is snooping the contents of the CPU cache, 4127 * we do not need to manually clear the CPU cache lines. However, 4128 * the caches are only snooped when the render cache is 4129 * flushed/invalidated. As we always have to emit invalidations 4130 * and flushes when moving into and out of the RENDER domain, correct 4131 * snooping behaviour occurs naturally as the result of our domain 4132 * tracking. 4133 */ 4134 if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) { 4135 obj->cache_dirty = true; 4136 return false; 4137 } 4138 4139 trace_i915_gem_object_clflush(obj); 4140 drm_clflush_sg(obj->pages); 4141 obj->cache_dirty = false; 4142 4143 return true; 4144 } 4145 4146 /** Flushes the GTT write domain for the object if it's dirty. */ 4147 static void 4148 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 4149 { 4150 uint32_t old_write_domain; 4151 4152 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 4153 return; 4154 4155 /* No actual flushing is required for the GTT write domain. Writes 4156 * to it immediately go to main memory as far as we know, so there's 4157 * no chipset flush. It also doesn't land in render cache. 4158 * 4159 * However, we do have to enforce the order so that all writes through 4160 * the GTT land before any writes to the device, such as updates to 4161 * the GATT itself. 4162 */ 4163 wmb(); 4164 4165 old_write_domain = obj->base.write_domain; 4166 obj->base.write_domain = 0; 4167 4168 intel_fb_obj_flush(obj, false, ORIGIN_GTT); 4169 4170 trace_i915_gem_object_change_domain(obj, 4171 obj->base.read_domains, 4172 old_write_domain); 4173 } 4174 4175 /** Flushes the CPU write domain for the object if it's dirty. */ 4176 static void 4177 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 4178 { 4179 uint32_t old_write_domain; 4180 4181 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 4182 return; 4183 4184 if (i915_gem_clflush_object(obj, obj->pin_display)) 4185 i915_gem_chipset_flush(to_i915(obj->base.dev)); 4186 4187 old_write_domain = obj->base.write_domain; 4188 obj->base.write_domain = 0; 4189 4190 intel_fb_obj_flush(obj, false, ORIGIN_CPU); 4191 4192 trace_i915_gem_object_change_domain(obj, 4193 obj->base.read_domains, 4194 old_write_domain); 4195 } 4196 4197 /** 4198 * Moves a single object to the GTT read, and possibly write domain. 4199 * @obj: object to act on 4200 * @write: ask for write access or read only 4201 * 4202 * This function returns when the move is complete, including waiting on 4203 * flushes to occur. 4204 */ 4205 int 4206 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 4207 { 4208 struct drm_device *dev = obj->base.dev; 4209 struct drm_i915_private *dev_priv = to_i915(dev); 4210 struct i915_ggtt *ggtt = &dev_priv->ggtt; 4211 uint32_t old_write_domain, old_read_domains; 4212 struct i915_vma *vma; 4213 int ret; 4214 4215 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 4216 return 0; 4217 4218 ret = i915_gem_object_wait_rendering(obj, !write); 4219 if (ret) 4220 return ret; 4221 4222 /* Flush and acquire obj->pages so that we are coherent through 4223 * direct access in memory with previous cached writes through 4224 * shmemfs and that our cache domain tracking remains valid. 4225 * For example, if the obj->filp was moved to swap without us 4226 * being notified and releasing the pages, we would mistakenly 4227 * continue to assume that the obj remained out of the CPU cached 4228 * domain. 4229 */ 4230 ret = i915_gem_object_get_pages(obj); 4231 if (ret) 4232 return ret; 4233 4234 i915_gem_object_flush_cpu_write_domain(obj); 4235 4236 /* Serialise direct access to this object with the barriers for 4237 * coherent writes from the GPU, by effectively invalidating the 4238 * GTT domain upon first access. 4239 */ 4240 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 4241 mb(); 4242 4243 old_write_domain = obj->base.write_domain; 4244 old_read_domains = obj->base.read_domains; 4245 4246 /* It should now be out of any other write domains, and we can update 4247 * the domain values for our changes. 4248 */ 4249 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 4250 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 4251 if (write) { 4252 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 4253 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 4254 obj->dirty = 1; 4255 } 4256 4257 trace_i915_gem_object_change_domain(obj, 4258 old_read_domains, 4259 old_write_domain); 4260 4261 /* And bump the LRU for this access */ 4262 vma = i915_gem_obj_to_ggtt(obj); 4263 if (vma && drm_mm_node_allocated(&vma->node) && !obj->active) 4264 list_move_tail(&vma->vm_link, 4265 &ggtt->base.inactive_list); 4266 4267 return 0; 4268 } 4269 4270 /** 4271 * Changes the cache-level of an object across all VMA. 4272 * @obj: object to act on 4273 * @cache_level: new cache level to set for the object 4274 * 4275 * After this function returns, the object will be in the new cache-level 4276 * across all GTT and the contents of the backing storage will be coherent, 4277 * with respect to the new cache-level. In order to keep the backing storage 4278 * coherent for all users, we only allow a single cache level to be set 4279 * globally on the object and prevent it from being changed whilst the 4280 * hardware is reading from the object. That is if the object is currently 4281 * on the scanout it will be set to uncached (or equivalent display 4282 * cache coherency) and all non-MOCS GPU access will also be uncached so 4283 * that all direct access to the scanout remains coherent. 4284 */ 4285 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 4286 enum i915_cache_level cache_level) 4287 { 4288 struct drm_device *dev = obj->base.dev; 4289 struct i915_vma *vma, *next; 4290 bool bound = false; 4291 int ret = 0; 4292 4293 if (obj->cache_level == cache_level) 4294 goto out; 4295 4296 /* Inspect the list of currently bound VMA and unbind any that would 4297 * be invalid given the new cache-level. This is principally to 4298 * catch the issue of the CS prefetch crossing page boundaries and 4299 * reading an invalid PTE on older architectures. 4300 */ 4301 list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) { 4302 if (!drm_mm_node_allocated(&vma->node)) 4303 continue; 4304 4305 if (vma->pin_count) { 4306 DRM_DEBUG("can not change the cache level of pinned objects\n"); 4307 return -EBUSY; 4308 } 4309 4310 if (!i915_gem_valid_gtt_space(vma, cache_level)) { 4311 ret = i915_vma_unbind(vma); 4312 if (ret) 4313 return ret; 4314 } else 4315 bound = true; 4316 } 4317 4318 /* We can reuse the existing drm_mm nodes but need to change the 4319 * cache-level on the PTE. We could simply unbind them all and 4320 * rebind with the correct cache-level on next use. However since 4321 * we already have a valid slot, dma mapping, pages etc, we may as 4322 * rewrite the PTE in the belief that doing so tramples upon less 4323 * state and so involves less work. 4324 */ 4325 if (bound) { 4326 /* Before we change the PTE, the GPU must not be accessing it. 4327 * If we wait upon the object, we know that all the bound 4328 * VMA are no longer active. 4329 */ 4330 ret = i915_gem_object_wait_rendering(obj, false); 4331 if (ret) 4332 return ret; 4333 4334 if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) { 4335 /* Access to snoopable pages through the GTT is 4336 * incoherent and on some machines causes a hard 4337 * lockup. Relinquish the CPU mmaping to force 4338 * userspace to refault in the pages and we can 4339 * then double check if the GTT mapping is still 4340 * valid for that pointer access. 4341 */ 4342 i915_gem_release_mmap(obj); 4343 4344 /* As we no longer need a fence for GTT access, 4345 * we can relinquish it now (and so prevent having 4346 * to steal a fence from someone else on the next 4347 * fence request). Note GPU activity would have 4348 * dropped the fence as all snoopable access is 4349 * supposed to be linear. 4350 */ 4351 ret = i915_gem_object_put_fence(obj); 4352 if (ret) 4353 return ret; 4354 } else { 4355 /* We either have incoherent backing store and 4356 * so no GTT access or the architecture is fully 4357 * coherent. In such cases, existing GTT mmaps 4358 * ignore the cache bit in the PTE and we can 4359 * rewrite it without confusing the GPU or having 4360 * to force userspace to fault back in its mmaps. 4361 */ 4362 } 4363 4364 list_for_each_entry(vma, &obj->vma_list, obj_link) { 4365 if (!drm_mm_node_allocated(&vma->node)) 4366 continue; 4367 4368 ret = i915_vma_bind(vma, cache_level, PIN_UPDATE); 4369 if (ret) 4370 return ret; 4371 } 4372 } 4373 4374 list_for_each_entry(vma, &obj->vma_list, obj_link) 4375 vma->node.color = cache_level; 4376 obj->cache_level = cache_level; 4377 4378 out: 4379 /* Flush the dirty CPU caches to the backing storage so that the 4380 * object is now coherent at its new cache level (with respect 4381 * to the access domain). 4382 */ 4383 if (obj->cache_dirty && cpu_write_needs_clflush(obj)) { 4384 if (i915_gem_clflush_object(obj, true)) 4385 i915_gem_chipset_flush(to_i915(obj->base.dev)); 4386 } 4387 4388 return 0; 4389 } 4390 4391 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, 4392 struct drm_file *file) 4393 { 4394 struct drm_i915_gem_caching *args = data; 4395 struct drm_i915_gem_object *obj; 4396 4397 obj = to_intel_bo(drm_gem_object_lookup(file, args->handle)); 4398 if (&obj->base == NULL) 4399 return -ENOENT; 4400 4401 switch (obj->cache_level) { 4402 case I915_CACHE_LLC: 4403 case I915_CACHE_L3_LLC: 4404 args->caching = I915_CACHING_CACHED; 4405 break; 4406 4407 case I915_CACHE_WT: 4408 args->caching = I915_CACHING_DISPLAY; 4409 break; 4410 4411 default: 4412 args->caching = I915_CACHING_NONE; 4413 break; 4414 } 4415 4416 drm_gem_object_unreference_unlocked(&obj->base); 4417 return 0; 4418 } 4419 4420 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, 4421 struct drm_file *file) 4422 { 4423 struct drm_i915_private *dev_priv = dev->dev_private; 4424 struct drm_i915_gem_caching *args = data; 4425 struct drm_i915_gem_object *obj; 4426 enum i915_cache_level level; 4427 int ret; 4428 4429 switch (args->caching) { 4430 case I915_CACHING_NONE: 4431 level = I915_CACHE_NONE; 4432 break; 4433 case I915_CACHING_CACHED: 4434 /* 4435 * Due to a HW issue on BXT A stepping, GPU stores via a 4436 * snooped mapping may leave stale data in a corresponding CPU 4437 * cacheline, whereas normally such cachelines would get 4438 * invalidated. 4439 */ 4440 if (!HAS_LLC(dev) && !HAS_SNOOP(dev)) 4441 return -ENODEV; 4442 4443 level = I915_CACHE_LLC; 4444 break; 4445 case I915_CACHING_DISPLAY: 4446 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE; 4447 break; 4448 default: 4449 return -EINVAL; 4450 } 4451 4452 intel_runtime_pm_get(dev_priv); 4453 4454 ret = i915_mutex_lock_interruptible(dev); 4455 if (ret) 4456 goto rpm_put; 4457 4458 obj = to_intel_bo(drm_gem_object_lookup(file, args->handle)); 4459 if (&obj->base == NULL) { 4460 ret = -ENOENT; 4461 goto unlock; 4462 } 4463 4464 ret = i915_gem_object_set_cache_level(obj, level); 4465 4466 drm_gem_object_unreference(&obj->base); 4467 unlock: 4468 mutex_unlock(&dev->struct_mutex); 4469 rpm_put: 4470 intel_runtime_pm_put(dev_priv); 4471 4472 return ret; 4473 } 4474 4475 /* 4476 * Prepare buffer for display plane (scanout, cursors, etc). 4477 * Can be called from an uninterruptible phase (modesetting) and allows 4478 * any flushes to be pipelined (for pageflips). 4479 */ 4480 int 4481 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 4482 u32 alignment, 4483 const struct i915_ggtt_view *view) 4484 { 4485 u32 old_read_domains, old_write_domain; 4486 int ret; 4487 4488 /* Mark the pin_display early so that we account for the 4489 * display coherency whilst setting up the cache domains. 4490 */ 4491 obj->pin_display++; 4492 4493 /* The display engine is not coherent with the LLC cache on gen6. As 4494 * a result, we make sure that the pinning that is about to occur is 4495 * done with uncached PTEs. This is lowest common denominator for all 4496 * chipsets. 4497 * 4498 * However for gen6+, we could do better by using the GFDT bit instead 4499 * of uncaching, which would allow us to flush all the LLC-cached data 4500 * with that bit in the PTE to main memory with just one PIPE_CONTROL. 4501 */ 4502 ret = i915_gem_object_set_cache_level(obj, 4503 HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE); 4504 if (ret) 4505 goto err_unpin_display; 4506 4507 /* As the user may map the buffer once pinned in the display plane 4508 * (e.g. libkms for the bootup splash), we have to ensure that we 4509 * always use map_and_fenceable for all scanout buffers. 4510 */ 4511 ret = i915_gem_object_ggtt_pin(obj, view, alignment, 4512 view->type == I915_GGTT_VIEW_NORMAL ? 4513 PIN_MAPPABLE : 0); 4514 if (ret) 4515 goto err_unpin_display; 4516 4517 i915_gem_object_flush_cpu_write_domain(obj); 4518 4519 old_write_domain = obj->base.write_domain; 4520 old_read_domains = obj->base.read_domains; 4521 4522 /* It should now be out of any other write domains, and we can update 4523 * the domain values for our changes. 4524 */ 4525 obj->base.write_domain = 0; 4526 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 4527 4528 trace_i915_gem_object_change_domain(obj, 4529 old_read_domains, 4530 old_write_domain); 4531 4532 return 0; 4533 4534 err_unpin_display: 4535 obj->pin_display--; 4536 return ret; 4537 } 4538 4539 void 4540 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj, 4541 const struct i915_ggtt_view *view) 4542 { 4543 if (WARN_ON(obj->pin_display == 0)) 4544 return; 4545 4546 i915_gem_object_ggtt_unpin_view(obj, view); 4547 4548 obj->pin_display--; 4549 } 4550 4551 /** 4552 * Moves a single object to the CPU read, and possibly write domain. 4553 * @obj: object to act on 4554 * @write: requesting write or read-only access 4555 * 4556 * This function returns when the move is complete, including waiting on 4557 * flushes to occur. 4558 */ 4559 int 4560 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 4561 { 4562 uint32_t old_write_domain, old_read_domains; 4563 int ret; 4564 4565 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 4566 return 0; 4567 4568 ret = i915_gem_object_wait_rendering(obj, !write); 4569 if (ret) 4570 return ret; 4571 4572 i915_gem_object_flush_gtt_write_domain(obj); 4573 4574 old_write_domain = obj->base.write_domain; 4575 old_read_domains = obj->base.read_domains; 4576 4577 /* Flush the CPU cache if it's still invalid. */ 4578 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 4579 i915_gem_clflush_object(obj, false); 4580 4581 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 4582 } 4583 4584 /* It should now be out of any other write domains, and we can update 4585 * the domain values for our changes. 4586 */ 4587 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 4588 4589 /* If we're writing through the CPU, then the GPU read domains will 4590 * need to be invalidated at next use. 4591 */ 4592 if (write) { 4593 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 4594 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 4595 } 4596 4597 trace_i915_gem_object_change_domain(obj, 4598 old_read_domains, 4599 old_write_domain); 4600 4601 return 0; 4602 } 4603 4604 /* Throttle our rendering by waiting until the ring has completed our requests 4605 * emitted over 20 msec ago. 4606 * 4607 * Note that if we were to use the current jiffies each time around the loop, 4608 * we wouldn't escape the function with any frames outstanding if the time to 4609 * render a frame was over 20ms. 4610 * 4611 * This should get us reasonable parallelism between CPU and GPU but also 4612 * relatively low latency when blocking on a particular request to finish. 4613 */ 4614 static int 4615 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 4616 { 4617 struct drm_i915_private *dev_priv = dev->dev_private; 4618 struct drm_i915_file_private *file_priv = file->driver_priv; 4619 unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES; 4620 struct drm_i915_gem_request *request, *target = NULL; 4621 int ret; 4622 4623 ret = i915_gem_wait_for_error(&dev_priv->gpu_error); 4624 if (ret) 4625 return ret; 4626 4627 /* ABI: return -EIO if already wedged */ 4628 if (i915_terminally_wedged(&dev_priv->gpu_error)) 4629 return -EIO; 4630 4631 spin_lock(&file_priv->mm.lock); 4632 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 4633 if (time_after_eq(request->emitted_jiffies, recent_enough)) 4634 break; 4635 4636 /* 4637 * Note that the request might not have been submitted yet. 4638 * In which case emitted_jiffies will be zero. 4639 */ 4640 if (!request->emitted_jiffies) 4641 continue; 4642 4643 target = request; 4644 } 4645 if (target) 4646 i915_gem_request_reference(target); 4647 spin_unlock(&file_priv->mm.lock); 4648 4649 if (target == NULL) 4650 return 0; 4651 4652 ret = __i915_wait_request(target, true, NULL, NULL); 4653 if (ret == 0) 4654 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); 4655 4656 i915_gem_request_unreference(target); 4657 4658 return ret; 4659 } 4660 4661 static bool 4662 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags) 4663 { 4664 struct drm_i915_gem_object *obj = vma->obj; 4665 4666 if (alignment && 4667 vma->node.start & (alignment - 1)) 4668 return true; 4669 4670 if (flags & PIN_MAPPABLE && !obj->map_and_fenceable) 4671 return true; 4672 4673 if (flags & PIN_OFFSET_BIAS && 4674 vma->node.start < (flags & PIN_OFFSET_MASK)) 4675 return true; 4676 4677 if (flags & PIN_OFFSET_FIXED && 4678 vma->node.start != (flags & PIN_OFFSET_MASK)) 4679 return true; 4680 4681 return false; 4682 } 4683 4684 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) 4685 { 4686 struct drm_i915_gem_object *obj = vma->obj; 4687 bool mappable, fenceable; 4688 u32 fence_size, fence_alignment; 4689 4690 fence_size = i915_gem_get_gtt_size(obj->base.dev, 4691 obj->base.size, 4692 obj->tiling_mode); 4693 fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev, 4694 obj->base.size, 4695 obj->tiling_mode, 4696 true); 4697 4698 fenceable = (vma->node.size == fence_size && 4699 (vma->node.start & (fence_alignment - 1)) == 0); 4700 4701 mappable = (vma->node.start + fence_size <= 4702 to_i915(obj->base.dev)->ggtt.mappable_end); 4703 4704 obj->map_and_fenceable = mappable && fenceable; 4705 } 4706 4707 static int 4708 i915_gem_object_do_pin(struct drm_i915_gem_object *obj, 4709 struct i915_address_space *vm, 4710 const struct i915_ggtt_view *ggtt_view, 4711 uint32_t alignment, 4712 uint64_t flags) 4713 { 4714 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 4715 struct i915_vma *vma; 4716 unsigned bound; 4717 int ret; 4718 4719 if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base)) 4720 return -ENODEV; 4721 4722 if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm))) 4723 return -EINVAL; 4724 4725 if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE)) 4726 return -EINVAL; 4727 4728 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view)) 4729 return -EINVAL; 4730 4731 vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) : 4732 i915_gem_obj_to_vma(obj, vm); 4733 4734 if (vma) { 4735 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) 4736 return -EBUSY; 4737 4738 if (i915_vma_misplaced(vma, alignment, flags)) { 4739 WARN(vma->pin_count, 4740 "bo is already pinned in %s with incorrect alignment:" 4741 " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d," 4742 " obj->map_and_fenceable=%d\n", 4743 ggtt_view ? "ggtt" : "ppgtt", 4744 upper_32_bits(vma->node.start), 4745 lower_32_bits(vma->node.start), 4746 alignment, 4747 !!(flags & PIN_MAPPABLE), 4748 obj->map_and_fenceable); 4749 ret = i915_vma_unbind(vma); 4750 if (ret) 4751 return ret; 4752 4753 vma = NULL; 4754 } 4755 } 4756 4757 bound = vma ? vma->bound : 0; 4758 if (vma == NULL || !drm_mm_node_allocated(&vma->node)) { 4759 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment, 4760 flags); 4761 if (IS_ERR(vma)) 4762 return PTR_ERR(vma); 4763 } else { 4764 ret = i915_vma_bind(vma, obj->cache_level, flags); 4765 if (ret) 4766 return ret; 4767 } 4768 4769 if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL && 4770 (bound ^ vma->bound) & GLOBAL_BIND) { 4771 __i915_vma_set_map_and_fenceable(vma); 4772 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable); 4773 } 4774 4775 vma->pin_count++; 4776 return 0; 4777 } 4778 4779 int 4780 i915_gem_object_pin(struct drm_i915_gem_object *obj, 4781 struct i915_address_space *vm, 4782 uint32_t alignment, 4783 uint64_t flags) 4784 { 4785 return i915_gem_object_do_pin(obj, vm, 4786 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL, 4787 alignment, flags); 4788 } 4789 4790 int 4791 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, 4792 const struct i915_ggtt_view *view, 4793 uint32_t alignment, 4794 uint64_t flags) 4795 { 4796 struct drm_device *dev = obj->base.dev; 4797 struct drm_i915_private *dev_priv = to_i915(dev); 4798 struct i915_ggtt *ggtt = &dev_priv->ggtt; 4799 4800 BUG_ON(!view); 4801 4802 return i915_gem_object_do_pin(obj, &ggtt->base, view, 4803 alignment, flags | PIN_GLOBAL); 4804 } 4805 4806 void 4807 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj, 4808 const struct i915_ggtt_view *view) 4809 { 4810 struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view); 4811 4812 WARN_ON(vma->pin_count == 0); 4813 WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view)); 4814 4815 --vma->pin_count; 4816 } 4817 4818 int 4819 i915_gem_busy_ioctl(struct drm_device *dev, void *data, 4820 struct drm_file *file) 4821 { 4822 struct drm_i915_gem_busy *args = data; 4823 struct drm_i915_gem_object *obj; 4824 int ret; 4825 4826 ret = i915_mutex_lock_interruptible(dev); 4827 if (ret) 4828 return ret; 4829 4830 obj = to_intel_bo(drm_gem_object_lookup(file, args->handle)); 4831 if (&obj->base == NULL) { 4832 ret = -ENOENT; 4833 goto unlock; 4834 } 4835 4836 /* Count all active objects as busy, even if they are currently not used 4837 * by the gpu. Users of this interface expect objects to eventually 4838 * become non-busy without any further actions, therefore emit any 4839 * necessary flushes here. 4840 */ 4841 ret = i915_gem_object_flush_active(obj); 4842 if (ret) 4843 goto unref; 4844 4845 args->busy = 0; 4846 if (obj->active) { 4847 int i; 4848 4849 for (i = 0; i < I915_NUM_ENGINES; i++) { 4850 struct drm_i915_gem_request *req; 4851 4852 req = obj->last_read_req[i]; 4853 if (req) 4854 args->busy |= 1 << (16 + req->engine->exec_id); 4855 } 4856 if (obj->last_write_req) 4857 args->busy |= obj->last_write_req->engine->exec_id; 4858 } 4859 4860 unref: 4861 drm_gem_object_unreference(&obj->base); 4862 unlock: 4863 mutex_unlock(&dev->struct_mutex); 4864 return ret; 4865 } 4866 4867 int 4868 i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 4869 struct drm_file *file_priv) 4870 { 4871 return i915_gem_ring_throttle(dev, file_priv); 4872 } 4873 4874 int 4875 i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 4876 struct drm_file *file_priv) 4877 { 4878 struct drm_i915_private *dev_priv = dev->dev_private; 4879 struct drm_i915_gem_madvise *args = data; 4880 struct drm_i915_gem_object *obj; 4881 int ret; 4882 4883 switch (args->madv) { 4884 case I915_MADV_DONTNEED: 4885 case I915_MADV_WILLNEED: 4886 break; 4887 default: 4888 return -EINVAL; 4889 } 4890 4891 ret = i915_mutex_lock_interruptible(dev); 4892 if (ret) 4893 return ret; 4894 4895 obj = to_intel_bo(drm_gem_object_lookup(file_priv, args->handle)); 4896 if (&obj->base == NULL) { 4897 ret = -ENOENT; 4898 goto unlock; 4899 } 4900 4901 if (i915_gem_obj_is_pinned(obj)) { 4902 ret = -EINVAL; 4903 goto out; 4904 } 4905 4906 if (obj->pages && 4907 obj->tiling_mode != I915_TILING_NONE && 4908 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) { 4909 if (obj->madv == I915_MADV_WILLNEED) 4910 i915_gem_object_unpin_pages(obj); 4911 if (args->madv == I915_MADV_WILLNEED) 4912 i915_gem_object_pin_pages(obj); 4913 } 4914 4915 if (obj->madv != __I915_MADV_PURGED) 4916 obj->madv = args->madv; 4917 4918 /* if the object is no longer attached, discard its backing storage */ 4919 if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL) 4920 i915_gem_object_truncate(obj); 4921 4922 args->retained = obj->madv != __I915_MADV_PURGED; 4923 4924 out: 4925 drm_gem_object_unreference(&obj->base); 4926 unlock: 4927 mutex_unlock(&dev->struct_mutex); 4928 return ret; 4929 } 4930 4931 void i915_gem_object_init(struct drm_i915_gem_object *obj, 4932 const struct drm_i915_gem_object_ops *ops) 4933 { 4934 int i; 4935 4936 INIT_LIST_HEAD(&obj->global_list); 4937 for (i = 0; i < I915_NUM_ENGINES; i++) 4938 INIT_LIST_HEAD(&obj->engine_list[i]); 4939 INIT_LIST_HEAD(&obj->obj_exec_link); 4940 INIT_LIST_HEAD(&obj->vma_list); 4941 INIT_LIST_HEAD(&obj->batch_pool_link); 4942 4943 obj->ops = ops; 4944 4945 obj->fence_reg = I915_FENCE_REG_NONE; 4946 obj->madv = I915_MADV_WILLNEED; 4947 4948 i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size); 4949 } 4950 4951 static const struct drm_i915_gem_object_ops i915_gem_object_ops = { 4952 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE, 4953 .get_pages = i915_gem_object_get_pages_gtt, 4954 .put_pages = i915_gem_object_put_pages_gtt, 4955 }; 4956 4957 struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev, 4958 size_t size) 4959 { 4960 struct drm_i915_gem_object *obj; 4961 #if 0 4962 struct address_space *mapping; 4963 gfp_t mask; 4964 #endif 4965 int ret; 4966 4967 obj = i915_gem_object_alloc(dev); 4968 if (obj == NULL) 4969 return ERR_PTR(-ENOMEM); 4970 4971 ret = drm_gem_object_init(dev, &obj->base, size); 4972 if (ret) 4973 goto fail; 4974 4975 #if 0 4976 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 4977 if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) { 4978 /* 965gm cannot relocate objects above 4GiB. */ 4979 mask &= ~__GFP_HIGHMEM; 4980 mask |= __GFP_DMA32; 4981 } 4982 4983 mapping = file_inode(obj->base.filp)->i_mapping; 4984 mapping_set_gfp_mask(mapping, mask); 4985 #endif 4986 4987 i915_gem_object_init(obj, &i915_gem_object_ops); 4988 4989 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 4990 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 4991 4992 if (HAS_LLC(dev)) { 4993 /* On some devices, we can have the GPU use the LLC (the CPU 4994 * cache) for about a 10% performance improvement 4995 * compared to uncached. Graphics requests other than 4996 * display scanout are coherent with the CPU in 4997 * accessing this cache. This means in this mode we 4998 * don't need to clflush on the CPU side, and on the 4999 * GPU side we only need to flush internal caches to 5000 * get data visible to the CPU. 5001 * 5002 * However, we maintain the display planes as UC, and so 5003 * need to rebind when first used as such. 5004 */ 5005 obj->cache_level = I915_CACHE_LLC; 5006 } else 5007 obj->cache_level = I915_CACHE_NONE; 5008 5009 trace_i915_gem_object_create(obj); 5010 5011 return obj; 5012 5013 fail: 5014 i915_gem_object_free(obj); 5015 5016 return ERR_PTR(ret); 5017 } 5018 5019 static bool discard_backing_storage(struct drm_i915_gem_object *obj) 5020 { 5021 /* If we are the last user of the backing storage (be it shmemfs 5022 * pages or stolen etc), we know that the pages are going to be 5023 * immediately released. In this case, we can then skip copying 5024 * back the contents from the GPU. 5025 */ 5026 5027 if (obj->madv != I915_MADV_WILLNEED) 5028 return false; 5029 5030 if (obj->base.filp == NULL) 5031 return true; 5032 5033 /* At first glance, this looks racy, but then again so would be 5034 * userspace racing mmap against close. However, the first external 5035 * reference to the filp can only be obtained through the 5036 * i915_gem_mmap_ioctl() which safeguards us against the user 5037 * acquiring such a reference whilst we are in the middle of 5038 * freeing the object. 5039 */ 5040 #if 0 5041 return atomic_long_read(&obj->base.filp->f_count) == 1; 5042 #else 5043 return false; 5044 #endif 5045 } 5046 5047 void i915_gem_free_object(struct drm_gem_object *gem_obj) 5048 { 5049 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 5050 struct drm_device *dev = obj->base.dev; 5051 struct drm_i915_private *dev_priv = dev->dev_private; 5052 struct i915_vma *vma, *next; 5053 5054 intel_runtime_pm_get(dev_priv); 5055 5056 trace_i915_gem_object_destroy(obj); 5057 5058 list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) { 5059 int ret; 5060 5061 vma->pin_count = 0; 5062 ret = i915_vma_unbind(vma); 5063 if (WARN_ON(ret == -ERESTARTSYS)) { 5064 bool was_interruptible; 5065 5066 was_interruptible = dev_priv->mm.interruptible; 5067 dev_priv->mm.interruptible = false; 5068 5069 WARN_ON(i915_vma_unbind(vma)); 5070 5071 dev_priv->mm.interruptible = was_interruptible; 5072 } 5073 } 5074 5075 /* Stolen objects don't hold a ref, but do hold pin count. Fix that up 5076 * before progressing. */ 5077 if (obj->stolen) 5078 i915_gem_object_unpin_pages(obj); 5079 5080 WARN_ON(obj->frontbuffer_bits); 5081 5082 if (obj->pages && obj->madv == I915_MADV_WILLNEED && 5083 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES && 5084 obj->tiling_mode != I915_TILING_NONE) 5085 i915_gem_object_unpin_pages(obj); 5086 5087 if (WARN_ON(obj->pages_pin_count)) 5088 obj->pages_pin_count = 0; 5089 if (discard_backing_storage(obj)) 5090 obj->madv = I915_MADV_DONTNEED; 5091 i915_gem_object_put_pages(obj); 5092 i915_gem_object_free_mmap_offset(obj); 5093 5094 BUG_ON(obj->pages); 5095 5096 #if 0 5097 if (obj->base.import_attach) 5098 drm_prime_gem_destroy(&obj->base, NULL); 5099 #endif 5100 5101 if (obj->ops->release) 5102 obj->ops->release(obj); 5103 5104 drm_gem_object_release(&obj->base); 5105 i915_gem_info_remove_obj(dev_priv, obj->base.size); 5106 5107 kfree(obj->bit_17); 5108 i915_gem_object_free(obj); 5109 5110 intel_runtime_pm_put(dev_priv); 5111 } 5112 5113 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, 5114 struct i915_address_space *vm) 5115 { 5116 struct i915_vma *vma; 5117 list_for_each_entry(vma, &obj->vma_list, obj_link) { 5118 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL && 5119 vma->vm == vm) 5120 return vma; 5121 } 5122 return NULL; 5123 } 5124 5125 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj, 5126 const struct i915_ggtt_view *view) 5127 { 5128 struct i915_vma *vma; 5129 5130 GEM_BUG_ON(!view); 5131 5132 list_for_each_entry(vma, &obj->vma_list, obj_link) 5133 if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view)) 5134 return vma; 5135 return NULL; 5136 } 5137 5138 void i915_gem_vma_destroy(struct i915_vma *vma) 5139 { 5140 WARN_ON(vma->node.allocated); 5141 5142 /* Keep the vma as a placeholder in the execbuffer reservation lists */ 5143 if (!list_empty(&vma->exec_list)) 5144 return; 5145 5146 if (!vma->is_ggtt) 5147 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm)); 5148 5149 list_del(&vma->obj_link); 5150 5151 kfree(vma); 5152 } 5153 5154 static void 5155 i915_gem_stop_engines(struct drm_device *dev) 5156 { 5157 struct drm_i915_private *dev_priv = dev->dev_private; 5158 struct intel_engine_cs *engine; 5159 5160 for_each_engine(engine, dev_priv) 5161 dev_priv->gt.stop_engine(engine); 5162 } 5163 5164 int 5165 i915_gem_suspend(struct drm_device *dev) 5166 { 5167 struct drm_i915_private *dev_priv = dev->dev_private; 5168 int ret = 0; 5169 5170 mutex_lock(&dev->struct_mutex); 5171 ret = i915_gem_wait_for_idle(dev_priv); 5172 if (ret) 5173 goto err; 5174 5175 i915_gem_retire_requests(dev_priv); 5176 5177 i915_gem_stop_engines(dev); 5178 i915_gem_context_lost(dev_priv); 5179 mutex_unlock(&dev->struct_mutex); 5180 5181 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); 5182 cancel_delayed_work_sync(&dev_priv->mm.retire_work); 5183 flush_delayed_work(&dev_priv->mm.idle_work); 5184 5185 /* Assert that we sucessfully flushed all the work and 5186 * reset the GPU back to its idle, low power state. 5187 */ 5188 WARN_ON(dev_priv->mm.busy); 5189 5190 return 0; 5191 5192 err: 5193 mutex_unlock(&dev->struct_mutex); 5194 return ret; 5195 } 5196 5197 void i915_gem_init_swizzling(struct drm_device *dev) 5198 { 5199 struct drm_i915_private *dev_priv = dev->dev_private; 5200 5201 if (INTEL_INFO(dev)->gen < 5 || 5202 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 5203 return; 5204 5205 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 5206 DISP_TILE_SURFACE_SWIZZLING); 5207 5208 if (IS_GEN5(dev)) 5209 return; 5210 5211 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 5212 if (IS_GEN6(dev)) 5213 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); 5214 else if (IS_GEN7(dev)) 5215 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); 5216 else if (IS_GEN8(dev)) 5217 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW)); 5218 else 5219 BUG(); 5220 } 5221 5222 static void init_unused_ring(struct drm_device *dev, u32 base) 5223 { 5224 struct drm_i915_private *dev_priv = dev->dev_private; 5225 5226 I915_WRITE(RING_CTL(base), 0); 5227 I915_WRITE(RING_HEAD(base), 0); 5228 I915_WRITE(RING_TAIL(base), 0); 5229 I915_WRITE(RING_START(base), 0); 5230 } 5231 5232 static void init_unused_rings(struct drm_device *dev) 5233 { 5234 if (IS_I830(dev)) { 5235 init_unused_ring(dev, PRB1_BASE); 5236 init_unused_ring(dev, SRB0_BASE); 5237 init_unused_ring(dev, SRB1_BASE); 5238 init_unused_ring(dev, SRB2_BASE); 5239 init_unused_ring(dev, SRB3_BASE); 5240 } else if (IS_GEN2(dev)) { 5241 init_unused_ring(dev, SRB0_BASE); 5242 init_unused_ring(dev, SRB1_BASE); 5243 } else if (IS_GEN3(dev)) { 5244 init_unused_ring(dev, PRB1_BASE); 5245 init_unused_ring(dev, PRB2_BASE); 5246 } 5247 } 5248 5249 int i915_gem_init_engines(struct drm_device *dev) 5250 { 5251 struct drm_i915_private *dev_priv = dev->dev_private; 5252 int ret; 5253 5254 ret = intel_init_render_ring_buffer(dev); 5255 if (ret) 5256 return ret; 5257 5258 if (HAS_BSD(dev)) { 5259 ret = intel_init_bsd_ring_buffer(dev); 5260 if (ret) 5261 goto cleanup_render_ring; 5262 } 5263 5264 if (HAS_BLT(dev)) { 5265 ret = intel_init_blt_ring_buffer(dev); 5266 if (ret) 5267 goto cleanup_bsd_ring; 5268 } 5269 5270 if (HAS_VEBOX(dev)) { 5271 ret = intel_init_vebox_ring_buffer(dev); 5272 if (ret) 5273 goto cleanup_blt_ring; 5274 } 5275 5276 if (HAS_BSD2(dev)) { 5277 ret = intel_init_bsd2_ring_buffer(dev); 5278 if (ret) 5279 goto cleanup_vebox_ring; 5280 } 5281 5282 return 0; 5283 5284 cleanup_vebox_ring: 5285 intel_cleanup_engine(&dev_priv->engine[VECS]); 5286 cleanup_blt_ring: 5287 intel_cleanup_engine(&dev_priv->engine[BCS]); 5288 cleanup_bsd_ring: 5289 intel_cleanup_engine(&dev_priv->engine[VCS]); 5290 cleanup_render_ring: 5291 intel_cleanup_engine(&dev_priv->engine[RCS]); 5292 5293 return ret; 5294 } 5295 5296 int 5297 i915_gem_init_hw(struct drm_device *dev) 5298 { 5299 struct drm_i915_private *dev_priv = dev->dev_private; 5300 struct intel_engine_cs *engine; 5301 int ret; 5302 5303 /* Double layer security blanket, see i915_gem_init() */ 5304 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 5305 5306 if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9) 5307 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf)); 5308 5309 if (IS_HASWELL(dev)) 5310 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ? 5311 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED); 5312 5313 if (HAS_PCH_NOP(dev)) { 5314 if (IS_IVYBRIDGE(dev)) { 5315 u32 temp = I915_READ(GEN7_MSG_CTL); 5316 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK); 5317 I915_WRITE(GEN7_MSG_CTL, temp); 5318 } else if (INTEL_INFO(dev)->gen >= 7) { 5319 u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT); 5320 temp &= ~RESET_PCH_HANDSHAKE_ENABLE; 5321 I915_WRITE(HSW_NDE_RSTWRN_OPT, temp); 5322 } 5323 } 5324 5325 i915_gem_init_swizzling(dev); 5326 5327 /* 5328 * At least 830 can leave some of the unused rings 5329 * "active" (ie. head != tail) after resume which 5330 * will prevent c3 entry. Makes sure all unused rings 5331 * are totally idle. 5332 */ 5333 init_unused_rings(dev); 5334 5335 BUG_ON(!dev_priv->kernel_context); 5336 5337 ret = i915_ppgtt_init_hw(dev); 5338 if (ret) { 5339 DRM_ERROR("PPGTT enable HW failed %d\n", ret); 5340 goto out; 5341 } 5342 5343 /* Need to do basic initialisation of all rings first: */ 5344 for_each_engine(engine, dev_priv) { 5345 ret = engine->init_hw(engine); 5346 if (ret) 5347 goto out; 5348 } 5349 5350 intel_mocs_init_l3cc_table(dev); 5351 5352 /* We can't enable contexts until all firmware is loaded */ 5353 ret = intel_guc_setup(dev); 5354 if (ret) 5355 goto out; 5356 5357 /* 5358 * Increment the next seqno by 0x100 so we have a visible break 5359 * on re-initialisation 5360 */ 5361 ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100); 5362 5363 out: 5364 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5365 return ret; 5366 } 5367 5368 int i915_gem_init(struct drm_device *dev) 5369 { 5370 struct drm_i915_private *dev_priv = dev->dev_private; 5371 int ret; 5372 5373 mutex_lock(&dev->struct_mutex); 5374 5375 if (!i915.enable_execlists) { 5376 dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission; 5377 dev_priv->gt.init_engines = i915_gem_init_engines; 5378 dev_priv->gt.cleanup_engine = intel_cleanup_engine; 5379 dev_priv->gt.stop_engine = intel_stop_engine; 5380 } else { 5381 dev_priv->gt.execbuf_submit = intel_execlists_submission; 5382 dev_priv->gt.init_engines = intel_logical_rings_init; 5383 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup; 5384 dev_priv->gt.stop_engine = intel_logical_ring_stop; 5385 } 5386 5387 /* This is just a security blanket to placate dragons. 5388 * On some systems, we very sporadically observe that the first TLBs 5389 * used by the CS may be stale, despite us poking the TLB reset. If 5390 * we hold the forcewake during initialisation these problems 5391 * just magically go away. 5392 */ 5393 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 5394 5395 i915_gem_init_userptr(dev_priv); 5396 i915_gem_init_ggtt(dev); 5397 5398 ret = i915_gem_context_init(dev); 5399 if (ret) 5400 goto out_unlock; 5401 5402 ret = dev_priv->gt.init_engines(dev); 5403 if (ret) 5404 goto out_unlock; 5405 5406 ret = i915_gem_init_hw(dev); 5407 if (ret == -EIO) { 5408 /* Allow ring initialisation to fail by marking the GPU as 5409 * wedged. But we only want to do this where the GPU is angry, 5410 * for all other failure, such as an allocation failure, bail. 5411 */ 5412 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n"); 5413 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter); 5414 ret = 0; 5415 } 5416 5417 out_unlock: 5418 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5419 mutex_unlock(&dev->struct_mutex); 5420 5421 return ret; 5422 } 5423 5424 void 5425 i915_gem_cleanup_engines(struct drm_device *dev) 5426 { 5427 struct drm_i915_private *dev_priv = dev->dev_private; 5428 struct intel_engine_cs *engine; 5429 5430 for_each_engine(engine, dev_priv) 5431 dev_priv->gt.cleanup_engine(engine); 5432 } 5433 5434 static void 5435 init_engine_lists(struct intel_engine_cs *engine) 5436 { 5437 INIT_LIST_HEAD(&engine->active_list); 5438 INIT_LIST_HEAD(&engine->request_list); 5439 } 5440 5441 void 5442 i915_gem_load_init_fences(struct drm_i915_private *dev_priv) 5443 { 5444 struct drm_device *dev = dev_priv->dev; 5445 5446 if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) && 5447 !IS_CHERRYVIEW(dev_priv)) 5448 dev_priv->num_fence_regs = 32; 5449 else if (INTEL_INFO(dev_priv)->gen >= 4 || IS_I945G(dev_priv) || 5450 IS_I945GM(dev_priv) || IS_G33(dev_priv)) 5451 dev_priv->num_fence_regs = 16; 5452 else 5453 dev_priv->num_fence_regs = 8; 5454 5455 if (intel_vgpu_active(dev_priv)) 5456 dev_priv->num_fence_regs = 5457 I915_READ(vgtif_reg(avail_rs.fence_num)); 5458 5459 /* Initialize fence registers to zero */ 5460 i915_gem_restore_fences(dev); 5461 5462 i915_gem_detect_bit_6_swizzle(dev); 5463 } 5464 5465 void 5466 i915_gem_load_init(struct drm_device *dev) 5467 { 5468 struct drm_i915_private *dev_priv = dev->dev_private; 5469 int i; 5470 5471 INIT_LIST_HEAD(&dev_priv->vm_list); 5472 INIT_LIST_HEAD(&dev_priv->context_list); 5473 INIT_LIST_HEAD(&dev_priv->mm.unbound_list); 5474 INIT_LIST_HEAD(&dev_priv->mm.bound_list); 5475 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 5476 for (i = 0; i < I915_NUM_ENGINES; i++) 5477 init_engine_lists(&dev_priv->engine[i]); 5478 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 5479 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 5480 INIT_DELAYED_WORK(&dev_priv->mm.retire_work, 5481 i915_gem_retire_work_handler); 5482 INIT_DELAYED_WORK(&dev_priv->mm.idle_work, 5483 i915_gem_idle_work_handler); 5484 init_waitqueue_head(&dev_priv->gpu_error.wait_queue); 5485 init_waitqueue_head(&dev_priv->gpu_error.reset_queue); 5486 5487 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 5488 5489 /* 5490 * Set initial sequence number for requests. 5491 * Using this number allows the wraparound to happen early, 5492 * catching any obvious problems. 5493 */ 5494 dev_priv->next_seqno = ((u32)~0 - 0x1100); 5495 dev_priv->last_seqno = ((u32)~0 - 0x1101); 5496 5497 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 5498 5499 init_waitqueue_head(&dev_priv->pending_flip_queue); 5500 5501 dev_priv->mm.interruptible = true; 5502 5503 lockinit(&dev_priv->fb_tracking.lock, "drmftl", 0, LK_CANRECURSE); 5504 } 5505 5506 void i915_gem_load_cleanup(struct drm_device *dev) 5507 { 5508 #if 0 5509 struct drm_i915_private *dev_priv = to_i915(dev); 5510 5511 kmem_cache_destroy(dev_priv->requests); 5512 kmem_cache_destroy(dev_priv->vmas); 5513 kmem_cache_destroy(dev_priv->objects); 5514 #endif 5515 } 5516 5517 int i915_gem_freeze_late(struct drm_i915_private *dev_priv) 5518 { 5519 struct drm_i915_gem_object *obj; 5520 5521 /* Called just before we write the hibernation image. 5522 * 5523 * We need to update the domain tracking to reflect that the CPU 5524 * will be accessing all the pages to create and restore from the 5525 * hibernation, and so upon restoration those pages will be in the 5526 * CPU domain. 5527 * 5528 * To make sure the hibernation image contains the latest state, 5529 * we update that state just before writing out the image. 5530 */ 5531 5532 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { 5533 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 5534 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 5535 } 5536 5537 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 5538 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 5539 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 5540 } 5541 5542 return 0; 5543 } 5544 5545 void i915_gem_release(struct drm_device *dev, struct drm_file *file) 5546 { 5547 struct drm_i915_file_private *file_priv = file->driver_priv; 5548 5549 /* Clean up our request list when the client is going away, so that 5550 * later retire_requests won't dereference our soon-to-be-gone 5551 * file_priv. 5552 */ 5553 spin_lock(&file_priv->mm.lock); 5554 while (!list_empty(&file_priv->mm.request_list)) { 5555 struct drm_i915_gem_request *request; 5556 5557 request = list_first_entry(&file_priv->mm.request_list, 5558 struct drm_i915_gem_request, 5559 client_list); 5560 list_del(&request->client_list); 5561 request->file_priv = NULL; 5562 } 5563 spin_unlock(&file_priv->mm.lock); 5564 5565 if (!list_empty(&file_priv->rps.link)) { 5566 lockmgr(&to_i915(dev)->rps.client_lock, LK_EXCLUSIVE); 5567 list_del(&file_priv->rps.link); 5568 lockmgr(&to_i915(dev)->rps.client_lock, LK_RELEASE); 5569 } 5570 } 5571 5572 int 5573 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 5574 vm_ooffset_t foff, struct ucred *cred, u_short *color) 5575 { 5576 *color = 0; /* XXXKIB */ 5577 return (0); 5578 } 5579 5580 void 5581 i915_gem_pager_dtor(void *handle) 5582 { 5583 struct drm_gem_object *obj; 5584 struct drm_device *dev; 5585 5586 obj = handle; 5587 dev = obj->dev; 5588 5589 mutex_lock(&dev->struct_mutex); 5590 drm_gem_free_mmap_offset(obj); 5591 i915_gem_release_mmap(to_intel_bo(obj)); 5592 drm_gem_object_unreference(obj); 5593 mutex_unlock(&dev->struct_mutex); 5594 } 5595 5596 int i915_gem_open(struct drm_device *dev, struct drm_file *file) 5597 { 5598 struct drm_i915_file_private *file_priv; 5599 int ret; 5600 5601 DRM_DEBUG_DRIVER("\n"); 5602 5603 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL); 5604 if (!file_priv) 5605 return -ENOMEM; 5606 5607 file->driver_priv = file_priv; 5608 file_priv->dev_priv = dev->dev_private; 5609 file_priv->file = file; 5610 INIT_LIST_HEAD(&file_priv->rps.link); 5611 5612 spin_init(&file_priv->mm.lock, "i915_priv"); 5613 INIT_LIST_HEAD(&file_priv->mm.request_list); 5614 5615 file_priv->bsd_ring = -1; 5616 5617 ret = i915_gem_context_open(dev, file); 5618 if (ret) 5619 kfree(file_priv); 5620 5621 return ret; 5622 } 5623 5624 /** 5625 * i915_gem_track_fb - update frontbuffer tracking 5626 * @old: current GEM buffer for the frontbuffer slots 5627 * @new: new GEM buffer for the frontbuffer slots 5628 * @frontbuffer_bits: bitmask of frontbuffer slots 5629 * 5630 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them 5631 * from @old and setting them in @new. Both @old and @new can be NULL. 5632 */ 5633 void i915_gem_track_fb(struct drm_i915_gem_object *old, 5634 struct drm_i915_gem_object *new, 5635 unsigned frontbuffer_bits) 5636 { 5637 if (old) { 5638 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex)); 5639 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits)); 5640 old->frontbuffer_bits &= ~frontbuffer_bits; 5641 } 5642 5643 if (new) { 5644 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex)); 5645 WARN_ON(new->frontbuffer_bits & frontbuffer_bits); 5646 new->frontbuffer_bits |= frontbuffer_bits; 5647 } 5648 } 5649 5650 /* All the new VM stuff */ 5651 u64 i915_gem_obj_offset(struct drm_i915_gem_object *o, 5652 struct i915_address_space *vm) 5653 { 5654 struct drm_i915_private *dev_priv = o->base.dev->dev_private; 5655 struct i915_vma *vma; 5656 5657 WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base); 5658 5659 list_for_each_entry(vma, &o->vma_list, obj_link) { 5660 if (vma->is_ggtt && 5661 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) 5662 continue; 5663 if (vma->vm == vm) 5664 return vma->node.start; 5665 } 5666 5667 WARN(1, "%s vma for this object not found.\n", 5668 i915_is_ggtt(vm) ? "global" : "ppgtt"); 5669 return -1; 5670 } 5671 5672 u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o, 5673 const struct i915_ggtt_view *view) 5674 { 5675 struct i915_vma *vma; 5676 5677 list_for_each_entry(vma, &o->vma_list, obj_link) 5678 if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view)) 5679 return vma->node.start; 5680 5681 WARN(1, "global vma for this object not found. (view=%u)\n", view->type); 5682 return -1; 5683 } 5684 5685 bool i915_gem_obj_bound(struct drm_i915_gem_object *o, 5686 struct i915_address_space *vm) 5687 { 5688 struct i915_vma *vma; 5689 5690 list_for_each_entry(vma, &o->vma_list, obj_link) { 5691 if (vma->is_ggtt && 5692 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) 5693 continue; 5694 if (vma->vm == vm && drm_mm_node_allocated(&vma->node)) 5695 return true; 5696 } 5697 5698 return false; 5699 } 5700 5701 bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o, 5702 const struct i915_ggtt_view *view) 5703 { 5704 struct i915_vma *vma; 5705 5706 list_for_each_entry(vma, &o->vma_list, obj_link) 5707 if (vma->is_ggtt && 5708 i915_ggtt_view_equal(&vma->ggtt_view, view) && 5709 drm_mm_node_allocated(&vma->node)) 5710 return true; 5711 5712 return false; 5713 } 5714 5715 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o) 5716 { 5717 struct i915_vma *vma; 5718 5719 list_for_each_entry(vma, &o->vma_list, obj_link) 5720 if (drm_mm_node_allocated(&vma->node)) 5721 return true; 5722 5723 return false; 5724 } 5725 5726 unsigned long i915_gem_obj_ggtt_size(struct drm_i915_gem_object *o) 5727 { 5728 struct i915_vma *vma; 5729 5730 GEM_BUG_ON(list_empty(&o->vma_list)); 5731 5732 list_for_each_entry(vma, &o->vma_list, obj_link) { 5733 if (vma->is_ggtt && 5734 vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) 5735 return vma->node.size; 5736 } 5737 5738 return 0; 5739 } 5740 5741 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj) 5742 { 5743 struct i915_vma *vma; 5744 list_for_each_entry(vma, &obj->vma_list, obj_link) 5745 if (vma->pin_count > 0) 5746 return true; 5747 5748 return false; 5749 } 5750 5751 /* Like i915_gem_object_get_page(), but mark the returned page dirty */ 5752 struct page * 5753 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n) 5754 { 5755 struct page *page; 5756 5757 /* Only default objects have per-page dirty tracking */ 5758 if (WARN_ON(!i915_gem_object_has_struct_page(obj))) 5759 return NULL; 5760 5761 page = i915_gem_object_get_page(obj, n); 5762 set_page_dirty(page); 5763 return page; 5764 } 5765 5766 /* Allocate a new GEM object and fill it with the supplied data */ 5767 struct drm_i915_gem_object * 5768 i915_gem_object_create_from_data(struct drm_device *dev, 5769 const void *data, size_t size) 5770 { 5771 struct drm_i915_gem_object *obj; 5772 struct sg_table *sg; 5773 size_t bytes; 5774 int ret; 5775 5776 obj = i915_gem_object_create(dev, round_up(size, PAGE_SIZE)); 5777 if (IS_ERR(obj)) 5778 return obj; 5779 5780 ret = i915_gem_object_set_to_cpu_domain(obj, true); 5781 if (ret) 5782 goto fail; 5783 5784 ret = i915_gem_object_get_pages(obj); 5785 if (ret) 5786 goto fail; 5787 5788 i915_gem_object_pin_pages(obj); 5789 sg = obj->pages; 5790 bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size); 5791 obj->dirty = 1; /* Backing store is now out of date */ 5792 i915_gem_object_unpin_pages(obj); 5793 5794 if (WARN_ON(bytes != size)) { 5795 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size); 5796 ret = -EFAULT; 5797 goto fail; 5798 } 5799 5800 return obj; 5801 5802 fail: 5803 drm_gem_object_unreference(&obj->base); 5804 return ERR_PTR(ret); 5805 } 5806