1 /* 2 * Copyright © 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 * Copyright (c) 2011 The FreeBSD Foundation 27 * All rights reserved. 28 * 29 * This software was developed by Konstantin Belousov under sponsorship from 30 * the FreeBSD Foundation. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51 * SUCH DAMAGE. 52 * 53 */ 54 55 #include <machine/md_var.h> 56 57 #include <drm/drmP.h> 58 #include <drm/drm_vma_manager.h> 59 #include <drm/i915_drm.h> 60 #include "i915_drv.h" 61 #include "i915_trace.h" 62 #include "intel_drv.h" 63 #include <linux/shmem_fs.h> 64 #include <linux/slab.h> 65 #include <linux/swap.h> 66 #include <linux/pci.h> 67 68 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); 69 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj, 70 bool force); 71 static __must_check int 72 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, 73 bool readonly); 74 static __must_check int 75 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, 76 struct i915_address_space *vm, 77 unsigned alignment, 78 bool map_and_fenceable, 79 bool nonblocking); 80 static int i915_gem_phys_pwrite(struct drm_device *dev, 81 struct drm_i915_gem_object *obj, 82 struct drm_i915_gem_pwrite *args, 83 struct drm_file *file); 84 85 static void i915_gem_write_fence(struct drm_device *dev, int reg, 86 struct drm_i915_gem_object *obj); 87 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, 88 struct drm_i915_fence_reg *fence, 89 bool enable); 90 91 static unsigned long i915_gem_purge(struct drm_i915_private *dev_priv, long target); 92 static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv); 93 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 94 95 static bool cpu_cache_is_coherent(struct drm_device *dev, 96 enum i915_cache_level level) 97 { 98 return HAS_LLC(dev) || level != I915_CACHE_NONE; 99 } 100 101 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) 102 { 103 if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) 104 return true; 105 106 return obj->pin_display; 107 } 108 109 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) 110 { 111 if (obj->tiling_mode) 112 i915_gem_release_mmap(obj); 113 114 /* As we do not have an associated fence register, we will force 115 * a tiling change if we ever need to acquire one. 116 */ 117 obj->fence_dirty = false; 118 obj->fence_reg = I915_FENCE_REG_NONE; 119 } 120 121 /* some bookkeeping */ 122 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, 123 size_t size) 124 { 125 spin_lock(&dev_priv->mm.object_stat_lock); 126 dev_priv->mm.object_count++; 127 dev_priv->mm.object_memory += size; 128 spin_unlock(&dev_priv->mm.object_stat_lock); 129 } 130 131 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, 132 size_t size) 133 { 134 spin_lock(&dev_priv->mm.object_stat_lock); 135 dev_priv->mm.object_count--; 136 dev_priv->mm.object_memory -= size; 137 spin_unlock(&dev_priv->mm.object_stat_lock); 138 } 139 140 static int 141 i915_gem_wait_for_error(struct i915_gpu_error *error) 142 { 143 int ret; 144 145 #define EXIT_COND (!i915_reset_in_progress(error) || \ 146 i915_terminally_wedged(error)) 147 if (EXIT_COND) 148 return 0; 149 150 /* 151 * Only wait 10 seconds for the gpu reset to complete to avoid hanging 152 * userspace. If it takes that long something really bad is going on and 153 * we should simply try to bail out and fail as gracefully as possible. 154 */ 155 ret = wait_event_interruptible_timeout(error->reset_queue, 156 EXIT_COND, 157 10*HZ); 158 if (ret == 0) { 159 DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); 160 return -EIO; 161 } else if (ret < 0) { 162 return ret; 163 } 164 #undef EXIT_COND 165 166 return 0; 167 } 168 169 int i915_mutex_lock_interruptible(struct drm_device *dev) 170 { 171 struct drm_i915_private *dev_priv = dev->dev_private; 172 int ret; 173 174 ret = i915_gem_wait_for_error(&dev_priv->gpu_error); 175 if (ret) 176 return ret; 177 178 ret = lockmgr(&dev->struct_mutex, LK_EXCLUSIVE|LK_SLEEPFAIL); 179 if (ret) 180 return -EINTR; 181 182 WARN_ON(i915_verify_lists(dev)); 183 return 0; 184 } 185 186 static inline bool 187 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 188 { 189 return i915_gem_obj_bound_any(obj) && !obj->active; 190 } 191 192 int 193 i915_gem_init_ioctl(struct drm_device *dev, void *data, 194 struct drm_file *file) 195 { 196 struct drm_i915_private *dev_priv = dev->dev_private; 197 struct drm_i915_gem_init *args = data; 198 199 if (drm_core_check_feature(dev, DRIVER_MODESET)) 200 return -ENODEV; 201 202 if (args->gtt_start >= args->gtt_end || 203 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 204 return -EINVAL; 205 206 /* GEM with user mode setting was never supported on ilk and later. */ 207 if (INTEL_INFO(dev)->gen >= 5) 208 return -ENODEV; 209 210 mutex_lock(&dev->struct_mutex); 211 dev_priv->gtt.mappable_end = args->gtt_end; 212 kprintf("INITGLOBALGTT GTT_START %016jx\n", (uintmax_t)args->gtt_start); 213 i915_gem_init_global_gtt(dev); 214 #if 0 215 i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end, 216 args->gtt_end); 217 #endif 218 mutex_unlock(&dev->struct_mutex); 219 220 return 0; 221 } 222 223 int 224 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 225 struct drm_file *file) 226 { 227 struct drm_i915_private *dev_priv = dev->dev_private; 228 struct drm_i915_gem_get_aperture *args = data; 229 struct drm_i915_gem_object *obj; 230 size_t pinned; 231 232 pinned = 0; 233 mutex_lock(&dev->struct_mutex); 234 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) 235 if (obj->pin_count) 236 pinned += i915_gem_obj_ggtt_size(obj); 237 mutex_unlock(&dev->struct_mutex); 238 239 args->aper_size = dev_priv->gtt.base.total; 240 args->aper_available_size = args->aper_size - pinned; 241 242 return 0; 243 } 244 245 void *i915_gem_object_alloc(struct drm_device *dev) 246 { 247 return kmalloc(sizeof(struct drm_i915_gem_object), 248 M_DRM, M_WAITOK | M_ZERO); 249 } 250 251 void i915_gem_object_free(struct drm_i915_gem_object *obj) 252 { 253 kfree(obj); 254 } 255 256 static int 257 i915_gem_create(struct drm_file *file, 258 struct drm_device *dev, 259 uint64_t size, 260 uint32_t *handle_p) 261 { 262 struct drm_i915_gem_object *obj; 263 int ret; 264 u32 handle; 265 266 size = roundup(size, PAGE_SIZE); 267 if (size == 0) 268 return -EINVAL; 269 270 /* Allocate the new object */ 271 obj = i915_gem_alloc_object(dev, size); 272 if (obj == NULL) 273 return -ENOMEM; 274 275 ret = drm_gem_handle_create(file, &obj->base, &handle); 276 /* drop reference from allocate - handle holds it now */ 277 drm_gem_object_unreference_unlocked(&obj->base); 278 if (ret) 279 return ret; 280 281 *handle_p = handle; 282 return 0; 283 } 284 285 int 286 i915_gem_dumb_create(struct drm_file *file, 287 struct drm_device *dev, 288 struct drm_mode_create_dumb *args) 289 { 290 /* have to work out size/pitch and return them */ 291 args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64); 292 args->size = args->pitch * args->height; 293 return i915_gem_create(file, dev, 294 args->size, &args->handle); 295 } 296 297 /** 298 * Creates a new mm object and returns a handle to it. 299 */ 300 int 301 i915_gem_create_ioctl(struct drm_device *dev, void *data, 302 struct drm_file *file) 303 { 304 struct drm_i915_gem_create *args = data; 305 306 return i915_gem_create(file, dev, 307 args->size, &args->handle); 308 } 309 310 static inline int 311 __copy_to_user_swizzled(char __user *cpu_vaddr, 312 const char *gpu_vaddr, int gpu_offset, 313 int length) 314 { 315 int ret, cpu_offset = 0; 316 317 while (length > 0) { 318 int cacheline_end = ALIGN(gpu_offset + 1, 64); 319 int this_length = min(cacheline_end - gpu_offset, length); 320 int swizzled_gpu_offset = gpu_offset ^ 64; 321 322 ret = __copy_to_user(cpu_vaddr + cpu_offset, 323 gpu_vaddr + swizzled_gpu_offset, 324 this_length); 325 if (ret) 326 return ret + length; 327 328 cpu_offset += this_length; 329 gpu_offset += this_length; 330 length -= this_length; 331 } 332 333 return 0; 334 } 335 336 static inline int 337 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, 338 const char __user *cpu_vaddr, 339 int length) 340 { 341 int ret, cpu_offset = 0; 342 343 while (length > 0) { 344 int cacheline_end = ALIGN(gpu_offset + 1, 64); 345 int this_length = min(cacheline_end - gpu_offset, length); 346 int swizzled_gpu_offset = gpu_offset ^ 64; 347 348 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, 349 cpu_vaddr + cpu_offset, 350 this_length); 351 if (ret) 352 return ret + length; 353 354 cpu_offset += this_length; 355 gpu_offset += this_length; 356 length -= this_length; 357 } 358 359 return 0; 360 } 361 362 /* Per-page copy function for the shmem pread fastpath. 363 * Flushes invalid cachelines before reading the target if 364 * needs_clflush is set. */ 365 static int 366 shmem_pread_fast(struct vm_page *page, int shmem_page_offset, int page_length, 367 char __user *user_data, 368 bool page_do_bit17_swizzling, bool needs_clflush) 369 { 370 char *vaddr; 371 int ret; 372 373 if (unlikely(page_do_bit17_swizzling)) 374 return -EINVAL; 375 376 vaddr = kmap_atomic(page); 377 if (needs_clflush) 378 drm_clflush_virt_range(vaddr + shmem_page_offset, 379 page_length); 380 ret = __copy_to_user_inatomic(user_data, 381 vaddr + shmem_page_offset, 382 page_length); 383 kunmap_atomic(vaddr); 384 385 return ret ? -EFAULT : 0; 386 } 387 388 static void 389 shmem_clflush_swizzled_range(char *addr, unsigned long length, 390 bool swizzled) 391 { 392 if (unlikely(swizzled)) { 393 unsigned long start = (unsigned long) addr; 394 unsigned long end = (unsigned long) addr + length; 395 396 /* For swizzling simply ensure that we always flush both 397 * channels. Lame, but simple and it works. Swizzled 398 * pwrite/pread is far from a hotpath - current userspace 399 * doesn't use it at all. */ 400 start = round_down(start, 128); 401 end = round_up(end, 128); 402 403 drm_clflush_virt_range((void *)start, end - start); 404 } else { 405 drm_clflush_virt_range(addr, length); 406 } 407 408 } 409 410 /* Only difference to the fast-path function is that this can handle bit17 411 * and uses non-atomic copy and kmap functions. */ 412 static int 413 shmem_pread_slow(struct vm_page *page, int shmem_page_offset, int page_length, 414 char __user *user_data, 415 bool page_do_bit17_swizzling, bool needs_clflush) 416 { 417 char *vaddr; 418 int ret; 419 420 vaddr = kmap(page); 421 if (needs_clflush) 422 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 423 page_length, 424 page_do_bit17_swizzling); 425 426 if (page_do_bit17_swizzling) 427 ret = __copy_to_user_swizzled(user_data, 428 vaddr, shmem_page_offset, 429 page_length); 430 else 431 ret = __copy_to_user(user_data, 432 vaddr + shmem_page_offset, 433 page_length); 434 kunmap(page); 435 436 return ret ? - EFAULT : 0; 437 } 438 439 static int 440 i915_gem_shmem_pread(struct drm_device *dev, 441 struct drm_i915_gem_object *obj, 442 struct drm_i915_gem_pread *args, 443 struct drm_file *file) 444 { 445 char __user *user_data; 446 ssize_t remain; 447 loff_t offset; 448 int shmem_page_offset, page_length, ret = 0; 449 int obj_do_bit17_swizzling, page_do_bit17_swizzling; 450 int prefaulted = 0; 451 int needs_clflush = 0; 452 int i; 453 454 user_data = to_user_ptr(args->data_ptr); 455 remain = args->size; 456 457 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 458 459 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { 460 /* If we're not in the cpu read domain, set ourself into the gtt 461 * read domain and manually flush cachelines (if required). This 462 * optimizes for the case when the gpu will dirty the data 463 * anyway again before the next pread happens. */ 464 needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level); 465 ret = i915_gem_object_wait_rendering(obj, true); 466 if (ret) 467 return ret; 468 } 469 470 ret = i915_gem_object_get_pages(obj); 471 if (ret) 472 return ret; 473 474 i915_gem_object_pin_pages(obj); 475 476 offset = args->offset; 477 478 for (i = 0; i < (obj->base.size >> PAGE_SHIFT); i++) { 479 struct vm_page *page = obj->pages[i]; 480 481 if (remain <= 0) 482 break; 483 484 /* Operation in this page 485 * 486 * shmem_page_offset = offset within page in shmem file 487 * page_length = bytes to copy for this page 488 */ 489 shmem_page_offset = offset_in_page(offset); 490 page_length = remain; 491 if ((shmem_page_offset + page_length) > PAGE_SIZE) 492 page_length = PAGE_SIZE - shmem_page_offset; 493 494 page_do_bit17_swizzling = obj_do_bit17_swizzling && 495 (page_to_phys(page) & (1 << 17)) != 0; 496 497 ret = shmem_pread_fast(page, shmem_page_offset, page_length, 498 user_data, page_do_bit17_swizzling, 499 needs_clflush); 500 if (ret == 0) 501 goto next_page; 502 503 mutex_unlock(&dev->struct_mutex); 504 505 if (likely(!i915_prefault_disable) && !prefaulted) { 506 ret = fault_in_multipages_writeable(user_data, remain); 507 /* Userspace is tricking us, but we've already clobbered 508 * its pages with the prefault and promised to write the 509 * data up to the first fault. Hence ignore any errors 510 * and just continue. */ 511 (void)ret; 512 prefaulted = 1; 513 } 514 515 ret = shmem_pread_slow(page, shmem_page_offset, page_length, 516 user_data, page_do_bit17_swizzling, 517 needs_clflush); 518 519 mutex_lock(&dev->struct_mutex); 520 521 next_page: 522 mark_page_accessed(page); 523 524 if (ret) 525 goto out; 526 527 remain -= page_length; 528 user_data += page_length; 529 offset += page_length; 530 } 531 532 out: 533 i915_gem_object_unpin_pages(obj); 534 535 return ret; 536 } 537 538 /** 539 * Reads data from the object referenced by handle. 540 * 541 * On error, the contents of *data are undefined. 542 */ 543 int 544 i915_gem_pread_ioctl(struct drm_device *dev, void *data, 545 struct drm_file *file) 546 { 547 struct drm_i915_gem_pread *args = data; 548 struct drm_i915_gem_object *obj; 549 int ret = 0; 550 551 if (args->size == 0) 552 return 0; 553 554 ret = i915_mutex_lock_interruptible(dev); 555 if (ret) 556 return ret; 557 558 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 559 if (&obj->base == NULL) { 560 ret = -ENOENT; 561 goto unlock; 562 } 563 564 /* Bounds check source. */ 565 if (args->offset > obj->base.size || 566 args->size > obj->base.size - args->offset) { 567 ret = -EINVAL; 568 goto out; 569 } 570 571 trace_i915_gem_object_pread(obj, args->offset, args->size); 572 573 ret = i915_gem_shmem_pread(dev, obj, args, file); 574 575 out: 576 drm_gem_object_unreference(&obj->base); 577 unlock: 578 mutex_unlock(&dev->struct_mutex); 579 return ret; 580 } 581 582 /* This is the fast write path which cannot handle 583 * page faults in the source data 584 */ 585 586 #if 0 /* XXX: buggy on core2 machines */ 587 static inline int 588 fast_user_write(struct io_mapping *mapping, 589 loff_t page_base, int page_offset, 590 char __user *user_data, 591 int length) 592 { 593 void __iomem *vaddr_atomic; 594 void *vaddr; 595 unsigned long unwritten; 596 597 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base); 598 /* We can use the cpu mem copy function because this is X86. */ 599 vaddr = (char __force*)vaddr_atomic + page_offset; 600 unwritten = __copy_from_user_inatomic_nocache(vaddr, 601 user_data, length); 602 io_mapping_unmap_atomic(vaddr_atomic); 603 return unwritten; 604 } 605 #endif 606 607 static int 608 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj, 609 uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file) 610 { 611 vm_offset_t mkva; 612 int ret; 613 614 /* 615 * Pass the unaligned physical address and size to pmap_mapdev_attr() 616 * so it can properly calculate whether an extra page needs to be 617 * mapped or not to cover the requested range. The function will 618 * add the page offset into the returned mkva for us. 619 */ 620 mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + 621 i915_gem_obj_ggtt_offset(obj) + offset, size, PAT_WRITE_COMBINING); 622 ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size); 623 pmap_unmapdev(mkva, size); 624 return ret; 625 } 626 627 /** 628 * This is the fast pwrite path, where we copy the data directly from the 629 * user into the GTT, uncached. 630 */ 631 static int 632 i915_gem_gtt_pwrite_fast(struct drm_device *dev, 633 struct drm_i915_gem_object *obj, 634 struct drm_i915_gem_pwrite *args, 635 struct drm_file *file) 636 { 637 ssize_t remain; 638 loff_t offset, page_base; 639 char __user *user_data; 640 int page_offset, page_length, ret; 641 642 ret = i915_gem_obj_ggtt_pin(obj, 0, true, true); 643 if (ret) 644 goto out; 645 646 ret = i915_gem_object_set_to_gtt_domain(obj, true); 647 if (ret) 648 goto out_unpin; 649 650 ret = i915_gem_object_put_fence(obj); 651 if (ret) 652 goto out_unpin; 653 654 user_data = to_user_ptr(args->data_ptr); 655 remain = args->size; 656 657 offset = i915_gem_obj_ggtt_offset(obj) + args->offset; 658 659 while (remain > 0) { 660 /* Operation in this page 661 * 662 * page_base = page offset within aperture 663 * page_offset = offset within page 664 * page_length = bytes to copy for this page 665 */ 666 page_base = offset & ~PAGE_MASK; 667 page_offset = offset_in_page(offset); 668 page_length = remain; 669 if ((page_offset + remain) > PAGE_SIZE) 670 page_length = PAGE_SIZE - page_offset; 671 672 /* If we get a fault while copying data, then (presumably) our 673 * source page isn't available. Return the error and we'll 674 * retry in the slow path. 675 */ 676 #if 0 677 if (fast_user_write(dev_priv->gtt.mappable, page_base, 678 page_offset, user_data, page_length)) { 679 #else 680 if (i915_gem_gtt_write(dev, obj, args->data_ptr, args->size, args->offset, file)) { 681 #endif 682 ret = -EFAULT; 683 goto out_unpin; 684 } 685 686 remain -= page_length; 687 user_data += page_length; 688 offset += page_length; 689 } 690 691 out_unpin: 692 i915_gem_object_unpin(obj); 693 out: 694 return ret; 695 } 696 697 /* Per-page copy function for the shmem pwrite fastpath. 698 * Flushes invalid cachelines before writing to the target if 699 * needs_clflush_before is set and flushes out any written cachelines after 700 * writing if needs_clflush is set. */ 701 static int 702 shmem_pwrite_fast(struct vm_page *page, int shmem_page_offset, int page_length, 703 char __user *user_data, 704 bool page_do_bit17_swizzling, 705 bool needs_clflush_before, 706 bool needs_clflush_after) 707 { 708 char *vaddr; 709 int ret; 710 711 if (unlikely(page_do_bit17_swizzling)) 712 return -EINVAL; 713 714 vaddr = kmap_atomic(page); 715 if (needs_clflush_before) 716 drm_clflush_virt_range(vaddr + shmem_page_offset, 717 page_length); 718 ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset, 719 user_data, 720 page_length); 721 if (needs_clflush_after) 722 drm_clflush_virt_range(vaddr + shmem_page_offset, 723 page_length); 724 kunmap_atomic(vaddr); 725 726 return ret ? -EFAULT : 0; 727 } 728 729 /* Only difference to the fast-path function is that this can handle bit17 730 * and uses non-atomic copy and kmap functions. */ 731 static int 732 shmem_pwrite_slow(struct vm_page *page, int shmem_page_offset, int page_length, 733 char __user *user_data, 734 bool page_do_bit17_swizzling, 735 bool needs_clflush_before, 736 bool needs_clflush_after) 737 { 738 char *vaddr; 739 int ret; 740 741 vaddr = kmap(page); 742 if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) 743 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 744 page_length, 745 page_do_bit17_swizzling); 746 if (page_do_bit17_swizzling) 747 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, 748 user_data, 749 page_length); 750 else 751 ret = __copy_from_user(vaddr + shmem_page_offset, 752 user_data, 753 page_length); 754 if (needs_clflush_after) 755 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 756 page_length, 757 page_do_bit17_swizzling); 758 kunmap(page); 759 760 return ret ? -EFAULT : 0; 761 } 762 763 static int 764 i915_gem_shmem_pwrite(struct drm_device *dev, 765 struct drm_i915_gem_object *obj, 766 struct drm_i915_gem_pwrite *args, 767 struct drm_file *file) 768 { 769 ssize_t remain; 770 loff_t offset; 771 char __user *user_data; 772 int shmem_page_offset, page_length, ret = 0; 773 int obj_do_bit17_swizzling, page_do_bit17_swizzling; 774 int hit_slowpath = 0; 775 int needs_clflush_after = 0; 776 int needs_clflush_before = 0; 777 int i; 778 779 user_data = to_user_ptr(args->data_ptr); 780 remain = args->size; 781 782 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 783 784 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 785 /* If we're not in the cpu write domain, set ourself into the gtt 786 * write domain and manually flush cachelines (if required). This 787 * optimizes for the case when the gpu will use the data 788 * right away and we therefore have to clflush anyway. */ 789 needs_clflush_after = cpu_write_needs_clflush(obj); 790 ret = i915_gem_object_wait_rendering(obj, false); 791 if (ret) 792 return ret; 793 } 794 /* Same trick applies to invalidate partially written cachelines read 795 * before writing. */ 796 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 797 needs_clflush_before = 798 !cpu_cache_is_coherent(dev, obj->cache_level); 799 800 ret = i915_gem_object_get_pages(obj); 801 if (ret) 802 return ret; 803 804 i915_gem_object_pin_pages(obj); 805 806 offset = args->offset; 807 obj->dirty = 1; 808 809 VM_OBJECT_LOCK(obj->base.vm_obj); 810 vm_object_pip_add(obj->base.vm_obj, 1); 811 for (i = 0; i < (obj->base.size >> PAGE_SHIFT); i++) { 812 struct vm_page *page = obj->pages[i]; 813 int partial_cacheline_write; 814 815 if (i < offset >> PAGE_SHIFT) 816 continue; 817 818 if (remain <= 0) 819 break; 820 821 /* Operation in this page 822 * 823 * shmem_page_offset = offset within page in shmem file 824 * page_length = bytes to copy for this page 825 */ 826 shmem_page_offset = offset_in_page(offset); 827 828 page_length = remain; 829 if ((shmem_page_offset + page_length) > PAGE_SIZE) 830 page_length = PAGE_SIZE - shmem_page_offset; 831 832 /* If we don't overwrite a cacheline completely we need to be 833 * careful to have up-to-date data by first clflushing. Don't 834 * overcomplicate things and flush the entire patch. */ 835 partial_cacheline_write = needs_clflush_before && 836 ((shmem_page_offset | page_length) 837 & (cpu_clflush_line_size - 1)); 838 839 page_do_bit17_swizzling = obj_do_bit17_swizzling && 840 (page_to_phys(page) & (1 << 17)) != 0; 841 842 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, 843 user_data, page_do_bit17_swizzling, 844 partial_cacheline_write, 845 needs_clflush_after); 846 if (ret == 0) 847 goto next_page; 848 849 hit_slowpath = 1; 850 mutex_unlock(&dev->struct_mutex); 851 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, 852 user_data, page_do_bit17_swizzling, 853 partial_cacheline_write, 854 needs_clflush_after); 855 856 mutex_lock(&dev->struct_mutex); 857 858 next_page: 859 set_page_dirty(page); 860 mark_page_accessed(page); 861 862 if (ret) 863 goto out; 864 865 remain -= page_length; 866 user_data += page_length; 867 offset += page_length; 868 } 869 vm_object_pip_wakeup(obj->base.vm_obj); 870 VM_OBJECT_UNLOCK(obj->base.vm_obj); 871 872 out: 873 i915_gem_object_unpin_pages(obj); 874 875 if (hit_slowpath) { 876 /* 877 * Fixup: Flush cpu caches in case we didn't flush the dirty 878 * cachelines in-line while writing and the object moved 879 * out of the cpu write domain while we've dropped the lock. 880 */ 881 if (!needs_clflush_after && 882 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 883 if (i915_gem_clflush_object(obj, obj->pin_display)) 884 i915_gem_chipset_flush(dev); 885 } 886 } 887 888 if (needs_clflush_after) 889 i915_gem_chipset_flush(dev); 890 891 return ret; 892 } 893 894 /** 895 * Writes data to the object referenced by handle. 896 * 897 * On error, the contents of the buffer that were to be modified are undefined. 898 */ 899 int 900 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 901 struct drm_file *file) 902 { 903 struct drm_i915_gem_pwrite *args = data; 904 struct drm_i915_gem_object *obj; 905 int ret; 906 907 if (args->size == 0) 908 return 0; 909 910 if (likely(!i915_prefault_disable)) { 911 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr), 912 args->size); 913 if (ret) 914 return -EFAULT; 915 } 916 917 ret = i915_mutex_lock_interruptible(dev); 918 if (ret) 919 return ret; 920 921 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 922 if (&obj->base == NULL) { 923 ret = -ENOENT; 924 goto unlock; 925 } 926 927 /* Bounds check destination. */ 928 if (args->offset > obj->base.size || 929 args->size > obj->base.size - args->offset) { 930 ret = -EINVAL; 931 goto out; 932 } 933 934 trace_i915_gem_object_pwrite(obj, args->offset, args->size); 935 936 ret = -EFAULT; 937 /* We can only do the GTT pwrite on untiled buffers, as otherwise 938 * it would end up going through the fenced access, and we'll get 939 * different detiling behavior between reading and writing. 940 * pread/pwrite currently are reading and writing from the CPU 941 * perspective, requiring manual detiling by the client. 942 */ 943 if (obj->phys_obj) { 944 ret = i915_gem_phys_pwrite(dev, obj, args, file); 945 goto out; 946 } 947 948 if (obj->tiling_mode == I915_TILING_NONE && 949 obj->base.write_domain != I915_GEM_DOMAIN_CPU && 950 cpu_write_needs_clflush(obj)) { 951 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); 952 /* Note that the gtt paths might fail with non-page-backed user 953 * pointers (e.g. gtt mappings when moving data between 954 * textures). Fallback to the shmem path in that case. */ 955 } 956 957 if (ret == -EFAULT || ret == -ENOSPC) 958 ret = i915_gem_shmem_pwrite(dev, obj, args, file); 959 960 out: 961 drm_gem_object_unreference(&obj->base); 962 unlock: 963 mutex_unlock(&dev->struct_mutex); 964 return ret; 965 } 966 967 int 968 i915_gem_check_wedge(struct i915_gpu_error *error, 969 bool interruptible) 970 { 971 if (i915_reset_in_progress(error)) { 972 /* Non-interruptible callers can't handle -EAGAIN, hence return 973 * -EIO unconditionally for these. */ 974 if (!interruptible) 975 return -EIO; 976 977 /* Recovery complete, but the reset failed ... */ 978 if (i915_terminally_wedged(error)) 979 return -EIO; 980 981 return -EAGAIN; 982 } 983 984 return 0; 985 } 986 987 /* 988 * Compare seqno against outstanding lazy request. Emit a request if they are 989 * equal. 990 */ 991 static int 992 i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno) 993 { 994 int ret; 995 996 BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex)); 997 998 ret = 0; 999 if (seqno == ring->outstanding_lazy_seqno) 1000 ret = i915_add_request(ring, NULL); 1001 1002 return ret; 1003 } 1004 1005 #if 0 1006 static void fake_irq(unsigned long data) 1007 { 1008 wake_up_process((struct task_struct *)data); 1009 } 1010 1011 static bool missed_irq(struct drm_i915_private *dev_priv, 1012 struct intel_ring_buffer *ring) 1013 { 1014 return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings); 1015 } 1016 1017 static bool can_wait_boost(struct drm_i915_file_private *file_priv) 1018 { 1019 if (file_priv == NULL) 1020 return true; 1021 1022 return !atomic_xchg(&file_priv->rps_wait_boost, true); 1023 } 1024 #endif 1025 1026 /** 1027 * __wait_seqno - wait until execution of seqno has finished 1028 * @ring: the ring expected to report seqno 1029 * @seqno: duh! 1030 * @reset_counter: reset sequence associated with the given seqno 1031 * @interruptible: do an interruptible wait (normally yes) 1032 * @timeout: in - how long to wait (NULL forever); out - how much time remaining 1033 * 1034 * Note: It is of utmost importance that the passed in seqno and reset_counter 1035 * values have been read by the caller in an smp safe manner. Where read-side 1036 * locks are involved, it is sufficient to read the reset_counter before 1037 * unlocking the lock that protects the seqno. For lockless tricks, the 1038 * reset_counter _must_ be read before, and an appropriate smp_rmb must be 1039 * inserted. 1040 * 1041 * Returns 0 if the seqno was found within the alloted time. Else returns the 1042 * errno with remaining time filled in timeout argument. 1043 */ 1044 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, 1045 unsigned reset_counter, 1046 bool interruptible, struct timespec *timeout) 1047 { 1048 drm_i915_private_t *dev_priv = ring->dev->dev_private; 1049 struct timespec before, now, wait_time={1,0}; 1050 unsigned long timeout_jiffies; 1051 long end; 1052 bool wait_forever = true; 1053 int ret; 1054 1055 WARN(dev_priv->pc8.irqs_disabled, "IRQs disabled\n"); 1056 1057 if (i915_seqno_passed(ring->get_seqno(ring, true), seqno)) 1058 return 0; 1059 1060 trace_i915_gem_request_wait_begin(ring, seqno); 1061 1062 if (timeout != NULL) { 1063 wait_time = *timeout; 1064 wait_forever = false; 1065 } 1066 1067 timeout_jiffies = timespec_to_jiffies_timeout(&wait_time); 1068 1069 if (WARN_ON(!ring->irq_get(ring))) 1070 return -ENODEV; 1071 1072 /* Record current time in case interrupted by signal, or wedged * */ 1073 getrawmonotonic(&before); 1074 1075 #define EXIT_COND \ 1076 (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \ 1077 i915_reset_in_progress(&dev_priv->gpu_error) || \ 1078 reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) 1079 do { 1080 if (interruptible) 1081 end = wait_event_interruptible_timeout(ring->irq_queue, 1082 EXIT_COND, 1083 timeout_jiffies); 1084 else 1085 end = wait_event_timeout(ring->irq_queue, EXIT_COND, 1086 timeout_jiffies); 1087 1088 /* We need to check whether any gpu reset happened in between 1089 * the caller grabbing the seqno and now ... */ 1090 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) 1091 end = -EAGAIN; 1092 1093 /* ... but upgrade the -EGAIN to an -EIO if the gpu is truely 1094 * gone. */ 1095 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible); 1096 if (ret) 1097 end = ret; 1098 } while (end == 0 && wait_forever); 1099 1100 getrawmonotonic(&now); 1101 1102 ring->irq_put(ring); 1103 trace_i915_gem_request_wait_end(ring, seqno); 1104 #undef EXIT_COND 1105 1106 if (timeout) { 1107 struct timespec sleep_time = timespec_sub(now, before); 1108 *timeout = timespec_sub(*timeout, sleep_time); 1109 if (!timespec_valid(timeout)) /* i.e. negative time remains */ 1110 set_normalized_timespec(timeout, 0, 0); 1111 } 1112 1113 switch (end) { 1114 case -EIO: 1115 case -EAGAIN: /* Wedged */ 1116 case -ERESTARTSYS: /* Signal */ 1117 return (int)end; 1118 case 0: /* Timeout */ 1119 return -ETIMEDOUT; /* -ETIME on Linux */ 1120 default: /* Completed */ 1121 WARN_ON(end < 0); /* We're not aware of other errors */ 1122 return 0; 1123 } 1124 } 1125 1126 /** 1127 * Waits for a sequence number to be signaled, and cleans up the 1128 * request and object lists appropriately for that event. 1129 */ 1130 int 1131 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno) 1132 { 1133 struct drm_device *dev = ring->dev; 1134 struct drm_i915_private *dev_priv = dev->dev_private; 1135 bool interruptible = dev_priv->mm.interruptible; 1136 int ret; 1137 1138 BUG_ON(!mutex_is_locked(&dev->struct_mutex)); 1139 BUG_ON(seqno == 0); 1140 1141 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible); 1142 if (ret) 1143 return ret; 1144 1145 ret = i915_gem_check_olr(ring, seqno); 1146 if (ret) 1147 return ret; 1148 1149 return __wait_seqno(ring, seqno, 1150 atomic_read(&dev_priv->gpu_error.reset_counter), 1151 interruptible, NULL); 1152 } 1153 1154 static int 1155 i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj, 1156 struct intel_ring_buffer *ring) 1157 { 1158 i915_gem_retire_requests_ring(ring); 1159 1160 /* Manually manage the write flush as we may have not yet 1161 * retired the buffer. 1162 * 1163 * Note that the last_write_seqno is always the earlier of 1164 * the two (read/write) seqno, so if we haved successfully waited, 1165 * we know we have passed the last write. 1166 */ 1167 obj->last_write_seqno = 0; 1168 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; 1169 1170 return 0; 1171 } 1172 1173 /** 1174 * Ensures that all rendering to the object has completed and the object is 1175 * safe to unbind from the GTT or access from the CPU. 1176 */ 1177 static __must_check int 1178 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, 1179 bool readonly) 1180 { 1181 struct intel_ring_buffer *ring = obj->ring; 1182 u32 seqno; 1183 int ret; 1184 1185 seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; 1186 if (seqno == 0) 1187 return 0; 1188 1189 ret = i915_wait_seqno(ring, seqno); 1190 if (ret) 1191 return ret; 1192 1193 return i915_gem_object_wait_rendering__tail(obj, ring); 1194 } 1195 1196 /* A nonblocking variant of the above wait. This is a highly dangerous routine 1197 * as the object state may change during this call. 1198 */ 1199 static __must_check int 1200 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, 1201 bool readonly) 1202 { 1203 struct drm_device *dev = obj->base.dev; 1204 struct drm_i915_private *dev_priv = dev->dev_private; 1205 struct intel_ring_buffer *ring = obj->ring; 1206 unsigned reset_counter; 1207 u32 seqno; 1208 int ret; 1209 1210 BUG_ON(!mutex_is_locked(&dev->struct_mutex)); 1211 BUG_ON(!dev_priv->mm.interruptible); 1212 1213 seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; 1214 if (seqno == 0) 1215 return 0; 1216 1217 ret = i915_gem_check_wedge(&dev_priv->gpu_error, true); 1218 if (ret) 1219 return ret; 1220 1221 ret = i915_gem_check_olr(ring, seqno); 1222 if (ret) 1223 return ret; 1224 1225 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); 1226 mutex_unlock(&dev->struct_mutex); 1227 ret = __wait_seqno(ring, seqno, reset_counter, true, NULL); 1228 mutex_lock(&dev->struct_mutex); 1229 if (ret) 1230 return ret; 1231 1232 return i915_gem_object_wait_rendering__tail(obj, ring); 1233 } 1234 1235 /** 1236 * Called when user space prepares to use an object with the CPU, either 1237 * through the mmap ioctl's mapping or a GTT mapping. 1238 */ 1239 int 1240 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1241 struct drm_file *file) 1242 { 1243 struct drm_i915_gem_set_domain *args = data; 1244 struct drm_i915_gem_object *obj; 1245 uint32_t read_domains = args->read_domains; 1246 uint32_t write_domain = args->write_domain; 1247 int ret; 1248 1249 /* Only handle setting domains to types used by the CPU. */ 1250 if (write_domain & I915_GEM_GPU_DOMAINS) 1251 return -EINVAL; 1252 1253 if (read_domains & I915_GEM_GPU_DOMAINS) 1254 return -EINVAL; 1255 1256 /* Having something in the write domain implies it's in the read 1257 * domain, and only that read domain. Enforce that in the request. 1258 */ 1259 if (write_domain != 0 && read_domains != write_domain) 1260 return -EINVAL; 1261 1262 ret = i915_mutex_lock_interruptible(dev); 1263 if (ret) 1264 return ret; 1265 1266 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1267 if (&obj->base == NULL) { 1268 ret = -ENOENT; 1269 goto unlock; 1270 } 1271 1272 /* Try to flush the object off the GPU without holding the lock. 1273 * We will repeat the flush holding the lock in the normal manner 1274 * to catch cases where we are gazumped. 1275 */ 1276 ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain); 1277 if (ret) 1278 goto unref; 1279 1280 if (read_domains & I915_GEM_DOMAIN_GTT) { 1281 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1282 1283 /* Silently promote "you're not bound, there was nothing to do" 1284 * to success, since the client was just asking us to 1285 * make sure everything was done. 1286 */ 1287 if (ret == -EINVAL) 1288 ret = 0; 1289 } else { 1290 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1291 } 1292 1293 unref: 1294 drm_gem_object_unreference(&obj->base); 1295 unlock: 1296 mutex_unlock(&dev->struct_mutex); 1297 return ret; 1298 } 1299 1300 /** 1301 * Called when user space has done writes to this buffer 1302 */ 1303 int 1304 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1305 struct drm_file *file) 1306 { 1307 struct drm_i915_gem_sw_finish *args = data; 1308 struct drm_i915_gem_object *obj; 1309 int ret = 0; 1310 1311 ret = i915_mutex_lock_interruptible(dev); 1312 if (ret) 1313 return ret; 1314 1315 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1316 if (&obj->base == NULL) { 1317 ret = -ENOENT; 1318 goto unlock; 1319 } 1320 1321 /* Pinned buffers may be scanout, so flush the cache */ 1322 if (obj->pin_display) 1323 i915_gem_object_flush_cpu_write_domain(obj, true); 1324 1325 drm_gem_object_unreference(&obj->base); 1326 unlock: 1327 mutex_unlock(&dev->struct_mutex); 1328 return ret; 1329 } 1330 1331 /** 1332 * Maps the contents of an object, returning the address it is mapped 1333 * into. 1334 * 1335 * While the mapping holds a reference on the contents of the object, it doesn't 1336 * imply a ref on the object itself. 1337 */ 1338 int 1339 i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1340 struct drm_file *file) 1341 { 1342 struct drm_i915_gem_mmap *args = data; 1343 struct drm_gem_object *obj; 1344 unsigned long addr; 1345 struct proc *p = curproc; 1346 vm_map_t map = &p->p_vmspace->vm_map; 1347 vm_size_t size; 1348 int error = 0, rv; 1349 1350 obj = drm_gem_object_lookup(dev, file, args->handle); 1351 if (obj == NULL) 1352 return -ENOENT; 1353 1354 if (args->size == 0) 1355 goto out; 1356 1357 size = round_page(args->size); 1358 if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) { 1359 error = -ENOMEM; 1360 goto out; 1361 } 1362 1363 /* 1364 * Call hint to ensure that NULL is not returned as a valid address 1365 * and to reduce vm_map traversals. XXX causes instability, use a 1366 * fixed low address as the start point instead to avoid the NULL 1367 * return issue. 1368 */ 1369 addr = PAGE_SIZE; 1370 1371 /* 1372 * Use 256KB alignment. It is unclear why this matters for a 1373 * virtual address but it appears to fix a number of application/X 1374 * crashes and kms console switching is much faster. 1375 */ 1376 vm_object_hold(obj->vm_obj); 1377 vm_object_reference_locked(obj->vm_obj); 1378 vm_object_drop(obj->vm_obj); 1379 1380 rv = vm_map_find(map, obj->vm_obj, NULL, 1381 args->offset, &addr, args->size, 1382 256 * 1024, /* align */ 1383 TRUE, /* fitit */ 1384 VM_MAPTYPE_NORMAL, /* maptype */ 1385 VM_PROT_READ | VM_PROT_WRITE, /* prot */ 1386 VM_PROT_READ | VM_PROT_WRITE, /* max */ 1387 MAP_SHARED /* cow */); 1388 if (rv != KERN_SUCCESS) { 1389 vm_object_deallocate(obj->vm_obj); 1390 error = -vm_mmap_to_errno(rv); 1391 } else { 1392 args->addr_ptr = (uint64_t)addr; 1393 } 1394 out: 1395 drm_gem_object_unreference(obj); 1396 return (error); 1397 } 1398 1399 /** 1400 * i915_gem_fault - fault a page into the GTT 1401 * vma: VMA in question 1402 * vmf: fault info 1403 * 1404 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped 1405 * from userspace. The fault handler takes care of binding the object to 1406 * the GTT (if needed), allocating and programming a fence register (again, 1407 * only if needed based on whether the old reg is still valid or the object 1408 * is tiled) and inserting a new PTE into the faulting process. 1409 * 1410 * Note that the faulting process may involve evicting existing objects 1411 * from the GTT and/or fence registers to make room. So performance may 1412 * suffer if the GTT working set is large or there are few fence registers 1413 * left. 1414 */ 1415 int i915_gem_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, vm_page_t *mres) 1416 { 1417 struct drm_i915_gem_object *obj = to_intel_bo(vm_obj->handle); 1418 struct drm_device *dev = obj->base.dev; 1419 drm_i915_private_t *dev_priv = dev->dev_private; 1420 unsigned long page_offset; 1421 vm_page_t m, oldm = NULL; 1422 int ret = 0; 1423 bool write = !!(prot & VM_PROT_WRITE); 1424 1425 intel_runtime_pm_get(dev_priv); 1426 1427 /* We don't use vmf->pgoff since that has the fake offset */ 1428 page_offset = (unsigned long)offset; 1429 1430 /* Magic FreeBSD VM stuff */ 1431 vm_object_pip_add(vm_obj, 1); 1432 1433 /* 1434 * Remove the placeholder page inserted by vm_fault() from the 1435 * object before dropping the object lock. If 1436 * i915_gem_release_mmap() is active in parallel on this gem 1437 * object, then it owns the drm device sx and might find the 1438 * placeholder already. Then, since the page is busy, 1439 * i915_gem_release_mmap() sleeps waiting for the busy state 1440 * of the page cleared. We will be not able to acquire drm 1441 * device lock until i915_gem_release_mmap() is able to make a 1442 * progress. 1443 */ 1444 if (*mres != NULL) { 1445 oldm = *mres; 1446 vm_page_remove(oldm); 1447 *mres = NULL; 1448 } else 1449 oldm = NULL; 1450 retry: 1451 VM_OBJECT_UNLOCK(vm_obj); 1452 unlocked_vmobj: 1453 ret = 0; 1454 m = NULL; 1455 1456 mutex_lock(&dev->struct_mutex); 1457 1458 /* 1459 * Since the object lock was dropped, other thread might have 1460 * faulted on the same GTT address and instantiated the 1461 * mapping for the page. Recheck. 1462 */ 1463 VM_OBJECT_LOCK(vm_obj); 1464 m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 1465 if (m != NULL) { 1466 if ((m->flags & PG_BUSY) != 0) { 1467 mutex_unlock(&dev->struct_mutex); 1468 goto retry; 1469 } 1470 goto have_page; 1471 } else 1472 VM_OBJECT_UNLOCK(vm_obj); 1473 /* End magic VM stuff */ 1474 1475 trace_i915_gem_object_fault(obj, page_offset, true, write); 1476 1477 /* Access to snoopable pages through the GTT is incoherent. */ 1478 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) { 1479 ret = -EINVAL; 1480 goto unlock; 1481 } 1482 1483 /* Now bind it into the GTT if needed */ 1484 ret = i915_gem_obj_ggtt_pin(obj, 0, true, false); 1485 if (ret) 1486 goto unlock; 1487 1488 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1489 if (ret) 1490 goto unpin; 1491 1492 ret = i915_gem_object_get_fence(obj); 1493 if (ret) 1494 goto unpin; 1495 1496 obj->fault_mappable = true; 1497 1498 VM_OBJECT_LOCK(vm_obj); 1499 m = vm_phys_fictitious_to_vm_page(dev->agp->base + 1500 i915_gem_obj_ggtt_offset(obj) + offset); 1501 if (m == NULL) { 1502 ret = -EFAULT; 1503 goto unpin; 1504 } 1505 KASSERT((m->flags & PG_FICTITIOUS) != 0, 1506 ("not fictitious %p", m)); 1507 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 1508 1509 if ((m->flags & PG_BUSY) != 0) { 1510 i915_gem_object_unpin(obj); 1511 mutex_unlock(&dev->struct_mutex); 1512 goto retry; 1513 } 1514 m->valid = VM_PAGE_BITS_ALL; 1515 1516 /* Finally, remap it using the new GTT offset */ 1517 vm_page_insert(m, vm_obj, OFF_TO_IDX(offset)); 1518 have_page: 1519 *mres = m; 1520 vm_page_busy_try(m, false); 1521 1522 i915_gem_object_unpin(obj); 1523 mutex_unlock(&dev->struct_mutex); 1524 if (oldm != NULL) { 1525 vm_page_free(oldm); 1526 } 1527 vm_object_pip_wakeup(vm_obj); 1528 return (VM_PAGER_OK); 1529 1530 unpin: 1531 i915_gem_object_unpin(obj); 1532 unlock: 1533 mutex_unlock(&dev->struct_mutex); 1534 1535 KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 1536 switch (ret) { 1537 case -EIO: 1538 case -EAGAIN: 1539 case -EINTR: 1540 goto unlocked_vmobj; 1541 default: 1542 VM_OBJECT_LOCK(vm_obj); 1543 vm_object_pip_wakeup(vm_obj); 1544 ret = VM_PAGER_ERROR; 1545 } 1546 1547 intel_runtime_pm_put(dev_priv); 1548 return ret; 1549 } 1550 1551 void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv) 1552 { 1553 struct i915_vma *vma; 1554 1555 /* 1556 * Only the global gtt is relevant for gtt memory mappings, so restrict 1557 * list traversal to objects bound into the global address space. Note 1558 * that the active list should be empty, but better safe than sorry. 1559 */ 1560 WARN_ON(!list_empty(&dev_priv->gtt.base.active_list)); 1561 list_for_each_entry(vma, &dev_priv->gtt.base.active_list, mm_list) 1562 i915_gem_release_mmap(vma->obj); 1563 list_for_each_entry(vma, &dev_priv->gtt.base.inactive_list, mm_list) 1564 i915_gem_release_mmap(vma->obj); 1565 } 1566 1567 /** 1568 * i915_gem_release_mmap - remove physical page mappings 1569 * @obj: obj in question 1570 * 1571 * Preserve the reservation of the mmapping with the DRM core code, but 1572 * relinquish ownership of the pages back to the system. 1573 * 1574 * It is vital that we remove the page mapping if we have mapped a tiled 1575 * object through the GTT and then lose the fence register due to 1576 * resource pressure. Similarly if the object has been moved out of the 1577 * aperture, than pages mapped into userspace must be revoked. Removing the 1578 * mapping will then trigger a page fault on the next user access, allowing 1579 * fixup by i915_gem_fault(). 1580 */ 1581 void 1582 i915_gem_release_mmap(struct drm_i915_gem_object *obj) 1583 { 1584 vm_object_t devobj; 1585 vm_page_t m; 1586 int i, page_count; 1587 1588 if (!obj->fault_mappable) 1589 return; 1590 1591 devobj = cdev_pager_lookup(obj); 1592 if (devobj != NULL) { 1593 page_count = OFF_TO_IDX(obj->base.size); 1594 1595 VM_OBJECT_LOCK(devobj); 1596 for (i = 0; i < page_count; i++) { 1597 m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm"); 1598 if (m == NULL) 1599 continue; 1600 cdev_pager_free_page(devobj, m); 1601 } 1602 VM_OBJECT_UNLOCK(devobj); 1603 vm_object_deallocate(devobj); 1604 } 1605 1606 obj->fault_mappable = false; 1607 } 1608 1609 uint32_t 1610 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 1611 { 1612 uint32_t gtt_size; 1613 1614 if (INTEL_INFO(dev)->gen >= 4 || 1615 tiling_mode == I915_TILING_NONE) 1616 return size; 1617 1618 /* Previous chips need a power-of-two fence region when tiling */ 1619 if (INTEL_INFO(dev)->gen == 3) 1620 gtt_size = 1024*1024; 1621 else 1622 gtt_size = 512*1024; 1623 1624 while (gtt_size < size) 1625 gtt_size <<= 1; 1626 1627 return gtt_size; 1628 } 1629 1630 /** 1631 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 1632 * @obj: object to check 1633 * 1634 * Return the required GTT alignment for an object, taking into account 1635 * potential fence register mapping. 1636 */ 1637 uint32_t 1638 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, 1639 int tiling_mode, bool fenced) 1640 { 1641 /* 1642 * Minimum alignment is 4k (GTT page size), but might be greater 1643 * if a fence register is needed for the object. 1644 */ 1645 if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) || 1646 tiling_mode == I915_TILING_NONE) 1647 return 4096; 1648 1649 /* 1650 * Previous chips need to be aligned to the size of the smallest 1651 * fence register that can contain the object. 1652 */ 1653 return i915_gem_get_gtt_size(dev, size, tiling_mode); 1654 } 1655 1656 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj) 1657 { 1658 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 1659 int ret; 1660 1661 #if 0 1662 if (drm_vma_node_has_offset(&obj->base.vma_node)) 1663 return 0; 1664 #endif 1665 1666 dev_priv->mm.shrinker_no_lock_stealing = true; 1667 1668 ret = drm_gem_create_mmap_offset(&obj->base); 1669 if (ret != -ENOSPC) 1670 goto out; 1671 1672 /* Badly fragmented mmap space? The only way we can recover 1673 * space is by destroying unwanted objects. We can't randomly release 1674 * mmap_offsets as userspace expects them to be persistent for the 1675 * lifetime of the objects. The closest we can is to release the 1676 * offsets on purgeable objects by truncating it and marking it purged, 1677 * which prevents userspace from ever using that object again. 1678 */ 1679 i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT); 1680 ret = drm_gem_create_mmap_offset(&obj->base); 1681 if (ret != -ENOSPC) 1682 goto out; 1683 1684 i915_gem_shrink_all(dev_priv); 1685 ret = drm_gem_create_mmap_offset(&obj->base); 1686 out: 1687 dev_priv->mm.shrinker_no_lock_stealing = false; 1688 1689 return ret; 1690 } 1691 1692 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj) 1693 { 1694 drm_gem_free_mmap_offset(&obj->base); 1695 } 1696 1697 int 1698 i915_gem_mmap_gtt(struct drm_file *file, 1699 struct drm_device *dev, 1700 uint32_t handle, 1701 uint64_t *offset) 1702 { 1703 struct drm_i915_private *dev_priv = dev->dev_private; 1704 struct drm_i915_gem_object *obj; 1705 int ret; 1706 1707 ret = i915_mutex_lock_interruptible(dev); 1708 if (ret) 1709 return ret; 1710 1711 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1712 if (&obj->base == NULL) { 1713 ret = -ENOENT; 1714 goto unlock; 1715 } 1716 1717 if (obj->base.size > dev_priv->gtt.mappable_end) { 1718 ret = -E2BIG; 1719 goto out; 1720 } 1721 1722 if (obj->madv != I915_MADV_WILLNEED) { 1723 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1724 ret = -EINVAL; 1725 goto out; 1726 } 1727 1728 ret = i915_gem_object_create_mmap_offset(obj); 1729 if (ret) 1730 goto out; 1731 1732 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 1733 DRM_GEM_MAPPING_KEY; 1734 1735 out: 1736 drm_gem_object_unreference(&obj->base); 1737 unlock: 1738 mutex_unlock(&dev->struct_mutex); 1739 return ret; 1740 } 1741 1742 /** 1743 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 1744 * @dev: DRM device 1745 * @data: GTT mapping ioctl data 1746 * @file: GEM object info 1747 * 1748 * Simply returns the fake offset to userspace so it can mmap it. 1749 * The mmap call will end up in drm_gem_mmap(), which will set things 1750 * up so we can get faults in the handler above. 1751 * 1752 * The fault handler will take care of binding the object into the GTT 1753 * (since it may have been evicted to make room for something), allocating 1754 * a fence register, and mapping the appropriate aperture address into 1755 * userspace. 1756 */ 1757 int 1758 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1759 struct drm_file *file) 1760 { 1761 struct drm_i915_gem_mmap_gtt *args = data; 1762 1763 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); 1764 } 1765 1766 /* Immediately discard the backing storage */ 1767 static void 1768 i915_gem_object_truncate(struct drm_i915_gem_object *obj) 1769 { 1770 vm_object_t vm_obj; 1771 1772 vm_obj = obj->base.vm_obj; 1773 VM_OBJECT_LOCK(vm_obj); 1774 vm_object_page_remove(vm_obj, 0, 0, false); 1775 VM_OBJECT_UNLOCK(vm_obj); 1776 1777 obj->madv = __I915_MADV_PURGED; 1778 } 1779 1780 static inline int 1781 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 1782 { 1783 return obj->madv == I915_MADV_DONTNEED; 1784 } 1785 1786 static void 1787 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 1788 { 1789 int page_count = obj->base.size / PAGE_SIZE; 1790 int i, ret; 1791 1792 if (!obj->pages) 1793 return; 1794 1795 BUG_ON(obj->madv == __I915_MADV_PURGED); 1796 1797 ret = i915_gem_object_set_to_cpu_domain(obj, true); 1798 if (ret) { 1799 /* In the event of a disaster, abandon all caches and 1800 * hope for the best. 1801 */ 1802 WARN_ON(ret != -EIO); 1803 i915_gem_clflush_object(obj, true); 1804 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1805 } 1806 1807 if (i915_gem_object_needs_bit17_swizzle(obj)) 1808 i915_gem_object_save_bit_17_swizzle(obj); 1809 1810 if (obj->madv == I915_MADV_DONTNEED) 1811 obj->dirty = 0; 1812 1813 for (i = 0; i < page_count; i++) { 1814 struct vm_page *page = obj->pages[i]; 1815 1816 if (obj->dirty) 1817 set_page_dirty(page); 1818 1819 if (obj->madv == I915_MADV_WILLNEED) 1820 mark_page_accessed(page); 1821 1822 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem"); 1823 vm_page_unwire(obj->pages[i], 1); 1824 vm_page_wakeup(obj->pages[i]); 1825 } 1826 obj->dirty = 0; 1827 1828 kfree(obj->pages); 1829 obj->pages = NULL; 1830 } 1831 1832 int 1833 i915_gem_object_put_pages(struct drm_i915_gem_object *obj) 1834 { 1835 const struct drm_i915_gem_object_ops *ops = obj->ops; 1836 1837 if (obj->pages == NULL) 1838 return 0; 1839 1840 if (obj->pages_pin_count) 1841 return -EBUSY; 1842 1843 BUG_ON(i915_gem_obj_bound_any(obj)); 1844 1845 /* ->put_pages might need to allocate memory for the bit17 swizzle 1846 * array, hence protect them from being reaped by removing them from gtt 1847 * lists early. */ 1848 list_del(&obj->global_list); 1849 1850 ops->put_pages(obj); 1851 obj->pages = NULL; 1852 1853 if (i915_gem_object_is_purgeable(obj)) 1854 i915_gem_object_truncate(obj); 1855 1856 return 0; 1857 } 1858 1859 static unsigned long 1860 __i915_gem_shrink(struct drm_i915_private *dev_priv, long target, 1861 bool purgeable_only) 1862 { 1863 struct drm_i915_gem_object *obj, *next; 1864 unsigned long count = 0; 1865 1866 list_for_each_entry_safe(obj, next, 1867 &dev_priv->mm.unbound_list, 1868 global_list) { 1869 if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) && 1870 i915_gem_object_put_pages(obj) == 0) { 1871 count += obj->base.size >> PAGE_SHIFT; 1872 if (count >= target) 1873 return count; 1874 } 1875 } 1876 1877 list_for_each_entry_safe(obj, next, &dev_priv->mm.bound_list, 1878 global_list) { 1879 struct i915_vma *vma, *v; 1880 1881 if (!i915_gem_object_is_purgeable(obj) && purgeable_only) 1882 continue; 1883 1884 list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link) 1885 if (i915_vma_unbind(vma)) 1886 break; 1887 1888 if (!i915_gem_object_put_pages(obj)) { 1889 count += obj->base.size >> PAGE_SHIFT; 1890 if (count >= target) 1891 return count; 1892 } 1893 } 1894 1895 return count; 1896 } 1897 1898 static unsigned long 1899 i915_gem_purge(struct drm_i915_private *dev_priv, long target) 1900 { 1901 return __i915_gem_shrink(dev_priv, target, true); 1902 } 1903 1904 static unsigned long 1905 i915_gem_shrink_all(struct drm_i915_private *dev_priv) 1906 { 1907 struct drm_i915_gem_object *obj, *next; 1908 long freed = 0; 1909 1910 i915_gem_evict_everything(dev_priv->dev); 1911 1912 list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, 1913 global_list) { 1914 if (i915_gem_object_put_pages(obj) == 0) 1915 freed += obj->base.size >> PAGE_SHIFT; 1916 } 1917 return freed; 1918 } 1919 1920 static int 1921 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) 1922 { 1923 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 1924 int page_count, i, j; 1925 vm_object_t vm_obj; 1926 struct vm_page *page; 1927 1928 /* Assert that the object is not currently in any GPU domain. As it 1929 * wasn't in the GTT, there shouldn't be any way it could have been in 1930 * a GPU cache 1931 */ 1932 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); 1933 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); 1934 1935 page_count = obj->base.size / PAGE_SIZE; 1936 obj->pages = kmalloc(page_count * sizeof(vm_page_t), M_DRM, 1937 M_WAITOK); 1938 1939 /* Get the list of pages out of our struct file. They'll be pinned 1940 * at this point until we release them. 1941 * 1942 * Fail silently without starting the shrinker 1943 */ 1944 vm_obj = obj->base.vm_obj; 1945 VM_OBJECT_LOCK(vm_obj); 1946 for (i = 0; i < page_count; i++) { 1947 page = shmem_read_mapping_page(vm_obj, i); 1948 if (IS_ERR(page)) { 1949 i915_gem_purge(dev_priv, page_count); 1950 page = shmem_read_mapping_page(vm_obj, i); 1951 } 1952 if (IS_ERR(page)) { 1953 /* We've tried hard to allocate the memory by reaping 1954 * our own buffer, now let the real VM do its job and 1955 * go down in flames if truly OOM. 1956 */ 1957 1958 i915_gem_shrink_all(dev_priv); 1959 page = shmem_read_mapping_page(vm_obj, i); 1960 if (IS_ERR(page)) 1961 goto err_pages; 1962 } 1963 #ifdef CONFIG_SWIOTLB 1964 if (swiotlb_nr_tbl()) { 1965 st->nents++; 1966 sg_set_page(sg, page, PAGE_SIZE, 0); 1967 sg = sg_next(sg); 1968 continue; 1969 } 1970 #endif 1971 obj->pages[i] = page; 1972 } 1973 #ifdef CONFIG_SWIOTLB 1974 if (!swiotlb_nr_tbl()) 1975 #endif 1976 VM_OBJECT_UNLOCK(vm_obj); 1977 1978 if (i915_gem_object_needs_bit17_swizzle(obj)) 1979 i915_gem_object_do_bit_17_swizzle(obj); 1980 1981 return 0; 1982 1983 err_pages: 1984 for (j = 0; j < i; j++) { 1985 page = obj->pages[j]; 1986 vm_page_busy_wait(page, FALSE, "i915gem"); 1987 vm_page_unwire(page, 0); 1988 vm_page_wakeup(page); 1989 } 1990 VM_OBJECT_UNLOCK(vm_obj); 1991 kfree(obj->pages); 1992 obj->pages = NULL; 1993 return (-EIO); 1994 } 1995 1996 /* Ensure that the associated pages are gathered from the backing storage 1997 * and pinned into our object. i915_gem_object_get_pages() may be called 1998 * multiple times before they are released by a single call to 1999 * i915_gem_object_put_pages() - once the pages are no longer referenced 2000 * either as a result of memory pressure (reaping pages under the shrinker) 2001 * or as the object is itself released. 2002 */ 2003 int 2004 i915_gem_object_get_pages(struct drm_i915_gem_object *obj) 2005 { 2006 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 2007 const struct drm_i915_gem_object_ops *ops = obj->ops; 2008 int ret; 2009 2010 if (obj->pages) 2011 return 0; 2012 2013 if (obj->madv != I915_MADV_WILLNEED) { 2014 DRM_ERROR("Attempting to obtain a purgeable object\n"); 2015 return -EINVAL; 2016 } 2017 2018 BUG_ON(obj->pages_pin_count); 2019 2020 ret = ops->get_pages(obj); 2021 if (ret) 2022 return ret; 2023 2024 list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list); 2025 return 0; 2026 } 2027 2028 static void 2029 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 2030 struct intel_ring_buffer *ring) 2031 { 2032 struct drm_device *dev = obj->base.dev; 2033 struct drm_i915_private *dev_priv = dev->dev_private; 2034 u32 seqno = intel_ring_get_seqno(ring); 2035 2036 BUG_ON(ring == NULL); 2037 if (obj->ring != ring && obj->last_write_seqno) { 2038 /* Keep the seqno relative to the current ring */ 2039 obj->last_write_seqno = seqno; 2040 } 2041 obj->ring = ring; 2042 2043 /* Add a reference if we're newly entering the active list. */ 2044 if (!obj->active) { 2045 drm_gem_object_reference(&obj->base); 2046 obj->active = 1; 2047 } 2048 2049 list_move_tail(&obj->ring_list, &ring->active_list); 2050 2051 obj->last_read_seqno = seqno; 2052 2053 if (obj->fenced_gpu_access) { 2054 obj->last_fenced_seqno = seqno; 2055 2056 /* Bump MRU to take account of the delayed flush */ 2057 if (obj->fence_reg != I915_FENCE_REG_NONE) { 2058 struct drm_i915_fence_reg *reg; 2059 2060 reg = &dev_priv->fence_regs[obj->fence_reg]; 2061 list_move_tail(®->lru_list, 2062 &dev_priv->mm.fence_list); 2063 } 2064 } 2065 } 2066 2067 void i915_vma_move_to_active(struct i915_vma *vma, 2068 struct intel_ring_buffer *ring) 2069 { 2070 list_move_tail(&vma->mm_list, &vma->vm->active_list); 2071 return i915_gem_object_move_to_active(vma->obj, ring); 2072 } 2073 2074 static void 2075 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 2076 { 2077 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 2078 struct i915_address_space *ggtt_vm = &dev_priv->gtt.base; 2079 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); 2080 2081 BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS); 2082 BUG_ON(!obj->active); 2083 2084 list_move_tail(&vma->mm_list, &ggtt_vm->inactive_list); 2085 2086 list_del_init(&obj->ring_list); 2087 obj->ring = NULL; 2088 2089 obj->last_read_seqno = 0; 2090 obj->last_write_seqno = 0; 2091 obj->base.write_domain = 0; 2092 2093 obj->last_fenced_seqno = 0; 2094 obj->fenced_gpu_access = false; 2095 2096 obj->active = 0; 2097 drm_gem_object_unreference(&obj->base); 2098 2099 WARN_ON(i915_verify_lists(dev)); 2100 } 2101 2102 static int 2103 i915_gem_init_seqno(struct drm_device *dev, u32 seqno) 2104 { 2105 struct drm_i915_private *dev_priv = dev->dev_private; 2106 struct intel_ring_buffer *ring; 2107 int ret, i, j; 2108 2109 /* Carefully retire all requests without writing to the rings */ 2110 for_each_ring(ring, dev_priv, i) { 2111 ret = intel_ring_idle(ring); 2112 if (ret) 2113 return ret; 2114 } 2115 i915_gem_retire_requests(dev); 2116 2117 /* Finally reset hw state */ 2118 for_each_ring(ring, dev_priv, i) { 2119 intel_ring_init_seqno(ring, seqno); 2120 2121 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++) 2122 ring->sync_seqno[j] = 0; 2123 } 2124 2125 return 0; 2126 } 2127 2128 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) 2129 { 2130 struct drm_i915_private *dev_priv = dev->dev_private; 2131 int ret; 2132 2133 if (seqno == 0) 2134 return -EINVAL; 2135 2136 /* HWS page needs to be set less than what we 2137 * will inject to ring 2138 */ 2139 ret = i915_gem_init_seqno(dev, seqno - 1); 2140 if (ret) 2141 return ret; 2142 2143 /* Carefully set the last_seqno value so that wrap 2144 * detection still works 2145 */ 2146 dev_priv->next_seqno = seqno; 2147 dev_priv->last_seqno = seqno - 1; 2148 if (dev_priv->last_seqno == 0) 2149 dev_priv->last_seqno--; 2150 2151 return 0; 2152 } 2153 2154 int 2155 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno) 2156 { 2157 struct drm_i915_private *dev_priv = dev->dev_private; 2158 2159 /* reserve 0 for non-seqno */ 2160 if (dev_priv->next_seqno == 0) { 2161 int ret = i915_gem_init_seqno(dev, 0); 2162 if (ret) 2163 return ret; 2164 2165 dev_priv->next_seqno = 1; 2166 } 2167 2168 *seqno = dev_priv->last_seqno = dev_priv->next_seqno++; 2169 return 0; 2170 } 2171 2172 int __i915_add_request(struct intel_ring_buffer *ring, 2173 struct drm_file *file, 2174 struct drm_i915_gem_object *obj, 2175 u32 *out_seqno) 2176 { 2177 drm_i915_private_t *dev_priv = ring->dev->dev_private; 2178 struct drm_i915_gem_request *request; 2179 u32 request_ring_position, request_start; 2180 int was_empty; 2181 int ret; 2182 2183 request_start = intel_ring_get_tail(ring); 2184 /* 2185 * Emit any outstanding flushes - execbuf can fail to emit the flush 2186 * after having emitted the batchbuffer command. Hence we need to fix 2187 * things up similar to emitting the lazy request. The difference here 2188 * is that the flush _must_ happen before the next request, no matter 2189 * what. 2190 */ 2191 ret = intel_ring_flush_all_caches(ring); 2192 if (ret) 2193 return ret; 2194 2195 request = ring->preallocated_lazy_request; 2196 if (WARN_ON(request == NULL)) 2197 return -ENOMEM; 2198 2199 /* Record the position of the start of the request so that 2200 * should we detect the updated seqno part-way through the 2201 * GPU processing the request, we never over-estimate the 2202 * position of the head. 2203 */ 2204 request_ring_position = intel_ring_get_tail(ring); 2205 2206 ret = ring->add_request(ring); 2207 if (ret) 2208 return ret; 2209 2210 request->seqno = intel_ring_get_seqno(ring); 2211 request->ring = ring; 2212 request->head = request_start; 2213 request->tail = request_ring_position; 2214 2215 /* Whilst this request exists, batch_obj will be on the 2216 * active_list, and so will hold the active reference. Only when this 2217 * request is retired will the the batch_obj be moved onto the 2218 * inactive_list and lose its active reference. Hence we do not need 2219 * to explicitly hold another reference here. 2220 */ 2221 request->batch_obj = obj; 2222 2223 /* Hold a reference to the current context so that we can inspect 2224 * it later in case a hangcheck error event fires. 2225 */ 2226 request->ctx = ring->last_context; 2227 if (request->ctx) 2228 i915_gem_context_reference(request->ctx); 2229 2230 request->emitted_jiffies = jiffies; 2231 was_empty = list_empty(&ring->request_list); 2232 list_add_tail(&request->list, &ring->request_list); 2233 request->file_priv = NULL; 2234 2235 if (file) { 2236 struct drm_i915_file_private *file_priv = file->driver_priv; 2237 2238 spin_lock(&file_priv->mm.lock); 2239 request->file_priv = file_priv; 2240 list_add_tail(&request->client_list, 2241 &file_priv->mm.request_list); 2242 spin_unlock(&file_priv->mm.lock); 2243 } 2244 2245 trace_i915_gem_request_add(ring, request->seqno); 2246 ring->outstanding_lazy_seqno = 0; 2247 ring->preallocated_lazy_request = NULL; 2248 2249 if (!dev_priv->ums.mm_suspended) { 2250 i915_queue_hangcheck(ring->dev); 2251 2252 if (was_empty) { 2253 cancel_delayed_work_sync(&dev_priv->mm.idle_work); 2254 queue_delayed_work(dev_priv->wq, 2255 &dev_priv->mm.retire_work, 2256 round_jiffies_up_relative(HZ)); 2257 intel_mark_busy(dev_priv->dev); 2258 } 2259 } 2260 2261 if (out_seqno) 2262 *out_seqno = request->seqno; 2263 return 0; 2264 } 2265 2266 static inline void 2267 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 2268 { 2269 struct drm_i915_file_private *file_priv = request->file_priv; 2270 2271 if (!file_priv) 2272 return; 2273 2274 spin_lock(&file_priv->mm.lock); 2275 if (request->file_priv) { 2276 list_del(&request->client_list); 2277 request->file_priv = NULL; 2278 } 2279 spin_unlock(&file_priv->mm.lock); 2280 } 2281 2282 static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj, 2283 struct i915_address_space *vm) 2284 { 2285 if (acthd >= i915_gem_obj_offset(obj, vm) && 2286 acthd < i915_gem_obj_offset(obj, vm) + obj->base.size) 2287 return true; 2288 2289 return false; 2290 } 2291 2292 static bool i915_head_inside_request(const u32 acthd_unmasked, 2293 const u32 request_start, 2294 const u32 request_end) 2295 { 2296 const u32 acthd = acthd_unmasked & HEAD_ADDR; 2297 2298 if (request_start < request_end) { 2299 if (acthd >= request_start && acthd < request_end) 2300 return true; 2301 } else if (request_start > request_end) { 2302 if (acthd >= request_start || acthd < request_end) 2303 return true; 2304 } 2305 2306 return false; 2307 } 2308 2309 static struct i915_address_space * 2310 request_to_vm(struct drm_i915_gem_request *request) 2311 { 2312 struct drm_i915_private *dev_priv = request->ring->dev->dev_private; 2313 struct i915_address_space *vm; 2314 2315 vm = &dev_priv->gtt.base; 2316 2317 return vm; 2318 } 2319 2320 static bool i915_request_guilty(struct drm_i915_gem_request *request, 2321 const u32 acthd, bool *inside) 2322 { 2323 /* There is a possibility that unmasked head address 2324 * pointing inside the ring, matches the batch_obj address range. 2325 * However this is extremely unlikely. 2326 */ 2327 if (request->batch_obj) { 2328 if (i915_head_inside_object(acthd, request->batch_obj, 2329 request_to_vm(request))) { 2330 *inside = true; 2331 return true; 2332 } 2333 } 2334 2335 if (i915_head_inside_request(acthd, request->head, request->tail)) { 2336 *inside = false; 2337 return true; 2338 } 2339 2340 return false; 2341 } 2342 2343 static bool i915_context_is_banned(const struct i915_ctx_hang_stats *hs) 2344 { 2345 const unsigned long elapsed = get_seconds() - hs->guilty_ts; 2346 2347 if (hs->banned) 2348 return true; 2349 2350 if (elapsed <= DRM_I915_CTX_BAN_PERIOD) { 2351 DRM_ERROR("context hanging too fast, declaring banned!\n"); 2352 return true; 2353 } 2354 2355 return false; 2356 } 2357 2358 static void i915_set_reset_status(struct intel_ring_buffer *ring, 2359 struct drm_i915_gem_request *request, 2360 u32 acthd) 2361 { 2362 struct i915_ctx_hang_stats *hs = NULL; 2363 bool inside, guilty; 2364 unsigned long offset = 0; 2365 2366 /* Innocent until proven guilty */ 2367 guilty = false; 2368 2369 if (request->batch_obj) 2370 offset = i915_gem_obj_offset(request->batch_obj, 2371 request_to_vm(request)); 2372 2373 if (ring->hangcheck.action != HANGCHECK_WAIT && 2374 i915_request_guilty(request, acthd, &inside)) { 2375 DRM_DEBUG("%s hung %s bo (0x%lx ctx %d) at 0x%x\n", 2376 ring->name, 2377 inside ? "inside" : "flushing", 2378 offset, 2379 request->ctx ? request->ctx->id : 0, 2380 acthd); 2381 2382 guilty = true; 2383 } 2384 2385 /* If contexts are disabled or this is the default context, use 2386 * file_priv->reset_state 2387 */ 2388 if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID) 2389 hs = &request->ctx->hang_stats; 2390 else if (request->file_priv) 2391 hs = &request->file_priv->hang_stats; 2392 2393 if (hs) { 2394 if (guilty) { 2395 hs->banned = i915_context_is_banned(hs); 2396 hs->batch_active++; 2397 hs->guilty_ts = get_seconds(); 2398 } else { 2399 hs->batch_pending++; 2400 } 2401 } 2402 } 2403 2404 static void i915_gem_free_request(struct drm_i915_gem_request *request) 2405 { 2406 list_del(&request->list); 2407 i915_gem_request_remove_from_client(request); 2408 2409 if (request->ctx) 2410 i915_gem_context_unreference(request->ctx); 2411 2412 kfree(request); 2413 } 2414 2415 static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv, 2416 struct intel_ring_buffer *ring) 2417 { 2418 u32 completed_seqno = ring->get_seqno(ring, false); 2419 u32 acthd = intel_ring_get_active_head(ring); 2420 struct drm_i915_gem_request *request; 2421 2422 list_for_each_entry(request, &ring->request_list, list) { 2423 if (i915_seqno_passed(completed_seqno, request->seqno)) 2424 continue; 2425 2426 i915_set_reset_status(ring, request, acthd); 2427 } 2428 } 2429 2430 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv, 2431 struct intel_ring_buffer *ring) 2432 { 2433 while (!list_empty(&ring->active_list)) { 2434 struct drm_i915_gem_object *obj; 2435 2436 obj = list_first_entry(&ring->active_list, 2437 struct drm_i915_gem_object, 2438 ring_list); 2439 2440 i915_gem_object_move_to_inactive(obj); 2441 } 2442 2443 /* 2444 * We must free the requests after all the corresponding objects have 2445 * been moved off active lists. Which is the same order as the normal 2446 * retire_requests function does. This is important if object hold 2447 * implicit references on things like e.g. ppgtt address spaces through 2448 * the request. 2449 */ 2450 while (!list_empty(&ring->request_list)) { 2451 struct drm_i915_gem_request *request; 2452 2453 request = list_first_entry(&ring->request_list, 2454 struct drm_i915_gem_request, 2455 list); 2456 2457 i915_gem_free_request(request); 2458 } 2459 } 2460 2461 void i915_gem_restore_fences(struct drm_device *dev) 2462 { 2463 struct drm_i915_private *dev_priv = dev->dev_private; 2464 int i; 2465 2466 for (i = 0; i < dev_priv->num_fence_regs; i++) { 2467 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2468 2469 /* 2470 * Commit delayed tiling changes if we have an object still 2471 * attached to the fence, otherwise just clear the fence. 2472 */ 2473 if (reg->obj) { 2474 i915_gem_object_update_fence(reg->obj, reg, 2475 reg->obj->tiling_mode); 2476 } else { 2477 i915_gem_write_fence(dev, i, NULL); 2478 } 2479 } 2480 } 2481 2482 void i915_gem_reset(struct drm_device *dev) 2483 { 2484 struct drm_i915_private *dev_priv = dev->dev_private; 2485 struct intel_ring_buffer *ring; 2486 int i; 2487 2488 /* 2489 * Before we free the objects from the requests, we need to inspect 2490 * them for finding the guilty party. As the requests only borrow 2491 * their reference to the objects, the inspection must be done first. 2492 */ 2493 for_each_ring(ring, dev_priv, i) 2494 i915_gem_reset_ring_status(dev_priv, ring); 2495 2496 for_each_ring(ring, dev_priv, i) 2497 i915_gem_reset_ring_cleanup(dev_priv, ring); 2498 2499 i915_gem_cleanup_ringbuffer(dev); 2500 2501 i915_gem_restore_fences(dev); 2502 } 2503 2504 /** 2505 * This function clears the request list as sequence numbers are passed. 2506 */ 2507 void 2508 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 2509 { 2510 uint32_t seqno; 2511 2512 if (list_empty(&ring->request_list)) 2513 return; 2514 2515 WARN_ON(i915_verify_lists(ring->dev)); 2516 2517 seqno = ring->get_seqno(ring, true); 2518 2519 while (!list_empty(&ring->request_list)) { 2520 struct drm_i915_gem_request *request; 2521 2522 request = list_first_entry(&ring->request_list, 2523 struct drm_i915_gem_request, 2524 list); 2525 2526 if (!i915_seqno_passed(seqno, request->seqno)) 2527 break; 2528 2529 trace_i915_gem_request_retire(ring, request->seqno); 2530 /* We know the GPU must have read the request to have 2531 * sent us the seqno + interrupt, so use the position 2532 * of tail of the request to update the last known position 2533 * of the GPU head. 2534 */ 2535 ring->last_retired_head = request->tail; 2536 2537 i915_gem_free_request(request); 2538 } 2539 2540 /* Move any buffers on the active list that are no longer referenced 2541 * by the ringbuffer to the flushing/inactive lists as appropriate. 2542 */ 2543 while (!list_empty(&ring->active_list)) { 2544 struct drm_i915_gem_object *obj; 2545 2546 obj = list_first_entry(&ring->active_list, 2547 struct drm_i915_gem_object, 2548 ring_list); 2549 2550 if (!i915_seqno_passed(seqno, obj->last_read_seqno)) 2551 break; 2552 2553 i915_gem_object_move_to_inactive(obj); 2554 } 2555 2556 if (unlikely(ring->trace_irq_seqno && 2557 i915_seqno_passed(seqno, ring->trace_irq_seqno))) { 2558 ring->irq_put(ring); 2559 ring->trace_irq_seqno = 0; 2560 } 2561 2562 WARN_ON(i915_verify_lists(ring->dev)); 2563 } 2564 2565 bool 2566 i915_gem_retire_requests(struct drm_device *dev) 2567 { 2568 drm_i915_private_t *dev_priv = dev->dev_private; 2569 struct intel_ring_buffer *ring; 2570 bool idle = true; 2571 int i; 2572 2573 for_each_ring(ring, dev_priv, i) { 2574 i915_gem_retire_requests_ring(ring); 2575 idle &= list_empty(&ring->request_list); 2576 } 2577 2578 return idle; 2579 } 2580 2581 static void 2582 i915_gem_retire_work_handler(struct work_struct *work) 2583 { 2584 drm_i915_private_t *dev_priv; 2585 struct drm_device *dev; 2586 struct intel_ring_buffer *ring; 2587 bool idle; 2588 int i; 2589 2590 dev_priv = container_of(work, drm_i915_private_t, 2591 mm.retire_work.work); 2592 dev = dev_priv->dev; 2593 2594 /* Come back later if the device is busy... */ 2595 if (lockmgr(&dev->struct_mutex, LK_EXCLUSIVE|LK_NOWAIT)) { 2596 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 2597 round_jiffies_up_relative(HZ)); 2598 return; 2599 } 2600 2601 i915_gem_retire_requests(dev); 2602 2603 /* Send a periodic flush down the ring so we don't hold onto GEM 2604 * objects indefinitely. 2605 */ 2606 idle = true; 2607 for_each_ring(ring, dev_priv, i) { 2608 if (ring->gpu_caches_dirty) 2609 i915_add_request(ring, NULL); 2610 2611 idle &= list_empty(&ring->request_list); 2612 } 2613 2614 if (!dev_priv->ums.mm_suspended && !idle) 2615 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 2616 round_jiffies_up_relative(HZ)); 2617 if (idle) 2618 intel_mark_idle(dev); 2619 2620 mutex_unlock(&dev->struct_mutex); 2621 } 2622 2623 static void 2624 i915_gem_idle_work_handler(struct work_struct *work) 2625 { 2626 struct drm_i915_private *dev_priv = 2627 container_of(work, typeof(*dev_priv), mm.idle_work.work); 2628 2629 intel_mark_idle(dev_priv->dev); 2630 } 2631 2632 /** 2633 * Ensures that an object will eventually get non-busy by flushing any required 2634 * write domains, emitting any outstanding lazy request and retiring and 2635 * completed requests. 2636 */ 2637 static int 2638 i915_gem_object_flush_active(struct drm_i915_gem_object *obj) 2639 { 2640 int ret; 2641 2642 if (obj->active) { 2643 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno); 2644 if (ret) 2645 return ret; 2646 2647 i915_gem_retire_requests_ring(obj->ring); 2648 } 2649 2650 return 0; 2651 } 2652 2653 /** 2654 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT 2655 * @DRM_IOCTL_ARGS: standard ioctl arguments 2656 * 2657 * Returns 0 if successful, else an error is returned with the remaining time in 2658 * the timeout parameter. 2659 * -ETIME: object is still busy after timeout 2660 * -ERESTARTSYS: signal interrupted the wait 2661 * -ENONENT: object doesn't exist 2662 * Also possible, but rare: 2663 * -EAGAIN: GPU wedged 2664 * -ENOMEM: damn 2665 * -ENODEV: Internal IRQ fail 2666 * -E?: The add request failed 2667 * 2668 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any 2669 * non-zero timeout parameter the wait ioctl will wait for the given number of 2670 * nanoseconds on an object becoming unbusy. Since the wait itself does so 2671 * without holding struct_mutex the object may become re-busied before this 2672 * function completes. A similar but shorter * race condition exists in the busy 2673 * ioctl 2674 */ 2675 int 2676 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 2677 { 2678 drm_i915_private_t *dev_priv = dev->dev_private; 2679 struct drm_i915_gem_wait *args = data; 2680 struct drm_i915_gem_object *obj; 2681 struct intel_ring_buffer *ring = NULL; 2682 struct timespec timeout_stack, *timeout = NULL; 2683 unsigned reset_counter; 2684 u32 seqno = 0; 2685 int ret = 0; 2686 2687 if (args->timeout_ns >= 0) { 2688 timeout_stack = ns_to_timespec(args->timeout_ns); 2689 timeout = &timeout_stack; 2690 } 2691 2692 ret = i915_mutex_lock_interruptible(dev); 2693 if (ret) 2694 return ret; 2695 2696 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle)); 2697 if (&obj->base == NULL) { 2698 mutex_unlock(&dev->struct_mutex); 2699 return -ENOENT; 2700 } 2701 2702 /* Need to make sure the object gets inactive eventually. */ 2703 ret = i915_gem_object_flush_active(obj); 2704 if (ret) 2705 goto out; 2706 2707 if (obj->active) { 2708 seqno = obj->last_read_seqno; 2709 ring = obj->ring; 2710 } 2711 2712 if (seqno == 0) 2713 goto out; 2714 2715 /* Do this after OLR check to make sure we make forward progress polling 2716 * on this IOCTL with a 0 timeout (like busy ioctl) 2717 */ 2718 if (!args->timeout_ns) { 2719 ret = -ETIMEDOUT; 2720 goto out; 2721 } 2722 2723 drm_gem_object_unreference(&obj->base); 2724 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); 2725 mutex_unlock(&dev->struct_mutex); 2726 2727 ret = __wait_seqno(ring, seqno, reset_counter, true, timeout); 2728 if (timeout) 2729 args->timeout_ns = timespec_to_ns(timeout); 2730 return ret; 2731 2732 out: 2733 drm_gem_object_unreference(&obj->base); 2734 mutex_unlock(&dev->struct_mutex); 2735 return ret; 2736 } 2737 2738 /** 2739 * i915_gem_object_sync - sync an object to a ring. 2740 * 2741 * @obj: object which may be in use on another ring. 2742 * @to: ring we wish to use the object on. May be NULL. 2743 * 2744 * This code is meant to abstract object synchronization with the GPU. 2745 * Calling with NULL implies synchronizing the object with the CPU 2746 * rather than a particular GPU ring. 2747 * 2748 * Returns 0 if successful, else propagates up the lower layer error. 2749 */ 2750 int 2751 i915_gem_object_sync(struct drm_i915_gem_object *obj, 2752 struct intel_ring_buffer *to) 2753 { 2754 struct intel_ring_buffer *from = obj->ring; 2755 u32 seqno; 2756 int ret, idx; 2757 2758 if (from == NULL || to == from) 2759 return 0; 2760 2761 if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) 2762 return i915_gem_object_wait_rendering(obj, false); 2763 2764 idx = intel_ring_sync_index(from, to); 2765 2766 seqno = obj->last_read_seqno; 2767 if (seqno <= from->sync_seqno[idx]) 2768 return 0; 2769 2770 ret = i915_gem_check_olr(obj->ring, seqno); 2771 if (ret) 2772 return ret; 2773 2774 trace_i915_gem_ring_sync_to(from, to, seqno); 2775 ret = to->sync_to(to, from, seqno); 2776 if (!ret) 2777 /* We use last_read_seqno because sync_to() 2778 * might have just caused seqno wrap under 2779 * the radar. 2780 */ 2781 from->sync_seqno[idx] = obj->last_read_seqno; 2782 2783 return ret; 2784 } 2785 2786 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2787 { 2788 u32 old_write_domain, old_read_domains; 2789 2790 /* Force a pagefault for domain tracking on next user access */ 2791 i915_gem_release_mmap(obj); 2792 2793 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2794 return; 2795 2796 /* Wait for any direct GTT access to complete */ 2797 mb(); 2798 2799 old_read_domains = obj->base.read_domains; 2800 old_write_domain = obj->base.write_domain; 2801 2802 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2803 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2804 2805 trace_i915_gem_object_change_domain(obj, 2806 old_read_domains, 2807 old_write_domain); 2808 } 2809 2810 int i915_vma_unbind(struct i915_vma *vma) 2811 { 2812 struct drm_i915_gem_object *obj = vma->obj; 2813 drm_i915_private_t *dev_priv = obj->base.dev->dev_private; 2814 int ret; 2815 2816 /* For now we only ever use 1 vma per object */ 2817 #if 0 2818 WARN_ON(!list_is_singular(&obj->vma_list)); 2819 #endif 2820 2821 if (list_empty(&vma->vma_link)) 2822 return 0; 2823 2824 if (!drm_mm_node_allocated(&vma->node)) { 2825 i915_gem_vma_destroy(vma); 2826 2827 return 0; 2828 } 2829 2830 if (obj->pin_count) 2831 return -EBUSY; 2832 2833 BUG_ON(obj->pages == NULL); 2834 2835 ret = i915_gem_object_finish_gpu(obj); 2836 if (ret) 2837 return ret; 2838 /* Continue on if we fail due to EIO, the GPU is hung so we 2839 * should be safe and we need to cleanup or else we might 2840 * cause memory corruption through use-after-free. 2841 */ 2842 2843 i915_gem_object_finish_gtt(obj); 2844 2845 /* release the fence reg _after_ flushing */ 2846 ret = i915_gem_object_put_fence(obj); 2847 if (ret) 2848 return ret; 2849 2850 trace_i915_vma_unbind(vma); 2851 2852 if (obj->has_global_gtt_mapping) 2853 i915_gem_gtt_unbind_object(obj); 2854 if (obj->has_aliasing_ppgtt_mapping) { 2855 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2856 obj->has_aliasing_ppgtt_mapping = 0; 2857 } 2858 i915_gem_gtt_finish_object(obj); 2859 2860 list_del(&vma->mm_list); 2861 /* Avoid an unnecessary call to unbind on rebind. */ 2862 if (i915_is_ggtt(vma->vm)) 2863 obj->map_and_fenceable = true; 2864 2865 drm_mm_remove_node(&vma->node); 2866 i915_gem_vma_destroy(vma); 2867 2868 /* Since the unbound list is global, only move to that list if 2869 * no more VMAs exist. */ 2870 if (list_empty(&obj->vma_list)) 2871 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list); 2872 2873 /* And finally now the object is completely decoupled from this vma, 2874 * we can drop its hold on the backing storage and allow it to be 2875 * reaped by the shrinker. 2876 */ 2877 i915_gem_object_unpin_pages(obj); 2878 2879 return 0; 2880 } 2881 2882 /** 2883 * Unbinds an object from the global GTT aperture. 2884 */ 2885 int 2886 i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj) 2887 { 2888 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 2889 struct i915_address_space *ggtt = &dev_priv->gtt.base; 2890 2891 if (!i915_gem_obj_ggtt_bound(obj)) 2892 return 0; 2893 2894 if (obj->pin_count) 2895 return -EBUSY; 2896 2897 BUG_ON(obj->pages == NULL); 2898 2899 return i915_vma_unbind(i915_gem_obj_to_vma(obj, ggtt)); 2900 } 2901 2902 int i915_gpu_idle(struct drm_device *dev) 2903 { 2904 drm_i915_private_t *dev_priv = dev->dev_private; 2905 struct intel_ring_buffer *ring; 2906 int ret, i; 2907 2908 /* Flush everything onto the inactive list. */ 2909 for_each_ring(ring, dev_priv, i) { 2910 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID); 2911 if (ret) 2912 return ret; 2913 2914 ret = intel_ring_idle(ring); 2915 if (ret) 2916 return ret; 2917 } 2918 2919 return 0; 2920 } 2921 2922 static void i965_write_fence_reg(struct drm_device *dev, int reg, 2923 struct drm_i915_gem_object *obj) 2924 { 2925 drm_i915_private_t *dev_priv = dev->dev_private; 2926 int fence_reg; 2927 int fence_pitch_shift; 2928 2929 if (INTEL_INFO(dev)->gen >= 6) { 2930 fence_reg = FENCE_REG_SANDYBRIDGE_0; 2931 fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT; 2932 } else { 2933 fence_reg = FENCE_REG_965_0; 2934 fence_pitch_shift = I965_FENCE_PITCH_SHIFT; 2935 } 2936 2937 fence_reg += reg * 8; 2938 2939 /* To w/a incoherency with non-atomic 64-bit register updates, 2940 * we split the 64-bit update into two 32-bit writes. In order 2941 * for a partial fence not to be evaluated between writes, we 2942 * precede the update with write to turn off the fence register, 2943 * and only enable the fence as the last step. 2944 * 2945 * For extra levels of paranoia, we make sure each step lands 2946 * before applying the next step. 2947 */ 2948 I915_WRITE(fence_reg, 0); 2949 POSTING_READ(fence_reg); 2950 2951 if (obj) { 2952 u32 size = i915_gem_obj_ggtt_size(obj); 2953 uint64_t val; 2954 2955 val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) & 2956 0xfffff000) << 32; 2957 val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000; 2958 val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift; 2959 if (obj->tiling_mode == I915_TILING_Y) 2960 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2961 val |= I965_FENCE_REG_VALID; 2962 2963 I915_WRITE(fence_reg + 4, val >> 32); 2964 POSTING_READ(fence_reg + 4); 2965 2966 I915_WRITE(fence_reg + 0, val); 2967 POSTING_READ(fence_reg); 2968 } else { 2969 I915_WRITE(fence_reg + 4, 0); 2970 POSTING_READ(fence_reg + 4); 2971 } 2972 } 2973 2974 static void i915_write_fence_reg(struct drm_device *dev, int reg, 2975 struct drm_i915_gem_object *obj) 2976 { 2977 drm_i915_private_t *dev_priv = dev->dev_private; 2978 u32 val; 2979 2980 if (obj) { 2981 u32 size = i915_gem_obj_ggtt_size(obj); 2982 int pitch_val; 2983 int tile_width; 2984 2985 WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) || 2986 (size & -size) != size || 2987 (i915_gem_obj_ggtt_offset(obj) & (size - 1)), 2988 "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 2989 i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size); 2990 2991 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 2992 tile_width = 128; 2993 else 2994 tile_width = 512; 2995 2996 /* Note: pitch better be a power of two tile widths */ 2997 pitch_val = obj->stride / tile_width; 2998 pitch_val = ffs(pitch_val) - 1; 2999 3000 val = i915_gem_obj_ggtt_offset(obj); 3001 if (obj->tiling_mode == I915_TILING_Y) 3002 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3003 val |= I915_FENCE_SIZE_BITS(size); 3004 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3005 val |= I830_FENCE_REG_VALID; 3006 } else 3007 val = 0; 3008 3009 if (reg < 8) 3010 reg = FENCE_REG_830_0 + reg * 4; 3011 else 3012 reg = FENCE_REG_945_8 + (reg - 8) * 4; 3013 3014 I915_WRITE(reg, val); 3015 POSTING_READ(reg); 3016 } 3017 3018 static void i830_write_fence_reg(struct drm_device *dev, int reg, 3019 struct drm_i915_gem_object *obj) 3020 { 3021 drm_i915_private_t *dev_priv = dev->dev_private; 3022 uint32_t val; 3023 3024 if (obj) { 3025 u32 size = i915_gem_obj_ggtt_size(obj); 3026 uint32_t pitch_val; 3027 3028 WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) || 3029 (size & -size) != size || 3030 (i915_gem_obj_ggtt_offset(obj) & (size - 1)), 3031 "object 0x%08lx not 512K or pot-size 0x%08x aligned\n", 3032 i915_gem_obj_ggtt_offset(obj), size); 3033 3034 pitch_val = obj->stride / 128; 3035 pitch_val = ffs(pitch_val) - 1; 3036 3037 val = i915_gem_obj_ggtt_offset(obj); 3038 if (obj->tiling_mode == I915_TILING_Y) 3039 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3040 val |= I830_FENCE_SIZE_BITS(size); 3041 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3042 val |= I830_FENCE_REG_VALID; 3043 } else 3044 val = 0; 3045 3046 I915_WRITE(FENCE_REG_830_0 + reg * 4, val); 3047 POSTING_READ(FENCE_REG_830_0 + reg * 4); 3048 } 3049 3050 inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj) 3051 { 3052 return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT; 3053 } 3054 3055 static void i915_gem_write_fence(struct drm_device *dev, int reg, 3056 struct drm_i915_gem_object *obj) 3057 { 3058 struct drm_i915_private *dev_priv = dev->dev_private; 3059 3060 /* Ensure that all CPU reads are completed before installing a fence 3061 * and all writes before removing the fence. 3062 */ 3063 if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj)) 3064 mb(); 3065 3066 WARN(obj && (!obj->stride || !obj->tiling_mode), 3067 "bogus fence setup with stride: 0x%x, tiling mode: %i\n", 3068 obj->stride, obj->tiling_mode); 3069 3070 switch (INTEL_INFO(dev)->gen) { 3071 case 8: 3072 case 7: 3073 case 6: 3074 case 5: 3075 case 4: i965_write_fence_reg(dev, reg, obj); break; 3076 case 3: i915_write_fence_reg(dev, reg, obj); break; 3077 case 2: i830_write_fence_reg(dev, reg, obj); break; 3078 default: BUG(); 3079 } 3080 3081 /* And similarly be paranoid that no direct access to this region 3082 * is reordered to before the fence is installed. 3083 */ 3084 if (i915_gem_object_needs_mb(obj)) 3085 mb(); 3086 } 3087 3088 static inline int fence_number(struct drm_i915_private *dev_priv, 3089 struct drm_i915_fence_reg *fence) 3090 { 3091 return fence - dev_priv->fence_regs; 3092 } 3093 3094 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, 3095 struct drm_i915_fence_reg *fence, 3096 bool enable) 3097 { 3098 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3099 int reg = fence_number(dev_priv, fence); 3100 3101 i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); 3102 3103 if (enable) { 3104 obj->fence_reg = reg; 3105 fence->obj = obj; 3106 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); 3107 } else { 3108 obj->fence_reg = I915_FENCE_REG_NONE; 3109 fence->obj = NULL; 3110 list_del_init(&fence->lru_list); 3111 } 3112 obj->fence_dirty = false; 3113 } 3114 3115 static int 3116 i915_gem_object_wait_fence(struct drm_i915_gem_object *obj) 3117 { 3118 if (obj->last_fenced_seqno) { 3119 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno); 3120 if (ret) 3121 return ret; 3122 3123 obj->last_fenced_seqno = 0; 3124 } 3125 3126 obj->fenced_gpu_access = false; 3127 return 0; 3128 } 3129 3130 int 3131 i915_gem_object_put_fence(struct drm_i915_gem_object *obj) 3132 { 3133 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3134 struct drm_i915_fence_reg *fence; 3135 int ret; 3136 3137 ret = i915_gem_object_wait_fence(obj); 3138 if (ret) 3139 return ret; 3140 3141 if (obj->fence_reg == I915_FENCE_REG_NONE) 3142 return 0; 3143 3144 fence = &dev_priv->fence_regs[obj->fence_reg]; 3145 3146 i915_gem_object_fence_lost(obj); 3147 i915_gem_object_update_fence(obj, fence, false); 3148 3149 return 0; 3150 } 3151 3152 static struct drm_i915_fence_reg * 3153 i915_find_fence_reg(struct drm_device *dev) 3154 { 3155 struct drm_i915_private *dev_priv = dev->dev_private; 3156 struct drm_i915_fence_reg *reg, *avail; 3157 int i; 3158 3159 /* First try to find a free reg */ 3160 avail = NULL; 3161 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 3162 reg = &dev_priv->fence_regs[i]; 3163 if (!reg->obj) 3164 return reg; 3165 3166 if (!reg->pin_count) 3167 avail = reg; 3168 } 3169 3170 if (avail == NULL) 3171 goto deadlock; 3172 3173 /* None available, try to steal one or wait for a user to finish */ 3174 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 3175 if (reg->pin_count) 3176 continue; 3177 3178 return reg; 3179 } 3180 3181 deadlock: 3182 /* Wait for completion of pending flips which consume fences */ 3183 if (intel_has_pending_fb_unpin(dev)) 3184 return ERR_PTR(-EAGAIN); 3185 3186 return ERR_PTR(-EDEADLK); 3187 } 3188 3189 /** 3190 * i915_gem_object_get_fence - set up fencing for an object 3191 * @obj: object to map through a fence reg 3192 * 3193 * When mapping objects through the GTT, userspace wants to be able to write 3194 * to them without having to worry about swizzling if the object is tiled. 3195 * This function walks the fence regs looking for a free one for @obj, 3196 * stealing one if it can't find any. 3197 * 3198 * It then sets up the reg based on the object's properties: address, pitch 3199 * and tiling format. 3200 * 3201 * For an untiled surface, this removes any existing fence. 3202 */ 3203 int 3204 i915_gem_object_get_fence(struct drm_i915_gem_object *obj) 3205 { 3206 struct drm_device *dev = obj->base.dev; 3207 struct drm_i915_private *dev_priv = dev->dev_private; 3208 bool enable = obj->tiling_mode != I915_TILING_NONE; 3209 struct drm_i915_fence_reg *reg; 3210 int ret; 3211 3212 /* Have we updated the tiling parameters upon the object and so 3213 * will need to serialise the write to the associated fence register? 3214 */ 3215 if (obj->fence_dirty) { 3216 ret = i915_gem_object_wait_fence(obj); 3217 if (ret) 3218 return ret; 3219 } 3220 3221 /* Just update our place in the LRU if our fence is getting reused. */ 3222 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3223 reg = &dev_priv->fence_regs[obj->fence_reg]; 3224 if (!obj->fence_dirty) { 3225 list_move_tail(®->lru_list, 3226 &dev_priv->mm.fence_list); 3227 return 0; 3228 } 3229 } else if (enable) { 3230 reg = i915_find_fence_reg(dev); 3231 if (IS_ERR(reg)) 3232 return PTR_ERR(reg); 3233 3234 if (reg->obj) { 3235 struct drm_i915_gem_object *old = reg->obj; 3236 3237 ret = i915_gem_object_wait_fence(old); 3238 if (ret) 3239 return ret; 3240 3241 i915_gem_object_fence_lost(old); 3242 } 3243 } else 3244 return 0; 3245 3246 i915_gem_object_update_fence(obj, reg, enable); 3247 3248 return 0; 3249 } 3250 3251 static bool i915_gem_valid_gtt_space(struct drm_device *dev, 3252 struct drm_mm_node *gtt_space, 3253 unsigned long cache_level) 3254 { 3255 struct drm_mm_node *other; 3256 3257 /* On non-LLC machines we have to be careful when putting differing 3258 * types of snoopable memory together to avoid the prefetcher 3259 * crossing memory domains and dying. 3260 */ 3261 if (HAS_LLC(dev)) 3262 return true; 3263 3264 if (!drm_mm_node_allocated(gtt_space)) 3265 return true; 3266 3267 if (list_empty(>t_space->node_list)) 3268 return true; 3269 3270 other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list); 3271 if (other->allocated && !other->hole_follows && other->color != cache_level) 3272 return false; 3273 3274 other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list); 3275 if (other->allocated && !gtt_space->hole_follows && other->color != cache_level) 3276 return false; 3277 3278 return true; 3279 } 3280 3281 static void i915_gem_verify_gtt(struct drm_device *dev) 3282 { 3283 #if WATCH_GTT 3284 struct drm_i915_private *dev_priv = dev->dev_private; 3285 struct drm_i915_gem_object *obj; 3286 int err = 0; 3287 3288 list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) { 3289 if (obj->gtt_space == NULL) { 3290 printk(KERN_ERR "object found on GTT list with no space reserved\n"); 3291 err++; 3292 continue; 3293 } 3294 3295 if (obj->cache_level != obj->gtt_space->color) { 3296 printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n", 3297 i915_gem_obj_ggtt_offset(obj), 3298 i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj), 3299 obj->cache_level, 3300 obj->gtt_space->color); 3301 err++; 3302 continue; 3303 } 3304 3305 if (!i915_gem_valid_gtt_space(dev, 3306 obj->gtt_space, 3307 obj->cache_level)) { 3308 printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n", 3309 i915_gem_obj_ggtt_offset(obj), 3310 i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj), 3311 obj->cache_level); 3312 err++; 3313 continue; 3314 } 3315 } 3316 3317 WARN_ON(err); 3318 #endif 3319 } 3320 3321 /** 3322 * Finds free space in the GTT aperture and binds the object there. 3323 */ 3324 static int 3325 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, 3326 struct i915_address_space *vm, 3327 unsigned alignment, 3328 bool map_and_fenceable, 3329 bool nonblocking) 3330 { 3331 struct drm_device *dev = obj->base.dev; 3332 drm_i915_private_t *dev_priv = dev->dev_private; 3333 u32 size, fence_size, fence_alignment, unfenced_alignment; 3334 size_t gtt_max = 3335 map_and_fenceable ? dev_priv->gtt.mappable_end : vm->total; 3336 struct i915_vma *vma; 3337 int ret; 3338 3339 fence_size = i915_gem_get_gtt_size(dev, 3340 obj->base.size, 3341 obj->tiling_mode); 3342 fence_alignment = i915_gem_get_gtt_alignment(dev, 3343 obj->base.size, 3344 obj->tiling_mode, true); 3345 unfenced_alignment = 3346 i915_gem_get_gtt_alignment(dev, 3347 obj->base.size, 3348 obj->tiling_mode, false); 3349 3350 if (alignment == 0) 3351 alignment = map_and_fenceable ? fence_alignment : 3352 unfenced_alignment; 3353 if (map_and_fenceable && alignment & (fence_alignment - 1)) { 3354 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 3355 return -EINVAL; 3356 } 3357 3358 size = map_and_fenceable ? fence_size : obj->base.size; 3359 3360 /* If the object is bigger than the entire aperture, reject it early 3361 * before evicting everything in a vain attempt to find space. 3362 */ 3363 if (obj->base.size > gtt_max) { 3364 DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n", 3365 obj->base.size, 3366 map_and_fenceable ? "mappable" : "total", 3367 gtt_max); 3368 return -E2BIG; 3369 } 3370 3371 ret = i915_gem_object_get_pages(obj); 3372 if (ret) 3373 return ret; 3374 3375 i915_gem_object_pin_pages(obj); 3376 3377 BUG_ON(!i915_is_ggtt(vm)); 3378 3379 vma = i915_gem_obj_lookup_or_create_vma(obj, vm); 3380 if (IS_ERR(vma)) { 3381 ret = PTR_ERR(vma); 3382 goto err_unpin; 3383 } 3384 3385 /* For now we only ever use 1 vma per object */ 3386 #if 0 3387 WARN_ON(!list_is_singular(&obj->vma_list)); 3388 #endif 3389 3390 search_free: 3391 ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node, 3392 size, alignment, 3393 obj->cache_level, 0, gtt_max, 3394 DRM_MM_SEARCH_DEFAULT); 3395 if (ret) { 3396 ret = i915_gem_evict_something(dev, vm, size, alignment, 3397 obj->cache_level, 3398 map_and_fenceable, 3399 nonblocking); 3400 if (ret == 0) 3401 goto search_free; 3402 3403 goto err_free_vma; 3404 } 3405 if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node, 3406 obj->cache_level))) { 3407 ret = -EINVAL; 3408 goto err_remove_node; 3409 } 3410 3411 ret = i915_gem_gtt_prepare_object(obj); 3412 if (ret) 3413 goto err_remove_node; 3414 3415 list_move_tail(&obj->global_list, &dev_priv->mm.bound_list); 3416 list_add_tail(&vma->mm_list, &vm->inactive_list); 3417 3418 if (i915_is_ggtt(vm)) { 3419 bool mappable, fenceable; 3420 3421 fenceable = (vma->node.size == fence_size && 3422 (vma->node.start & (fence_alignment - 1)) == 0); 3423 3424 mappable = (vma->node.start + obj->base.size <= 3425 dev_priv->gtt.mappable_end); 3426 3427 obj->map_and_fenceable = mappable && fenceable; 3428 } 3429 3430 WARN_ON(map_and_fenceable && !obj->map_and_fenceable); 3431 3432 trace_i915_vma_bind(vma, map_and_fenceable); 3433 i915_gem_verify_gtt(dev); 3434 return 0; 3435 3436 err_remove_node: 3437 drm_mm_remove_node(&vma->node); 3438 err_free_vma: 3439 i915_gem_vma_destroy(vma); 3440 err_unpin: 3441 i915_gem_object_unpin_pages(obj); 3442 return ret; 3443 } 3444 3445 bool 3446 i915_gem_clflush_object(struct drm_i915_gem_object *obj, 3447 bool force) 3448 { 3449 /* If we don't have a page list set up, then we're not pinned 3450 * to GPU, and we can ignore the cache flush because it'll happen 3451 * again at bind time. 3452 */ 3453 if (obj->pages == NULL) 3454 return false; 3455 3456 /* 3457 * Stolen memory is always coherent with the GPU as it is explicitly 3458 * marked as wc by the system, or the system is cache-coherent. 3459 */ 3460 if (obj->stolen) 3461 return false; 3462 3463 /* If the GPU is snooping the contents of the CPU cache, 3464 * we do not need to manually clear the CPU cache lines. However, 3465 * the caches are only snooped when the render cache is 3466 * flushed/invalidated. As we always have to emit invalidations 3467 * and flushes when moving into and out of the RENDER domain, correct 3468 * snooping behaviour occurs naturally as the result of our domain 3469 * tracking. 3470 */ 3471 if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) 3472 return false; 3473 3474 trace_i915_gem_object_clflush(obj); 3475 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 3476 3477 return true; 3478 } 3479 3480 /** Flushes the GTT write domain for the object if it's dirty. */ 3481 static void 3482 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 3483 { 3484 uint32_t old_write_domain; 3485 3486 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 3487 return; 3488 3489 /* No actual flushing is required for the GTT write domain. Writes 3490 * to it immediately go to main memory as far as we know, so there's 3491 * no chipset flush. It also doesn't land in render cache. 3492 * 3493 * However, we do have to enforce the order so that all writes through 3494 * the GTT land before any writes to the device, such as updates to 3495 * the GATT itself. 3496 */ 3497 wmb(); 3498 3499 old_write_domain = obj->base.write_domain; 3500 obj->base.write_domain = 0; 3501 3502 trace_i915_gem_object_change_domain(obj, 3503 obj->base.read_domains, 3504 old_write_domain); 3505 } 3506 3507 /** Flushes the CPU write domain for the object if it's dirty. */ 3508 static void 3509 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj, 3510 bool force) 3511 { 3512 uint32_t old_write_domain; 3513 3514 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 3515 return; 3516 3517 if (i915_gem_clflush_object(obj, force)) 3518 i915_gem_chipset_flush(obj->base.dev); 3519 3520 old_write_domain = obj->base.write_domain; 3521 obj->base.write_domain = 0; 3522 3523 trace_i915_gem_object_change_domain(obj, 3524 obj->base.read_domains, 3525 old_write_domain); 3526 } 3527 3528 /** 3529 * Moves a single object to the GTT read, and possibly write domain. 3530 * 3531 * This function returns when the move is complete, including waiting on 3532 * flushes to occur. 3533 */ 3534 int 3535 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 3536 { 3537 drm_i915_private_t *dev_priv = obj->base.dev->dev_private; 3538 uint32_t old_write_domain, old_read_domains; 3539 int ret; 3540 3541 /* Not valid to be called on unbound objects. */ 3542 if (!i915_gem_obj_bound_any(obj)) 3543 return -EINVAL; 3544 3545 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 3546 return 0; 3547 3548 ret = i915_gem_object_wait_rendering(obj, !write); 3549 if (ret) 3550 return ret; 3551 3552 i915_gem_object_flush_cpu_write_domain(obj, false); 3553 3554 /* Serialise direct access to this object with the barriers for 3555 * coherent writes from the GPU, by effectively invalidating the 3556 * GTT domain upon first access. 3557 */ 3558 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 3559 mb(); 3560 3561 old_write_domain = obj->base.write_domain; 3562 old_read_domains = obj->base.read_domains; 3563 3564 /* It should now be out of any other write domains, and we can update 3565 * the domain values for our changes. 3566 */ 3567 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 3568 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 3569 if (write) { 3570 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 3571 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 3572 obj->dirty = 1; 3573 } 3574 3575 trace_i915_gem_object_change_domain(obj, 3576 old_read_domains, 3577 old_write_domain); 3578 3579 /* And bump the LRU for this access */ 3580 if (i915_gem_object_is_inactive(obj)) { 3581 struct i915_vma *vma = i915_gem_obj_to_ggtt(obj); 3582 if (vma) 3583 list_move_tail(&vma->mm_list, 3584 &dev_priv->gtt.base.inactive_list); 3585 3586 } 3587 3588 return 0; 3589 } 3590 3591 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 3592 enum i915_cache_level cache_level) 3593 { 3594 struct drm_device *dev = obj->base.dev; 3595 drm_i915_private_t *dev_priv = dev->dev_private; 3596 struct i915_vma *vma; 3597 int ret; 3598 3599 if (obj->cache_level == cache_level) 3600 return 0; 3601 3602 if (obj->pin_count) { 3603 DRM_DEBUG("can not change the cache level of pinned objects\n"); 3604 return -EBUSY; 3605 } 3606 3607 list_for_each_entry(vma, &obj->vma_list, vma_link) { 3608 if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) { 3609 ret = i915_vma_unbind(vma); 3610 if (ret) 3611 return ret; 3612 3613 break; 3614 } 3615 } 3616 3617 if (i915_gem_obj_bound_any(obj)) { 3618 ret = i915_gem_object_finish_gpu(obj); 3619 if (ret) 3620 return ret; 3621 3622 i915_gem_object_finish_gtt(obj); 3623 3624 /* Before SandyBridge, you could not use tiling or fence 3625 * registers with snooped memory, so relinquish any fences 3626 * currently pointing to our region in the aperture. 3627 */ 3628 if (INTEL_INFO(dev)->gen < 6) { 3629 ret = i915_gem_object_put_fence(obj); 3630 if (ret) 3631 return ret; 3632 } 3633 3634 if (obj->has_global_gtt_mapping) 3635 i915_gem_gtt_bind_object(obj, cache_level); 3636 if (obj->has_aliasing_ppgtt_mapping) 3637 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 3638 obj, cache_level); 3639 } 3640 3641 list_for_each_entry(vma, &obj->vma_list, vma_link) 3642 vma->node.color = cache_level; 3643 obj->cache_level = cache_level; 3644 3645 if (cpu_write_needs_clflush(obj)) { 3646 u32 old_read_domains, old_write_domain; 3647 3648 /* If we're coming from LLC cached, then we haven't 3649 * actually been tracking whether the data is in the 3650 * CPU cache or not, since we only allow one bit set 3651 * in obj->write_domain and have been skipping the clflushes. 3652 * Just set it to the CPU cache for now. 3653 */ 3654 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU); 3655 3656 old_read_domains = obj->base.read_domains; 3657 old_write_domain = obj->base.write_domain; 3658 3659 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 3660 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 3661 3662 trace_i915_gem_object_change_domain(obj, 3663 old_read_domains, 3664 old_write_domain); 3665 } 3666 3667 i915_gem_verify_gtt(dev); 3668 return 0; 3669 } 3670 3671 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, 3672 struct drm_file *file) 3673 { 3674 struct drm_i915_gem_caching *args = data; 3675 struct drm_i915_gem_object *obj; 3676 int ret; 3677 3678 ret = i915_mutex_lock_interruptible(dev); 3679 if (ret) 3680 return ret; 3681 3682 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3683 if (&obj->base == NULL) { 3684 ret = -ENOENT; 3685 goto unlock; 3686 } 3687 3688 switch (obj->cache_level) { 3689 case I915_CACHE_LLC: 3690 case I915_CACHE_L3_LLC: 3691 args->caching = I915_CACHING_CACHED; 3692 break; 3693 3694 case I915_CACHE_WT: 3695 args->caching = I915_CACHING_DISPLAY; 3696 break; 3697 3698 default: 3699 args->caching = I915_CACHING_NONE; 3700 break; 3701 } 3702 3703 drm_gem_object_unreference(&obj->base); 3704 unlock: 3705 mutex_unlock(&dev->struct_mutex); 3706 return ret; 3707 } 3708 3709 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, 3710 struct drm_file *file) 3711 { 3712 struct drm_i915_gem_caching *args = data; 3713 struct drm_i915_gem_object *obj; 3714 enum i915_cache_level level; 3715 int ret; 3716 3717 switch (args->caching) { 3718 case I915_CACHING_NONE: 3719 level = I915_CACHE_NONE; 3720 break; 3721 case I915_CACHING_CACHED: 3722 level = I915_CACHE_LLC; 3723 break; 3724 case I915_CACHING_DISPLAY: 3725 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE; 3726 break; 3727 default: 3728 return -EINVAL; 3729 } 3730 3731 ret = i915_mutex_lock_interruptible(dev); 3732 if (ret) 3733 return ret; 3734 3735 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3736 if (&obj->base == NULL) { 3737 ret = -ENOENT; 3738 goto unlock; 3739 } 3740 3741 ret = i915_gem_object_set_cache_level(obj, level); 3742 3743 drm_gem_object_unreference(&obj->base); 3744 unlock: 3745 mutex_unlock(&dev->struct_mutex); 3746 return ret; 3747 } 3748 3749 static bool is_pin_display(struct drm_i915_gem_object *obj) 3750 { 3751 /* There are 3 sources that pin objects: 3752 * 1. The display engine (scanouts, sprites, cursors); 3753 * 2. Reservations for execbuffer; 3754 * 3. The user. 3755 * 3756 * We can ignore reservations as we hold the struct_mutex and 3757 * are only called outside of the reservation path. The user 3758 * can only increment pin_count once, and so if after 3759 * subtracting the potential reference by the user, any pin_count 3760 * remains, it must be due to another use by the display engine. 3761 */ 3762 return obj->pin_count - !!obj->user_pin_count; 3763 } 3764 3765 /* 3766 * Prepare buffer for display plane (scanout, cursors, etc). 3767 * Can be called from an uninterruptible phase (modesetting) and allows 3768 * any flushes to be pipelined (for pageflips). 3769 */ 3770 int 3771 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 3772 u32 alignment, 3773 struct intel_ring_buffer *pipelined) 3774 { 3775 u32 old_read_domains, old_write_domain; 3776 int ret; 3777 3778 if (pipelined != obj->ring) { 3779 ret = i915_gem_object_sync(obj, pipelined); 3780 if (ret) 3781 return ret; 3782 } 3783 3784 /* Mark the pin_display early so that we account for the 3785 * display coherency whilst setting up the cache domains. 3786 */ 3787 obj->pin_display = true; 3788 3789 /* The display engine is not coherent with the LLC cache on gen6. As 3790 * a result, we make sure that the pinning that is about to occur is 3791 * done with uncached PTEs. This is lowest common denominator for all 3792 * chipsets. 3793 * 3794 * However for gen6+, we could do better by using the GFDT bit instead 3795 * of uncaching, which would allow us to flush all the LLC-cached data 3796 * with that bit in the PTE to main memory with just one PIPE_CONTROL. 3797 */ 3798 ret = i915_gem_object_set_cache_level(obj, 3799 HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE); 3800 if (ret) 3801 goto err_unpin_display; 3802 3803 /* As the user may map the buffer once pinned in the display plane 3804 * (e.g. libkms for the bootup splash), we have to ensure that we 3805 * always use map_and_fenceable for all scanout buffers. 3806 */ 3807 ret = i915_gem_obj_ggtt_pin(obj, alignment, true, false); 3808 if (ret) 3809 goto err_unpin_display; 3810 3811 i915_gem_object_flush_cpu_write_domain(obj, true); 3812 3813 old_write_domain = obj->base.write_domain; 3814 old_read_domains = obj->base.read_domains; 3815 3816 /* It should now be out of any other write domains, and we can update 3817 * the domain values for our changes. 3818 */ 3819 obj->base.write_domain = 0; 3820 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 3821 3822 trace_i915_gem_object_change_domain(obj, 3823 old_read_domains, 3824 old_write_domain); 3825 3826 return 0; 3827 3828 err_unpin_display: 3829 obj->pin_display = is_pin_display(obj); 3830 return ret; 3831 } 3832 3833 void 3834 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj) 3835 { 3836 i915_gem_object_unpin(obj); 3837 obj->pin_display = is_pin_display(obj); 3838 } 3839 3840 int 3841 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 3842 { 3843 int ret; 3844 3845 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 3846 return 0; 3847 3848 ret = i915_gem_object_wait_rendering(obj, false); 3849 if (ret) 3850 return ret; 3851 3852 /* Ensure that we invalidate the GPU's caches and TLBs. */ 3853 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 3854 return 0; 3855 } 3856 3857 /** 3858 * Moves a single object to the CPU read, and possibly write domain. 3859 * 3860 * This function returns when the move is complete, including waiting on 3861 * flushes to occur. 3862 */ 3863 int 3864 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 3865 { 3866 uint32_t old_write_domain, old_read_domains; 3867 int ret; 3868 3869 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 3870 return 0; 3871 3872 ret = i915_gem_object_wait_rendering(obj, !write); 3873 if (ret) 3874 return ret; 3875 3876 i915_gem_object_flush_gtt_write_domain(obj); 3877 3878 old_write_domain = obj->base.write_domain; 3879 old_read_domains = obj->base.read_domains; 3880 3881 /* Flush the CPU cache if it's still invalid. */ 3882 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 3883 i915_gem_clflush_object(obj, false); 3884 3885 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 3886 } 3887 3888 /* It should now be out of any other write domains, and we can update 3889 * the domain values for our changes. 3890 */ 3891 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 3892 3893 /* If we're writing through the CPU, then the GPU read domains will 3894 * need to be invalidated at next use. 3895 */ 3896 if (write) { 3897 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 3898 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 3899 } 3900 3901 trace_i915_gem_object_change_domain(obj, 3902 old_read_domains, 3903 old_write_domain); 3904 3905 return 0; 3906 } 3907 3908 /* Throttle our rendering by waiting until the ring has completed our requests 3909 * emitted over 20 msec ago. 3910 * 3911 * Note that if we were to use the current jiffies each time around the loop, 3912 * we wouldn't escape the function with any frames outstanding if the time to 3913 * render a frame was over 20ms. 3914 * 3915 * This should get us reasonable parallelism between CPU and GPU but also 3916 * relatively low latency when blocking on a particular request to finish. 3917 */ 3918 static int 3919 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 3920 { 3921 struct drm_i915_private *dev_priv = dev->dev_private; 3922 struct drm_i915_file_private *file_priv = file->driver_priv; 3923 unsigned long recent_enough = jiffies - msecs_to_jiffies(20); 3924 struct drm_i915_gem_request *request; 3925 struct intel_ring_buffer *ring = NULL; 3926 unsigned reset_counter; 3927 u32 seqno = 0; 3928 int ret; 3929 3930 ret = i915_gem_wait_for_error(&dev_priv->gpu_error); 3931 if (ret) 3932 return ret; 3933 3934 ret = i915_gem_check_wedge(&dev_priv->gpu_error, false); 3935 if (ret) 3936 return ret; 3937 3938 spin_lock(&file_priv->mm.lock); 3939 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 3940 if (time_after_eq(request->emitted_jiffies, recent_enough)) 3941 break; 3942 3943 ring = request->ring; 3944 seqno = request->seqno; 3945 } 3946 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); 3947 spin_unlock(&file_priv->mm.lock); 3948 3949 if (seqno == 0) 3950 return 0; 3951 3952 ret = __wait_seqno(ring, seqno, reset_counter, true, NULL); 3953 if (ret == 0) 3954 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); 3955 3956 return ret; 3957 } 3958 3959 int 3960 i915_gem_object_pin(struct drm_i915_gem_object *obj, 3961 struct i915_address_space *vm, 3962 uint32_t alignment, 3963 bool map_and_fenceable, 3964 bool nonblocking) 3965 { 3966 struct i915_vma *vma; 3967 int ret; 3968 3969 if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) 3970 return -EBUSY; 3971 3972 WARN_ON(map_and_fenceable && !i915_is_ggtt(vm)); 3973 3974 vma = i915_gem_obj_to_vma(obj, vm); 3975 3976 if (vma) { 3977 if ((alignment && 3978 vma->node.start & (alignment - 1)) || 3979 (map_and_fenceable && !obj->map_and_fenceable)) { 3980 WARN(obj->pin_count, 3981 "bo is already pinned with incorrect alignment:" 3982 " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d," 3983 " obj->map_and_fenceable=%d\n", 3984 i915_gem_obj_offset(obj, vm), alignment, 3985 map_and_fenceable, 3986 obj->map_and_fenceable); 3987 ret = i915_vma_unbind(vma); 3988 if (ret) 3989 return ret; 3990 } 3991 } 3992 3993 if (!i915_gem_obj_bound(obj, vm)) { 3994 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3995 3996 ret = i915_gem_object_bind_to_vm(obj, vm, alignment, 3997 map_and_fenceable, 3998 nonblocking); 3999 if (ret) 4000 return ret; 4001 4002 if (!dev_priv->mm.aliasing_ppgtt) 4003 i915_gem_gtt_bind_object(obj, obj->cache_level); 4004 } 4005 4006 if (!obj->has_global_gtt_mapping && map_and_fenceable) 4007 i915_gem_gtt_bind_object(obj, obj->cache_level); 4008 4009 obj->pin_count++; 4010 obj->pin_mappable |= map_and_fenceable; 4011 4012 return 0; 4013 } 4014 4015 void 4016 i915_gem_object_unpin(struct drm_i915_gem_object *obj) 4017 { 4018 BUG_ON(obj->pin_count == 0); 4019 BUG_ON(!i915_gem_obj_bound_any(obj)); 4020 4021 if (--obj->pin_count == 0) 4022 obj->pin_mappable = false; 4023 } 4024 4025 int 4026 i915_gem_pin_ioctl(struct drm_device *dev, void *data, 4027 struct drm_file *file) 4028 { 4029 struct drm_i915_gem_pin *args = data; 4030 struct drm_i915_gem_object *obj; 4031 int ret; 4032 4033 ret = i915_mutex_lock_interruptible(dev); 4034 if (ret) 4035 return ret; 4036 4037 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 4038 if (&obj->base == NULL) { 4039 ret = -ENOENT; 4040 goto unlock; 4041 } 4042 4043 if (obj->madv != I915_MADV_WILLNEED) { 4044 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 4045 ret = -EINVAL; 4046 goto out; 4047 } 4048 4049 if (obj->pin_filp != NULL && obj->pin_filp != file) { 4050 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 4051 args->handle); 4052 ret = -EINVAL; 4053 goto out; 4054 } 4055 4056 if (obj->user_pin_count == ULONG_MAX) { 4057 ret = -EBUSY; 4058 goto out; 4059 } 4060 4061 if (obj->user_pin_count == 0) { 4062 ret = i915_gem_obj_ggtt_pin(obj, args->alignment, true, false); 4063 if (ret) 4064 goto out; 4065 } 4066 4067 obj->user_pin_count++; 4068 obj->pin_filp = file; 4069 4070 args->offset = i915_gem_obj_ggtt_offset(obj); 4071 out: 4072 drm_gem_object_unreference(&obj->base); 4073 unlock: 4074 mutex_unlock(&dev->struct_mutex); 4075 return ret; 4076 } 4077 4078 int 4079 i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 4080 struct drm_file *file) 4081 { 4082 struct drm_i915_gem_pin *args = data; 4083 struct drm_i915_gem_object *obj; 4084 int ret; 4085 4086 ret = i915_mutex_lock_interruptible(dev); 4087 if (ret) 4088 return ret; 4089 4090 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 4091 if (&obj->base == NULL) { 4092 ret = -ENOENT; 4093 goto unlock; 4094 } 4095 4096 if (obj->pin_filp != file) { 4097 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 4098 args->handle); 4099 ret = -EINVAL; 4100 goto out; 4101 } 4102 obj->user_pin_count--; 4103 if (obj->user_pin_count == 0) { 4104 obj->pin_filp = NULL; 4105 i915_gem_object_unpin(obj); 4106 } 4107 4108 out: 4109 drm_gem_object_unreference(&obj->base); 4110 unlock: 4111 mutex_unlock(&dev->struct_mutex); 4112 return ret; 4113 } 4114 4115 int 4116 i915_gem_busy_ioctl(struct drm_device *dev, void *data, 4117 struct drm_file *file) 4118 { 4119 struct drm_i915_gem_busy *args = data; 4120 struct drm_i915_gem_object *obj; 4121 int ret; 4122 4123 ret = i915_mutex_lock_interruptible(dev); 4124 if (ret) 4125 return ret; 4126 4127 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 4128 if (&obj->base == NULL) { 4129 ret = -ENOENT; 4130 goto unlock; 4131 } 4132 4133 /* Count all active objects as busy, even if they are currently not used 4134 * by the gpu. Users of this interface expect objects to eventually 4135 * become non-busy without any further actions, therefore emit any 4136 * necessary flushes here. 4137 */ 4138 ret = i915_gem_object_flush_active(obj); 4139 4140 args->busy = obj->active; 4141 if (obj->ring) { 4142 args->busy |= intel_ring_flag(obj->ring) << 16; 4143 } 4144 4145 drm_gem_object_unreference(&obj->base); 4146 unlock: 4147 mutex_unlock(&dev->struct_mutex); 4148 return ret; 4149 } 4150 4151 int 4152 i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 4153 struct drm_file *file_priv) 4154 { 4155 return i915_gem_ring_throttle(dev, file_priv); 4156 } 4157 4158 int 4159 i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 4160 struct drm_file *file_priv) 4161 { 4162 struct drm_i915_gem_madvise *args = data; 4163 struct drm_i915_gem_object *obj; 4164 int ret; 4165 4166 switch (args->madv) { 4167 case I915_MADV_DONTNEED: 4168 case I915_MADV_WILLNEED: 4169 break; 4170 default: 4171 return -EINVAL; 4172 } 4173 4174 ret = i915_mutex_lock_interruptible(dev); 4175 if (ret) 4176 return ret; 4177 4178 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 4179 if (&obj->base == NULL) { 4180 ret = -ENOENT; 4181 goto unlock; 4182 } 4183 4184 if (obj->pin_count) { 4185 ret = -EINVAL; 4186 goto out; 4187 } 4188 4189 if (obj->madv != __I915_MADV_PURGED) 4190 obj->madv = args->madv; 4191 4192 /* if the object is no longer attached, discard its backing storage */ 4193 if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL) 4194 i915_gem_object_truncate(obj); 4195 4196 args->retained = obj->madv != __I915_MADV_PURGED; 4197 4198 out: 4199 drm_gem_object_unreference(&obj->base); 4200 unlock: 4201 mutex_unlock(&dev->struct_mutex); 4202 return ret; 4203 } 4204 4205 void i915_gem_object_init(struct drm_i915_gem_object *obj, 4206 const struct drm_i915_gem_object_ops *ops) 4207 { 4208 INIT_LIST_HEAD(&obj->global_list); 4209 INIT_LIST_HEAD(&obj->ring_list); 4210 INIT_LIST_HEAD(&obj->obj_exec_link); 4211 INIT_LIST_HEAD(&obj->vma_list); 4212 4213 obj->ops = ops; 4214 4215 obj->fence_reg = I915_FENCE_REG_NONE; 4216 obj->madv = I915_MADV_WILLNEED; 4217 /* Avoid an unnecessary call to unbind on the first bind. */ 4218 obj->map_and_fenceable = true; 4219 4220 i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size); 4221 } 4222 4223 static const struct drm_i915_gem_object_ops i915_gem_object_ops = { 4224 .get_pages = i915_gem_object_get_pages_gtt, 4225 .put_pages = i915_gem_object_put_pages_gtt, 4226 }; 4227 4228 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, 4229 size_t size) 4230 { 4231 struct drm_i915_gem_object *obj; 4232 #if 0 4233 struct address_space *mapping; 4234 gfp_t mask; 4235 #endif 4236 4237 obj = i915_gem_object_alloc(dev); 4238 if (obj == NULL) 4239 return NULL; 4240 4241 if (drm_gem_object_init(dev, &obj->base, size) != 0) { 4242 i915_gem_object_free(obj); 4243 return NULL; 4244 } 4245 4246 #if 0 4247 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 4248 if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) { 4249 /* 965gm cannot relocate objects above 4GiB. */ 4250 mask &= ~__GFP_HIGHMEM; 4251 mask |= __GFP_DMA32; 4252 } 4253 4254 mapping = file_inode(obj->base.filp)->i_mapping; 4255 mapping_set_gfp_mask(mapping, mask); 4256 #endif 4257 4258 i915_gem_object_init(obj, &i915_gem_object_ops); 4259 4260 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 4261 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 4262 4263 if (HAS_LLC(dev)) { 4264 /* On some devices, we can have the GPU use the LLC (the CPU 4265 * cache) for about a 10% performance improvement 4266 * compared to uncached. Graphics requests other than 4267 * display scanout are coherent with the CPU in 4268 * accessing this cache. This means in this mode we 4269 * don't need to clflush on the CPU side, and on the 4270 * GPU side we only need to flush internal caches to 4271 * get data visible to the CPU. 4272 * 4273 * However, we maintain the display planes as UC, and so 4274 * need to rebind when first used as such. 4275 */ 4276 obj->cache_level = I915_CACHE_LLC; 4277 } else 4278 obj->cache_level = I915_CACHE_NONE; 4279 4280 trace_i915_gem_object_create(obj); 4281 4282 return obj; 4283 } 4284 4285 void i915_gem_free_object(struct drm_gem_object *gem_obj) 4286 { 4287 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 4288 struct drm_device *dev = obj->base.dev; 4289 drm_i915_private_t *dev_priv = dev->dev_private; 4290 struct i915_vma *vma, *next; 4291 4292 intel_runtime_pm_get(dev_priv); 4293 4294 trace_i915_gem_object_destroy(obj); 4295 4296 if (obj->phys_obj) 4297 i915_gem_detach_phys_object(dev, obj); 4298 4299 obj->pin_count = 0; 4300 /* NB: 0 or 1 elements */ 4301 #if 0 4302 WARN_ON(!list_empty(&obj->vma_list) && 4303 !list_is_singular(&obj->vma_list)); 4304 #endif 4305 list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { 4306 int ret = i915_vma_unbind(vma); 4307 if (WARN_ON(ret == -ERESTARTSYS)) { 4308 bool was_interruptible; 4309 4310 was_interruptible = dev_priv->mm.interruptible; 4311 dev_priv->mm.interruptible = false; 4312 4313 WARN_ON(i915_vma_unbind(vma)); 4314 4315 dev_priv->mm.interruptible = was_interruptible; 4316 } 4317 } 4318 4319 /* Stolen objects don't hold a ref, but do hold pin count. Fix that up 4320 * before progressing. */ 4321 if (obj->stolen) 4322 i915_gem_object_unpin_pages(obj); 4323 4324 if (WARN_ON(obj->pages_pin_count)) 4325 obj->pages_pin_count = 0; 4326 i915_gem_object_put_pages(obj); 4327 i915_gem_object_free_mmap_offset(obj); 4328 4329 BUG_ON(obj->pages); 4330 4331 #if 0 4332 if (obj->base.import_attach) 4333 drm_prime_gem_destroy(&obj->base, NULL); 4334 #endif 4335 4336 drm_gem_object_release(&obj->base); 4337 i915_gem_info_remove_obj(dev_priv, obj->base.size); 4338 4339 kfree(obj->bit_17); 4340 i915_gem_object_free(obj); 4341 4342 intel_runtime_pm_put(dev_priv); 4343 } 4344 4345 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, 4346 struct i915_address_space *vm) 4347 { 4348 struct i915_vma *vma; 4349 list_for_each_entry(vma, &obj->vma_list, vma_link) 4350 if (vma->vm == vm) 4351 return vma; 4352 4353 return NULL; 4354 } 4355 4356 static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj, 4357 struct i915_address_space *vm) 4358 { 4359 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); 4360 if (vma == NULL) 4361 return ERR_PTR(-ENOMEM); 4362 4363 INIT_LIST_HEAD(&vma->vma_link); 4364 INIT_LIST_HEAD(&vma->mm_list); 4365 INIT_LIST_HEAD(&vma->exec_list); 4366 vma->vm = vm; 4367 vma->obj = obj; 4368 4369 /* Keep GGTT vmas first to make debug easier */ 4370 if (i915_is_ggtt(vm)) 4371 list_add(&vma->vma_link, &obj->vma_list); 4372 else 4373 list_add_tail(&vma->vma_link, &obj->vma_list); 4374 4375 return vma; 4376 } 4377 4378 struct i915_vma * 4379 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj, 4380 struct i915_address_space *vm) 4381 { 4382 struct i915_vma *vma; 4383 4384 vma = i915_gem_obj_to_vma(obj, vm); 4385 if (!vma) 4386 vma = __i915_gem_vma_create(obj, vm); 4387 4388 return vma; 4389 } 4390 4391 void i915_gem_vma_destroy(struct i915_vma *vma) 4392 { 4393 WARN_ON(vma->node.allocated); 4394 4395 /* Keep the vma as a placeholder in the execbuffer reservation lists */ 4396 if (!list_empty(&vma->exec_list)) 4397 return; 4398 4399 list_del(&vma->vma_link); 4400 4401 kfree(vma); 4402 } 4403 4404 int 4405 i915_gem_suspend(struct drm_device *dev) 4406 { 4407 drm_i915_private_t *dev_priv = dev->dev_private; 4408 int ret = 0; 4409 4410 mutex_lock(&dev->struct_mutex); 4411 if (dev_priv->ums.mm_suspended) 4412 goto err; 4413 4414 ret = i915_gpu_idle(dev); 4415 if (ret) 4416 goto err; 4417 4418 i915_gem_retire_requests(dev); 4419 4420 /* Under UMS, be paranoid and evict. */ 4421 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 4422 i915_gem_evict_everything(dev); 4423 4424 i915_kernel_lost_context(dev); 4425 i915_gem_cleanup_ringbuffer(dev); 4426 4427 /* Hack! Don't let anybody do execbuf while we don't control the chip. 4428 * We need to replace this with a semaphore, or something. 4429 * And not confound ums.mm_suspended! 4430 */ 4431 dev_priv->ums.mm_suspended = !drm_core_check_feature(dev, 4432 DRIVER_MODESET); 4433 mutex_unlock(&dev->struct_mutex); 4434 4435 del_timer_sync(&dev_priv->gpu_error.hangcheck_timer); 4436 cancel_delayed_work_sync(&dev_priv->mm.retire_work); 4437 cancel_delayed_work_sync(&dev_priv->mm.idle_work); 4438 4439 return 0; 4440 4441 err: 4442 mutex_unlock(&dev->struct_mutex); 4443 return ret; 4444 } 4445 4446 int i915_gem_l3_remap(struct intel_ring_buffer *ring, int slice) 4447 { 4448 struct drm_device *dev = ring->dev; 4449 drm_i915_private_t *dev_priv = dev->dev_private; 4450 u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200); 4451 u32 *remap_info = dev_priv->l3_parity.remap_info[slice]; 4452 int i, ret; 4453 4454 if (!HAS_L3_DPF(dev) || !remap_info) 4455 return 0; 4456 4457 ret = intel_ring_begin(ring, GEN7_L3LOG_SIZE / 4 * 3); 4458 if (ret) 4459 return ret; 4460 4461 /* 4462 * Note: We do not worry about the concurrent register cacheline hang 4463 * here because no other code should access these registers other than 4464 * at initialization time. 4465 */ 4466 for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) { 4467 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 4468 intel_ring_emit(ring, reg_base + i); 4469 intel_ring_emit(ring, remap_info[i/4]); 4470 } 4471 4472 intel_ring_advance(ring); 4473 4474 return ret; 4475 } 4476 4477 void i915_gem_init_swizzling(struct drm_device *dev) 4478 { 4479 drm_i915_private_t *dev_priv = dev->dev_private; 4480 4481 if (INTEL_INFO(dev)->gen < 5 || 4482 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 4483 return; 4484 4485 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 4486 DISP_TILE_SURFACE_SWIZZLING); 4487 4488 if (IS_GEN5(dev)) 4489 return; 4490 4491 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 4492 if (IS_GEN6(dev)) 4493 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); 4494 else if (IS_GEN7(dev)) 4495 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); 4496 else if (IS_GEN8(dev)) 4497 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW)); 4498 else 4499 BUG(); 4500 } 4501 4502 static bool 4503 intel_enable_blt(struct drm_device *dev) 4504 { 4505 int revision; 4506 4507 if (!HAS_BLT(dev)) 4508 return false; 4509 4510 /* The blitter was dysfunctional on early prototypes */ 4511 revision = pci_read_config(dev->dev, PCIR_REVID, 1); 4512 if (IS_GEN6(dev) && revision < 8) { 4513 DRM_INFO("BLT not supported on this pre-production hardware;" 4514 " graphics performance will be degraded.\n"); 4515 return false; 4516 } 4517 4518 return true; 4519 } 4520 4521 static int i915_gem_init_rings(struct drm_device *dev) 4522 { 4523 struct drm_i915_private *dev_priv = dev->dev_private; 4524 int ret; 4525 4526 ret = intel_init_render_ring_buffer(dev); 4527 if (ret) 4528 return ret; 4529 4530 if (HAS_BSD(dev)) { 4531 ret = intel_init_bsd_ring_buffer(dev); 4532 if (ret) 4533 goto cleanup_render_ring; 4534 } 4535 4536 if (intel_enable_blt(dev)) { 4537 ret = intel_init_blt_ring_buffer(dev); 4538 if (ret) 4539 goto cleanup_bsd_ring; 4540 } 4541 4542 if (HAS_VEBOX(dev)) { 4543 ret = intel_init_vebox_ring_buffer(dev); 4544 if (ret) 4545 goto cleanup_blt_ring; 4546 } 4547 4548 4549 ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000)); 4550 if (ret) 4551 goto cleanup_vebox_ring; 4552 4553 return 0; 4554 4555 cleanup_vebox_ring: 4556 intel_cleanup_ring_buffer(&dev_priv->ring[VECS]); 4557 cleanup_blt_ring: 4558 intel_cleanup_ring_buffer(&dev_priv->ring[BCS]); 4559 cleanup_bsd_ring: 4560 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]); 4561 cleanup_render_ring: 4562 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]); 4563 4564 return ret; 4565 } 4566 4567 int 4568 i915_gem_init_hw(struct drm_device *dev) 4569 { 4570 drm_i915_private_t *dev_priv = dev->dev_private; 4571 int ret, i; 4572 4573 #if 0 4574 if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) 4575 return -EIO; 4576 #endif 4577 4578 if (dev_priv->ellc_size) 4579 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf)); 4580 4581 if (IS_HASWELL(dev)) 4582 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ? 4583 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED); 4584 4585 if (HAS_PCH_NOP(dev)) { 4586 u32 temp = I915_READ(GEN7_MSG_CTL); 4587 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK); 4588 I915_WRITE(GEN7_MSG_CTL, temp); 4589 } 4590 4591 i915_gem_init_swizzling(dev); 4592 4593 ret = i915_gem_init_rings(dev); 4594 if (ret) 4595 return ret; 4596 4597 for (i = 0; i < NUM_L3_SLICES(dev); i++) 4598 i915_gem_l3_remap(&dev_priv->ring[RCS], i); 4599 4600 /* 4601 * XXX: There was some w/a described somewhere suggesting loading 4602 * contexts before PPGTT. 4603 */ 4604 ret = i915_gem_context_init(dev); 4605 if (ret) { 4606 i915_gem_cleanup_ringbuffer(dev); 4607 DRM_ERROR("Context initialization failed %d\n", ret); 4608 return ret; 4609 } 4610 4611 if (dev_priv->mm.aliasing_ppgtt) { 4612 ret = dev_priv->mm.aliasing_ppgtt->enable(dev); 4613 if (ret) { 4614 i915_gem_cleanup_aliasing_ppgtt(dev); 4615 DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n"); 4616 } 4617 } 4618 4619 return 0; 4620 } 4621 4622 int i915_gem_init(struct drm_device *dev) 4623 { 4624 struct drm_i915_private *dev_priv = dev->dev_private; 4625 int ret; 4626 4627 mutex_lock(&dev->struct_mutex); 4628 4629 if (IS_VALLEYVIEW(dev)) { 4630 /* VLVA0 (potential hack), BIOS isn't actually waking us */ 4631 I915_WRITE(VLV_GTLC_WAKE_CTRL, 1); 4632 if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10)) 4633 DRM_DEBUG_DRIVER("allow wake ack timed out\n"); 4634 } 4635 4636 i915_gem_init_global_gtt(dev); 4637 4638 ret = i915_gem_init_hw(dev); 4639 mutex_unlock(&dev->struct_mutex); 4640 if (ret) { 4641 i915_gem_cleanup_aliasing_ppgtt(dev); 4642 return ret; 4643 } 4644 4645 /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */ 4646 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 4647 dev_priv->dri1.allow_batchbuffer = 1; 4648 return 0; 4649 } 4650 4651 void 4652 i915_gem_cleanup_ringbuffer(struct drm_device *dev) 4653 { 4654 drm_i915_private_t *dev_priv = dev->dev_private; 4655 struct intel_ring_buffer *ring; 4656 int i; 4657 4658 for_each_ring(ring, dev_priv, i) 4659 intel_cleanup_ring_buffer(ring); 4660 } 4661 4662 int 4663 i915_gem_entervt_ioctl(struct drm_device *dev, void *data, 4664 struct drm_file *file_priv) 4665 { 4666 struct drm_i915_private *dev_priv = dev->dev_private; 4667 int ret; 4668 4669 if (drm_core_check_feature(dev, DRIVER_MODESET)) 4670 return 0; 4671 4672 if (i915_reset_in_progress(&dev_priv->gpu_error)) { 4673 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 4674 atomic_set(&dev_priv->gpu_error.reset_counter, 0); 4675 } 4676 4677 mutex_lock(&dev->struct_mutex); 4678 dev_priv->ums.mm_suspended = 0; 4679 4680 ret = i915_gem_init_hw(dev); 4681 if (ret != 0) { 4682 mutex_unlock(&dev->struct_mutex); 4683 return ret; 4684 } 4685 4686 BUG_ON(!list_empty(&dev_priv->gtt.base.active_list)); 4687 mutex_unlock(&dev->struct_mutex); 4688 4689 ret = drm_irq_install(dev); 4690 if (ret) 4691 goto cleanup_ringbuffer; 4692 4693 return 0; 4694 4695 cleanup_ringbuffer: 4696 mutex_lock(&dev->struct_mutex); 4697 i915_gem_cleanup_ringbuffer(dev); 4698 dev_priv->ums.mm_suspended = 1; 4699 mutex_unlock(&dev->struct_mutex); 4700 4701 return ret; 4702 } 4703 4704 int 4705 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 4706 struct drm_file *file_priv) 4707 { 4708 if (drm_core_check_feature(dev, DRIVER_MODESET)) 4709 return 0; 4710 4711 drm_irq_uninstall(dev); 4712 4713 return i915_gem_suspend(dev); 4714 } 4715 4716 void 4717 i915_gem_lastclose(struct drm_device *dev) 4718 { 4719 int ret; 4720 4721 if (drm_core_check_feature(dev, DRIVER_MODESET)) 4722 return; 4723 4724 ret = i915_gem_suspend(dev); 4725 if (ret) 4726 DRM_ERROR("failed to idle hardware: %d\n", ret); 4727 } 4728 4729 static void 4730 init_ring_lists(struct intel_ring_buffer *ring) 4731 { 4732 INIT_LIST_HEAD(&ring->active_list); 4733 INIT_LIST_HEAD(&ring->request_list); 4734 } 4735 4736 static void i915_init_vm(struct drm_i915_private *dev_priv, 4737 struct i915_address_space *vm) 4738 { 4739 vm->dev = dev_priv->dev; 4740 INIT_LIST_HEAD(&vm->active_list); 4741 INIT_LIST_HEAD(&vm->inactive_list); 4742 INIT_LIST_HEAD(&vm->global_link); 4743 list_add(&vm->global_link, &dev_priv->vm_list); 4744 } 4745 4746 void 4747 i915_gem_load(struct drm_device *dev) 4748 { 4749 drm_i915_private_t *dev_priv = dev->dev_private; 4750 int i; 4751 4752 INIT_LIST_HEAD(&dev_priv->vm_list); 4753 i915_init_vm(dev_priv, &dev_priv->gtt.base); 4754 4755 INIT_LIST_HEAD(&dev_priv->context_list); 4756 INIT_LIST_HEAD(&dev_priv->mm.unbound_list); 4757 INIT_LIST_HEAD(&dev_priv->mm.bound_list); 4758 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 4759 for (i = 0; i < I915_NUM_RINGS; i++) 4760 init_ring_lists(&dev_priv->ring[i]); 4761 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 4762 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 4763 INIT_DELAYED_WORK(&dev_priv->mm.retire_work, 4764 i915_gem_retire_work_handler); 4765 INIT_DELAYED_WORK(&dev_priv->mm.idle_work, 4766 i915_gem_idle_work_handler); 4767 init_waitqueue_head(&dev_priv->gpu_error.reset_queue); 4768 4769 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 4770 if (IS_GEN3(dev)) { 4771 I915_WRITE(MI_ARB_STATE, 4772 _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); 4773 } 4774 4775 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 4776 4777 /* Old X drivers will take 0-2 for front, back, depth buffers */ 4778 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 4779 dev_priv->fence_reg_start = 3; 4780 4781 if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev)) 4782 dev_priv->num_fence_regs = 32; 4783 else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) 4784 dev_priv->num_fence_regs = 16; 4785 else 4786 dev_priv->num_fence_regs = 8; 4787 4788 /* Initialize fence registers to zero */ 4789 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 4790 i915_gem_restore_fences(dev); 4791 4792 i915_gem_detect_bit_6_swizzle(dev); 4793 init_waitqueue_head(&dev_priv->pending_flip_queue); 4794 4795 dev_priv->mm.interruptible = true; 4796 4797 #if 0 4798 dev_priv->mm.inactive_shrinker.scan_objects = i915_gem_inactive_scan; 4799 dev_priv->mm.inactive_shrinker.count_objects = i915_gem_inactive_count; 4800 dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS; 4801 register_shrinker(&dev_priv->mm.inactive_shrinker); 4802 /* Old FreeBSD code */ 4803 dev_priv->mm.inactive_shrinker = EVENTHANDLER_REGISTER(vm_lowmem, 4804 i915_gem_inactive_shrink, dev, EVENTHANDLER_PRI_ANY); 4805 #endif 4806 } 4807 4808 /* 4809 * Create a physically contiguous memory object for this object 4810 * e.g. for cursor + overlay regs 4811 */ 4812 static int i915_gem_init_phys_object(struct drm_device *dev, 4813 int id, int size, int align) 4814 { 4815 drm_i915_private_t *dev_priv = dev->dev_private; 4816 struct drm_i915_gem_phys_object *phys_obj; 4817 int ret; 4818 4819 if (dev_priv->mm.phys_objs[id - 1] || !size) 4820 return 0; 4821 4822 phys_obj = kzalloc(sizeof(*phys_obj), GFP_KERNEL); 4823 if (!phys_obj) 4824 return -ENOMEM; 4825 4826 phys_obj->id = id; 4827 4828 phys_obj->handle = drm_pci_alloc(dev, size, align); 4829 if (!phys_obj->handle) { 4830 ret = -ENOMEM; 4831 goto kfree_obj; 4832 } 4833 #ifdef CONFIG_X86 4834 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE); 4835 #endif 4836 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 4837 size / PAGE_SIZE, PAT_WRITE_COMBINING); 4838 4839 dev_priv->mm.phys_objs[id - 1] = phys_obj; 4840 4841 return 0; 4842 kfree_obj: 4843 kfree(phys_obj); 4844 return ret; 4845 } 4846 4847 static void i915_gem_free_phys_object(struct drm_device *dev, int id) 4848 { 4849 drm_i915_private_t *dev_priv = dev->dev_private; 4850 struct drm_i915_gem_phys_object *phys_obj; 4851 4852 if (!dev_priv->mm.phys_objs[id - 1]) 4853 return; 4854 4855 phys_obj = dev_priv->mm.phys_objs[id - 1]; 4856 if (phys_obj->cur_obj) { 4857 i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 4858 } 4859 4860 #ifdef CONFIG_X86 4861 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE); 4862 #endif 4863 drm_pci_free(dev, phys_obj->handle); 4864 kfree(phys_obj); 4865 dev_priv->mm.phys_objs[id - 1] = NULL; 4866 } 4867 4868 void i915_gem_free_all_phys_object(struct drm_device *dev) 4869 { 4870 int i; 4871 4872 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 4873 i915_gem_free_phys_object(dev, i); 4874 } 4875 4876 void i915_gem_detach_phys_object(struct drm_device *dev, 4877 struct drm_i915_gem_object *obj) 4878 { 4879 struct vm_object *mapping = obj->base.vm_obj; 4880 char *vaddr; 4881 int i; 4882 int page_count; 4883 4884 if (!obj->phys_obj) 4885 return; 4886 vaddr = obj->phys_obj->handle->vaddr; 4887 4888 page_count = obj->base.size / PAGE_SIZE; 4889 for (i = 0; i < page_count; i++) { 4890 struct vm_page *page = shmem_read_mapping_page(mapping, i); 4891 if (!IS_ERR(page)) { 4892 char *dst = kmap_atomic(page); 4893 memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE); 4894 kunmap_atomic(dst); 4895 4896 drm_clflush_pages(&page, 1); 4897 4898 set_page_dirty(page); 4899 mark_page_accessed(page); 4900 #if 0 4901 page_cache_release(page); 4902 #endif 4903 vm_page_busy_wait(page, FALSE, "i915gem"); 4904 vm_page_unwire(page, 0); 4905 vm_page_wakeup(page); 4906 } 4907 } 4908 i915_gem_chipset_flush(dev); 4909 4910 obj->phys_obj->cur_obj = NULL; 4911 obj->phys_obj = NULL; 4912 } 4913 4914 int 4915 i915_gem_attach_phys_object(struct drm_device *dev, 4916 struct drm_i915_gem_object *obj, 4917 int id, 4918 int align) 4919 { 4920 struct vm_object *mapping = obj->base.vm_obj; 4921 drm_i915_private_t *dev_priv = dev->dev_private; 4922 int ret = 0; 4923 int page_count; 4924 int i; 4925 4926 if (id > I915_MAX_PHYS_OBJECT) 4927 return -EINVAL; 4928 4929 if (obj->phys_obj) { 4930 if (obj->phys_obj->id == id) 4931 return 0; 4932 i915_gem_detach_phys_object(dev, obj); 4933 } 4934 4935 /* create a new object */ 4936 if (!dev_priv->mm.phys_objs[id - 1]) { 4937 ret = i915_gem_init_phys_object(dev, id, 4938 obj->base.size, align); 4939 if (ret) { 4940 DRM_ERROR("failed to init phys object %d size: %zu\n", 4941 id, obj->base.size); 4942 return ret; 4943 } 4944 } 4945 4946 /* bind to the object */ 4947 obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 4948 obj->phys_obj->cur_obj = obj; 4949 4950 page_count = obj->base.size / PAGE_SIZE; 4951 4952 for (i = 0; i < page_count; i++) { 4953 struct vm_page *page; 4954 char *dst, *src; 4955 4956 page = shmem_read_mapping_page(mapping, i); 4957 if (IS_ERR(page)) 4958 return PTR_ERR(page); 4959 4960 src = kmap_atomic(page); 4961 dst = (char*)obj->phys_obj->handle->vaddr + (i * PAGE_SIZE); 4962 memcpy(dst, src, PAGE_SIZE); 4963 kunmap_atomic(src); 4964 4965 mark_page_accessed(page); 4966 #if 0 4967 page_cache_release(page); 4968 #endif 4969 vm_page_busy_wait(page, FALSE, "i915gem"); 4970 vm_page_unwire(page, 0); 4971 vm_page_wakeup(page); 4972 } 4973 4974 return 0; 4975 } 4976 4977 static int 4978 i915_gem_phys_pwrite(struct drm_device *dev, 4979 struct drm_i915_gem_object *obj, 4980 struct drm_i915_gem_pwrite *args, 4981 struct drm_file *file_priv) 4982 { 4983 void *vaddr = (char *)obj->phys_obj->handle->vaddr + args->offset; 4984 char __user *user_data = to_user_ptr(args->data_ptr); 4985 4986 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { 4987 unsigned long unwritten; 4988 4989 /* The physical object once assigned is fixed for the lifetime 4990 * of the obj, so we can safely drop the lock and continue 4991 * to access vaddr. 4992 */ 4993 mutex_unlock(&dev->struct_mutex); 4994 unwritten = copy_from_user(vaddr, user_data, args->size); 4995 mutex_lock(&dev->struct_mutex); 4996 if (unwritten) 4997 return -EFAULT; 4998 } 4999 5000 i915_gem_chipset_flush(dev); 5001 return 0; 5002 } 5003 5004 void i915_gem_release(struct drm_device *dev, struct drm_file *file) 5005 { 5006 struct drm_i915_file_private *file_priv = file->driver_priv; 5007 5008 /* Clean up our request list when the client is going away, so that 5009 * later retire_requests won't dereference our soon-to-be-gone 5010 * file_priv. 5011 */ 5012 spin_lock(&file_priv->mm.lock); 5013 while (!list_empty(&file_priv->mm.request_list)) { 5014 struct drm_i915_gem_request *request; 5015 5016 request = list_first_entry(&file_priv->mm.request_list, 5017 struct drm_i915_gem_request, 5018 client_list); 5019 list_del(&request->client_list); 5020 request->file_priv = NULL; 5021 } 5022 spin_unlock(&file_priv->mm.lock); 5023 } 5024 5025 int 5026 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 5027 vm_ooffset_t foff, struct ucred *cred, u_short *color) 5028 { 5029 *color = 0; /* XXXKIB */ 5030 return (0); 5031 } 5032 5033 void 5034 i915_gem_pager_dtor(void *handle) 5035 { 5036 struct drm_gem_object *obj; 5037 struct drm_device *dev; 5038 5039 obj = handle; 5040 dev = obj->dev; 5041 5042 mutex_lock(&dev->struct_mutex); 5043 drm_gem_free_mmap_offset(obj); 5044 i915_gem_release_mmap(to_intel_bo(obj)); 5045 drm_gem_object_unreference(obj); 5046 mutex_unlock(&dev->struct_mutex); 5047 } 5048 5049 static void 5050 i915_gem_file_idle_work_handler(struct work_struct *work) 5051 { 5052 struct drm_i915_file_private *file_priv = 5053 container_of(work, typeof(*file_priv), mm.idle_work.work); 5054 5055 atomic_set(&file_priv->rps_wait_boost, false); 5056 } 5057 5058 int i915_gem_open(struct drm_device *dev, struct drm_file *file) 5059 { 5060 struct drm_i915_file_private *file_priv; 5061 5062 DRM_DEBUG_DRIVER("\n"); 5063 5064 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL); 5065 if (!file_priv) 5066 return -ENOMEM; 5067 5068 file->driver_priv = file_priv; 5069 file_priv->dev_priv = dev->dev_private; 5070 5071 spin_init(&file_priv->mm.lock, "i915_priv"); 5072 INIT_LIST_HEAD(&file_priv->mm.request_list); 5073 INIT_DELAYED_WORK(&file_priv->mm.idle_work, 5074 i915_gem_file_idle_work_handler); 5075 5076 idr_init(&file_priv->context_idr); 5077 5078 return 0; 5079 } 5080 5081 #if 0 5082 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) 5083 { 5084 if (!mutex_is_locked(mutex)) 5085 return false; 5086 5087 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES) 5088 return mutex->owner == task; 5089 #else 5090 /* Since UP may be pre-empted, we cannot assume that we own the lock */ 5091 return false; 5092 #endif 5093 } 5094 #endif 5095 5096 #if 0 5097 static unsigned long 5098 i915_gem_inactive_count(struct shrinker *shrinker, struct shrink_control *sc) 5099 { 5100 struct drm_i915_private *dev_priv = 5101 container_of(shrinker, 5102 struct drm_i915_private, 5103 mm.inactive_shrinker); 5104 struct drm_device *dev = dev_priv->dev; 5105 struct drm_i915_gem_object *obj; 5106 bool unlock = true; 5107 unsigned long count; 5108 5109 if (!mutex_trylock(&dev->struct_mutex)) { 5110 if (!mutex_is_locked_by(&dev->struct_mutex, current)) 5111 return 0; 5112 5113 if (dev_priv->mm.shrinker_no_lock_stealing) 5114 return 0; 5115 5116 unlock = false; 5117 } 5118 5119 count = 0; 5120 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) 5121 if (obj->pages_pin_count == 0) 5122 count += obj->base.size >> PAGE_SHIFT; 5123 5124 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 5125 if (obj->active) 5126 continue; 5127 5128 if (obj->pin_count == 0 && obj->pages_pin_count == 0) 5129 count += obj->base.size >> PAGE_SHIFT; 5130 } 5131 5132 if (unlock) 5133 mutex_unlock(&dev->struct_mutex); 5134 5135 return count; 5136 } 5137 #endif 5138 5139 /* All the new VM stuff */ 5140 unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o, 5141 struct i915_address_space *vm) 5142 { 5143 struct drm_i915_private *dev_priv = o->base.dev->dev_private; 5144 struct i915_vma *vma; 5145 5146 if (vm == &dev_priv->mm.aliasing_ppgtt->base) 5147 vm = &dev_priv->gtt.base; 5148 5149 BUG_ON(list_empty(&o->vma_list)); 5150 list_for_each_entry(vma, &o->vma_list, vma_link) { 5151 if (vma->vm == vm) 5152 return vma->node.start; 5153 5154 } 5155 return -1; 5156 } 5157 5158 bool i915_gem_obj_bound(struct drm_i915_gem_object *o, 5159 struct i915_address_space *vm) 5160 { 5161 struct i915_vma *vma; 5162 5163 list_for_each_entry(vma, &o->vma_list, vma_link) 5164 if (vma->vm == vm && drm_mm_node_allocated(&vma->node)) 5165 return true; 5166 5167 return false; 5168 } 5169 5170 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o) 5171 { 5172 struct i915_vma *vma; 5173 5174 list_for_each_entry(vma, &o->vma_list, vma_link) 5175 if (drm_mm_node_allocated(&vma->node)) 5176 return true; 5177 5178 return false; 5179 } 5180 5181 unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o, 5182 struct i915_address_space *vm) 5183 { 5184 struct drm_i915_private *dev_priv = o->base.dev->dev_private; 5185 struct i915_vma *vma; 5186 5187 if (vm == &dev_priv->mm.aliasing_ppgtt->base) 5188 vm = &dev_priv->gtt.base; 5189 5190 BUG_ON(list_empty(&o->vma_list)); 5191 5192 list_for_each_entry(vma, &o->vma_list, vma_link) 5193 if (vma->vm == vm) 5194 return vma->node.size; 5195 5196 return 0; 5197 } 5198 5199 #if 0 5200 static unsigned long 5201 i915_gem_inactive_scan(struct shrinker *shrinker, struct shrink_control *sc) 5202 { 5203 struct drm_i915_private *dev_priv = 5204 container_of(shrinker, 5205 struct drm_i915_private, 5206 mm.inactive_shrinker); 5207 struct drm_device *dev = dev_priv->dev; 5208 unsigned long freed; 5209 bool unlock = true; 5210 5211 if (!mutex_trylock(&dev->struct_mutex)) { 5212 if (!mutex_is_locked_by(&dev->struct_mutex, current)) 5213 return SHRINK_STOP; 5214 5215 if (dev_priv->mm.shrinker_no_lock_stealing) 5216 return SHRINK_STOP; 5217 5218 unlock = false; 5219 } 5220 5221 freed = i915_gem_purge(dev_priv, sc->nr_to_scan); 5222 if (freed < sc->nr_to_scan) 5223 freed += __i915_gem_shrink(dev_priv, 5224 sc->nr_to_scan - freed, 5225 false); 5226 if (freed < sc->nr_to_scan) 5227 freed += i915_gem_shrink_all(dev_priv); 5228 5229 if (unlock) 5230 mutex_unlock(&dev->struct_mutex); 5231 5232 return freed; 5233 } 5234 #endif 5235 5236 struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj) 5237 { 5238 struct i915_vma *vma; 5239 5240 if (WARN_ON(list_empty(&obj->vma_list))) 5241 return NULL; 5242 5243 vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link); 5244 if (WARN_ON(vma->vm != obj_to_ggtt(obj))) 5245 return NULL; 5246 5247 return vma; 5248 } 5249