1 /*- 2 * Copyright © 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 * Copyright (c) 2011 The FreeBSD Foundation 27 * All rights reserved. 28 * 29 * This software was developed by Konstantin Belousov under sponsorship from 30 * the FreeBSD Foundation. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51 * SUCH DAMAGE. 52 * 53 * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $ 54 */ 55 56 #include <sys/resourcevar.h> 57 #include <sys/sfbuf.h> 58 59 #include <drm/drmP.h> 60 #include <drm/i915_drm.h> 61 #include "i915_drv.h" 62 #include "intel_drv.h" 63 #include "intel_ringbuffer.h" 64 65 static void i915_gem_object_flush_cpu_write_domain( 66 struct drm_i915_gem_object *obj); 67 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, 68 int tiling_mode); 69 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, 70 uint32_t size, int tiling_mode); 71 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 72 unsigned alignment, bool map_and_fenceable); 73 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 74 int flags); 75 static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj); 76 static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, 77 bool write); 78 static void i915_gem_object_set_to_full_cpu_read_domain( 79 struct drm_i915_gem_object *obj); 80 static int i915_gem_object_set_cpu_read_domain_range( 81 struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size); 82 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj); 83 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 84 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj); 85 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj); 86 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj); 87 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex); 88 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 89 uint32_t flush_domains); 90 static void i915_gem_clear_fence_reg(struct drm_device *dev, 91 struct drm_i915_fence_reg *reg); 92 static void i915_gem_reset_fences(struct drm_device *dev); 93 static void i915_gem_retire_task_handler(void *arg, int pending); 94 static int i915_gem_phys_pwrite(struct drm_device *dev, 95 struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset, 96 uint64_t size, struct drm_file *file_priv); 97 static void i915_gem_lowmem(void *arg); 98 99 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 100 long i915_gem_wired_pages_cnt; 101 102 static void 103 i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) 104 { 105 106 dev_priv->mm.object_count++; 107 dev_priv->mm.object_memory += size; 108 } 109 110 static void 111 i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size) 112 { 113 114 dev_priv->mm.object_count--; 115 dev_priv->mm.object_memory -= size; 116 } 117 118 static int 119 i915_gem_wait_for_error(struct drm_device *dev) 120 { 121 struct drm_i915_private *dev_priv; 122 int ret; 123 124 dev_priv = dev->dev_private; 125 if (!atomic_load_acq_int(&dev_priv->mm.wedged)) 126 return (0); 127 128 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 129 while (dev_priv->error_completion == 0) { 130 ret = -lksleep(&dev_priv->error_completion, 131 &dev_priv->error_completion_lock, PCATCH, "915wco", 0); 132 if (ret != 0) { 133 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 134 return (ret); 135 } 136 } 137 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 138 139 if (atomic_load_acq_int(&dev_priv->mm.wedged)) { 140 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 141 dev_priv->error_completion++; 142 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 143 } 144 return (0); 145 } 146 147 int 148 i915_mutex_lock_interruptible(struct drm_device *dev) 149 { 150 struct drm_i915_private *dev_priv; 151 int ret; 152 153 dev_priv = dev->dev_private; 154 ret = i915_gem_wait_for_error(dev); 155 if (ret != 0) 156 return (ret); 157 158 /* 159 * interruptible shall it be. might indeed be if dev_lock is 160 * changed to sx 161 */ 162 ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL); 163 if (ret != 0) 164 return (-ret); 165 166 return (0); 167 } 168 169 170 static void 171 i915_gem_free_object_tail(struct drm_i915_gem_object *obj) 172 { 173 struct drm_device *dev; 174 drm_i915_private_t *dev_priv; 175 int ret; 176 177 dev = obj->base.dev; 178 dev_priv = dev->dev_private; 179 180 ret = i915_gem_object_unbind(obj); 181 if (ret == -ERESTART) { 182 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list); 183 return; 184 } 185 186 drm_gem_free_mmap_offset(&obj->base); 187 drm_gem_object_release(&obj->base); 188 i915_gem_info_remove_obj(dev_priv, obj->base.size); 189 190 drm_free(obj->page_cpu_valid, DRM_I915_GEM); 191 drm_free(obj->bit_17, DRM_I915_GEM); 192 drm_free(obj, DRM_I915_GEM); 193 } 194 195 void 196 i915_gem_free_object(struct drm_gem_object *gem_obj) 197 { 198 struct drm_i915_gem_object *obj; 199 struct drm_device *dev; 200 201 obj = to_intel_bo(gem_obj); 202 dev = obj->base.dev; 203 204 while (obj->pin_count > 0) 205 i915_gem_object_unpin(obj); 206 207 if (obj->phys_obj != NULL) 208 i915_gem_detach_phys_object(dev, obj); 209 210 i915_gem_free_object_tail(obj); 211 } 212 213 static void 214 init_ring_lists(struct intel_ring_buffer *ring) 215 { 216 217 INIT_LIST_HEAD(&ring->active_list); 218 INIT_LIST_HEAD(&ring->request_list); 219 INIT_LIST_HEAD(&ring->gpu_write_list); 220 } 221 222 void 223 i915_gem_load(struct drm_device *dev) 224 { 225 drm_i915_private_t *dev_priv; 226 int i; 227 228 dev_priv = dev->dev_private; 229 230 INIT_LIST_HEAD(&dev_priv->mm.active_list); 231 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 232 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 233 INIT_LIST_HEAD(&dev_priv->mm.pinned_list); 234 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 235 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 236 INIT_LIST_HEAD(&dev_priv->mm.gtt_list); 237 for (i = 0; i < I915_NUM_RINGS; i++) 238 init_ring_lists(&dev_priv->rings[i]); 239 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 240 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 241 TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0, 242 i915_gem_retire_task_handler, dev_priv); 243 dev_priv->error_completion = 0; 244 245 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 246 if (IS_GEN3(dev)) { 247 u32 tmp = I915_READ(MI_ARB_STATE); 248 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) { 249 /* 250 * arb state is a masked write, so set bit + 251 * bit in mask. 252 */ 253 tmp = MI_ARB_C3_LP_WRITE_ENABLE | 254 (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT); 255 I915_WRITE(MI_ARB_STATE, tmp); 256 } 257 } 258 259 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 260 261 /* Old X drivers will take 0-2 for front, back, depth buffers */ 262 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 263 dev_priv->fence_reg_start = 3; 264 265 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || 266 IS_G33(dev)) 267 dev_priv->num_fence_regs = 16; 268 else 269 dev_priv->num_fence_regs = 8; 270 271 /* Initialize fence registers to zero */ 272 for (i = 0; i < dev_priv->num_fence_regs; i++) { 273 i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]); 274 } 275 i915_gem_detect_bit_6_swizzle(dev); 276 dev_priv->mm.interruptible = true; 277 278 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, 279 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); 280 } 281 282 int 283 i915_gem_do_init(struct drm_device *dev, unsigned long start, 284 unsigned long mappable_end, unsigned long end) 285 { 286 drm_i915_private_t *dev_priv; 287 unsigned long mappable; 288 int error; 289 290 dev_priv = dev->dev_private; 291 mappable = min(end, mappable_end) - start; 292 293 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); 294 295 dev_priv->mm.gtt_start = start; 296 dev_priv->mm.gtt_mappable_end = mappable_end; 297 dev_priv->mm.gtt_end = end; 298 dev_priv->mm.gtt_total = end - start; 299 dev_priv->mm.mappable_gtt_total = mappable; 300 301 /* Take over this portion of the GTT */ 302 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); 303 device_printf(dev->dev, 304 "taking over the fictitious range 0x%lx-0x%lx\n", 305 dev->agp->base + start, dev->agp->base + start + mappable); 306 error = -vm_phys_fictitious_reg_range(dev->agp->base + start, 307 dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING); 308 return (error); 309 } 310 311 int 312 i915_gem_init_ioctl(struct drm_device *dev, void *data, 313 struct drm_file *file) 314 { 315 struct drm_i915_gem_init *args; 316 drm_i915_private_t *dev_priv; 317 318 dev_priv = dev->dev_private; 319 args = data; 320 321 if (args->gtt_start >= args->gtt_end || 322 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 323 return (-EINVAL); 324 325 /* 326 * XXXKIB. The second-time initialization should be guarded 327 * against. 328 */ 329 lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE); 330 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end); 331 lockmgr(&dev->dev_lock, LK_RELEASE); 332 333 return 0; 334 } 335 336 int 337 i915_gem_idle(struct drm_device *dev) 338 { 339 drm_i915_private_t *dev_priv; 340 int ret; 341 342 dev_priv = dev->dev_private; 343 if (dev_priv->mm.suspended) 344 return (0); 345 346 ret = i915_gpu_idle(dev, true); 347 if (ret != 0) 348 return (ret); 349 350 /* Under UMS, be paranoid and evict. */ 351 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 352 ret = i915_gem_evict_inactive(dev, false); 353 if (ret != 0) 354 return ret; 355 } 356 357 i915_gem_reset_fences(dev); 358 359 /* Hack! Don't let anybody do execbuf while we don't control the chip. 360 * We need to replace this with a semaphore, or something. 361 * And not confound mm.suspended! 362 */ 363 dev_priv->mm.suspended = 1; 364 callout_stop(&dev_priv->hangcheck_timer); 365 366 i915_kernel_lost_context(dev); 367 i915_gem_cleanup_ringbuffer(dev); 368 369 /* Cancel the retire work handler, which should be idle now. */ 370 taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->mm.retire_task, NULL); 371 return (ret); 372 } 373 374 void 375 i915_gem_init_swizzling(struct drm_device *dev) 376 { 377 drm_i915_private_t *dev_priv; 378 379 dev_priv = dev->dev_private; 380 381 if (INTEL_INFO(dev)->gen < 5 || 382 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 383 return; 384 385 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 386 DISP_TILE_SURFACE_SWIZZLING); 387 388 if (IS_GEN5(dev)) 389 return; 390 391 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 392 if (IS_GEN6(dev)) 393 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_SNB)); 394 else 395 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_IVB)); 396 } 397 398 void 399 i915_gem_init_ppgtt(struct drm_device *dev) 400 { 401 drm_i915_private_t *dev_priv; 402 struct i915_hw_ppgtt *ppgtt; 403 uint32_t pd_offset, pd_entry; 404 vm_paddr_t pt_addr; 405 struct intel_ring_buffer *ring; 406 u_int first_pd_entry_in_global_pt, i; 407 408 dev_priv = dev->dev_private; 409 ppgtt = dev_priv->mm.aliasing_ppgtt; 410 if (ppgtt == NULL) 411 return; 412 413 first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; 414 for (i = 0; i < ppgtt->num_pd_entries; i++) { 415 pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); 416 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); 417 pd_entry |= GEN6_PDE_VALID; 418 intel_gtt_write(first_pd_entry_in_global_pt + i, pd_entry); 419 } 420 intel_gtt_read_pte(first_pd_entry_in_global_pt); 421 422 pd_offset = ppgtt->pd_offset; 423 pd_offset /= 64; /* in cachelines, */ 424 pd_offset <<= 16; 425 426 if (INTEL_INFO(dev)->gen == 6) { 427 uint32_t ecochk = I915_READ(GAM_ECOCHK); 428 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | 429 ECOCHK_PPGTT_CACHE64B); 430 I915_WRITE(GFX_MODE, GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 431 } else if (INTEL_INFO(dev)->gen >= 7) { 432 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); 433 /* GFX_MODE is per-ring on gen7+ */ 434 } 435 436 for (i = 0; i < I915_NUM_RINGS; i++) { 437 ring = &dev_priv->rings[i]; 438 439 if (INTEL_INFO(dev)->gen >= 7) 440 I915_WRITE(RING_MODE_GEN7(ring), 441 GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 442 443 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 444 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); 445 } 446 } 447 448 int 449 i915_gem_init_hw(struct drm_device *dev) 450 { 451 drm_i915_private_t *dev_priv; 452 int ret; 453 454 dev_priv = dev->dev_private; 455 456 i915_gem_init_swizzling(dev); 457 458 ret = intel_init_render_ring_buffer(dev); 459 if (ret != 0) 460 return (ret); 461 462 if (HAS_BSD(dev)) { 463 ret = intel_init_bsd_ring_buffer(dev); 464 if (ret != 0) 465 goto cleanup_render_ring; 466 } 467 468 if (HAS_BLT(dev)) { 469 ret = intel_init_blt_ring_buffer(dev); 470 if (ret != 0) 471 goto cleanup_bsd_ring; 472 } 473 474 dev_priv->next_seqno = 1; 475 i915_gem_init_ppgtt(dev); 476 return (0); 477 478 cleanup_bsd_ring: 479 intel_cleanup_ring_buffer(&dev_priv->rings[VCS]); 480 cleanup_render_ring: 481 intel_cleanup_ring_buffer(&dev_priv->rings[RCS]); 482 return (ret); 483 } 484 485 int 486 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 487 struct drm_file *file) 488 { 489 struct drm_i915_private *dev_priv; 490 struct drm_i915_gem_get_aperture *args; 491 struct drm_i915_gem_object *obj; 492 size_t pinned; 493 494 dev_priv = dev->dev_private; 495 args = data; 496 497 if (!(dev->driver->driver_features & DRIVER_GEM)) 498 return (-ENODEV); 499 500 pinned = 0; 501 DRM_LOCK(dev); 502 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) 503 pinned += obj->gtt_space->size; 504 DRM_UNLOCK(dev); 505 506 args->aper_size = dev_priv->mm.gtt_total; 507 args->aper_available_size = args->aper_size - pinned; 508 509 return (0); 510 } 511 512 int 513 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, 514 bool map_and_fenceable) 515 { 516 struct drm_device *dev; 517 struct drm_i915_private *dev_priv; 518 int ret; 519 520 dev = obj->base.dev; 521 dev_priv = dev->dev_private; 522 523 KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT, 524 ("Max pin count")); 525 526 if (obj->gtt_space != NULL) { 527 if ((alignment && obj->gtt_offset & (alignment - 1)) || 528 (map_and_fenceable && !obj->map_and_fenceable)) { 529 DRM_DEBUG("bo is already pinned with incorrect alignment:" 530 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 531 " obj->map_and_fenceable=%d\n", 532 obj->gtt_offset, alignment, 533 map_and_fenceable, 534 obj->map_and_fenceable); 535 ret = i915_gem_object_unbind(obj); 536 if (ret != 0) 537 return (ret); 538 } 539 } 540 541 if (obj->gtt_space == NULL) { 542 ret = i915_gem_object_bind_to_gtt(obj, alignment, 543 map_and_fenceable); 544 if (ret) 545 return (ret); 546 } 547 548 if (obj->pin_count++ == 0 && !obj->active) 549 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 550 obj->pin_mappable |= map_and_fenceable; 551 552 #if 1 553 KIB_NOTYET(); 554 #else 555 WARN_ON(i915_verify_lists(dev)); 556 #endif 557 return (0); 558 } 559 560 void 561 i915_gem_object_unpin(struct drm_i915_gem_object *obj) 562 { 563 struct drm_device *dev; 564 drm_i915_private_t *dev_priv; 565 566 dev = obj->base.dev; 567 dev_priv = dev->dev_private; 568 569 #if 1 570 KIB_NOTYET(); 571 #else 572 WARN_ON(i915_verify_lists(dev)); 573 #endif 574 575 KASSERT(obj->pin_count != 0, ("zero pin count")); 576 KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); 577 578 if (--obj->pin_count == 0) { 579 if (!obj->active) 580 list_move_tail(&obj->mm_list, 581 &dev_priv->mm.inactive_list); 582 obj->pin_mappable = false; 583 } 584 #if 1 585 KIB_NOTYET(); 586 #else 587 WARN_ON(i915_verify_lists(dev)); 588 #endif 589 } 590 591 int 592 i915_gem_pin_ioctl(struct drm_device *dev, void *data, 593 struct drm_file *file) 594 { 595 struct drm_i915_gem_pin *args; 596 struct drm_i915_gem_object *obj; 597 struct drm_gem_object *gobj; 598 int ret; 599 600 args = data; 601 602 ret = i915_mutex_lock_interruptible(dev); 603 if (ret != 0) 604 return ret; 605 606 gobj = drm_gem_object_lookup(dev, file, args->handle); 607 if (gobj == NULL) { 608 ret = -ENOENT; 609 goto unlock; 610 } 611 obj = to_intel_bo(gobj); 612 613 if (obj->madv != I915_MADV_WILLNEED) { 614 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 615 ret = -EINVAL; 616 goto out; 617 } 618 619 if (obj->pin_filp != NULL && obj->pin_filp != file) { 620 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 621 args->handle); 622 ret = -EINVAL; 623 goto out; 624 } 625 626 obj->user_pin_count++; 627 obj->pin_filp = file; 628 if (obj->user_pin_count == 1) { 629 ret = i915_gem_object_pin(obj, args->alignment, true); 630 if (ret != 0) 631 goto out; 632 } 633 634 /* XXX - flush the CPU caches for pinned objects 635 * as the X server doesn't manage domains yet 636 */ 637 i915_gem_object_flush_cpu_write_domain(obj); 638 args->offset = obj->gtt_offset; 639 out: 640 drm_gem_object_unreference(&obj->base); 641 unlock: 642 DRM_UNLOCK(dev); 643 return (ret); 644 } 645 646 int 647 i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 648 struct drm_file *file) 649 { 650 struct drm_i915_gem_pin *args; 651 struct drm_i915_gem_object *obj; 652 int ret; 653 654 args = data; 655 ret = i915_mutex_lock_interruptible(dev); 656 if (ret != 0) 657 return (ret); 658 659 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 660 if (&obj->base == NULL) { 661 ret = -ENOENT; 662 goto unlock; 663 } 664 665 if (obj->pin_filp != file) { 666 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 667 args->handle); 668 ret = -EINVAL; 669 goto out; 670 } 671 obj->user_pin_count--; 672 if (obj->user_pin_count == 0) { 673 obj->pin_filp = NULL; 674 i915_gem_object_unpin(obj); 675 } 676 677 out: 678 drm_gem_object_unreference(&obj->base); 679 unlock: 680 DRM_UNLOCK(dev); 681 return (ret); 682 } 683 684 int 685 i915_gem_busy_ioctl(struct drm_device *dev, void *data, 686 struct drm_file *file) 687 { 688 struct drm_i915_gem_busy *args; 689 struct drm_i915_gem_object *obj; 690 struct drm_i915_gem_request *request; 691 int ret; 692 693 args = data; 694 695 ret = i915_mutex_lock_interruptible(dev); 696 if (ret != 0) 697 return ret; 698 699 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 700 if (&obj->base == NULL) { 701 ret = -ENOENT; 702 goto unlock; 703 } 704 705 args->busy = obj->active; 706 if (args->busy) { 707 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 708 ret = i915_gem_flush_ring(obj->ring, 709 0, obj->base.write_domain); 710 } else if (obj->ring->outstanding_lazy_request == 711 obj->last_rendering_seqno) { 712 request = kmalloc(sizeof(*request), DRM_I915_GEM, 713 M_WAITOK | M_ZERO); 714 ret = i915_add_request(obj->ring, NULL, request); 715 if (ret != 0) 716 drm_free(request, DRM_I915_GEM); 717 } 718 719 i915_gem_retire_requests_ring(obj->ring); 720 args->busy = obj->active; 721 } 722 723 drm_gem_object_unreference(&obj->base); 724 unlock: 725 DRM_UNLOCK(dev); 726 return (ret); 727 } 728 729 /* Throttle our rendering by waiting until the ring has completed our requests 730 * emitted over 20 msec ago. 731 * 732 * Note that if we were to use the current jiffies each time around the loop, 733 * we wouldn't escape the function with any frames outstanding if the time to 734 * render a frame was over 20ms. 735 * 736 * This should get us reasonable parallelism between CPU and GPU but also 737 * relatively low latency when blocking on a particular request to finish. 738 */ 739 static int 740 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 741 { 742 struct drm_i915_private *dev_priv = dev->dev_private; 743 struct drm_i915_file_private *file_priv = file->driver_priv; 744 unsigned long recent_enough = ticks - (20 * hz / 1000); 745 struct drm_i915_gem_request *request; 746 struct intel_ring_buffer *ring = NULL; 747 u32 seqno = 0; 748 int ret; 749 750 dev_priv = dev->dev_private; 751 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 752 return (-EIO); 753 754 recent_enough = ticks - (20 * hz / 1000); 755 ring = NULL; 756 seqno = 0; 757 758 spin_lock(&file_priv->mm.lock); 759 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 760 if (time_after_eq(request->emitted_jiffies, recent_enough)) 761 break; 762 763 ring = request->ring; 764 seqno = request->seqno; 765 } 766 spin_unlock(&file_priv->mm.lock); 767 768 if (seqno == 0) 769 return 0; 770 771 ret = 0; 772 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 773 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 774 if (ring->irq_get(ring)) { 775 while (ret == 0 && 776 !(i915_seqno_passed(ring->get_seqno(ring), seqno) || 777 atomic_load_acq_int(&dev_priv->mm.wedged))) 778 ret = -lksleep(ring, &ring->irq_lock, PCATCH, 779 "915thr", 0); 780 ring->irq_put(ring); 781 if (ret == 0 && atomic_load_acq_int(&dev_priv->mm.wedged)) 782 ret = -EIO; 783 } else if (_intel_wait_for(dev, 784 i915_seqno_passed(ring->get_seqno(ring), seqno) || 785 atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 0, "915rtr")) { 786 ret = -EBUSY; 787 } 788 } 789 lockmgr(&ring->irq_lock, LK_RELEASE); 790 791 if (ret == 0) 792 taskqueue_enqueue_timeout(dev_priv->tq, 793 &dev_priv->mm.retire_task, 0); 794 795 return ret; 796 } 797 798 int 799 i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 800 struct drm_file *file_priv) 801 { 802 803 return (i915_gem_ring_throttle(dev, file_priv)); 804 } 805 806 int 807 i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 808 struct drm_file *file_priv) 809 { 810 struct drm_i915_gem_madvise *args; 811 struct drm_i915_gem_object *obj; 812 int ret; 813 814 args = data; 815 switch (args->madv) { 816 case I915_MADV_DONTNEED: 817 case I915_MADV_WILLNEED: 818 break; 819 default: 820 return (-EINVAL); 821 } 822 823 ret = i915_mutex_lock_interruptible(dev); 824 if (ret != 0) 825 return (ret); 826 827 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 828 if (&obj->base == NULL) { 829 ret = -ENOENT; 830 goto unlock; 831 } 832 833 if (obj->pin_count != 0) { 834 ret = -EINVAL; 835 goto out; 836 } 837 838 if (obj->madv != I915_MADV_PURGED_INTERNAL) 839 obj->madv = args->madv; 840 if (i915_gem_object_is_purgeable(obj) && obj->gtt_space == NULL) 841 i915_gem_object_truncate(obj); 842 args->retained = obj->madv != I915_MADV_PURGED_INTERNAL; 843 844 out: 845 drm_gem_object_unreference(&obj->base); 846 unlock: 847 DRM_UNLOCK(dev); 848 return (ret); 849 } 850 851 void 852 i915_gem_cleanup_ringbuffer(struct drm_device *dev) 853 { 854 drm_i915_private_t *dev_priv; 855 int i; 856 857 dev_priv = dev->dev_private; 858 for (i = 0; i < I915_NUM_RINGS; i++) 859 intel_cleanup_ring_buffer(&dev_priv->rings[i]); 860 } 861 862 int 863 i915_gem_entervt_ioctl(struct drm_device *dev, void *data, 864 struct drm_file *file_priv) 865 { 866 drm_i915_private_t *dev_priv; 867 int ret, i; 868 869 if (drm_core_check_feature(dev, DRIVER_MODESET)) 870 return (0); 871 dev_priv = dev->dev_private; 872 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 873 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 874 atomic_store_rel_int(&dev_priv->mm.wedged, 0); 875 } 876 877 dev_priv->mm.suspended = 0; 878 879 ret = i915_gem_init_hw(dev); 880 if (ret != 0) { 881 return (ret); 882 } 883 884 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list")); 885 KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list")); 886 KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list")); 887 for (i = 0; i < I915_NUM_RINGS; i++) { 888 KASSERT(list_empty(&dev_priv->rings[i].active_list), 889 ("ring %d active list", i)); 890 KASSERT(list_empty(&dev_priv->rings[i].request_list), 891 ("ring %d request list", i)); 892 } 893 894 DRM_UNLOCK(dev); 895 ret = drm_irq_install(dev); 896 DRM_LOCK(dev); 897 if (ret) 898 goto cleanup_ringbuffer; 899 900 return (0); 901 902 cleanup_ringbuffer: 903 i915_gem_cleanup_ringbuffer(dev); 904 dev_priv->mm.suspended = 1; 905 906 return (ret); 907 } 908 909 int 910 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 911 struct drm_file *file_priv) 912 { 913 914 if (drm_core_check_feature(dev, DRIVER_MODESET)) 915 return 0; 916 917 drm_irq_uninstall(dev); 918 return (i915_gem_idle(dev)); 919 } 920 921 int 922 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, 923 uint32_t *handle_p) 924 { 925 struct drm_i915_gem_object *obj; 926 uint32_t handle; 927 int ret; 928 929 size = roundup(size, PAGE_SIZE); 930 if (size == 0) 931 return (-EINVAL); 932 933 obj = i915_gem_alloc_object(dev, size); 934 if (obj == NULL) 935 return (-ENOMEM); 936 937 handle = 0; 938 ret = drm_gem_handle_create(file, &obj->base, &handle); 939 if (ret != 0) { 940 drm_gem_object_release(&obj->base); 941 i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 942 drm_free(obj, DRM_I915_GEM); 943 return (-ret); 944 } 945 946 /* drop reference from allocate - handle holds it now */ 947 drm_gem_object_unreference(&obj->base); 948 *handle_p = handle; 949 return (0); 950 } 951 952 int 953 i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 954 struct drm_mode_create_dumb *args) 955 { 956 957 /* have to work out size/pitch and return them */ 958 args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 959 args->size = args->pitch * args->height; 960 return (i915_gem_create(file, dev, args->size, &args->handle)); 961 } 962 963 int 964 i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, 965 uint32_t handle) 966 { 967 968 return (drm_gem_handle_delete(file, handle)); 969 } 970 971 int 972 i915_gem_create_ioctl(struct drm_device *dev, void *data, 973 struct drm_file *file) 974 { 975 struct drm_i915_gem_create *args = data; 976 977 return (i915_gem_create(file, dev, args->size, &args->handle)); 978 } 979 980 static inline void vm_page_reference(vm_page_t m) 981 { 982 vm_page_flag_set(m, PG_REFERENCED); 983 } 984 985 static int 986 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj, 987 uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw, 988 struct drm_file *file) 989 { 990 vm_object_t vm_obj; 991 vm_page_t m; 992 struct sf_buf *sf; 993 vm_offset_t mkva; 994 vm_pindex_t obj_pi; 995 int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po; 996 997 if (obj->gtt_offset != 0 && rw == UIO_READ) 998 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 999 else 1000 do_bit17_swizzling = 0; 1001 1002 obj->dirty = 1; 1003 vm_obj = obj->base.vm_obj; 1004 ret = 0; 1005 1006 VM_OBJECT_LOCK(vm_obj); 1007 vm_object_pip_add(vm_obj, 1); 1008 while (size > 0) { 1009 obj_pi = OFF_TO_IDX(offset); 1010 obj_po = offset & PAGE_MASK; 1011 1012 m = i915_gem_wire_page(vm_obj, obj_pi); 1013 VM_OBJECT_UNLOCK(vm_obj); 1014 1015 sf = sf_buf_alloc(m); 1016 mkva = sf_buf_kva(sf); 1017 length = min(size, PAGE_SIZE - obj_po); 1018 while (length > 0) { 1019 if (do_bit17_swizzling && 1020 (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) { 1021 cnt = roundup2(obj_po + 1, 64); 1022 cnt = min(cnt - obj_po, length); 1023 swizzled_po = obj_po ^ 64; 1024 } else { 1025 cnt = length; 1026 swizzled_po = obj_po; 1027 } 1028 if (rw == UIO_READ) 1029 ret = -copyout_nofault( 1030 (char *)mkva + swizzled_po, 1031 (void *)(uintptr_t)data_ptr, cnt); 1032 else 1033 ret = -copyin_nofault( 1034 (void *)(uintptr_t)data_ptr, 1035 (char *)mkva + swizzled_po, cnt); 1036 if (ret != 0) 1037 break; 1038 data_ptr += cnt; 1039 size -= cnt; 1040 length -= cnt; 1041 offset += cnt; 1042 obj_po += cnt; 1043 } 1044 sf_buf_free(sf); 1045 VM_OBJECT_LOCK(vm_obj); 1046 if (rw == UIO_WRITE) 1047 vm_page_dirty(m); 1048 vm_page_reference(m); 1049 vm_page_busy_wait(m, FALSE, "i915gem"); 1050 vm_page_unwire(m, 1); 1051 vm_page_wakeup(m); 1052 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 1053 1054 if (ret != 0) 1055 break; 1056 } 1057 vm_object_pip_wakeup(vm_obj); 1058 VM_OBJECT_UNLOCK(vm_obj); 1059 1060 return (ret); 1061 } 1062 1063 static int 1064 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj, 1065 uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file) 1066 { 1067 vm_offset_t mkva; 1068 int ret; 1069 1070 /* 1071 * Pass the unaligned physical address and size to pmap_mapdev_attr() 1072 * so it can properly calculate whether an extra page needs to be 1073 * mapped or not to cover the requested range. The function will 1074 * add the page offset into the returned mkva for us. 1075 */ 1076 mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset + 1077 offset, size, PAT_WRITE_COMBINING); 1078 ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size); 1079 pmap_unmapdev(mkva, size); 1080 return (ret); 1081 } 1082 1083 static int 1084 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr, 1085 uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file) 1086 { 1087 struct drm_i915_gem_object *obj; 1088 vm_page_t *ma; 1089 vm_offset_t start, end; 1090 int npages, ret; 1091 1092 if (size == 0) 1093 return (0); 1094 start = trunc_page(data_ptr); 1095 end = round_page(data_ptr + size); 1096 npages = howmany(end - start, PAGE_SIZE); 1097 ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK | 1098 M_ZERO); 1099 npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, 1100 (vm_offset_t)data_ptr, size, 1101 (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages); 1102 if (npages == -1) { 1103 ret = -EFAULT; 1104 goto free_ma; 1105 } 1106 1107 ret = i915_mutex_lock_interruptible(dev); 1108 if (ret != 0) 1109 goto unlocked; 1110 1111 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1112 if (&obj->base == NULL) { 1113 ret = -ENOENT; 1114 goto unlock; 1115 } 1116 if (offset > obj->base.size || size > obj->base.size - offset) { 1117 ret = -EINVAL; 1118 goto out; 1119 } 1120 1121 if (rw == UIO_READ) { 1122 ret = i915_gem_object_set_cpu_read_domain_range(obj, 1123 offset, size); 1124 if (ret != 0) 1125 goto out; 1126 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1127 UIO_READ, file); 1128 } else { 1129 if (obj->phys_obj) { 1130 ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset, 1131 size, file); 1132 } else if (obj->gtt_space && 1133 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1134 ret = i915_gem_object_pin(obj, 0, true); 1135 if (ret != 0) 1136 goto out; 1137 ret = i915_gem_object_set_to_gtt_domain(obj, true); 1138 if (ret != 0) 1139 goto out_unpin; 1140 ret = i915_gem_object_put_fence(obj); 1141 if (ret != 0) 1142 goto out_unpin; 1143 ret = i915_gem_gtt_write(dev, obj, data_ptr, size, 1144 offset, file); 1145 out_unpin: 1146 i915_gem_object_unpin(obj); 1147 } else { 1148 ret = i915_gem_object_set_to_cpu_domain(obj, true); 1149 if (ret != 0) 1150 goto out; 1151 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1152 UIO_WRITE, file); 1153 } 1154 } 1155 out: 1156 drm_gem_object_unreference(&obj->base); 1157 unlock: 1158 DRM_UNLOCK(dev); 1159 unlocked: 1160 vm_page_unhold_pages(ma, npages); 1161 free_ma: 1162 drm_free(ma, DRM_I915_GEM); 1163 return (ret); 1164 } 1165 1166 int 1167 i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1168 { 1169 struct drm_i915_gem_pread *args; 1170 1171 args = data; 1172 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1173 args->offset, UIO_READ, file)); 1174 } 1175 1176 int 1177 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1178 { 1179 struct drm_i915_gem_pwrite *args; 1180 1181 args = data; 1182 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1183 args->offset, UIO_WRITE, file)); 1184 } 1185 1186 int 1187 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1188 struct drm_file *file) 1189 { 1190 struct drm_i915_gem_set_domain *args; 1191 struct drm_i915_gem_object *obj; 1192 uint32_t read_domains; 1193 uint32_t write_domain; 1194 int ret; 1195 1196 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1197 return (-ENODEV); 1198 1199 args = data; 1200 read_domains = args->read_domains; 1201 write_domain = args->write_domain; 1202 1203 if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 || 1204 (read_domains & I915_GEM_GPU_DOMAINS) != 0 || 1205 (write_domain != 0 && read_domains != write_domain)) 1206 return (-EINVAL); 1207 1208 ret = i915_mutex_lock_interruptible(dev); 1209 if (ret != 0) 1210 return (ret); 1211 1212 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1213 if (&obj->base == NULL) { 1214 ret = -ENOENT; 1215 goto unlock; 1216 } 1217 1218 if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) { 1219 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1220 if (ret == -EINVAL) 1221 ret = 0; 1222 } else 1223 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1224 1225 drm_gem_object_unreference(&obj->base); 1226 unlock: 1227 DRM_UNLOCK(dev); 1228 return (ret); 1229 } 1230 1231 int 1232 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1233 struct drm_file *file) 1234 { 1235 struct drm_i915_gem_sw_finish *args; 1236 struct drm_i915_gem_object *obj; 1237 int ret; 1238 1239 args = data; 1240 ret = 0; 1241 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1242 return (ENODEV); 1243 ret = i915_mutex_lock_interruptible(dev); 1244 if (ret != 0) 1245 return (ret); 1246 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1247 if (&obj->base == NULL) { 1248 ret = -ENOENT; 1249 goto unlock; 1250 } 1251 if (obj->pin_count != 0) 1252 i915_gem_object_flush_cpu_write_domain(obj); 1253 drm_gem_object_unreference(&obj->base); 1254 unlock: 1255 DRM_UNLOCK(dev); 1256 return (ret); 1257 } 1258 1259 int 1260 i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1261 struct drm_file *file) 1262 { 1263 struct drm_i915_gem_mmap *args; 1264 struct drm_gem_object *obj; 1265 struct proc *p; 1266 vm_map_t map; 1267 vm_offset_t addr; 1268 vm_size_t size; 1269 int error, rv; 1270 1271 args = data; 1272 1273 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1274 return (-ENODEV); 1275 1276 obj = drm_gem_object_lookup(dev, file, args->handle); 1277 if (obj == NULL) 1278 return (-ENOENT); 1279 error = 0; 1280 if (args->size == 0) 1281 goto out; 1282 p = curproc; 1283 map = &p->p_vmspace->vm_map; 1284 size = round_page(args->size); 1285 PROC_LOCK(p); 1286 if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) { 1287 PROC_UNLOCK(p); 1288 error = ENOMEM; 1289 goto out; 1290 } 1291 PROC_UNLOCK(p); 1292 1293 addr = 0; 1294 vm_object_hold(obj->vm_obj); 1295 vm_object_reference_locked(obj->vm_obj); 1296 vm_object_drop(obj->vm_obj); 1297 DRM_UNLOCK(dev); 1298 rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 1299 PAGE_SIZE, /* align */ 1300 TRUE, /* fitit */ 1301 VM_MAPTYPE_NORMAL, /* maptype */ 1302 VM_PROT_READ | VM_PROT_WRITE, /* prot */ 1303 VM_PROT_READ | VM_PROT_WRITE, /* max */ 1304 MAP_SHARED /* cow */); 1305 if (rv != KERN_SUCCESS) { 1306 vm_object_deallocate(obj->vm_obj); 1307 error = -vm_mmap_to_errno(rv); 1308 } else { 1309 args->addr_ptr = (uint64_t)addr; 1310 } 1311 DRM_LOCK(dev); 1312 out: 1313 drm_gem_object_unreference(obj); 1314 return (error); 1315 } 1316 1317 static int 1318 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 1319 vm_ooffset_t foff, struct ucred *cred, u_short *color) 1320 { 1321 1322 *color = 0; /* XXXKIB */ 1323 return (0); 1324 } 1325 1326 int i915_intr_pf; 1327 1328 static int 1329 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 1330 vm_page_t *mres) 1331 { 1332 struct drm_gem_object *gem_obj; 1333 struct drm_i915_gem_object *obj; 1334 struct drm_device *dev; 1335 drm_i915_private_t *dev_priv; 1336 vm_page_t m, oldm; 1337 int cause, ret; 1338 bool write; 1339 1340 gem_obj = vm_obj->handle; 1341 obj = to_intel_bo(gem_obj); 1342 dev = obj->base.dev; 1343 dev_priv = dev->dev_private; 1344 #if 0 1345 write = (prot & VM_PROT_WRITE) != 0; 1346 #else 1347 write = true; 1348 #endif 1349 vm_object_pip_add(vm_obj, 1); 1350 1351 /* 1352 * Remove the placeholder page inserted by vm_fault() from the 1353 * object before dropping the object lock. If 1354 * i915_gem_release_mmap() is active in parallel on this gem 1355 * object, then it owns the drm device sx and might find the 1356 * placeholder already. Then, since the page is busy, 1357 * i915_gem_release_mmap() sleeps waiting for the busy state 1358 * of the page cleared. We will be not able to acquire drm 1359 * device lock until i915_gem_release_mmap() is able to make a 1360 * progress. 1361 */ 1362 if (*mres != NULL) { 1363 oldm = *mres; 1364 vm_page_remove(oldm); 1365 *mres = NULL; 1366 } else 1367 oldm = NULL; 1368 retry: 1369 VM_OBJECT_UNLOCK(vm_obj); 1370 unlocked_vmobj: 1371 cause = ret = 0; 1372 m = NULL; 1373 1374 if (i915_intr_pf) { 1375 ret = i915_mutex_lock_interruptible(dev); 1376 if (ret != 0) { 1377 cause = 10; 1378 goto out; 1379 } 1380 } else 1381 DRM_LOCK(dev); 1382 1383 /* 1384 * Since the object lock was dropped, other thread might have 1385 * faulted on the same GTT address and instantiated the 1386 * mapping for the page. Recheck. 1387 */ 1388 VM_OBJECT_LOCK(vm_obj); 1389 m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 1390 if (m != NULL) { 1391 if ((m->flags & PG_BUSY) != 0) { 1392 DRM_UNLOCK(dev); 1393 #if 0 /* XXX */ 1394 vm_page_sleep(m, "915pee"); 1395 #endif 1396 goto retry; 1397 } 1398 goto have_page; 1399 } else 1400 VM_OBJECT_UNLOCK(vm_obj); 1401 1402 /* Now bind it into the GTT if needed */ 1403 if (!obj->map_and_fenceable) { 1404 ret = i915_gem_object_unbind(obj); 1405 if (ret != 0) { 1406 cause = 20; 1407 goto unlock; 1408 } 1409 } 1410 if (!obj->gtt_space) { 1411 ret = i915_gem_object_bind_to_gtt(obj, 0, true); 1412 if (ret != 0) { 1413 cause = 30; 1414 goto unlock; 1415 } 1416 1417 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1418 if (ret != 0) { 1419 cause = 40; 1420 goto unlock; 1421 } 1422 } 1423 1424 if (obj->tiling_mode == I915_TILING_NONE) 1425 ret = i915_gem_object_put_fence(obj); 1426 else 1427 ret = i915_gem_object_get_fence(obj, NULL); 1428 if (ret != 0) { 1429 cause = 50; 1430 goto unlock; 1431 } 1432 1433 if (i915_gem_object_is_inactive(obj)) 1434 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 1435 1436 obj->fault_mappable = true; 1437 VM_OBJECT_LOCK(vm_obj); 1438 m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset + 1439 offset); 1440 if (m == NULL) { 1441 cause = 60; 1442 ret = -EFAULT; 1443 goto unlock; 1444 } 1445 KASSERT((m->flags & PG_FICTITIOUS) != 0, 1446 ("not fictitious %p", m)); 1447 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 1448 1449 if ((m->flags & PG_BUSY) != 0) { 1450 DRM_UNLOCK(dev); 1451 #if 0 /* XXX */ 1452 vm_page_sleep(m, "915pbs"); 1453 #endif 1454 goto retry; 1455 } 1456 m->valid = VM_PAGE_BITS_ALL; 1457 vm_page_insert(m, vm_obj, OFF_TO_IDX(offset)); 1458 have_page: 1459 *mres = m; 1460 vm_page_busy_try(m, false); 1461 1462 DRM_UNLOCK(dev); 1463 if (oldm != NULL) { 1464 vm_page_free(oldm); 1465 } 1466 vm_object_pip_wakeup(vm_obj); 1467 return (VM_PAGER_OK); 1468 1469 unlock: 1470 DRM_UNLOCK(dev); 1471 out: 1472 KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 1473 if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 1474 goto unlocked_vmobj; 1475 } 1476 VM_OBJECT_LOCK(vm_obj); 1477 vm_object_pip_wakeup(vm_obj); 1478 return (VM_PAGER_ERROR); 1479 } 1480 1481 static void 1482 i915_gem_pager_dtor(void *handle) 1483 { 1484 struct drm_gem_object *obj; 1485 struct drm_device *dev; 1486 1487 obj = handle; 1488 dev = obj->dev; 1489 1490 DRM_LOCK(dev); 1491 drm_gem_free_mmap_offset(obj); 1492 i915_gem_release_mmap(to_intel_bo(obj)); 1493 drm_gem_object_unreference(obj); 1494 DRM_UNLOCK(dev); 1495 } 1496 1497 struct cdev_pager_ops i915_gem_pager_ops = { 1498 .cdev_pg_fault = i915_gem_pager_fault, 1499 .cdev_pg_ctor = i915_gem_pager_ctor, 1500 .cdev_pg_dtor = i915_gem_pager_dtor 1501 }; 1502 1503 int 1504 i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev, 1505 uint32_t handle, uint64_t *offset) 1506 { 1507 struct drm_i915_private *dev_priv; 1508 struct drm_i915_gem_object *obj; 1509 int ret; 1510 1511 if (!(dev->driver->driver_features & DRIVER_GEM)) 1512 return (-ENODEV); 1513 1514 dev_priv = dev->dev_private; 1515 1516 ret = i915_mutex_lock_interruptible(dev); 1517 if (ret != 0) 1518 return (ret); 1519 1520 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1521 if (&obj->base == NULL) { 1522 ret = -ENOENT; 1523 goto unlock; 1524 } 1525 1526 if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 1527 ret = -E2BIG; 1528 goto out; 1529 } 1530 1531 if (obj->madv != I915_MADV_WILLNEED) { 1532 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1533 ret = -EINVAL; 1534 goto out; 1535 } 1536 1537 ret = drm_gem_create_mmap_offset(&obj->base); 1538 if (ret != 0) 1539 goto out; 1540 1541 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 1542 DRM_GEM_MAPPING_KEY; 1543 out: 1544 drm_gem_object_unreference(&obj->base); 1545 unlock: 1546 DRM_UNLOCK(dev); 1547 return (ret); 1548 } 1549 1550 int 1551 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1552 struct drm_file *file) 1553 { 1554 struct drm_i915_private *dev_priv; 1555 struct drm_i915_gem_mmap_gtt *args; 1556 1557 dev_priv = dev->dev_private; 1558 args = data; 1559 1560 return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset)); 1561 } 1562 1563 struct drm_i915_gem_object * 1564 i915_gem_alloc_object(struct drm_device *dev, size_t size) 1565 { 1566 struct drm_i915_private *dev_priv; 1567 struct drm_i915_gem_object *obj; 1568 1569 dev_priv = dev->dev_private; 1570 1571 obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 1572 1573 if (drm_gem_object_init(dev, &obj->base, size) != 0) { 1574 drm_free(obj, DRM_I915_GEM); 1575 return (NULL); 1576 } 1577 1578 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1579 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1580 1581 if (HAS_LLC(dev)) 1582 obj->cache_level = I915_CACHE_LLC; 1583 else 1584 obj->cache_level = I915_CACHE_NONE; 1585 obj->base.driver_private = NULL; 1586 obj->fence_reg = I915_FENCE_REG_NONE; 1587 INIT_LIST_HEAD(&obj->mm_list); 1588 INIT_LIST_HEAD(&obj->gtt_list); 1589 INIT_LIST_HEAD(&obj->ring_list); 1590 INIT_LIST_HEAD(&obj->exec_list); 1591 INIT_LIST_HEAD(&obj->gpu_write_list); 1592 obj->madv = I915_MADV_WILLNEED; 1593 /* Avoid an unnecessary call to unbind on the first bind. */ 1594 obj->map_and_fenceable = true; 1595 1596 i915_gem_info_add_obj(dev_priv, size); 1597 1598 return (obj); 1599 } 1600 1601 void 1602 i915_gem_clflush_object(struct drm_i915_gem_object *obj) 1603 { 1604 1605 /* If we don't have a page list set up, then we're not pinned 1606 * to GPU, and we can ignore the cache flush because it'll happen 1607 * again at bind time. 1608 */ 1609 if (obj->pages == NULL) 1610 return; 1611 1612 /* If the GPU is snooping the contents of the CPU cache, 1613 * we do not need to manually clear the CPU cache lines. However, 1614 * the caches are only snooped when the render cache is 1615 * flushed/invalidated. As we always have to emit invalidations 1616 * and flushes when moving into and out of the RENDER domain, correct 1617 * snooping behaviour occurs naturally as the result of our domain 1618 * tracking. 1619 */ 1620 if (obj->cache_level != I915_CACHE_NONE) 1621 return; 1622 1623 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 1624 } 1625 1626 static void 1627 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 1628 { 1629 uint32_t old_write_domain; 1630 1631 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 1632 return; 1633 1634 i915_gem_clflush_object(obj); 1635 intel_gtt_chipset_flush(); 1636 old_write_domain = obj->base.write_domain; 1637 obj->base.write_domain = 0; 1638 } 1639 1640 static int 1641 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) 1642 { 1643 1644 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) 1645 return (0); 1646 return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain)); 1647 } 1648 1649 static void 1650 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 1651 { 1652 uint32_t old_write_domain; 1653 1654 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 1655 return; 1656 1657 cpu_sfence(); 1658 1659 old_write_domain = obj->base.write_domain; 1660 obj->base.write_domain = 0; 1661 } 1662 1663 int 1664 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 1665 { 1666 uint32_t old_write_domain, old_read_domains; 1667 int ret; 1668 1669 if (obj->gtt_space == NULL) 1670 return (-EINVAL); 1671 1672 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 1673 return 0; 1674 1675 ret = i915_gem_object_flush_gpu_write_domain(obj); 1676 if (ret != 0) 1677 return (ret); 1678 1679 if (obj->pending_gpu_write || write) { 1680 ret = i915_gem_object_wait_rendering(obj); 1681 if (ret != 0) 1682 return (ret); 1683 } 1684 1685 i915_gem_object_flush_cpu_write_domain(obj); 1686 1687 old_write_domain = obj->base.write_domain; 1688 old_read_domains = obj->base.read_domains; 1689 1690 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1691 ("In GTT write domain")); 1692 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1693 if (write) { 1694 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 1695 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 1696 obj->dirty = 1; 1697 } 1698 1699 return (0); 1700 } 1701 1702 int 1703 i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 1704 enum i915_cache_level cache_level) 1705 { 1706 struct drm_device *dev; 1707 drm_i915_private_t *dev_priv; 1708 int ret; 1709 1710 if (obj->cache_level == cache_level) 1711 return 0; 1712 1713 if (obj->pin_count) { 1714 DRM_DEBUG("can not change the cache level of pinned objects\n"); 1715 return (-EBUSY); 1716 } 1717 1718 dev = obj->base.dev; 1719 dev_priv = dev->dev_private; 1720 if (obj->gtt_space) { 1721 ret = i915_gem_object_finish_gpu(obj); 1722 if (ret != 0) 1723 return (ret); 1724 1725 i915_gem_object_finish_gtt(obj); 1726 1727 /* Before SandyBridge, you could not use tiling or fence 1728 * registers with snooped memory, so relinquish any fences 1729 * currently pointing to our region in the aperture. 1730 */ 1731 if (INTEL_INFO(obj->base.dev)->gen < 6) { 1732 ret = i915_gem_object_put_fence(obj); 1733 if (ret != 0) 1734 return (ret); 1735 } 1736 1737 i915_gem_gtt_rebind_object(obj, cache_level); 1738 if (obj->has_aliasing_ppgtt_mapping) 1739 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 1740 obj, cache_level); 1741 } 1742 1743 if (cache_level == I915_CACHE_NONE) { 1744 u32 old_read_domains, old_write_domain; 1745 1746 /* If we're coming from LLC cached, then we haven't 1747 * actually been tracking whether the data is in the 1748 * CPU cache or not, since we only allow one bit set 1749 * in obj->write_domain and have been skipping the clflushes. 1750 * Just set it to the CPU cache for now. 1751 */ 1752 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1753 ("obj %p in CPU write domain", obj)); 1754 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0, 1755 ("obj %p in CPU read domain", obj)); 1756 1757 old_read_domains = obj->base.read_domains; 1758 old_write_domain = obj->base.write_domain; 1759 1760 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1761 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1762 1763 } 1764 1765 obj->cache_level = cache_level; 1766 return (0); 1767 } 1768 1769 int 1770 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 1771 u32 alignment, struct intel_ring_buffer *pipelined) 1772 { 1773 u32 old_read_domains, old_write_domain; 1774 int ret; 1775 1776 ret = i915_gem_object_flush_gpu_write_domain(obj); 1777 if (ret != 0) 1778 return (ret); 1779 1780 if (pipelined != obj->ring) { 1781 ret = i915_gem_object_wait_rendering(obj); 1782 if (ret == -ERESTART || ret == -EINTR) 1783 return (ret); 1784 } 1785 1786 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 1787 if (ret != 0) 1788 return (ret); 1789 1790 ret = i915_gem_object_pin(obj, alignment, true); 1791 if (ret != 0) 1792 return (ret); 1793 1794 i915_gem_object_flush_cpu_write_domain(obj); 1795 1796 old_write_domain = obj->base.write_domain; 1797 old_read_domains = obj->base.read_domains; 1798 1799 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1800 ("obj %p in GTT write domain", obj)); 1801 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1802 1803 return (0); 1804 } 1805 1806 int 1807 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 1808 { 1809 int ret; 1810 1811 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 1812 return (0); 1813 1814 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 1815 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); 1816 if (ret != 0) 1817 return (ret); 1818 } 1819 1820 ret = i915_gem_object_wait_rendering(obj); 1821 if (ret != 0) 1822 return (ret); 1823 1824 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 1825 1826 return (0); 1827 } 1828 1829 static int 1830 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 1831 { 1832 uint32_t old_write_domain, old_read_domains; 1833 int ret; 1834 1835 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 1836 return 0; 1837 1838 ret = i915_gem_object_flush_gpu_write_domain(obj); 1839 if (ret != 0) 1840 return (ret); 1841 1842 ret = i915_gem_object_wait_rendering(obj); 1843 if (ret != 0) 1844 return (ret); 1845 1846 i915_gem_object_flush_gtt_write_domain(obj); 1847 i915_gem_object_set_to_full_cpu_read_domain(obj); 1848 1849 old_write_domain = obj->base.write_domain; 1850 old_read_domains = obj->base.read_domains; 1851 1852 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 1853 i915_gem_clflush_object(obj); 1854 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1855 } 1856 1857 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1858 ("In cpu write domain")); 1859 1860 if (write) { 1861 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1862 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1863 } 1864 1865 return (0); 1866 } 1867 1868 static void 1869 i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj) 1870 { 1871 int i; 1872 1873 if (obj->page_cpu_valid == NULL) 1874 return; 1875 1876 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) { 1877 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) { 1878 if (obj->page_cpu_valid[i] != 0) 1879 continue; 1880 drm_clflush_pages(obj->pages + i, 1); 1881 } 1882 } 1883 1884 drm_free(obj->page_cpu_valid, DRM_I915_GEM); 1885 obj->page_cpu_valid = NULL; 1886 } 1887 1888 static int 1889 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, 1890 uint64_t offset, uint64_t size) 1891 { 1892 uint32_t old_read_domains; 1893 int i, ret; 1894 1895 if (offset == 0 && size == obj->base.size) 1896 return (i915_gem_object_set_to_cpu_domain(obj, 0)); 1897 1898 ret = i915_gem_object_flush_gpu_write_domain(obj); 1899 if (ret != 0) 1900 return (ret); 1901 ret = i915_gem_object_wait_rendering(obj); 1902 if (ret != 0) 1903 return (ret); 1904 1905 i915_gem_object_flush_gtt_write_domain(obj); 1906 1907 if (obj->page_cpu_valid == NULL && 1908 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) 1909 return (0); 1910 1911 if (obj->page_cpu_valid == NULL) { 1912 obj->page_cpu_valid = kmalloc(obj->base.size / PAGE_SIZE, 1913 DRM_I915_GEM, M_WAITOK | M_ZERO); 1914 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 1915 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE); 1916 1917 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 1918 i++) { 1919 if (obj->page_cpu_valid[i]) 1920 continue; 1921 drm_clflush_pages(obj->pages + i, 1); 1922 obj->page_cpu_valid[i] = 1; 1923 } 1924 1925 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1926 ("In gpu write domain")); 1927 1928 old_read_domains = obj->base.read_domains; 1929 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1930 1931 return (0); 1932 } 1933 1934 static uint32_t 1935 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 1936 { 1937 uint32_t gtt_size; 1938 1939 if (INTEL_INFO(dev)->gen >= 4 || 1940 tiling_mode == I915_TILING_NONE) 1941 return (size); 1942 1943 /* Previous chips need a power-of-two fence region when tiling */ 1944 if (INTEL_INFO(dev)->gen == 3) 1945 gtt_size = 1024*1024; 1946 else 1947 gtt_size = 512*1024; 1948 1949 while (gtt_size < size) 1950 gtt_size <<= 1; 1951 1952 return (gtt_size); 1953 } 1954 1955 /** 1956 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 1957 * @obj: object to check 1958 * 1959 * Return the required GTT alignment for an object, taking into account 1960 * potential fence register mapping. 1961 */ 1962 static uint32_t 1963 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, 1964 int tiling_mode) 1965 { 1966 1967 /* 1968 * Minimum alignment is 4k (GTT page size), but might be greater 1969 * if a fence register is needed for the object. 1970 */ 1971 if (INTEL_INFO(dev)->gen >= 4 || 1972 tiling_mode == I915_TILING_NONE) 1973 return (4096); 1974 1975 /* 1976 * Previous chips need to be aligned to the size of the smallest 1977 * fence register that can contain the object. 1978 */ 1979 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 1980 } 1981 1982 uint32_t 1983 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, 1984 int tiling_mode) 1985 { 1986 1987 if (tiling_mode == I915_TILING_NONE) 1988 return (4096); 1989 1990 /* 1991 * Minimum alignment is 4k (GTT page size) for sane hw. 1992 */ 1993 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev)) 1994 return (4096); 1995 1996 /* 1997 * Previous hardware however needs to be aligned to a power-of-two 1998 * tile height. The simplest method for determining this is to reuse 1999 * the power-of-tile object size. 2000 */ 2001 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 2002 } 2003 2004 static int 2005 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 2006 unsigned alignment, bool map_and_fenceable) 2007 { 2008 struct drm_device *dev; 2009 struct drm_i915_private *dev_priv; 2010 struct drm_mm_node *free_space; 2011 uint32_t size, fence_size, fence_alignment, unfenced_alignment; 2012 bool mappable, fenceable; 2013 int ret; 2014 2015 dev = obj->base.dev; 2016 dev_priv = dev->dev_private; 2017 2018 if (obj->madv != I915_MADV_WILLNEED) { 2019 DRM_ERROR("Attempting to bind a purgeable object\n"); 2020 return (-EINVAL); 2021 } 2022 2023 fence_size = i915_gem_get_gtt_size(dev, obj->base.size, 2024 obj->tiling_mode); 2025 fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size, 2026 obj->tiling_mode); 2027 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev, 2028 obj->base.size, obj->tiling_mode); 2029 if (alignment == 0) 2030 alignment = map_and_fenceable ? fence_alignment : 2031 unfenced_alignment; 2032 if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) { 2033 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2034 return (-EINVAL); 2035 } 2036 2037 size = map_and_fenceable ? fence_size : obj->base.size; 2038 2039 /* If the object is bigger than the entire aperture, reject it early 2040 * before evicting everything in a vain attempt to find space. 2041 */ 2042 if (obj->base.size > (map_and_fenceable ? 2043 dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 2044 DRM_ERROR( 2045 "Attempting to bind an object larger than the aperture\n"); 2046 return (-E2BIG); 2047 } 2048 2049 search_free: 2050 if (map_and_fenceable) 2051 free_space = drm_mm_search_free_in_range( 2052 &dev_priv->mm.gtt_space, size, alignment, 0, 2053 dev_priv->mm.gtt_mappable_end, 0); 2054 else 2055 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2056 size, alignment, 0); 2057 if (free_space != NULL) { 2058 int color = 0; 2059 if (map_and_fenceable) 2060 obj->gtt_space = drm_mm_get_block_range_generic( 2061 free_space, size, alignment, color, 0, 2062 dev_priv->mm.gtt_mappable_end, 1); 2063 else 2064 obj->gtt_space = drm_mm_get_block_generic(free_space, 2065 size, alignment, color, 1); 2066 } 2067 if (obj->gtt_space == NULL) { 2068 ret = i915_gem_evict_something(dev, size, alignment, 2069 map_and_fenceable); 2070 if (ret != 0) 2071 return (ret); 2072 goto search_free; 2073 } 2074 ret = i915_gem_object_get_pages_gtt(obj, 0); 2075 if (ret != 0) { 2076 drm_mm_put_block(obj->gtt_space); 2077 obj->gtt_space = NULL; 2078 /* 2079 * i915_gem_object_get_pages_gtt() cannot return 2080 * ENOMEM, since we use vm_page_grab(VM_ALLOC_RETRY) 2081 * (which does not support operation without a flag 2082 * anyway). 2083 */ 2084 return (ret); 2085 } 2086 2087 ret = i915_gem_gtt_bind_object(obj); 2088 if (ret != 0) { 2089 i915_gem_object_put_pages_gtt(obj); 2090 drm_mm_put_block(obj->gtt_space); 2091 obj->gtt_space = NULL; 2092 if (i915_gem_evict_everything(dev, false)) 2093 return (ret); 2094 goto search_free; 2095 } 2096 2097 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); 2098 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2099 2100 KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0, 2101 ("Object in gpu read domain")); 2102 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2103 ("Object in gpu write domain")); 2104 2105 obj->gtt_offset = obj->gtt_space->start; 2106 2107 fenceable = 2108 obj->gtt_space->size == fence_size && 2109 (obj->gtt_space->start & (fence_alignment - 1)) == 0; 2110 2111 mappable = 2112 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 2113 obj->map_and_fenceable = mappable && fenceable; 2114 2115 return (0); 2116 } 2117 2118 static void 2119 i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2120 { 2121 u32 old_write_domain, old_read_domains; 2122 2123 /* Act a barrier for all accesses through the GTT */ 2124 cpu_mfence(); 2125 2126 /* Force a pagefault for domain tracking on next user access */ 2127 i915_gem_release_mmap(obj); 2128 2129 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2130 return; 2131 2132 old_read_domains = obj->base.read_domains; 2133 old_write_domain = obj->base.write_domain; 2134 2135 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2136 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2137 2138 } 2139 2140 int 2141 i915_gem_object_unbind(struct drm_i915_gem_object *obj) 2142 { 2143 drm_i915_private_t *dev_priv; 2144 int ret; 2145 2146 dev_priv = obj->base.dev->dev_private; 2147 ret = 0; 2148 if (obj->gtt_space == NULL) 2149 return (0); 2150 if (obj->pin_count != 0) { 2151 DRM_ERROR("Attempting to unbind pinned buffer\n"); 2152 return (-EINVAL); 2153 } 2154 2155 ret = i915_gem_object_finish_gpu(obj); 2156 if (ret == -ERESTART || ret == -EINTR) 2157 return (ret); 2158 2159 i915_gem_object_finish_gtt(obj); 2160 2161 if (ret == 0) 2162 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 2163 if (ret == -ERESTART || ret == -EINTR) 2164 return (ret); 2165 if (ret != 0) { 2166 i915_gem_clflush_object(obj); 2167 obj->base.read_domains = obj->base.write_domain = 2168 I915_GEM_DOMAIN_CPU; 2169 } 2170 2171 ret = i915_gem_object_put_fence(obj); 2172 if (ret == -ERESTART) 2173 return (ret); 2174 2175 i915_gem_gtt_unbind_object(obj); 2176 if (obj->has_aliasing_ppgtt_mapping) { 2177 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2178 obj->has_aliasing_ppgtt_mapping = 0; 2179 } 2180 i915_gem_object_put_pages_gtt(obj); 2181 2182 list_del_init(&obj->gtt_list); 2183 list_del_init(&obj->mm_list); 2184 obj->map_and_fenceable = true; 2185 2186 drm_mm_put_block(obj->gtt_space); 2187 obj->gtt_space = NULL; 2188 obj->gtt_offset = 0; 2189 2190 if (i915_gem_object_is_purgeable(obj)) 2191 i915_gem_object_truncate(obj); 2192 2193 return (ret); 2194 } 2195 2196 static int 2197 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 2198 int flags) 2199 { 2200 struct drm_device *dev; 2201 vm_object_t vm_obj; 2202 vm_page_t m; 2203 int page_count, i, j; 2204 2205 dev = obj->base.dev; 2206 KASSERT(obj->pages == NULL, ("Obj already has pages")); 2207 page_count = obj->base.size / PAGE_SIZE; 2208 obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 2209 M_WAITOK); 2210 vm_obj = obj->base.vm_obj; 2211 VM_OBJECT_LOCK(vm_obj); 2212 for (i = 0; i < page_count; i++) { 2213 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL) 2214 goto failed; 2215 } 2216 VM_OBJECT_UNLOCK(vm_obj); 2217 if (i915_gem_object_needs_bit17_swizzle(obj)) 2218 i915_gem_object_do_bit_17_swizzle(obj); 2219 return (0); 2220 2221 failed: 2222 for (j = 0; j < i; j++) { 2223 m = obj->pages[j]; 2224 vm_page_busy_wait(m, FALSE, "i915gem"); 2225 vm_page_unwire(m, 0); 2226 vm_page_wakeup(m); 2227 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2228 } 2229 VM_OBJECT_UNLOCK(vm_obj); 2230 drm_free(obj->pages, DRM_I915_GEM); 2231 obj->pages = NULL; 2232 return (-EIO); 2233 } 2234 2235 #define GEM_PARANOID_CHECK_GTT 0 2236 #if GEM_PARANOID_CHECK_GTT 2237 static void 2238 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 2239 int page_count) 2240 { 2241 struct drm_i915_private *dev_priv; 2242 vm_paddr_t pa; 2243 unsigned long start, end; 2244 u_int i; 2245 int j; 2246 2247 dev_priv = dev->dev_private; 2248 start = OFF_TO_IDX(dev_priv->mm.gtt_start); 2249 end = OFF_TO_IDX(dev_priv->mm.gtt_end); 2250 for (i = start; i < end; i++) { 2251 pa = intel_gtt_read_pte_paddr(i); 2252 for (j = 0; j < page_count; j++) { 2253 if (pa == VM_PAGE_TO_PHYS(ma[j])) { 2254 panic("Page %p in GTT pte index %d pte %x", 2255 ma[i], i, intel_gtt_read_pte(i)); 2256 } 2257 } 2258 } 2259 } 2260 #endif 2261 2262 static void 2263 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 2264 { 2265 vm_page_t m; 2266 int page_count, i; 2267 2268 KASSERT(obj->madv != I915_MADV_PURGED_INTERNAL, ("Purged object")); 2269 2270 if (obj->tiling_mode != I915_TILING_NONE) 2271 i915_gem_object_save_bit_17_swizzle(obj); 2272 if (obj->madv == I915_MADV_DONTNEED) 2273 obj->dirty = 0; 2274 page_count = obj->base.size / PAGE_SIZE; 2275 VM_OBJECT_LOCK(obj->base.vm_obj); 2276 #if GEM_PARANOID_CHECK_GTT 2277 i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 2278 #endif 2279 for (i = 0; i < page_count; i++) { 2280 m = obj->pages[i]; 2281 if (obj->dirty) 2282 vm_page_dirty(m); 2283 if (obj->madv == I915_MADV_WILLNEED) 2284 vm_page_reference(m); 2285 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem"); 2286 vm_page_unwire(obj->pages[i], 1); 2287 vm_page_wakeup(obj->pages[i]); 2288 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2289 } 2290 VM_OBJECT_UNLOCK(obj->base.vm_obj); 2291 obj->dirty = 0; 2292 drm_free(obj->pages, DRM_I915_GEM); 2293 obj->pages = NULL; 2294 } 2295 2296 void 2297 i915_gem_release_mmap(struct drm_i915_gem_object *obj) 2298 { 2299 vm_object_t devobj; 2300 vm_page_t m; 2301 int i, page_count; 2302 2303 if (!obj->fault_mappable) 2304 return; 2305 2306 devobj = cdev_pager_lookup(obj); 2307 if (devobj != NULL) { 2308 page_count = OFF_TO_IDX(obj->base.size); 2309 2310 VM_OBJECT_LOCK(devobj); 2311 for (i = 0; i < page_count; i++) { 2312 m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm"); 2313 if (m == NULL) 2314 continue; 2315 cdev_pager_free_page(devobj, m); 2316 } 2317 VM_OBJECT_UNLOCK(devobj); 2318 vm_object_deallocate(devobj); 2319 } 2320 2321 obj->fault_mappable = false; 2322 } 2323 2324 int 2325 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) 2326 { 2327 int ret; 2328 2329 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2330 ("In GPU write domain")); 2331 2332 if (obj->active) { 2333 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno, 2334 true); 2335 if (ret != 0) 2336 return (ret); 2337 } 2338 return (0); 2339 } 2340 2341 void 2342 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 2343 struct intel_ring_buffer *ring, uint32_t seqno) 2344 { 2345 struct drm_device *dev = obj->base.dev; 2346 struct drm_i915_private *dev_priv = dev->dev_private; 2347 struct drm_i915_fence_reg *reg; 2348 2349 obj->ring = ring; 2350 KASSERT(ring != NULL, ("NULL ring")); 2351 2352 /* Add a reference if we're newly entering the active list. */ 2353 if (!obj->active) { 2354 drm_gem_object_reference(&obj->base); 2355 obj->active = 1; 2356 } 2357 2358 /* Move from whatever list we were on to the tail of execution. */ 2359 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 2360 list_move_tail(&obj->ring_list, &ring->active_list); 2361 2362 obj->last_rendering_seqno = seqno; 2363 if (obj->fenced_gpu_access) { 2364 obj->last_fenced_seqno = seqno; 2365 obj->last_fenced_ring = ring; 2366 2367 /* Bump MRU to take account of the delayed flush */ 2368 if (obj->fence_reg != I915_FENCE_REG_NONE) { 2369 reg = &dev_priv->fence_regs[obj->fence_reg]; 2370 list_move_tail(®->lru_list, 2371 &dev_priv->mm.fence_list); 2372 } 2373 } 2374 } 2375 2376 static void 2377 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) 2378 { 2379 list_del_init(&obj->ring_list); 2380 obj->last_rendering_seqno = 0; 2381 obj->last_fenced_seqno = 0; 2382 } 2383 2384 static void 2385 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) 2386 { 2387 struct drm_device *dev = obj->base.dev; 2388 drm_i915_private_t *dev_priv = dev->dev_private; 2389 2390 KASSERT(obj->active, ("Object not active")); 2391 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); 2392 2393 i915_gem_object_move_off_active(obj); 2394 } 2395 2396 static void 2397 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 2398 { 2399 struct drm_device *dev = obj->base.dev; 2400 struct drm_i915_private *dev_priv = dev->dev_private; 2401 2402 if (obj->pin_count != 0) 2403 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 2404 else 2405 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2406 2407 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list")); 2408 KASSERT(obj->active, ("Object not active")); 2409 obj->ring = NULL; 2410 obj->last_fenced_ring = NULL; 2411 2412 i915_gem_object_move_off_active(obj); 2413 obj->fenced_gpu_access = false; 2414 2415 obj->active = 0; 2416 obj->pending_gpu_write = false; 2417 drm_gem_object_unreference(&obj->base); 2418 2419 #if 1 2420 KIB_NOTYET(); 2421 #else 2422 WARN_ON(i915_verify_lists(dev)); 2423 #endif 2424 } 2425 2426 static void 2427 i915_gem_object_truncate(struct drm_i915_gem_object *obj) 2428 { 2429 vm_object_t vm_obj; 2430 2431 vm_obj = obj->base.vm_obj; 2432 VM_OBJECT_LOCK(vm_obj); 2433 vm_object_page_remove(vm_obj, 0, 0, false); 2434 VM_OBJECT_UNLOCK(vm_obj); 2435 obj->madv = I915_MADV_PURGED_INTERNAL; 2436 } 2437 2438 static inline int 2439 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 2440 { 2441 2442 return (obj->madv == I915_MADV_DONTNEED); 2443 } 2444 2445 static void 2446 i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 2447 uint32_t flush_domains) 2448 { 2449 struct drm_i915_gem_object *obj, *next; 2450 uint32_t old_write_domain; 2451 2452 list_for_each_entry_safe(obj, next, &ring->gpu_write_list, 2453 gpu_write_list) { 2454 if (obj->base.write_domain & flush_domains) { 2455 old_write_domain = obj->base.write_domain; 2456 obj->base.write_domain = 0; 2457 list_del_init(&obj->gpu_write_list); 2458 i915_gem_object_move_to_active(obj, ring, 2459 i915_gem_next_request_seqno(ring)); 2460 } 2461 } 2462 } 2463 2464 static int 2465 i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 2466 { 2467 drm_i915_private_t *dev_priv; 2468 2469 dev_priv = obj->base.dev->dev_private; 2470 return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 2471 obj->tiling_mode != I915_TILING_NONE); 2472 } 2473 2474 #define VM_OBJECT_LOCK_ASSERT_OWNED(object) 2475 2476 static vm_page_t 2477 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex) 2478 { 2479 vm_page_t m; 2480 int rv; 2481 2482 VM_OBJECT_LOCK_ASSERT_OWNED(object); 2483 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 2484 if (m->valid != VM_PAGE_BITS_ALL) { 2485 if (vm_pager_has_page(object, pindex)) { 2486 rv = vm_pager_get_page(object, &m, 1); 2487 m = vm_page_lookup(object, pindex); 2488 if (m == NULL) 2489 return (NULL); 2490 if (rv != VM_PAGER_OK) { 2491 vm_page_free(m); 2492 return (NULL); 2493 } 2494 } else { 2495 pmap_zero_page(VM_PAGE_TO_PHYS(m)); 2496 m->valid = VM_PAGE_BITS_ALL; 2497 m->dirty = 0; 2498 } 2499 } 2500 vm_page_wire(m); 2501 vm_page_wakeup(m); 2502 atomic_add_long(&i915_gem_wired_pages_cnt, 1); 2503 return (m); 2504 } 2505 2506 int 2507 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, 2508 uint32_t flush_domains) 2509 { 2510 int ret; 2511 2512 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) 2513 return 0; 2514 2515 ret = ring->flush(ring, invalidate_domains, flush_domains); 2516 if (ret) 2517 return ret; 2518 2519 if (flush_domains & I915_GEM_GPU_DOMAINS) 2520 i915_gem_process_flushing_list(ring, flush_domains); 2521 return 0; 2522 } 2523 2524 static int 2525 i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire) 2526 { 2527 int ret; 2528 2529 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 2530 return 0; 2531 2532 if (!list_empty(&ring->gpu_write_list)) { 2533 ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS, 2534 I915_GEM_GPU_DOMAINS); 2535 if (ret != 0) 2536 return ret; 2537 } 2538 2539 return (i915_wait_request(ring, i915_gem_next_request_seqno(ring), 2540 do_retire)); 2541 } 2542 2543 int 2544 i915_gpu_idle(struct drm_device *dev, bool do_retire) 2545 { 2546 drm_i915_private_t *dev_priv = dev->dev_private; 2547 int ret, i; 2548 2549 /* Flush everything onto the inactive list. */ 2550 for (i = 0; i < I915_NUM_RINGS; i++) { 2551 ret = i915_ring_idle(&dev_priv->rings[i], do_retire); 2552 if (ret) 2553 return ret; 2554 } 2555 2556 return 0; 2557 } 2558 2559 int 2560 i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire) 2561 { 2562 drm_i915_private_t *dev_priv; 2563 struct drm_i915_gem_request *request; 2564 uint32_t ier; 2565 int flags, ret; 2566 bool recovery_complete; 2567 2568 KASSERT(seqno != 0, ("Zero seqno")); 2569 2570 dev_priv = ring->dev->dev_private; 2571 ret = 0; 2572 2573 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 2574 /* Give the error handler a chance to run. */ 2575 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 2576 recovery_complete = (&dev_priv->error_completion) > 0; 2577 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 2578 return (recovery_complete ? -EIO : -EAGAIN); 2579 } 2580 2581 if (seqno == ring->outstanding_lazy_request) { 2582 request = kmalloc(sizeof(*request), DRM_I915_GEM, 2583 M_WAITOK | M_ZERO); 2584 if (request == NULL) 2585 return (-ENOMEM); 2586 2587 ret = i915_add_request(ring, NULL, request); 2588 if (ret != 0) { 2589 drm_free(request, DRM_I915_GEM); 2590 return (ret); 2591 } 2592 2593 seqno = request->seqno; 2594 } 2595 2596 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 2597 if (HAS_PCH_SPLIT(ring->dev)) 2598 ier = I915_READ(DEIER) | I915_READ(GTIER); 2599 else 2600 ier = I915_READ(IER); 2601 if (!ier) { 2602 DRM_ERROR("something (likely vbetool) disabled " 2603 "interrupts, re-enabling\n"); 2604 ring->dev->driver->irq_preinstall(ring->dev); 2605 ring->dev->driver->irq_postinstall(ring->dev); 2606 } 2607 2608 ring->waiting_seqno = seqno; 2609 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 2610 if (ring->irq_get(ring)) { 2611 flags = dev_priv->mm.interruptible ? PCATCH : 0; 2612 while (!i915_seqno_passed(ring->get_seqno(ring), seqno) 2613 && !atomic_load_acq_int(&dev_priv->mm.wedged) && 2614 ret == 0) { 2615 ret = -lksleep(ring, &ring->irq_lock, flags, 2616 "915gwr", 0); 2617 } 2618 ring->irq_put(ring); 2619 lockmgr(&ring->irq_lock, LK_RELEASE); 2620 } else { 2621 lockmgr(&ring->irq_lock, LK_RELEASE); 2622 if (_intel_wait_for(ring->dev, 2623 i915_seqno_passed(ring->get_seqno(ring), seqno) || 2624 atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 2625 0, "i915wrq") != 0) 2626 ret = -EBUSY; 2627 } 2628 ring->waiting_seqno = 0; 2629 2630 } 2631 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 2632 ret = -EAGAIN; 2633 2634 /* Directly dispatch request retiring. While we have the work queue 2635 * to handle this, the waiter on a request often wants an associated 2636 * buffer to have made it to the inactive list, and we would need 2637 * a separate wait queue to handle that. 2638 */ 2639 if (ret == 0 && do_retire) 2640 i915_gem_retire_requests_ring(ring); 2641 2642 return (ret); 2643 } 2644 2645 static u32 2646 i915_gem_get_seqno(struct drm_device *dev) 2647 { 2648 drm_i915_private_t *dev_priv = dev->dev_private; 2649 u32 seqno = dev_priv->next_seqno; 2650 2651 /* reserve 0 for non-seqno */ 2652 if (++dev_priv->next_seqno == 0) 2653 dev_priv->next_seqno = 1; 2654 2655 return seqno; 2656 } 2657 2658 u32 2659 i915_gem_next_request_seqno(struct intel_ring_buffer *ring) 2660 { 2661 if (ring->outstanding_lazy_request == 0) 2662 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); 2663 2664 return ring->outstanding_lazy_request; 2665 } 2666 2667 int 2668 i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, 2669 struct drm_i915_gem_request *request) 2670 { 2671 drm_i915_private_t *dev_priv; 2672 struct drm_i915_file_private *file_priv; 2673 uint32_t seqno; 2674 u32 request_ring_position; 2675 int was_empty; 2676 int ret; 2677 2678 KASSERT(request != NULL, ("NULL request in add")); 2679 DRM_LOCK_ASSERT(ring->dev); 2680 dev_priv = ring->dev->dev_private; 2681 2682 seqno = i915_gem_next_request_seqno(ring); 2683 request_ring_position = intel_ring_get_tail(ring); 2684 2685 ret = ring->add_request(ring, &seqno); 2686 if (ret != 0) 2687 return ret; 2688 2689 request->seqno = seqno; 2690 request->ring = ring; 2691 request->tail = request_ring_position; 2692 request->emitted_jiffies = ticks; 2693 was_empty = list_empty(&ring->request_list); 2694 list_add_tail(&request->list, &ring->request_list); 2695 2696 if (file != NULL) { 2697 file_priv = file->driver_priv; 2698 2699 spin_lock(&file_priv->mm.lock); 2700 request->file_priv = file_priv; 2701 list_add_tail(&request->client_list, 2702 &file_priv->mm.request_list); 2703 spin_unlock(&file_priv->mm.lock); 2704 } 2705 2706 ring->outstanding_lazy_request = 0; 2707 2708 if (!dev_priv->mm.suspended) { 2709 if (i915_enable_hangcheck) { 2710 callout_reset(&dev_priv->hangcheck_timer, 2711 DRM_I915_HANGCHECK_PERIOD, i915_hangcheck_elapsed, ring->dev); 2712 } 2713 if (was_empty) 2714 taskqueue_enqueue_timeout(dev_priv->tq, 2715 &dev_priv->mm.retire_task, hz); 2716 } 2717 return (0); 2718 } 2719 2720 static inline void 2721 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 2722 { 2723 struct drm_i915_file_private *file_priv = request->file_priv; 2724 2725 if (!file_priv) 2726 return; 2727 2728 DRM_LOCK_ASSERT(request->ring->dev); 2729 2730 spin_lock(&file_priv->mm.lock); 2731 if (request->file_priv != NULL) { 2732 list_del(&request->client_list); 2733 request->file_priv = NULL; 2734 } 2735 spin_unlock(&file_priv->mm.lock); 2736 } 2737 2738 void 2739 i915_gem_release(struct drm_device *dev, struct drm_file *file) 2740 { 2741 struct drm_i915_file_private *file_priv; 2742 struct drm_i915_gem_request *request; 2743 2744 file_priv = file->driver_priv; 2745 2746 /* Clean up our request list when the client is going away, so that 2747 * later retire_requests won't dereference our soon-to-be-gone 2748 * file_priv. 2749 */ 2750 spin_lock(&file_priv->mm.lock); 2751 while (!list_empty(&file_priv->mm.request_list)) { 2752 request = list_first_entry(&file_priv->mm.request_list, 2753 struct drm_i915_gem_request, 2754 client_list); 2755 list_del(&request->client_list); 2756 request->file_priv = NULL; 2757 } 2758 spin_unlock(&file_priv->mm.lock); 2759 } 2760 2761 static void 2762 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 2763 struct intel_ring_buffer *ring) 2764 { 2765 2766 if (ring->dev != NULL) 2767 DRM_LOCK_ASSERT(ring->dev); 2768 2769 while (!list_empty(&ring->request_list)) { 2770 struct drm_i915_gem_request *request; 2771 2772 request = list_first_entry(&ring->request_list, 2773 struct drm_i915_gem_request, list); 2774 2775 list_del(&request->list); 2776 i915_gem_request_remove_from_client(request); 2777 drm_free(request, DRM_I915_GEM); 2778 } 2779 2780 while (!list_empty(&ring->active_list)) { 2781 struct drm_i915_gem_object *obj; 2782 2783 obj = list_first_entry(&ring->active_list, 2784 struct drm_i915_gem_object, ring_list); 2785 2786 obj->base.write_domain = 0; 2787 list_del_init(&obj->gpu_write_list); 2788 i915_gem_object_move_to_inactive(obj); 2789 } 2790 } 2791 2792 static void 2793 i915_gem_reset_fences(struct drm_device *dev) 2794 { 2795 struct drm_i915_private *dev_priv = dev->dev_private; 2796 int i; 2797 2798 for (i = 0; i < dev_priv->num_fence_regs; i++) { 2799 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2800 struct drm_i915_gem_object *obj = reg->obj; 2801 2802 if (!obj) 2803 continue; 2804 2805 if (obj->tiling_mode) 2806 i915_gem_release_mmap(obj); 2807 2808 reg->obj->fence_reg = I915_FENCE_REG_NONE; 2809 reg->obj->fenced_gpu_access = false; 2810 reg->obj->last_fenced_seqno = 0; 2811 reg->obj->last_fenced_ring = NULL; 2812 i915_gem_clear_fence_reg(dev, reg); 2813 } 2814 } 2815 2816 void 2817 i915_gem_reset(struct drm_device *dev) 2818 { 2819 struct drm_i915_private *dev_priv = dev->dev_private; 2820 struct drm_i915_gem_object *obj; 2821 int i; 2822 2823 for (i = 0; i < I915_NUM_RINGS; i++) 2824 i915_gem_reset_ring_lists(dev_priv, &dev_priv->rings[i]); 2825 2826 /* Remove anything from the flushing lists. The GPU cache is likely 2827 * to be lost on reset along with the data, so simply move the 2828 * lost bo to the inactive list. 2829 */ 2830 while (!list_empty(&dev_priv->mm.flushing_list)) { 2831 obj = list_first_entry(&dev_priv->mm.flushing_list, 2832 struct drm_i915_gem_object, 2833 mm_list); 2834 2835 obj->base.write_domain = 0; 2836 list_del_init(&obj->gpu_write_list); 2837 i915_gem_object_move_to_inactive(obj); 2838 } 2839 2840 /* Move everything out of the GPU domains to ensure we do any 2841 * necessary invalidation upon reuse. 2842 */ 2843 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { 2844 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2845 } 2846 2847 /* The fence registers are invalidated so clear them out */ 2848 i915_gem_reset_fences(dev); 2849 } 2850 2851 /** 2852 * This function clears the request list as sequence numbers are passed. 2853 */ 2854 void 2855 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 2856 { 2857 uint32_t seqno; 2858 int i; 2859 2860 if (list_empty(&ring->request_list)) 2861 return; 2862 2863 seqno = ring->get_seqno(ring); 2864 for (i = 0; i < DRM_ARRAY_SIZE(ring->sync_seqno); i++) 2865 if (seqno >= ring->sync_seqno[i]) 2866 ring->sync_seqno[i] = 0; 2867 2868 while (!list_empty(&ring->request_list)) { 2869 struct drm_i915_gem_request *request; 2870 2871 request = list_first_entry(&ring->request_list, 2872 struct drm_i915_gem_request, 2873 list); 2874 2875 if (!i915_seqno_passed(seqno, request->seqno)) 2876 break; 2877 2878 ring->last_retired_head = request->tail; 2879 2880 list_del(&request->list); 2881 i915_gem_request_remove_from_client(request); 2882 drm_free(request, DRM_I915_GEM); 2883 } 2884 2885 /* Move any buffers on the active list that are no longer referenced 2886 * by the ringbuffer to the flushing/inactive lists as appropriate. 2887 */ 2888 while (!list_empty(&ring->active_list)) { 2889 struct drm_i915_gem_object *obj; 2890 2891 obj = list_first_entry(&ring->active_list, 2892 struct drm_i915_gem_object, 2893 ring_list); 2894 2895 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) 2896 break; 2897 2898 if (obj->base.write_domain != 0) 2899 i915_gem_object_move_to_flushing(obj); 2900 else 2901 i915_gem_object_move_to_inactive(obj); 2902 } 2903 2904 if (ring->trace_irq_seqno && 2905 i915_seqno_passed(seqno, ring->trace_irq_seqno)) { 2906 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 2907 ring->irq_put(ring); 2908 lockmgr(&ring->irq_lock, LK_RELEASE); 2909 ring->trace_irq_seqno = 0; 2910 } 2911 } 2912 2913 void 2914 i915_gem_retire_requests(struct drm_device *dev) 2915 { 2916 drm_i915_private_t *dev_priv = dev->dev_private; 2917 struct drm_i915_gem_object *obj, *next; 2918 int i; 2919 2920 if (!list_empty(&dev_priv->mm.deferred_free_list)) { 2921 list_for_each_entry_safe(obj, next, 2922 &dev_priv->mm.deferred_free_list, mm_list) 2923 i915_gem_free_object_tail(obj); 2924 } 2925 2926 for (i = 0; i < I915_NUM_RINGS; i++) 2927 i915_gem_retire_requests_ring(&dev_priv->rings[i]); 2928 } 2929 2930 static int 2931 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj, 2932 struct intel_ring_buffer *pipelined) 2933 { 2934 struct drm_device *dev = obj->base.dev; 2935 drm_i915_private_t *dev_priv = dev->dev_private; 2936 u32 size = obj->gtt_space->size; 2937 int regnum = obj->fence_reg; 2938 uint64_t val; 2939 2940 val = (uint64_t)((obj->gtt_offset + size - 4096) & 2941 0xfffff000) << 32; 2942 val |= obj->gtt_offset & 0xfffff000; 2943 val |= (uint64_t)((obj->stride / 128) - 1) << 2944 SANDYBRIDGE_FENCE_PITCH_SHIFT; 2945 2946 if (obj->tiling_mode == I915_TILING_Y) 2947 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2948 val |= I965_FENCE_REG_VALID; 2949 2950 if (pipelined) { 2951 int ret = intel_ring_begin(pipelined, 6); 2952 if (ret) 2953 return ret; 2954 2955 intel_ring_emit(pipelined, MI_NOOP); 2956 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 2957 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8); 2958 intel_ring_emit(pipelined, (u32)val); 2959 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4); 2960 intel_ring_emit(pipelined, (u32)(val >> 32)); 2961 intel_ring_advance(pipelined); 2962 } else 2963 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val); 2964 2965 return 0; 2966 } 2967 2968 static int 2969 i965_write_fence_reg(struct drm_i915_gem_object *obj, 2970 struct intel_ring_buffer *pipelined) 2971 { 2972 struct drm_device *dev = obj->base.dev; 2973 drm_i915_private_t *dev_priv = dev->dev_private; 2974 u32 size = obj->gtt_space->size; 2975 int regnum = obj->fence_reg; 2976 uint64_t val; 2977 2978 val = (uint64_t)((obj->gtt_offset + size - 4096) & 2979 0xfffff000) << 32; 2980 val |= obj->gtt_offset & 0xfffff000; 2981 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 2982 if (obj->tiling_mode == I915_TILING_Y) 2983 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2984 val |= I965_FENCE_REG_VALID; 2985 2986 if (pipelined) { 2987 int ret = intel_ring_begin(pipelined, 6); 2988 if (ret) 2989 return ret; 2990 2991 intel_ring_emit(pipelined, MI_NOOP); 2992 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 2993 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8); 2994 intel_ring_emit(pipelined, (u32)val); 2995 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4); 2996 intel_ring_emit(pipelined, (u32)(val >> 32)); 2997 intel_ring_advance(pipelined); 2998 } else 2999 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val); 3000 3001 return 0; 3002 } 3003 3004 static int 3005 i915_write_fence_reg(struct drm_i915_gem_object *obj, 3006 struct intel_ring_buffer *pipelined) 3007 { 3008 struct drm_device *dev = obj->base.dev; 3009 drm_i915_private_t *dev_priv = dev->dev_private; 3010 u32 size = obj->gtt_space->size; 3011 u32 fence_reg, val, pitch_val; 3012 int tile_width; 3013 3014 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) || 3015 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3016 kprintf( 3017 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 3018 obj->gtt_offset, obj->map_and_fenceable, size); 3019 return -EINVAL; 3020 } 3021 3022 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 3023 tile_width = 128; 3024 else 3025 tile_width = 512; 3026 3027 /* Note: pitch better be a power of two tile widths */ 3028 pitch_val = obj->stride / tile_width; 3029 pitch_val = ffs(pitch_val) - 1; 3030 3031 val = obj->gtt_offset; 3032 if (obj->tiling_mode == I915_TILING_Y) 3033 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3034 val |= I915_FENCE_SIZE_BITS(size); 3035 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3036 val |= I830_FENCE_REG_VALID; 3037 3038 fence_reg = obj->fence_reg; 3039 if (fence_reg < 8) 3040 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3041 else 3042 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3043 3044 if (pipelined) { 3045 int ret = intel_ring_begin(pipelined, 4); 3046 if (ret) 3047 return ret; 3048 3049 intel_ring_emit(pipelined, MI_NOOP); 3050 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3051 intel_ring_emit(pipelined, fence_reg); 3052 intel_ring_emit(pipelined, val); 3053 intel_ring_advance(pipelined); 3054 } else 3055 I915_WRITE(fence_reg, val); 3056 3057 return 0; 3058 } 3059 3060 static int 3061 i830_write_fence_reg(struct drm_i915_gem_object *obj, 3062 struct intel_ring_buffer *pipelined) 3063 { 3064 struct drm_device *dev = obj->base.dev; 3065 drm_i915_private_t *dev_priv = dev->dev_private; 3066 u32 size = obj->gtt_space->size; 3067 int regnum = obj->fence_reg; 3068 uint32_t val; 3069 uint32_t pitch_val; 3070 3071 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) || 3072 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3073 kprintf( 3074 "object 0x%08x not 512K or pot-size 0x%08x aligned\n", 3075 obj->gtt_offset, size); 3076 return -EINVAL; 3077 } 3078 3079 pitch_val = obj->stride / 128; 3080 pitch_val = ffs(pitch_val) - 1; 3081 3082 val = obj->gtt_offset; 3083 if (obj->tiling_mode == I915_TILING_Y) 3084 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3085 val |= I830_FENCE_SIZE_BITS(size); 3086 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3087 val |= I830_FENCE_REG_VALID; 3088 3089 if (pipelined) { 3090 int ret = intel_ring_begin(pipelined, 4); 3091 if (ret) 3092 return ret; 3093 3094 intel_ring_emit(pipelined, MI_NOOP); 3095 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3096 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4); 3097 intel_ring_emit(pipelined, val); 3098 intel_ring_advance(pipelined); 3099 } else 3100 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val); 3101 3102 return 0; 3103 } 3104 3105 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno) 3106 { 3107 return i915_seqno_passed(ring->get_seqno(ring), seqno); 3108 } 3109 3110 static int 3111 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, 3112 struct intel_ring_buffer *pipelined) 3113 { 3114 int ret; 3115 3116 if (obj->fenced_gpu_access) { 3117 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 3118 ret = i915_gem_flush_ring(obj->last_fenced_ring, 0, 3119 obj->base.write_domain); 3120 if (ret) 3121 return ret; 3122 } 3123 3124 obj->fenced_gpu_access = false; 3125 } 3126 3127 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) { 3128 if (!ring_passed_seqno(obj->last_fenced_ring, 3129 obj->last_fenced_seqno)) { 3130 ret = i915_wait_request(obj->last_fenced_ring, 3131 obj->last_fenced_seqno, 3132 true); 3133 if (ret) 3134 return ret; 3135 } 3136 3137 obj->last_fenced_seqno = 0; 3138 obj->last_fenced_ring = NULL; 3139 } 3140 3141 /* Ensure that all CPU reads are completed before installing a fence 3142 * and all writes before removing the fence. 3143 */ 3144 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 3145 cpu_mfence(); 3146 3147 return 0; 3148 } 3149 3150 int 3151 i915_gem_object_put_fence(struct drm_i915_gem_object *obj) 3152 { 3153 int ret; 3154 3155 if (obj->tiling_mode) 3156 i915_gem_release_mmap(obj); 3157 3158 ret = i915_gem_object_flush_fence(obj, NULL); 3159 if (ret) 3160 return ret; 3161 3162 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3163 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3164 3165 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0) 3166 kprintf("%s: pin_count %d\n", __func__, 3167 dev_priv->fence_regs[obj->fence_reg].pin_count); 3168 i915_gem_clear_fence_reg(obj->base.dev, 3169 &dev_priv->fence_regs[obj->fence_reg]); 3170 3171 obj->fence_reg = I915_FENCE_REG_NONE; 3172 } 3173 3174 return 0; 3175 } 3176 3177 static struct drm_i915_fence_reg * 3178 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined) 3179 { 3180 struct drm_i915_private *dev_priv = dev->dev_private; 3181 struct drm_i915_fence_reg *reg, *first, *avail; 3182 int i; 3183 3184 /* First try to find a free reg */ 3185 avail = NULL; 3186 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 3187 reg = &dev_priv->fence_regs[i]; 3188 if (!reg->obj) 3189 return reg; 3190 3191 if (!reg->pin_count) 3192 avail = reg; 3193 } 3194 3195 if (avail == NULL) 3196 return NULL; 3197 3198 /* None available, try to steal one or wait for a user to finish */ 3199 avail = first = NULL; 3200 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 3201 if (reg->pin_count) 3202 continue; 3203 3204 if (first == NULL) 3205 first = reg; 3206 3207 if (!pipelined || 3208 !reg->obj->last_fenced_ring || 3209 reg->obj->last_fenced_ring == pipelined) { 3210 avail = reg; 3211 break; 3212 } 3213 } 3214 3215 if (avail == NULL) 3216 avail = first; 3217 3218 return avail; 3219 } 3220 3221 int 3222 i915_gem_object_get_fence(struct drm_i915_gem_object *obj, 3223 struct intel_ring_buffer *pipelined) 3224 { 3225 struct drm_device *dev = obj->base.dev; 3226 struct drm_i915_private *dev_priv = dev->dev_private; 3227 struct drm_i915_fence_reg *reg; 3228 int ret; 3229 3230 pipelined = NULL; 3231 ret = 0; 3232 3233 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3234 reg = &dev_priv->fence_regs[obj->fence_reg]; 3235 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3236 3237 if (obj->tiling_changed) { 3238 ret = i915_gem_object_flush_fence(obj, pipelined); 3239 if (ret) 3240 return ret; 3241 3242 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3243 pipelined = NULL; 3244 3245 if (pipelined) { 3246 reg->setup_seqno = 3247 i915_gem_next_request_seqno(pipelined); 3248 obj->last_fenced_seqno = reg->setup_seqno; 3249 obj->last_fenced_ring = pipelined; 3250 } 3251 3252 goto update; 3253 } 3254 3255 if (!pipelined) { 3256 if (reg->setup_seqno) { 3257 if (!ring_passed_seqno(obj->last_fenced_ring, 3258 reg->setup_seqno)) { 3259 ret = i915_wait_request( 3260 obj->last_fenced_ring, 3261 reg->setup_seqno, 3262 true); 3263 if (ret) 3264 return ret; 3265 } 3266 3267 reg->setup_seqno = 0; 3268 } 3269 } else if (obj->last_fenced_ring && 3270 obj->last_fenced_ring != pipelined) { 3271 ret = i915_gem_object_flush_fence(obj, pipelined); 3272 if (ret) 3273 return ret; 3274 } 3275 3276 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3277 pipelined = NULL; 3278 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined")); 3279 3280 if (obj->tiling_changed) { 3281 if (pipelined) { 3282 reg->setup_seqno = 3283 i915_gem_next_request_seqno(pipelined); 3284 obj->last_fenced_seqno = reg->setup_seqno; 3285 obj->last_fenced_ring = pipelined; 3286 } 3287 goto update; 3288 } 3289 3290 return 0; 3291 } 3292 3293 reg = i915_find_fence_reg(dev, pipelined); 3294 if (reg == NULL) 3295 return -EDEADLK; 3296 3297 ret = i915_gem_object_flush_fence(obj, pipelined); 3298 if (ret) 3299 return ret; 3300 3301 if (reg->obj) { 3302 struct drm_i915_gem_object *old = reg->obj; 3303 3304 drm_gem_object_reference(&old->base); 3305 3306 if (old->tiling_mode) 3307 i915_gem_release_mmap(old); 3308 3309 ret = i915_gem_object_flush_fence(old, pipelined); 3310 if (ret) { 3311 drm_gem_object_unreference(&old->base); 3312 return ret; 3313 } 3314 3315 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0) 3316 pipelined = NULL; 3317 3318 old->fence_reg = I915_FENCE_REG_NONE; 3319 old->last_fenced_ring = pipelined; 3320 old->last_fenced_seqno = 3321 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3322 3323 drm_gem_object_unreference(&old->base); 3324 } else if (obj->last_fenced_seqno == 0) 3325 pipelined = NULL; 3326 3327 reg->obj = obj; 3328 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3329 obj->fence_reg = reg - dev_priv->fence_regs; 3330 obj->last_fenced_ring = pipelined; 3331 3332 reg->setup_seqno = 3333 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3334 obj->last_fenced_seqno = reg->setup_seqno; 3335 3336 update: 3337 obj->tiling_changed = false; 3338 switch (INTEL_INFO(dev)->gen) { 3339 case 7: 3340 case 6: 3341 ret = sandybridge_write_fence_reg(obj, pipelined); 3342 break; 3343 case 5: 3344 case 4: 3345 ret = i965_write_fence_reg(obj, pipelined); 3346 break; 3347 case 3: 3348 ret = i915_write_fence_reg(obj, pipelined); 3349 break; 3350 case 2: 3351 ret = i830_write_fence_reg(obj, pipelined); 3352 break; 3353 } 3354 3355 return ret; 3356 } 3357 3358 static void 3359 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg) 3360 { 3361 drm_i915_private_t *dev_priv = dev->dev_private; 3362 uint32_t fence_reg = reg - dev_priv->fence_regs; 3363 3364 switch (INTEL_INFO(dev)->gen) { 3365 case 7: 3366 case 6: 3367 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0); 3368 break; 3369 case 5: 3370 case 4: 3371 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0); 3372 break; 3373 case 3: 3374 if (fence_reg >= 8) 3375 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3376 else 3377 case 2: 3378 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3379 3380 I915_WRITE(fence_reg, 0); 3381 break; 3382 } 3383 3384 list_del_init(®->lru_list); 3385 reg->obj = NULL; 3386 reg->setup_seqno = 0; 3387 reg->pin_count = 0; 3388 } 3389 3390 int 3391 i915_gem_init_object(struct drm_gem_object *obj) 3392 { 3393 3394 kprintf("i915_gem_init_object called\n"); 3395 return (0); 3396 } 3397 3398 static bool 3399 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 3400 { 3401 3402 return (obj->gtt_space && !obj->active && obj->pin_count == 0); 3403 } 3404 3405 static void 3406 i915_gem_retire_task_handler(void *arg, int pending) 3407 { 3408 drm_i915_private_t *dev_priv; 3409 struct drm_device *dev; 3410 bool idle; 3411 int i; 3412 3413 dev_priv = arg; 3414 dev = dev_priv->dev; 3415 3416 /* Come back later if the device is busy... */ 3417 if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) { 3418 taskqueue_enqueue_timeout(dev_priv->tq, 3419 &dev_priv->mm.retire_task, hz); 3420 return; 3421 } 3422 3423 i915_gem_retire_requests(dev); 3424 3425 /* Send a periodic flush down the ring so we don't hold onto GEM 3426 * objects indefinitely. 3427 */ 3428 idle = true; 3429 for (i = 0; i < I915_NUM_RINGS; i++) { 3430 struct intel_ring_buffer *ring = &dev_priv->rings[i]; 3431 3432 if (!list_empty(&ring->gpu_write_list)) { 3433 struct drm_i915_gem_request *request; 3434 int ret; 3435 3436 ret = i915_gem_flush_ring(ring, 3437 0, I915_GEM_GPU_DOMAINS); 3438 request = kmalloc(sizeof(*request), DRM_I915_GEM, 3439 M_WAITOK | M_ZERO); 3440 if (ret || request == NULL || 3441 i915_add_request(ring, NULL, request)) 3442 drm_free(request, DRM_I915_GEM); 3443 } 3444 3445 idle &= list_empty(&ring->request_list); 3446 } 3447 3448 if (!dev_priv->mm.suspended && !idle) 3449 taskqueue_enqueue_timeout(dev_priv->tq, 3450 &dev_priv->mm.retire_task, hz); 3451 3452 DRM_UNLOCK(dev); 3453 } 3454 3455 void 3456 i915_gem_lastclose(struct drm_device *dev) 3457 { 3458 int ret; 3459 3460 if (drm_core_check_feature(dev, DRIVER_MODESET)) 3461 return; 3462 3463 ret = i915_gem_idle(dev); 3464 if (ret != 0) 3465 DRM_ERROR("failed to idle hardware: %d\n", ret); 3466 } 3467 3468 static int 3469 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align) 3470 { 3471 drm_i915_private_t *dev_priv; 3472 struct drm_i915_gem_phys_object *phys_obj; 3473 int ret; 3474 3475 dev_priv = dev->dev_private; 3476 if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0) 3477 return (0); 3478 3479 phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM, 3480 M_WAITOK | M_ZERO); 3481 3482 phys_obj->id = id; 3483 3484 phys_obj->handle = drm_pci_alloc(dev, size, align, ~0); 3485 if (phys_obj->handle == NULL) { 3486 ret = -ENOMEM; 3487 goto free_obj; 3488 } 3489 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 3490 size / PAGE_SIZE, PAT_WRITE_COMBINING); 3491 3492 dev_priv->mm.phys_objs[id - 1] = phys_obj; 3493 3494 return (0); 3495 3496 free_obj: 3497 drm_free(phys_obj, DRM_I915_GEM); 3498 return (ret); 3499 } 3500 3501 static void 3502 i915_gem_free_phys_object(struct drm_device *dev, int id) 3503 { 3504 drm_i915_private_t *dev_priv; 3505 struct drm_i915_gem_phys_object *phys_obj; 3506 3507 dev_priv = dev->dev_private; 3508 if (dev_priv->mm.phys_objs[id - 1] == NULL) 3509 return; 3510 3511 phys_obj = dev_priv->mm.phys_objs[id - 1]; 3512 if (phys_obj->cur_obj != NULL) 3513 i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 3514 3515 drm_pci_free(dev, phys_obj->handle); 3516 drm_free(phys_obj, DRM_I915_GEM); 3517 dev_priv->mm.phys_objs[id - 1] = NULL; 3518 } 3519 3520 void 3521 i915_gem_free_all_phys_object(struct drm_device *dev) 3522 { 3523 int i; 3524 3525 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 3526 i915_gem_free_phys_object(dev, i); 3527 } 3528 3529 void 3530 i915_gem_detach_phys_object(struct drm_device *dev, 3531 struct drm_i915_gem_object *obj) 3532 { 3533 vm_page_t m; 3534 struct sf_buf *sf; 3535 char *vaddr, *dst; 3536 int i, page_count; 3537 3538 if (obj->phys_obj == NULL) 3539 return; 3540 vaddr = obj->phys_obj->handle->vaddr; 3541 3542 page_count = obj->base.size / PAGE_SIZE; 3543 VM_OBJECT_LOCK(obj->base.vm_obj); 3544 for (i = 0; i < page_count; i++) { 3545 m = i915_gem_wire_page(obj->base.vm_obj, i); 3546 if (m == NULL) 3547 continue; /* XXX */ 3548 3549 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3550 sf = sf_buf_alloc(m); 3551 if (sf != NULL) { 3552 dst = (char *)sf_buf_kva(sf); 3553 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 3554 sf_buf_free(sf); 3555 } 3556 drm_clflush_pages(&m, 1); 3557 3558 VM_OBJECT_LOCK(obj->base.vm_obj); 3559 vm_page_reference(m); 3560 vm_page_dirty(m); 3561 vm_page_busy_wait(m, FALSE, "i915gem"); 3562 vm_page_unwire(m, 0); 3563 vm_page_wakeup(m); 3564 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3565 } 3566 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3567 intel_gtt_chipset_flush(); 3568 3569 obj->phys_obj->cur_obj = NULL; 3570 obj->phys_obj = NULL; 3571 } 3572 3573 int 3574 i915_gem_attach_phys_object(struct drm_device *dev, 3575 struct drm_i915_gem_object *obj, int id, int align) 3576 { 3577 drm_i915_private_t *dev_priv; 3578 vm_page_t m; 3579 struct sf_buf *sf; 3580 char *dst, *src; 3581 int i, page_count, ret; 3582 3583 if (id > I915_MAX_PHYS_OBJECT) 3584 return (-EINVAL); 3585 3586 if (obj->phys_obj != NULL) { 3587 if (obj->phys_obj->id == id) 3588 return (0); 3589 i915_gem_detach_phys_object(dev, obj); 3590 } 3591 3592 dev_priv = dev->dev_private; 3593 if (dev_priv->mm.phys_objs[id - 1] == NULL) { 3594 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align); 3595 if (ret != 0) { 3596 DRM_ERROR("failed to init phys object %d size: %zu\n", 3597 id, obj->base.size); 3598 return (ret); 3599 } 3600 } 3601 3602 /* bind to the object */ 3603 obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 3604 obj->phys_obj->cur_obj = obj; 3605 3606 page_count = obj->base.size / PAGE_SIZE; 3607 3608 VM_OBJECT_LOCK(obj->base.vm_obj); 3609 ret = 0; 3610 for (i = 0; i < page_count; i++) { 3611 m = i915_gem_wire_page(obj->base.vm_obj, i); 3612 if (m == NULL) { 3613 ret = -EIO; 3614 break; 3615 } 3616 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3617 sf = sf_buf_alloc(m); 3618 src = (char *)sf_buf_kva(sf); 3619 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 3620 memcpy(dst, src, PAGE_SIZE); 3621 sf_buf_free(sf); 3622 3623 VM_OBJECT_LOCK(obj->base.vm_obj); 3624 3625 vm_page_reference(m); 3626 vm_page_busy_wait(m, FALSE, "i915gem"); 3627 vm_page_unwire(m, 0); 3628 vm_page_wakeup(m); 3629 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3630 } 3631 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3632 3633 return (0); 3634 } 3635 3636 static int 3637 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, 3638 uint64_t data_ptr, uint64_t offset, uint64_t size, 3639 struct drm_file *file_priv) 3640 { 3641 char *user_data, *vaddr; 3642 int ret; 3643 3644 vaddr = (char *)obj->phys_obj->handle->vaddr + offset; 3645 user_data = (char *)(uintptr_t)data_ptr; 3646 3647 if (copyin_nofault(user_data, vaddr, size) != 0) { 3648 /* The physical object once assigned is fixed for the lifetime 3649 * of the obj, so we can safely drop the lock and continue 3650 * to access vaddr. 3651 */ 3652 DRM_UNLOCK(dev); 3653 ret = -copyin(user_data, vaddr, size); 3654 DRM_LOCK(dev); 3655 if (ret != 0) 3656 return (ret); 3657 } 3658 3659 intel_gtt_chipset_flush(); 3660 return (0); 3661 } 3662 3663 static int 3664 i915_gpu_is_active(struct drm_device *dev) 3665 { 3666 drm_i915_private_t *dev_priv; 3667 3668 dev_priv = dev->dev_private; 3669 return (!list_empty(&dev_priv->mm.flushing_list) || 3670 !list_empty(&dev_priv->mm.active_list)); 3671 } 3672 3673 static void 3674 i915_gem_lowmem(void *arg) 3675 { 3676 struct drm_device *dev; 3677 struct drm_i915_private *dev_priv; 3678 struct drm_i915_gem_object *obj, *next; 3679 int cnt, cnt_fail, cnt_total; 3680 3681 dev = arg; 3682 dev_priv = dev->dev_private; 3683 3684 if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) 3685 return; 3686 3687 rescan: 3688 /* first scan for clean buffers */ 3689 i915_gem_retire_requests(dev); 3690 3691 cnt_total = cnt_fail = cnt = 0; 3692 3693 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3694 mm_list) { 3695 if (i915_gem_object_is_purgeable(obj)) { 3696 if (i915_gem_object_unbind(obj) != 0) 3697 cnt_total++; 3698 } else 3699 cnt_total++; 3700 } 3701 3702 /* second pass, evict/count anything still on the inactive list */ 3703 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3704 mm_list) { 3705 if (i915_gem_object_unbind(obj) == 0) 3706 cnt++; 3707 else 3708 cnt_fail++; 3709 } 3710 3711 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) { 3712 /* 3713 * We are desperate for pages, so as a last resort, wait 3714 * for the GPU to finish and discard whatever we can. 3715 * This has a dramatic impact to reduce the number of 3716 * OOM-killer events whilst running the GPU aggressively. 3717 */ 3718 if (i915_gpu_idle(dev, true) == 0) 3719 goto rescan; 3720 } 3721 DRM_UNLOCK(dev); 3722 } 3723 3724 void 3725 i915_gem_unload(struct drm_device *dev) 3726 { 3727 struct drm_i915_private *dev_priv; 3728 3729 dev_priv = dev->dev_private; 3730 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem); 3731 } 3732