1 /*- 2 * Copyright © 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 * Copyright (c) 2011 The FreeBSD Foundation 27 * All rights reserved. 28 * 29 * This software was developed by Konstantin Belousov under sponsorship from 30 * the FreeBSD Foundation. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51 * SUCH DAMAGE. 52 * 53 * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $ 54 */ 55 56 #include <sys/resourcevar.h> 57 #include <sys/sfbuf.h> 58 59 #include <drm/drmP.h> 60 #include "i915_drm.h" 61 #include "i915_drv.h" 62 #include "intel_drv.h" 63 #include "intel_ringbuffer.h" 64 65 static void i915_gem_object_flush_cpu_write_domain( 66 struct drm_i915_gem_object *obj); 67 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, 68 int tiling_mode); 69 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, 70 uint32_t size, int tiling_mode); 71 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 72 unsigned alignment, bool map_and_fenceable); 73 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 74 int flags); 75 static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj); 76 static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, 77 bool write); 78 static void i915_gem_object_set_to_full_cpu_read_domain( 79 struct drm_i915_gem_object *obj); 80 static int i915_gem_object_set_cpu_read_domain_range( 81 struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size); 82 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj); 83 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 84 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj); 85 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj); 86 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj); 87 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex); 88 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 89 uint32_t flush_domains); 90 static void i915_gem_clear_fence_reg(struct drm_device *dev, 91 struct drm_i915_fence_reg *reg); 92 static void i915_gem_reset_fences(struct drm_device *dev); 93 static void i915_gem_retire_task_handler(void *arg, int pending); 94 static int i915_gem_phys_pwrite(struct drm_device *dev, 95 struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset, 96 uint64_t size, struct drm_file *file_priv); 97 static void i915_gem_lowmem(void *arg); 98 99 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 100 long i915_gem_wired_pages_cnt; 101 102 static void 103 i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) 104 { 105 106 dev_priv->mm.object_count++; 107 dev_priv->mm.object_memory += size; 108 } 109 110 static void 111 i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size) 112 { 113 114 dev_priv->mm.object_count--; 115 dev_priv->mm.object_memory -= size; 116 } 117 118 static int 119 i915_gem_wait_for_error(struct drm_device *dev) 120 { 121 struct drm_i915_private *dev_priv; 122 int ret; 123 124 dev_priv = dev->dev_private; 125 if (!atomic_load_acq_int(&dev_priv->mm.wedged)) 126 return (0); 127 128 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 129 while (dev_priv->error_completion == 0) { 130 ret = -lksleep(&dev_priv->error_completion, 131 &dev_priv->error_completion_lock, PCATCH, "915wco", 0); 132 if (ret != 0) { 133 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 134 return (ret); 135 } 136 } 137 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 138 139 if (atomic_load_acq_int(&dev_priv->mm.wedged)) { 140 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 141 dev_priv->error_completion++; 142 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 143 } 144 return (0); 145 } 146 147 int 148 i915_mutex_lock_interruptible(struct drm_device *dev) 149 { 150 struct drm_i915_private *dev_priv; 151 int ret; 152 153 dev_priv = dev->dev_private; 154 ret = i915_gem_wait_for_error(dev); 155 if (ret != 0) 156 return (ret); 157 158 /* 159 * interruptible shall it be. might indeed be if dev_lock is 160 * changed to sx 161 */ 162 ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL); 163 if (ret != 0) 164 return (-ret); 165 166 return (0); 167 } 168 169 170 static void 171 i915_gem_free_object_tail(struct drm_i915_gem_object *obj) 172 { 173 struct drm_device *dev; 174 drm_i915_private_t *dev_priv; 175 int ret; 176 177 dev = obj->base.dev; 178 dev_priv = dev->dev_private; 179 180 ret = i915_gem_object_unbind(obj); 181 if (ret == -ERESTART) { 182 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list); 183 return; 184 } 185 186 drm_gem_free_mmap_offset(&obj->base); 187 drm_gem_object_release(&obj->base); 188 i915_gem_info_remove_obj(dev_priv, obj->base.size); 189 190 drm_free(obj->page_cpu_valid, DRM_I915_GEM); 191 drm_free(obj->bit_17, DRM_I915_GEM); 192 drm_free(obj, DRM_I915_GEM); 193 } 194 195 void 196 i915_gem_free_object(struct drm_gem_object *gem_obj) 197 { 198 struct drm_i915_gem_object *obj; 199 struct drm_device *dev; 200 201 obj = to_intel_bo(gem_obj); 202 dev = obj->base.dev; 203 204 while (obj->pin_count > 0) 205 i915_gem_object_unpin(obj); 206 207 if (obj->phys_obj != NULL) 208 i915_gem_detach_phys_object(dev, obj); 209 210 i915_gem_free_object_tail(obj); 211 } 212 213 static void 214 init_ring_lists(struct intel_ring_buffer *ring) 215 { 216 217 INIT_LIST_HEAD(&ring->active_list); 218 INIT_LIST_HEAD(&ring->request_list); 219 INIT_LIST_HEAD(&ring->gpu_write_list); 220 } 221 222 void 223 i915_gem_load(struct drm_device *dev) 224 { 225 drm_i915_private_t *dev_priv; 226 int i; 227 228 dev_priv = dev->dev_private; 229 230 INIT_LIST_HEAD(&dev_priv->mm.active_list); 231 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 232 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 233 INIT_LIST_HEAD(&dev_priv->mm.pinned_list); 234 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 235 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 236 INIT_LIST_HEAD(&dev_priv->mm.gtt_list); 237 for (i = 0; i < I915_NUM_RINGS; i++) 238 init_ring_lists(&dev_priv->rings[i]); 239 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 240 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 241 TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0, 242 i915_gem_retire_task_handler, dev_priv); 243 dev_priv->error_completion = 0; 244 245 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 246 if (IS_GEN3(dev)) { 247 u32 tmp = I915_READ(MI_ARB_STATE); 248 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) { 249 /* 250 * arb state is a masked write, so set bit + 251 * bit in mask. 252 */ 253 tmp = MI_ARB_C3_LP_WRITE_ENABLE | 254 (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT); 255 I915_WRITE(MI_ARB_STATE, tmp); 256 } 257 } 258 259 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 260 261 /* Old X drivers will take 0-2 for front, back, depth buffers */ 262 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 263 dev_priv->fence_reg_start = 3; 264 265 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || 266 IS_G33(dev)) 267 dev_priv->num_fence_regs = 16; 268 else 269 dev_priv->num_fence_regs = 8; 270 271 /* Initialize fence registers to zero */ 272 for (i = 0; i < dev_priv->num_fence_regs; i++) { 273 i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]); 274 } 275 i915_gem_detect_bit_6_swizzle(dev); 276 dev_priv->mm.interruptible = true; 277 278 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, 279 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); 280 } 281 282 int 283 i915_gem_do_init(struct drm_device *dev, unsigned long start, 284 unsigned long mappable_end, unsigned long end) 285 { 286 drm_i915_private_t *dev_priv; 287 unsigned long mappable; 288 int error; 289 290 dev_priv = dev->dev_private; 291 mappable = min(end, mappable_end) - start; 292 293 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); 294 295 dev_priv->mm.gtt_start = start; 296 dev_priv->mm.gtt_mappable_end = mappable_end; 297 dev_priv->mm.gtt_end = end; 298 dev_priv->mm.gtt_total = end - start; 299 dev_priv->mm.mappable_gtt_total = mappable; 300 301 /* Take over this portion of the GTT */ 302 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); 303 device_printf(dev->device, 304 "taking over the fictitious range 0x%lx-0x%lx\n", 305 dev->agp->base + start, dev->agp->base + start + mappable); 306 error = -vm_phys_fictitious_reg_range(dev->agp->base + start, 307 dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING); 308 return (error); 309 } 310 311 int 312 i915_gem_init_ioctl(struct drm_device *dev, void *data, 313 struct drm_file *file) 314 { 315 struct drm_i915_gem_init *args; 316 drm_i915_private_t *dev_priv; 317 318 dev_priv = dev->dev_private; 319 args = data; 320 321 if (args->gtt_start >= args->gtt_end || 322 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 323 return (-EINVAL); 324 325 /* 326 * XXXKIB. The second-time initialization should be guarded 327 * against. 328 */ 329 lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE); 330 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end); 331 lockmgr(&dev->dev_lock, LK_RELEASE); 332 333 return 0; 334 } 335 336 int 337 i915_gem_idle(struct drm_device *dev) 338 { 339 drm_i915_private_t *dev_priv; 340 int ret; 341 342 dev_priv = dev->dev_private; 343 if (dev_priv->mm.suspended) 344 return (0); 345 346 ret = i915_gpu_idle(dev, true); 347 if (ret != 0) 348 return (ret); 349 350 /* Under UMS, be paranoid and evict. */ 351 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 352 ret = i915_gem_evict_inactive(dev, false); 353 if (ret != 0) 354 return ret; 355 } 356 357 i915_gem_reset_fences(dev); 358 359 /* Hack! Don't let anybody do execbuf while we don't control the chip. 360 * We need to replace this with a semaphore, or something. 361 * And not confound mm.suspended! 362 */ 363 dev_priv->mm.suspended = 1; 364 callout_stop(&dev_priv->hangcheck_timer); 365 366 i915_kernel_lost_context(dev); 367 i915_gem_cleanup_ringbuffer(dev); 368 369 /* Cancel the retire work handler, which should be idle now. */ 370 taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->mm.retire_task, NULL); 371 return (ret); 372 } 373 374 void 375 i915_gem_init_swizzling(struct drm_device *dev) 376 { 377 drm_i915_private_t *dev_priv; 378 379 dev_priv = dev->dev_private; 380 381 if (INTEL_INFO(dev)->gen < 5 || 382 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 383 return; 384 385 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 386 DISP_TILE_SURFACE_SWIZZLING); 387 388 if (IS_GEN5(dev)) 389 return; 390 391 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 392 if (IS_GEN6(dev)) 393 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_SNB)); 394 else 395 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_IVB)); 396 } 397 398 void 399 i915_gem_init_ppgtt(struct drm_device *dev) 400 { 401 drm_i915_private_t *dev_priv; 402 struct i915_hw_ppgtt *ppgtt; 403 uint32_t pd_offset, pd_entry; 404 vm_paddr_t pt_addr; 405 struct intel_ring_buffer *ring; 406 u_int first_pd_entry_in_global_pt, i; 407 408 dev_priv = dev->dev_private; 409 ppgtt = dev_priv->mm.aliasing_ppgtt; 410 if (ppgtt == NULL) 411 return; 412 413 first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; 414 for (i = 0; i < ppgtt->num_pd_entries; i++) { 415 pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); 416 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); 417 pd_entry |= GEN6_PDE_VALID; 418 intel_gtt_write(first_pd_entry_in_global_pt + i, pd_entry); 419 } 420 intel_gtt_read_pte(first_pd_entry_in_global_pt); 421 422 pd_offset = ppgtt->pd_offset; 423 pd_offset /= 64; /* in cachelines, */ 424 pd_offset <<= 16; 425 426 if (INTEL_INFO(dev)->gen == 6) { 427 uint32_t ecochk = I915_READ(GAM_ECOCHK); 428 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | 429 ECOCHK_PPGTT_CACHE64B); 430 I915_WRITE(GFX_MODE, GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 431 } else if (INTEL_INFO(dev)->gen >= 7) { 432 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); 433 /* GFX_MODE is per-ring on gen7+ */ 434 } 435 436 for (i = 0; i < I915_NUM_RINGS; i++) { 437 ring = &dev_priv->rings[i]; 438 439 if (INTEL_INFO(dev)->gen >= 7) 440 I915_WRITE(RING_MODE_GEN7(ring), 441 GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 442 443 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 444 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); 445 } 446 } 447 448 int 449 i915_gem_init_hw(struct drm_device *dev) 450 { 451 drm_i915_private_t *dev_priv; 452 int ret; 453 454 dev_priv = dev->dev_private; 455 456 i915_gem_init_swizzling(dev); 457 458 ret = intel_init_render_ring_buffer(dev); 459 if (ret != 0) 460 return (ret); 461 462 if (HAS_BSD(dev)) { 463 ret = intel_init_bsd_ring_buffer(dev); 464 if (ret != 0) 465 goto cleanup_render_ring; 466 } 467 468 if (HAS_BLT(dev)) { 469 ret = intel_init_blt_ring_buffer(dev); 470 if (ret != 0) 471 goto cleanup_bsd_ring; 472 } 473 474 dev_priv->next_seqno = 1; 475 i915_gem_init_ppgtt(dev); 476 return (0); 477 478 cleanup_bsd_ring: 479 intel_cleanup_ring_buffer(&dev_priv->rings[VCS]); 480 cleanup_render_ring: 481 intel_cleanup_ring_buffer(&dev_priv->rings[RCS]); 482 return (ret); 483 } 484 485 int 486 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 487 struct drm_file *file) 488 { 489 struct drm_i915_private *dev_priv; 490 struct drm_i915_gem_get_aperture *args; 491 struct drm_i915_gem_object *obj; 492 size_t pinned; 493 494 dev_priv = dev->dev_private; 495 args = data; 496 497 if (!(dev->driver->driver_features & DRIVER_GEM)) 498 return (-ENODEV); 499 500 pinned = 0; 501 DRM_LOCK(dev); 502 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) 503 pinned += obj->gtt_space->size; 504 DRM_UNLOCK(dev); 505 506 args->aper_size = dev_priv->mm.gtt_total; 507 args->aper_available_size = args->aper_size - pinned; 508 509 return (0); 510 } 511 512 int 513 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, 514 bool map_and_fenceable) 515 { 516 struct drm_device *dev; 517 struct drm_i915_private *dev_priv; 518 int ret; 519 520 dev = obj->base.dev; 521 dev_priv = dev->dev_private; 522 523 KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT, 524 ("Max pin count")); 525 526 if (obj->gtt_space != NULL) { 527 if ((alignment && obj->gtt_offset & (alignment - 1)) || 528 (map_and_fenceable && !obj->map_and_fenceable)) { 529 DRM_DEBUG("bo is already pinned with incorrect alignment:" 530 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 531 " obj->map_and_fenceable=%d\n", 532 obj->gtt_offset, alignment, 533 map_and_fenceable, 534 obj->map_and_fenceable); 535 ret = i915_gem_object_unbind(obj); 536 if (ret != 0) 537 return (ret); 538 } 539 } 540 541 if (obj->gtt_space == NULL) { 542 ret = i915_gem_object_bind_to_gtt(obj, alignment, 543 map_and_fenceable); 544 if (ret) 545 return (ret); 546 } 547 548 if (obj->pin_count++ == 0 && !obj->active) 549 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 550 obj->pin_mappable |= map_and_fenceable; 551 552 #if 1 553 KIB_NOTYET(); 554 #else 555 WARN_ON(i915_verify_lists(dev)); 556 #endif 557 return (0); 558 } 559 560 void 561 i915_gem_object_unpin(struct drm_i915_gem_object *obj) 562 { 563 struct drm_device *dev; 564 drm_i915_private_t *dev_priv; 565 566 dev = obj->base.dev; 567 dev_priv = dev->dev_private; 568 569 #if 1 570 KIB_NOTYET(); 571 #else 572 WARN_ON(i915_verify_lists(dev)); 573 #endif 574 575 KASSERT(obj->pin_count != 0, ("zero pin count")); 576 KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); 577 578 if (--obj->pin_count == 0) { 579 if (!obj->active) 580 list_move_tail(&obj->mm_list, 581 &dev_priv->mm.inactive_list); 582 obj->pin_mappable = false; 583 } 584 #if 1 585 KIB_NOTYET(); 586 #else 587 WARN_ON(i915_verify_lists(dev)); 588 #endif 589 } 590 591 int 592 i915_gem_pin_ioctl(struct drm_device *dev, void *data, 593 struct drm_file *file) 594 { 595 struct drm_i915_gem_pin *args; 596 struct drm_i915_gem_object *obj; 597 struct drm_gem_object *gobj; 598 int ret; 599 600 args = data; 601 602 ret = i915_mutex_lock_interruptible(dev); 603 if (ret != 0) 604 return ret; 605 606 gobj = drm_gem_object_lookup(dev, file, args->handle); 607 if (gobj == NULL) { 608 ret = -ENOENT; 609 goto unlock; 610 } 611 obj = to_intel_bo(gobj); 612 613 if (obj->madv != I915_MADV_WILLNEED) { 614 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 615 ret = -EINVAL; 616 goto out; 617 } 618 619 if (obj->pin_filp != NULL && obj->pin_filp != file) { 620 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 621 args->handle); 622 ret = -EINVAL; 623 goto out; 624 } 625 626 obj->user_pin_count++; 627 obj->pin_filp = file; 628 if (obj->user_pin_count == 1) { 629 ret = i915_gem_object_pin(obj, args->alignment, true); 630 if (ret != 0) 631 goto out; 632 } 633 634 /* XXX - flush the CPU caches for pinned objects 635 * as the X server doesn't manage domains yet 636 */ 637 i915_gem_object_flush_cpu_write_domain(obj); 638 args->offset = obj->gtt_offset; 639 out: 640 drm_gem_object_unreference(&obj->base); 641 unlock: 642 DRM_UNLOCK(dev); 643 return (ret); 644 } 645 646 int 647 i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 648 struct drm_file *file) 649 { 650 struct drm_i915_gem_pin *args; 651 struct drm_i915_gem_object *obj; 652 int ret; 653 654 args = data; 655 ret = i915_mutex_lock_interruptible(dev); 656 if (ret != 0) 657 return (ret); 658 659 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 660 if (&obj->base == NULL) { 661 ret = -ENOENT; 662 goto unlock; 663 } 664 665 if (obj->pin_filp != file) { 666 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 667 args->handle); 668 ret = -EINVAL; 669 goto out; 670 } 671 obj->user_pin_count--; 672 if (obj->user_pin_count == 0) { 673 obj->pin_filp = NULL; 674 i915_gem_object_unpin(obj); 675 } 676 677 out: 678 drm_gem_object_unreference(&obj->base); 679 unlock: 680 DRM_UNLOCK(dev); 681 return (ret); 682 } 683 684 int 685 i915_gem_busy_ioctl(struct drm_device *dev, void *data, 686 struct drm_file *file) 687 { 688 struct drm_i915_gem_busy *args; 689 struct drm_i915_gem_object *obj; 690 struct drm_i915_gem_request *request; 691 int ret; 692 693 args = data; 694 695 ret = i915_mutex_lock_interruptible(dev); 696 if (ret != 0) 697 return ret; 698 699 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 700 if (&obj->base == NULL) { 701 ret = -ENOENT; 702 goto unlock; 703 } 704 705 args->busy = obj->active; 706 if (args->busy) { 707 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 708 ret = i915_gem_flush_ring(obj->ring, 709 0, obj->base.write_domain); 710 } else if (obj->ring->outstanding_lazy_request == 711 obj->last_rendering_seqno) { 712 request = kmalloc(sizeof(*request), DRM_I915_GEM, 713 M_WAITOK | M_ZERO); 714 ret = i915_add_request(obj->ring, NULL, request); 715 if (ret != 0) 716 drm_free(request, DRM_I915_GEM); 717 } 718 719 i915_gem_retire_requests_ring(obj->ring); 720 args->busy = obj->active; 721 } 722 723 drm_gem_object_unreference(&obj->base); 724 unlock: 725 DRM_UNLOCK(dev); 726 return (ret); 727 } 728 729 /* Throttle our rendering by waiting until the ring has completed our requests 730 * emitted over 20 msec ago. 731 * 732 * Note that if we were to use the current jiffies each time around the loop, 733 * we wouldn't escape the function with any frames outstanding if the time to 734 * render a frame was over 20ms. 735 * 736 * This should get us reasonable parallelism between CPU and GPU but also 737 * relatively low latency when blocking on a particular request to finish. 738 */ 739 static int 740 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 741 { 742 struct drm_i915_private *dev_priv = dev->dev_private; 743 struct drm_i915_file_private *file_priv = file->driver_priv; 744 unsigned long recent_enough = ticks - (20 * hz / 1000); 745 struct drm_i915_gem_request *request; 746 struct intel_ring_buffer *ring = NULL; 747 u32 seqno = 0; 748 int ret; 749 750 dev_priv = dev->dev_private; 751 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 752 return (-EIO); 753 754 recent_enough = ticks - (20 * hz / 1000); 755 ring = NULL; 756 seqno = 0; 757 758 spin_lock(&file_priv->mm.lock); 759 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 760 if (time_after_eq(request->emitted_jiffies, recent_enough)) 761 break; 762 763 ring = request->ring; 764 seqno = request->seqno; 765 } 766 spin_unlock(&file_priv->mm.lock); 767 768 if (seqno == 0) 769 return 0; 770 771 ret = 0; 772 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 773 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 774 if (ring->irq_get(ring)) { 775 while (ret == 0 && 776 !(i915_seqno_passed(ring->get_seqno(ring), seqno) || 777 atomic_load_acq_int(&dev_priv->mm.wedged))) 778 ret = -lksleep(ring, &ring->irq_lock, PCATCH, 779 "915thr", 0); 780 ring->irq_put(ring); 781 if (ret == 0 && atomic_load_acq_int(&dev_priv->mm.wedged)) 782 ret = -EIO; 783 } else if (_intel_wait_for(dev, 784 i915_seqno_passed(ring->get_seqno(ring), seqno) || 785 atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 0, "915rtr")) { 786 ret = -EBUSY; 787 } 788 } 789 lockmgr(&ring->irq_lock, LK_RELEASE); 790 791 if (ret == 0) 792 taskqueue_enqueue_timeout(dev_priv->tq, 793 &dev_priv->mm.retire_task, 0); 794 795 return ret; 796 } 797 798 int 799 i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 800 struct drm_file *file_priv) 801 { 802 803 return (i915_gem_ring_throttle(dev, file_priv)); 804 } 805 806 int 807 i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 808 struct drm_file *file_priv) 809 { 810 struct drm_i915_gem_madvise *args; 811 struct drm_i915_gem_object *obj; 812 int ret; 813 814 args = data; 815 switch (args->madv) { 816 case I915_MADV_DONTNEED: 817 case I915_MADV_WILLNEED: 818 break; 819 default: 820 return (-EINVAL); 821 } 822 823 ret = i915_mutex_lock_interruptible(dev); 824 if (ret != 0) 825 return (ret); 826 827 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 828 if (&obj->base == NULL) { 829 ret = -ENOENT; 830 goto unlock; 831 } 832 833 if (obj->pin_count != 0) { 834 ret = -EINVAL; 835 goto out; 836 } 837 838 if (obj->madv != I915_MADV_PURGED_INTERNAL) 839 obj->madv = args->madv; 840 if (i915_gem_object_is_purgeable(obj) && obj->gtt_space == NULL) 841 i915_gem_object_truncate(obj); 842 args->retained = obj->madv != I915_MADV_PURGED_INTERNAL; 843 844 out: 845 drm_gem_object_unreference(&obj->base); 846 unlock: 847 DRM_UNLOCK(dev); 848 return (ret); 849 } 850 851 void 852 i915_gem_cleanup_ringbuffer(struct drm_device *dev) 853 { 854 drm_i915_private_t *dev_priv; 855 int i; 856 857 dev_priv = dev->dev_private; 858 for (i = 0; i < I915_NUM_RINGS; i++) 859 intel_cleanup_ring_buffer(&dev_priv->rings[i]); 860 } 861 862 int 863 i915_gem_entervt_ioctl(struct drm_device *dev, void *data, 864 struct drm_file *file_priv) 865 { 866 drm_i915_private_t *dev_priv; 867 int ret, i; 868 869 if (drm_core_check_feature(dev, DRIVER_MODESET)) 870 return (0); 871 dev_priv = dev->dev_private; 872 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 873 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 874 atomic_store_rel_int(&dev_priv->mm.wedged, 0); 875 } 876 877 dev_priv->mm.suspended = 0; 878 879 ret = i915_gem_init_hw(dev); 880 if (ret != 0) { 881 return (ret); 882 } 883 884 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list")); 885 KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list")); 886 KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list")); 887 for (i = 0; i < I915_NUM_RINGS; i++) { 888 KASSERT(list_empty(&dev_priv->rings[i].active_list), 889 ("ring %d active list", i)); 890 KASSERT(list_empty(&dev_priv->rings[i].request_list), 891 ("ring %d request list", i)); 892 } 893 894 DRM_UNLOCK(dev); 895 ret = drm_irq_install(dev); 896 DRM_LOCK(dev); 897 if (ret) 898 goto cleanup_ringbuffer; 899 900 return (0); 901 902 cleanup_ringbuffer: 903 i915_gem_cleanup_ringbuffer(dev); 904 dev_priv->mm.suspended = 1; 905 906 return (ret); 907 } 908 909 int 910 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 911 struct drm_file *file_priv) 912 { 913 914 if (drm_core_check_feature(dev, DRIVER_MODESET)) 915 return 0; 916 917 drm_irq_uninstall(dev); 918 return (i915_gem_idle(dev)); 919 } 920 921 int 922 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, 923 uint32_t *handle_p) 924 { 925 struct drm_i915_gem_object *obj; 926 uint32_t handle; 927 int ret; 928 929 size = roundup(size, PAGE_SIZE); 930 if (size == 0) 931 return (-EINVAL); 932 933 obj = i915_gem_alloc_object(dev, size); 934 if (obj == NULL) 935 return (-ENOMEM); 936 937 handle = 0; 938 ret = drm_gem_handle_create(file, &obj->base, &handle); 939 if (ret != 0) { 940 drm_gem_object_release(&obj->base); 941 i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 942 drm_free(obj, DRM_I915_GEM); 943 return (-ret); 944 } 945 946 /* drop reference from allocate - handle holds it now */ 947 drm_gem_object_unreference(&obj->base); 948 *handle_p = handle; 949 return (0); 950 } 951 952 int 953 i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 954 struct drm_mode_create_dumb *args) 955 { 956 957 /* have to work out size/pitch and return them */ 958 args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 959 args->size = args->pitch * args->height; 960 return (i915_gem_create(file, dev, args->size, &args->handle)); 961 } 962 963 int 964 i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, 965 uint32_t handle) 966 { 967 968 return (drm_gem_handle_delete(file, handle)); 969 } 970 971 int 972 i915_gem_create_ioctl(struct drm_device *dev, void *data, 973 struct drm_file *file) 974 { 975 struct drm_i915_gem_create *args = data; 976 977 return (i915_gem_create(file, dev, args->size, &args->handle)); 978 } 979 980 static inline void vm_page_reference(vm_page_t m) 981 { 982 vm_page_flag_set(m, PG_REFERENCED); 983 } 984 985 static int 986 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj, 987 uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw, 988 struct drm_file *file) 989 { 990 vm_object_t vm_obj; 991 vm_page_t m; 992 struct sf_buf *sf; 993 vm_offset_t mkva; 994 vm_pindex_t obj_pi; 995 int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po; 996 997 if (obj->gtt_offset != 0 && rw == UIO_READ) 998 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 999 else 1000 do_bit17_swizzling = 0; 1001 1002 obj->dirty = 1; 1003 vm_obj = obj->base.vm_obj; 1004 ret = 0; 1005 1006 VM_OBJECT_LOCK(vm_obj); 1007 vm_object_pip_add(vm_obj, 1); 1008 while (size > 0) { 1009 obj_pi = OFF_TO_IDX(offset); 1010 obj_po = offset & PAGE_MASK; 1011 1012 m = i915_gem_wire_page(vm_obj, obj_pi); 1013 VM_OBJECT_UNLOCK(vm_obj); 1014 1015 sf = sf_buf_alloc(m); 1016 mkva = sf_buf_kva(sf); 1017 length = min(size, PAGE_SIZE - obj_po); 1018 while (length > 0) { 1019 if (do_bit17_swizzling && 1020 (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) { 1021 cnt = roundup2(obj_po + 1, 64); 1022 cnt = min(cnt - obj_po, length); 1023 swizzled_po = obj_po ^ 64; 1024 } else { 1025 cnt = length; 1026 swizzled_po = obj_po; 1027 } 1028 if (rw == UIO_READ) 1029 ret = -copyout_nofault( 1030 (char *)mkva + swizzled_po, 1031 (void *)(uintptr_t)data_ptr, cnt); 1032 else 1033 ret = -copyin_nofault( 1034 (void *)(uintptr_t)data_ptr, 1035 (char *)mkva + swizzled_po, cnt); 1036 if (ret != 0) 1037 break; 1038 data_ptr += cnt; 1039 size -= cnt; 1040 length -= cnt; 1041 offset += cnt; 1042 obj_po += cnt; 1043 } 1044 sf_buf_free(sf); 1045 VM_OBJECT_LOCK(vm_obj); 1046 if (rw == UIO_WRITE) 1047 vm_page_dirty(m); 1048 vm_page_reference(m); 1049 vm_page_busy_wait(m, FALSE, "i915gem"); 1050 vm_page_unwire(m, 1); 1051 vm_page_wakeup(m); 1052 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 1053 1054 if (ret != 0) 1055 break; 1056 } 1057 vm_object_pip_wakeup(vm_obj); 1058 VM_OBJECT_UNLOCK(vm_obj); 1059 1060 return (ret); 1061 } 1062 1063 static int 1064 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj, 1065 uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file) 1066 { 1067 vm_offset_t mkva; 1068 int ret; 1069 1070 /* 1071 * Pass the unaligned physical address and size to pmap_mapdev_attr() 1072 * so it can properly calculate whether an extra page needs to be 1073 * mapped or not to cover the requested range. The function will 1074 * add the page offset into the returned mkva for us. 1075 */ 1076 mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset + 1077 offset, size, PAT_WRITE_COMBINING); 1078 ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size); 1079 pmap_unmapdev(mkva, size); 1080 return (ret); 1081 } 1082 1083 static int 1084 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr, 1085 uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file) 1086 { 1087 struct drm_i915_gem_object *obj; 1088 vm_page_t *ma; 1089 vm_offset_t start, end; 1090 int npages, ret; 1091 1092 if (size == 0) 1093 return (0); 1094 start = trunc_page(data_ptr); 1095 end = round_page(data_ptr + size); 1096 npages = howmany(end - start, PAGE_SIZE); 1097 ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK | 1098 M_ZERO); 1099 npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, 1100 (vm_offset_t)data_ptr, size, 1101 (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages); 1102 if (npages == -1) { 1103 ret = -EFAULT; 1104 goto free_ma; 1105 } 1106 1107 ret = i915_mutex_lock_interruptible(dev); 1108 if (ret != 0) 1109 goto unlocked; 1110 1111 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1112 if (&obj->base == NULL) { 1113 ret = -ENOENT; 1114 goto unlock; 1115 } 1116 if (offset > obj->base.size || size > obj->base.size - offset) { 1117 ret = -EINVAL; 1118 goto out; 1119 } 1120 1121 if (rw == UIO_READ) { 1122 ret = i915_gem_object_set_cpu_read_domain_range(obj, 1123 offset, size); 1124 if (ret != 0) 1125 goto out; 1126 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1127 UIO_READ, file); 1128 } else { 1129 if (obj->phys_obj) { 1130 ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset, 1131 size, file); 1132 } else if (obj->gtt_space && 1133 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1134 ret = i915_gem_object_pin(obj, 0, true); 1135 if (ret != 0) 1136 goto out; 1137 ret = i915_gem_object_set_to_gtt_domain(obj, true); 1138 if (ret != 0) 1139 goto out_unpin; 1140 ret = i915_gem_object_put_fence(obj); 1141 if (ret != 0) 1142 goto out_unpin; 1143 ret = i915_gem_gtt_write(dev, obj, data_ptr, size, 1144 offset, file); 1145 out_unpin: 1146 i915_gem_object_unpin(obj); 1147 } else { 1148 ret = i915_gem_object_set_to_cpu_domain(obj, true); 1149 if (ret != 0) 1150 goto out; 1151 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1152 UIO_WRITE, file); 1153 } 1154 } 1155 out: 1156 drm_gem_object_unreference(&obj->base); 1157 unlock: 1158 DRM_UNLOCK(dev); 1159 unlocked: 1160 vm_page_unhold_pages(ma, npages); 1161 free_ma: 1162 drm_free(ma, DRM_I915_GEM); 1163 return (ret); 1164 } 1165 1166 int 1167 i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1168 { 1169 struct drm_i915_gem_pread *args; 1170 1171 args = data; 1172 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1173 args->offset, UIO_READ, file)); 1174 } 1175 1176 int 1177 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1178 { 1179 struct drm_i915_gem_pwrite *args; 1180 1181 args = data; 1182 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1183 args->offset, UIO_WRITE, file)); 1184 } 1185 1186 int 1187 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1188 struct drm_file *file) 1189 { 1190 struct drm_i915_gem_set_domain *args; 1191 struct drm_i915_gem_object *obj; 1192 uint32_t read_domains; 1193 uint32_t write_domain; 1194 int ret; 1195 1196 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1197 return (-ENODEV); 1198 1199 args = data; 1200 read_domains = args->read_domains; 1201 write_domain = args->write_domain; 1202 1203 if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 || 1204 (read_domains & I915_GEM_GPU_DOMAINS) != 0 || 1205 (write_domain != 0 && read_domains != write_domain)) 1206 return (-EINVAL); 1207 1208 ret = i915_mutex_lock_interruptible(dev); 1209 if (ret != 0) 1210 return (ret); 1211 1212 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1213 if (&obj->base == NULL) { 1214 ret = -ENOENT; 1215 goto unlock; 1216 } 1217 1218 if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) { 1219 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1220 if (ret == -EINVAL) 1221 ret = 0; 1222 } else 1223 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1224 1225 drm_gem_object_unreference(&obj->base); 1226 unlock: 1227 DRM_UNLOCK(dev); 1228 return (ret); 1229 } 1230 1231 int 1232 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1233 struct drm_file *file) 1234 { 1235 struct drm_i915_gem_sw_finish *args; 1236 struct drm_i915_gem_object *obj; 1237 int ret; 1238 1239 args = data; 1240 ret = 0; 1241 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1242 return (ENODEV); 1243 ret = i915_mutex_lock_interruptible(dev); 1244 if (ret != 0) 1245 return (ret); 1246 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1247 if (&obj->base == NULL) { 1248 ret = -ENOENT; 1249 goto unlock; 1250 } 1251 if (obj->pin_count != 0) 1252 i915_gem_object_flush_cpu_write_domain(obj); 1253 drm_gem_object_unreference(&obj->base); 1254 unlock: 1255 DRM_UNLOCK(dev); 1256 return (ret); 1257 } 1258 1259 int 1260 i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1261 struct drm_file *file) 1262 { 1263 struct drm_i915_gem_mmap *args; 1264 struct drm_gem_object *obj; 1265 struct proc *p; 1266 vm_map_t map; 1267 vm_offset_t addr; 1268 vm_size_t size; 1269 int error, rv; 1270 1271 args = data; 1272 1273 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1274 return (-ENODEV); 1275 1276 obj = drm_gem_object_lookup(dev, file, args->handle); 1277 if (obj == NULL) 1278 return (-ENOENT); 1279 error = 0; 1280 if (args->size == 0) 1281 goto out; 1282 p = curproc; 1283 map = &p->p_vmspace->vm_map; 1284 size = round_page(args->size); 1285 PROC_LOCK(p); 1286 if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) { 1287 PROC_UNLOCK(p); 1288 error = ENOMEM; 1289 goto out; 1290 } 1291 PROC_UNLOCK(p); 1292 1293 addr = 0; 1294 vm_object_hold(obj->vm_obj); 1295 vm_object_reference_locked(obj->vm_obj); 1296 vm_object_drop(obj->vm_obj); 1297 DRM_UNLOCK(dev); 1298 rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 1299 PAGE_SIZE, /* align */ 1300 TRUE, /* fitit */ 1301 VM_MAPTYPE_NORMAL, /* maptype */ 1302 VM_PROT_READ | VM_PROT_WRITE, /* prot */ 1303 VM_PROT_READ | VM_PROT_WRITE, /* max */ 1304 MAP_SHARED /* cow */); 1305 if (rv != KERN_SUCCESS) { 1306 vm_object_deallocate(obj->vm_obj); 1307 error = -vm_mmap_to_errno(rv); 1308 } else { 1309 args->addr_ptr = (uint64_t)addr; 1310 } 1311 DRM_LOCK(dev); 1312 out: 1313 drm_gem_object_unreference(obj); 1314 return (error); 1315 } 1316 1317 static int 1318 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 1319 vm_ooffset_t foff, struct ucred *cred, u_short *color) 1320 { 1321 1322 *color = 0; /* XXXKIB */ 1323 return (0); 1324 } 1325 1326 int i915_intr_pf; 1327 1328 static int 1329 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 1330 vm_page_t *mres) 1331 { 1332 struct drm_gem_object *gem_obj; 1333 struct drm_i915_gem_object *obj; 1334 struct drm_device *dev; 1335 drm_i915_private_t *dev_priv; 1336 vm_page_t m, oldm; 1337 int cause, ret; 1338 bool write; 1339 1340 gem_obj = vm_obj->handle; 1341 obj = to_intel_bo(gem_obj); 1342 dev = obj->base.dev; 1343 dev_priv = dev->dev_private; 1344 #if 0 1345 write = (prot & VM_PROT_WRITE) != 0; 1346 #else 1347 write = true; 1348 #endif 1349 vm_object_pip_add(vm_obj, 1); 1350 1351 /* 1352 * Remove the placeholder page inserted by vm_fault() from the 1353 * object before dropping the object lock. If 1354 * i915_gem_release_mmap() is active in parallel on this gem 1355 * object, then it owns the drm device sx and might find the 1356 * placeholder already. Then, since the page is busy, 1357 * i915_gem_release_mmap() sleeps waiting for the busy state 1358 * of the page cleared. We will be not able to acquire drm 1359 * device lock until i915_gem_release_mmap() is able to make a 1360 * progress. 1361 */ 1362 if (*mres != NULL) { 1363 oldm = *mres; 1364 vm_page_remove(oldm); 1365 *mres = NULL; 1366 } else 1367 oldm = NULL; 1368 retry: 1369 VM_OBJECT_UNLOCK(vm_obj); 1370 unlocked_vmobj: 1371 cause = ret = 0; 1372 m = NULL; 1373 1374 if (i915_intr_pf) { 1375 ret = i915_mutex_lock_interruptible(dev); 1376 if (ret != 0) { 1377 cause = 10; 1378 goto out; 1379 } 1380 } else 1381 DRM_LOCK(dev); 1382 1383 /* 1384 * Since the object lock was dropped, other thread might have 1385 * faulted on the same GTT address and instantiated the 1386 * mapping for the page. Recheck. 1387 */ 1388 VM_OBJECT_LOCK(vm_obj); 1389 m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 1390 if (m != NULL) { 1391 if ((m->flags & PG_BUSY) != 0) { 1392 DRM_UNLOCK(dev); 1393 #if 0 /* XXX */ 1394 vm_page_sleep(m, "915pee"); 1395 #endif 1396 goto retry; 1397 } 1398 goto have_page; 1399 } else 1400 VM_OBJECT_UNLOCK(vm_obj); 1401 1402 /* Now bind it into the GTT if needed */ 1403 if (!obj->map_and_fenceable) { 1404 ret = i915_gem_object_unbind(obj); 1405 if (ret != 0) { 1406 cause = 20; 1407 goto unlock; 1408 } 1409 } 1410 if (!obj->gtt_space) { 1411 ret = i915_gem_object_bind_to_gtt(obj, 0, true); 1412 if (ret != 0) { 1413 cause = 30; 1414 goto unlock; 1415 } 1416 1417 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1418 if (ret != 0) { 1419 cause = 40; 1420 goto unlock; 1421 } 1422 } 1423 1424 if (obj->tiling_mode == I915_TILING_NONE) 1425 ret = i915_gem_object_put_fence(obj); 1426 else 1427 ret = i915_gem_object_get_fence(obj, NULL); 1428 if (ret != 0) { 1429 cause = 50; 1430 goto unlock; 1431 } 1432 1433 if (i915_gem_object_is_inactive(obj)) 1434 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 1435 1436 obj->fault_mappable = true; 1437 VM_OBJECT_LOCK(vm_obj); 1438 m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset + 1439 offset); 1440 if (m == NULL) { 1441 cause = 60; 1442 ret = -EFAULT; 1443 goto unlock; 1444 } 1445 KASSERT((m->flags & PG_FICTITIOUS) != 0, 1446 ("not fictitious %p", m)); 1447 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 1448 1449 if ((m->flags & PG_BUSY) != 0) { 1450 DRM_UNLOCK(dev); 1451 #if 0 /* XXX */ 1452 vm_page_sleep(m, "915pbs"); 1453 #endif 1454 goto retry; 1455 } 1456 m->valid = VM_PAGE_BITS_ALL; 1457 vm_page_insert(m, vm_obj, OFF_TO_IDX(offset)); 1458 have_page: 1459 *mres = m; 1460 vm_page_busy_try(m, false); 1461 1462 DRM_UNLOCK(dev); 1463 if (oldm != NULL) { 1464 vm_page_free(oldm); 1465 } 1466 vm_object_pip_wakeup(vm_obj); 1467 return (VM_PAGER_OK); 1468 1469 unlock: 1470 DRM_UNLOCK(dev); 1471 out: 1472 KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 1473 if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 1474 goto unlocked_vmobj; 1475 } 1476 VM_OBJECT_LOCK(vm_obj); 1477 vm_object_pip_wakeup(vm_obj); 1478 return (VM_PAGER_ERROR); 1479 } 1480 1481 static void 1482 i915_gem_pager_dtor(void *handle) 1483 { 1484 struct drm_gem_object *obj; 1485 struct drm_device *dev; 1486 1487 obj = handle; 1488 dev = obj->dev; 1489 1490 DRM_LOCK(dev); 1491 drm_gem_free_mmap_offset(obj); 1492 i915_gem_release_mmap(to_intel_bo(obj)); 1493 drm_gem_object_unreference(obj); 1494 DRM_UNLOCK(dev); 1495 } 1496 1497 struct cdev_pager_ops i915_gem_pager_ops = { 1498 .cdev_pg_fault = i915_gem_pager_fault, 1499 .cdev_pg_ctor = i915_gem_pager_ctor, 1500 .cdev_pg_dtor = i915_gem_pager_dtor 1501 }; 1502 1503 int 1504 i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev, 1505 uint32_t handle, uint64_t *offset) 1506 { 1507 struct drm_i915_private *dev_priv; 1508 struct drm_i915_gem_object *obj; 1509 int ret; 1510 1511 if (!(dev->driver->driver_features & DRIVER_GEM)) 1512 return (-ENODEV); 1513 1514 dev_priv = dev->dev_private; 1515 1516 ret = i915_mutex_lock_interruptible(dev); 1517 if (ret != 0) 1518 return (ret); 1519 1520 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1521 if (&obj->base == NULL) { 1522 ret = -ENOENT; 1523 goto unlock; 1524 } 1525 1526 if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 1527 ret = -E2BIG; 1528 goto out; 1529 } 1530 1531 if (obj->madv != I915_MADV_WILLNEED) { 1532 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1533 ret = -EINVAL; 1534 goto out; 1535 } 1536 1537 ret = drm_gem_create_mmap_offset(&obj->base); 1538 if (ret != 0) 1539 goto out; 1540 1541 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 1542 DRM_GEM_MAPPING_KEY; 1543 out: 1544 drm_gem_object_unreference(&obj->base); 1545 unlock: 1546 DRM_UNLOCK(dev); 1547 return (ret); 1548 } 1549 1550 int 1551 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1552 struct drm_file *file) 1553 { 1554 struct drm_i915_private *dev_priv; 1555 struct drm_i915_gem_mmap_gtt *args; 1556 1557 dev_priv = dev->dev_private; 1558 args = data; 1559 1560 return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset)); 1561 } 1562 1563 struct drm_i915_gem_object * 1564 i915_gem_alloc_object(struct drm_device *dev, size_t size) 1565 { 1566 struct drm_i915_private *dev_priv; 1567 struct drm_i915_gem_object *obj; 1568 1569 dev_priv = dev->dev_private; 1570 1571 obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 1572 1573 if (drm_gem_object_init(dev, &obj->base, size) != 0) { 1574 drm_free(obj, DRM_I915_GEM); 1575 return (NULL); 1576 } 1577 1578 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1579 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1580 1581 if (HAS_LLC(dev)) 1582 obj->cache_level = I915_CACHE_LLC; 1583 else 1584 obj->cache_level = I915_CACHE_NONE; 1585 obj->base.driver_private = NULL; 1586 obj->fence_reg = I915_FENCE_REG_NONE; 1587 INIT_LIST_HEAD(&obj->mm_list); 1588 INIT_LIST_HEAD(&obj->gtt_list); 1589 INIT_LIST_HEAD(&obj->ring_list); 1590 INIT_LIST_HEAD(&obj->exec_list); 1591 INIT_LIST_HEAD(&obj->gpu_write_list); 1592 obj->madv = I915_MADV_WILLNEED; 1593 /* Avoid an unnecessary call to unbind on the first bind. */ 1594 obj->map_and_fenceable = true; 1595 1596 i915_gem_info_add_obj(dev_priv, size); 1597 1598 return (obj); 1599 } 1600 1601 void 1602 i915_gem_clflush_object(struct drm_i915_gem_object *obj) 1603 { 1604 1605 /* If we don't have a page list set up, then we're not pinned 1606 * to GPU, and we can ignore the cache flush because it'll happen 1607 * again at bind time. 1608 */ 1609 if (obj->pages == NULL) 1610 return; 1611 1612 /* If the GPU is snooping the contents of the CPU cache, 1613 * we do not need to manually clear the CPU cache lines. However, 1614 * the caches are only snooped when the render cache is 1615 * flushed/invalidated. As we always have to emit invalidations 1616 * and flushes when moving into and out of the RENDER domain, correct 1617 * snooping behaviour occurs naturally as the result of our domain 1618 * tracking. 1619 */ 1620 if (obj->cache_level != I915_CACHE_NONE) 1621 return; 1622 1623 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 1624 } 1625 1626 static void 1627 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 1628 { 1629 uint32_t old_write_domain; 1630 1631 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 1632 return; 1633 1634 i915_gem_clflush_object(obj); 1635 intel_gtt_chipset_flush(); 1636 old_write_domain = obj->base.write_domain; 1637 obj->base.write_domain = 0; 1638 } 1639 1640 static int 1641 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) 1642 { 1643 1644 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) 1645 return (0); 1646 return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain)); 1647 } 1648 1649 static void 1650 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 1651 { 1652 uint32_t old_write_domain; 1653 1654 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 1655 return; 1656 1657 cpu_sfence(); 1658 1659 old_write_domain = obj->base.write_domain; 1660 obj->base.write_domain = 0; 1661 } 1662 1663 int 1664 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 1665 { 1666 uint32_t old_write_domain, old_read_domains; 1667 int ret; 1668 1669 if (obj->gtt_space == NULL) 1670 return (-EINVAL); 1671 1672 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 1673 return 0; 1674 1675 ret = i915_gem_object_flush_gpu_write_domain(obj); 1676 if (ret != 0) 1677 return (ret); 1678 1679 if (obj->pending_gpu_write || write) { 1680 ret = i915_gem_object_wait_rendering(obj); 1681 if (ret != 0) 1682 return (ret); 1683 } 1684 1685 i915_gem_object_flush_cpu_write_domain(obj); 1686 1687 old_write_domain = obj->base.write_domain; 1688 old_read_domains = obj->base.read_domains; 1689 1690 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1691 ("In GTT write domain")); 1692 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1693 if (write) { 1694 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 1695 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 1696 obj->dirty = 1; 1697 } 1698 1699 return (0); 1700 } 1701 1702 int 1703 i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 1704 enum i915_cache_level cache_level) 1705 { 1706 struct drm_device *dev; 1707 drm_i915_private_t *dev_priv; 1708 int ret; 1709 1710 if (obj->cache_level == cache_level) 1711 return 0; 1712 1713 if (obj->pin_count) { 1714 DRM_DEBUG("can not change the cache level of pinned objects\n"); 1715 return (-EBUSY); 1716 } 1717 1718 dev = obj->base.dev; 1719 dev_priv = dev->dev_private; 1720 if (obj->gtt_space) { 1721 ret = i915_gem_object_finish_gpu(obj); 1722 if (ret != 0) 1723 return (ret); 1724 1725 i915_gem_object_finish_gtt(obj); 1726 1727 /* Before SandyBridge, you could not use tiling or fence 1728 * registers with snooped memory, so relinquish any fences 1729 * currently pointing to our region in the aperture. 1730 */ 1731 if (INTEL_INFO(obj->base.dev)->gen < 6) { 1732 ret = i915_gem_object_put_fence(obj); 1733 if (ret != 0) 1734 return (ret); 1735 } 1736 1737 i915_gem_gtt_rebind_object(obj, cache_level); 1738 if (obj->has_aliasing_ppgtt_mapping) 1739 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 1740 obj, cache_level); 1741 } 1742 1743 if (cache_level == I915_CACHE_NONE) { 1744 u32 old_read_domains, old_write_domain; 1745 1746 /* If we're coming from LLC cached, then we haven't 1747 * actually been tracking whether the data is in the 1748 * CPU cache or not, since we only allow one bit set 1749 * in obj->write_domain and have been skipping the clflushes. 1750 * Just set it to the CPU cache for now. 1751 */ 1752 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1753 ("obj %p in CPU write domain", obj)); 1754 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0, 1755 ("obj %p in CPU read domain", obj)); 1756 1757 old_read_domains = obj->base.read_domains; 1758 old_write_domain = obj->base.write_domain; 1759 1760 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1761 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1762 1763 } 1764 1765 obj->cache_level = cache_level; 1766 return (0); 1767 } 1768 1769 int 1770 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 1771 u32 alignment, struct intel_ring_buffer *pipelined) 1772 { 1773 u32 old_read_domains, old_write_domain; 1774 int ret; 1775 1776 ret = i915_gem_object_flush_gpu_write_domain(obj); 1777 if (ret != 0) 1778 return (ret); 1779 1780 if (pipelined != obj->ring) { 1781 ret = i915_gem_object_wait_rendering(obj); 1782 if (ret == -ERESTART || ret == -EINTR) 1783 return (ret); 1784 } 1785 1786 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 1787 if (ret != 0) 1788 return (ret); 1789 1790 ret = i915_gem_object_pin(obj, alignment, true); 1791 if (ret != 0) 1792 return (ret); 1793 1794 i915_gem_object_flush_cpu_write_domain(obj); 1795 1796 old_write_domain = obj->base.write_domain; 1797 old_read_domains = obj->base.read_domains; 1798 1799 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1800 ("obj %p in GTT write domain", obj)); 1801 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1802 1803 return (0); 1804 } 1805 1806 int 1807 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 1808 { 1809 int ret; 1810 1811 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 1812 return (0); 1813 1814 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 1815 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); 1816 if (ret != 0) 1817 return (ret); 1818 } 1819 1820 ret = i915_gem_object_wait_rendering(obj); 1821 if (ret != 0) 1822 return (ret); 1823 1824 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 1825 1826 return (0); 1827 } 1828 1829 static int 1830 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 1831 { 1832 uint32_t old_write_domain, old_read_domains; 1833 int ret; 1834 1835 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 1836 return 0; 1837 1838 ret = i915_gem_object_flush_gpu_write_domain(obj); 1839 if (ret != 0) 1840 return (ret); 1841 1842 ret = i915_gem_object_wait_rendering(obj); 1843 if (ret != 0) 1844 return (ret); 1845 1846 i915_gem_object_flush_gtt_write_domain(obj); 1847 i915_gem_object_set_to_full_cpu_read_domain(obj); 1848 1849 old_write_domain = obj->base.write_domain; 1850 old_read_domains = obj->base.read_domains; 1851 1852 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 1853 i915_gem_clflush_object(obj); 1854 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1855 } 1856 1857 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1858 ("In cpu write domain")); 1859 1860 if (write) { 1861 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1862 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1863 } 1864 1865 return (0); 1866 } 1867 1868 static void 1869 i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj) 1870 { 1871 int i; 1872 1873 if (obj->page_cpu_valid == NULL) 1874 return; 1875 1876 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) { 1877 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) { 1878 if (obj->page_cpu_valid[i] != 0) 1879 continue; 1880 drm_clflush_pages(obj->pages + i, 1); 1881 } 1882 } 1883 1884 drm_free(obj->page_cpu_valid, DRM_I915_GEM); 1885 obj->page_cpu_valid = NULL; 1886 } 1887 1888 static int 1889 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, 1890 uint64_t offset, uint64_t size) 1891 { 1892 uint32_t old_read_domains; 1893 int i, ret; 1894 1895 if (offset == 0 && size == obj->base.size) 1896 return (i915_gem_object_set_to_cpu_domain(obj, 0)); 1897 1898 ret = i915_gem_object_flush_gpu_write_domain(obj); 1899 if (ret != 0) 1900 return (ret); 1901 ret = i915_gem_object_wait_rendering(obj); 1902 if (ret != 0) 1903 return (ret); 1904 1905 i915_gem_object_flush_gtt_write_domain(obj); 1906 1907 if (obj->page_cpu_valid == NULL && 1908 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) 1909 return (0); 1910 1911 if (obj->page_cpu_valid == NULL) { 1912 obj->page_cpu_valid = kmalloc(obj->base.size / PAGE_SIZE, 1913 DRM_I915_GEM, M_WAITOK | M_ZERO); 1914 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 1915 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE); 1916 1917 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 1918 i++) { 1919 if (obj->page_cpu_valid[i]) 1920 continue; 1921 drm_clflush_pages(obj->pages + i, 1); 1922 obj->page_cpu_valid[i] = 1; 1923 } 1924 1925 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1926 ("In gpu write domain")); 1927 1928 old_read_domains = obj->base.read_domains; 1929 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1930 1931 return (0); 1932 } 1933 1934 static uint32_t 1935 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 1936 { 1937 uint32_t gtt_size; 1938 1939 if (INTEL_INFO(dev)->gen >= 4 || 1940 tiling_mode == I915_TILING_NONE) 1941 return (size); 1942 1943 /* Previous chips need a power-of-two fence region when tiling */ 1944 if (INTEL_INFO(dev)->gen == 3) 1945 gtt_size = 1024*1024; 1946 else 1947 gtt_size = 512*1024; 1948 1949 while (gtt_size < size) 1950 gtt_size <<= 1; 1951 1952 return (gtt_size); 1953 } 1954 1955 /** 1956 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 1957 * @obj: object to check 1958 * 1959 * Return the required GTT alignment for an object, taking into account 1960 * potential fence register mapping. 1961 */ 1962 static uint32_t 1963 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, 1964 int tiling_mode) 1965 { 1966 1967 /* 1968 * Minimum alignment is 4k (GTT page size), but might be greater 1969 * if a fence register is needed for the object. 1970 */ 1971 if (INTEL_INFO(dev)->gen >= 4 || 1972 tiling_mode == I915_TILING_NONE) 1973 return (4096); 1974 1975 /* 1976 * Previous chips need to be aligned to the size of the smallest 1977 * fence register that can contain the object. 1978 */ 1979 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 1980 } 1981 1982 uint32_t 1983 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, 1984 int tiling_mode) 1985 { 1986 1987 if (tiling_mode == I915_TILING_NONE) 1988 return (4096); 1989 1990 /* 1991 * Minimum alignment is 4k (GTT page size) for sane hw. 1992 */ 1993 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev)) 1994 return (4096); 1995 1996 /* 1997 * Previous hardware however needs to be aligned to a power-of-two 1998 * tile height. The simplest method for determining this is to reuse 1999 * the power-of-tile object size. 2000 */ 2001 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 2002 } 2003 2004 static int 2005 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 2006 unsigned alignment, bool map_and_fenceable) 2007 { 2008 struct drm_device *dev; 2009 struct drm_i915_private *dev_priv; 2010 struct drm_mm_node *free_space; 2011 uint32_t size, fence_size, fence_alignment, unfenced_alignment; 2012 bool mappable, fenceable; 2013 int ret; 2014 2015 dev = obj->base.dev; 2016 dev_priv = dev->dev_private; 2017 2018 if (obj->madv != I915_MADV_WILLNEED) { 2019 DRM_ERROR("Attempting to bind a purgeable object\n"); 2020 return (-EINVAL); 2021 } 2022 2023 fence_size = i915_gem_get_gtt_size(dev, obj->base.size, 2024 obj->tiling_mode); 2025 fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size, 2026 obj->tiling_mode); 2027 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev, 2028 obj->base.size, obj->tiling_mode); 2029 if (alignment == 0) 2030 alignment = map_and_fenceable ? fence_alignment : 2031 unfenced_alignment; 2032 if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) { 2033 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2034 return (-EINVAL); 2035 } 2036 2037 size = map_and_fenceable ? fence_size : obj->base.size; 2038 2039 /* If the object is bigger than the entire aperture, reject it early 2040 * before evicting everything in a vain attempt to find space. 2041 */ 2042 if (obj->base.size > (map_and_fenceable ? 2043 dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 2044 DRM_ERROR( 2045 "Attempting to bind an object larger than the aperture\n"); 2046 return (-E2BIG); 2047 } 2048 2049 search_free: 2050 if (map_and_fenceable) 2051 free_space = drm_mm_search_free_in_range( 2052 &dev_priv->mm.gtt_space, size, alignment, 0, 2053 dev_priv->mm.gtt_mappable_end, 0); 2054 else 2055 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2056 size, alignment, 0); 2057 if (free_space != NULL) { 2058 if (map_and_fenceable) 2059 obj->gtt_space = drm_mm_get_block_range_generic( 2060 free_space, size, alignment, 0, 2061 dev_priv->mm.gtt_mappable_end, 1); 2062 else 2063 obj->gtt_space = drm_mm_get_block_generic(free_space, 2064 size, alignment, 1); 2065 } 2066 if (obj->gtt_space == NULL) { 2067 ret = i915_gem_evict_something(dev, size, alignment, 2068 map_and_fenceable); 2069 if (ret != 0) 2070 return (ret); 2071 goto search_free; 2072 } 2073 ret = i915_gem_object_get_pages_gtt(obj, 0); 2074 if (ret != 0) { 2075 drm_mm_put_block(obj->gtt_space); 2076 obj->gtt_space = NULL; 2077 /* 2078 * i915_gem_object_get_pages_gtt() cannot return 2079 * ENOMEM, since we use vm_page_grab(VM_ALLOC_RETRY) 2080 * (which does not support operation without a flag 2081 * anyway). 2082 */ 2083 return (ret); 2084 } 2085 2086 ret = i915_gem_gtt_bind_object(obj); 2087 if (ret != 0) { 2088 i915_gem_object_put_pages_gtt(obj); 2089 drm_mm_put_block(obj->gtt_space); 2090 obj->gtt_space = NULL; 2091 if (i915_gem_evict_everything(dev, false)) 2092 return (ret); 2093 goto search_free; 2094 } 2095 2096 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); 2097 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2098 2099 KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0, 2100 ("Object in gpu read domain")); 2101 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2102 ("Object in gpu write domain")); 2103 2104 obj->gtt_offset = obj->gtt_space->start; 2105 2106 fenceable = 2107 obj->gtt_space->size == fence_size && 2108 (obj->gtt_space->start & (fence_alignment - 1)) == 0; 2109 2110 mappable = 2111 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 2112 obj->map_and_fenceable = mappable && fenceable; 2113 2114 return (0); 2115 } 2116 2117 static void 2118 i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2119 { 2120 u32 old_write_domain, old_read_domains; 2121 2122 /* Act a barrier for all accesses through the GTT */ 2123 cpu_mfence(); 2124 2125 /* Force a pagefault for domain tracking on next user access */ 2126 i915_gem_release_mmap(obj); 2127 2128 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2129 return; 2130 2131 old_read_domains = obj->base.read_domains; 2132 old_write_domain = obj->base.write_domain; 2133 2134 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2135 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2136 2137 } 2138 2139 int 2140 i915_gem_object_unbind(struct drm_i915_gem_object *obj) 2141 { 2142 drm_i915_private_t *dev_priv; 2143 int ret; 2144 2145 dev_priv = obj->base.dev->dev_private; 2146 ret = 0; 2147 if (obj->gtt_space == NULL) 2148 return (0); 2149 if (obj->pin_count != 0) { 2150 DRM_ERROR("Attempting to unbind pinned buffer\n"); 2151 return (-EINVAL); 2152 } 2153 2154 ret = i915_gem_object_finish_gpu(obj); 2155 if (ret == -ERESTART || ret == -EINTR) 2156 return (ret); 2157 2158 i915_gem_object_finish_gtt(obj); 2159 2160 if (ret == 0) 2161 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 2162 if (ret == -ERESTART || ret == -EINTR) 2163 return (ret); 2164 if (ret != 0) { 2165 i915_gem_clflush_object(obj); 2166 obj->base.read_domains = obj->base.write_domain = 2167 I915_GEM_DOMAIN_CPU; 2168 } 2169 2170 ret = i915_gem_object_put_fence(obj); 2171 if (ret == -ERESTART) 2172 return (ret); 2173 2174 i915_gem_gtt_unbind_object(obj); 2175 if (obj->has_aliasing_ppgtt_mapping) { 2176 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2177 obj->has_aliasing_ppgtt_mapping = 0; 2178 } 2179 i915_gem_object_put_pages_gtt(obj); 2180 2181 list_del_init(&obj->gtt_list); 2182 list_del_init(&obj->mm_list); 2183 obj->map_and_fenceable = true; 2184 2185 drm_mm_put_block(obj->gtt_space); 2186 obj->gtt_space = NULL; 2187 obj->gtt_offset = 0; 2188 2189 if (i915_gem_object_is_purgeable(obj)) 2190 i915_gem_object_truncate(obj); 2191 2192 return (ret); 2193 } 2194 2195 static int 2196 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 2197 int flags) 2198 { 2199 struct drm_device *dev; 2200 vm_object_t vm_obj; 2201 vm_page_t m; 2202 int page_count, i, j; 2203 2204 dev = obj->base.dev; 2205 KASSERT(obj->pages == NULL, ("Obj already has pages")); 2206 page_count = obj->base.size / PAGE_SIZE; 2207 obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 2208 M_WAITOK); 2209 vm_obj = obj->base.vm_obj; 2210 VM_OBJECT_LOCK(vm_obj); 2211 for (i = 0; i < page_count; i++) { 2212 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL) 2213 goto failed; 2214 } 2215 VM_OBJECT_UNLOCK(vm_obj); 2216 if (i915_gem_object_needs_bit17_swizzle(obj)) 2217 i915_gem_object_do_bit_17_swizzle(obj); 2218 return (0); 2219 2220 failed: 2221 for (j = 0; j < i; j++) { 2222 m = obj->pages[j]; 2223 vm_page_busy_wait(m, FALSE, "i915gem"); 2224 vm_page_unwire(m, 0); 2225 vm_page_wakeup(m); 2226 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2227 } 2228 VM_OBJECT_UNLOCK(vm_obj); 2229 drm_free(obj->pages, DRM_I915_GEM); 2230 obj->pages = NULL; 2231 return (-EIO); 2232 } 2233 2234 #define GEM_PARANOID_CHECK_GTT 0 2235 #if GEM_PARANOID_CHECK_GTT 2236 static void 2237 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 2238 int page_count) 2239 { 2240 struct drm_i915_private *dev_priv; 2241 vm_paddr_t pa; 2242 unsigned long start, end; 2243 u_int i; 2244 int j; 2245 2246 dev_priv = dev->dev_private; 2247 start = OFF_TO_IDX(dev_priv->mm.gtt_start); 2248 end = OFF_TO_IDX(dev_priv->mm.gtt_end); 2249 for (i = start; i < end; i++) { 2250 pa = intel_gtt_read_pte_paddr(i); 2251 for (j = 0; j < page_count; j++) { 2252 if (pa == VM_PAGE_TO_PHYS(ma[j])) { 2253 panic("Page %p in GTT pte index %d pte %x", 2254 ma[i], i, intel_gtt_read_pte(i)); 2255 } 2256 } 2257 } 2258 } 2259 #endif 2260 2261 static void 2262 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 2263 { 2264 vm_page_t m; 2265 int page_count, i; 2266 2267 KASSERT(obj->madv != I915_MADV_PURGED_INTERNAL, ("Purged object")); 2268 2269 if (obj->tiling_mode != I915_TILING_NONE) 2270 i915_gem_object_save_bit_17_swizzle(obj); 2271 if (obj->madv == I915_MADV_DONTNEED) 2272 obj->dirty = 0; 2273 page_count = obj->base.size / PAGE_SIZE; 2274 VM_OBJECT_LOCK(obj->base.vm_obj); 2275 #if GEM_PARANOID_CHECK_GTT 2276 i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 2277 #endif 2278 for (i = 0; i < page_count; i++) { 2279 m = obj->pages[i]; 2280 if (obj->dirty) 2281 vm_page_dirty(m); 2282 if (obj->madv == I915_MADV_WILLNEED) 2283 vm_page_reference(m); 2284 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem"); 2285 vm_page_unwire(obj->pages[i], 1); 2286 vm_page_wakeup(obj->pages[i]); 2287 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2288 } 2289 VM_OBJECT_UNLOCK(obj->base.vm_obj); 2290 obj->dirty = 0; 2291 drm_free(obj->pages, DRM_I915_GEM); 2292 obj->pages = NULL; 2293 } 2294 2295 void 2296 i915_gem_release_mmap(struct drm_i915_gem_object *obj) 2297 { 2298 vm_object_t devobj; 2299 vm_page_t m; 2300 int i, page_count; 2301 2302 if (!obj->fault_mappable) 2303 return; 2304 2305 devobj = cdev_pager_lookup(obj); 2306 if (devobj != NULL) { 2307 page_count = OFF_TO_IDX(obj->base.size); 2308 2309 VM_OBJECT_LOCK(devobj); 2310 for (i = 0; i < page_count; i++) { 2311 m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm"); 2312 if (m == NULL) 2313 continue; 2314 cdev_pager_free_page(devobj, m); 2315 } 2316 VM_OBJECT_UNLOCK(devobj); 2317 vm_object_deallocate(devobj); 2318 } 2319 2320 obj->fault_mappable = false; 2321 } 2322 2323 int 2324 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) 2325 { 2326 int ret; 2327 2328 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2329 ("In GPU write domain")); 2330 2331 if (obj->active) { 2332 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno, 2333 true); 2334 if (ret != 0) 2335 return (ret); 2336 } 2337 return (0); 2338 } 2339 2340 void 2341 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 2342 struct intel_ring_buffer *ring, uint32_t seqno) 2343 { 2344 struct drm_device *dev = obj->base.dev; 2345 struct drm_i915_private *dev_priv = dev->dev_private; 2346 struct drm_i915_fence_reg *reg; 2347 2348 obj->ring = ring; 2349 KASSERT(ring != NULL, ("NULL ring")); 2350 2351 /* Add a reference if we're newly entering the active list. */ 2352 if (!obj->active) { 2353 drm_gem_object_reference(&obj->base); 2354 obj->active = 1; 2355 } 2356 2357 /* Move from whatever list we were on to the tail of execution. */ 2358 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 2359 list_move_tail(&obj->ring_list, &ring->active_list); 2360 2361 obj->last_rendering_seqno = seqno; 2362 if (obj->fenced_gpu_access) { 2363 obj->last_fenced_seqno = seqno; 2364 obj->last_fenced_ring = ring; 2365 2366 /* Bump MRU to take account of the delayed flush */ 2367 if (obj->fence_reg != I915_FENCE_REG_NONE) { 2368 reg = &dev_priv->fence_regs[obj->fence_reg]; 2369 list_move_tail(®->lru_list, 2370 &dev_priv->mm.fence_list); 2371 } 2372 } 2373 } 2374 2375 static void 2376 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) 2377 { 2378 list_del_init(&obj->ring_list); 2379 obj->last_rendering_seqno = 0; 2380 obj->last_fenced_seqno = 0; 2381 } 2382 2383 static void 2384 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) 2385 { 2386 struct drm_device *dev = obj->base.dev; 2387 drm_i915_private_t *dev_priv = dev->dev_private; 2388 2389 KASSERT(obj->active, ("Object not active")); 2390 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); 2391 2392 i915_gem_object_move_off_active(obj); 2393 } 2394 2395 static void 2396 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 2397 { 2398 struct drm_device *dev = obj->base.dev; 2399 struct drm_i915_private *dev_priv = dev->dev_private; 2400 2401 if (obj->pin_count != 0) 2402 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 2403 else 2404 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2405 2406 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list")); 2407 KASSERT(obj->active, ("Object not active")); 2408 obj->ring = NULL; 2409 obj->last_fenced_ring = NULL; 2410 2411 i915_gem_object_move_off_active(obj); 2412 obj->fenced_gpu_access = false; 2413 2414 obj->active = 0; 2415 obj->pending_gpu_write = false; 2416 drm_gem_object_unreference(&obj->base); 2417 2418 #if 1 2419 KIB_NOTYET(); 2420 #else 2421 WARN_ON(i915_verify_lists(dev)); 2422 #endif 2423 } 2424 2425 static void 2426 i915_gem_object_truncate(struct drm_i915_gem_object *obj) 2427 { 2428 vm_object_t vm_obj; 2429 2430 vm_obj = obj->base.vm_obj; 2431 VM_OBJECT_LOCK(vm_obj); 2432 vm_object_page_remove(vm_obj, 0, 0, false); 2433 VM_OBJECT_UNLOCK(vm_obj); 2434 obj->madv = I915_MADV_PURGED_INTERNAL; 2435 } 2436 2437 static inline int 2438 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 2439 { 2440 2441 return (obj->madv == I915_MADV_DONTNEED); 2442 } 2443 2444 static void 2445 i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 2446 uint32_t flush_domains) 2447 { 2448 struct drm_i915_gem_object *obj, *next; 2449 uint32_t old_write_domain; 2450 2451 list_for_each_entry_safe(obj, next, &ring->gpu_write_list, 2452 gpu_write_list) { 2453 if (obj->base.write_domain & flush_domains) { 2454 old_write_domain = obj->base.write_domain; 2455 obj->base.write_domain = 0; 2456 list_del_init(&obj->gpu_write_list); 2457 i915_gem_object_move_to_active(obj, ring, 2458 i915_gem_next_request_seqno(ring)); 2459 } 2460 } 2461 } 2462 2463 static int 2464 i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 2465 { 2466 drm_i915_private_t *dev_priv; 2467 2468 dev_priv = obj->base.dev->dev_private; 2469 return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 2470 obj->tiling_mode != I915_TILING_NONE); 2471 } 2472 2473 #define VM_OBJECT_LOCK_ASSERT_OWNED(object) 2474 2475 static vm_page_t 2476 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex) 2477 { 2478 vm_page_t m; 2479 int rv; 2480 2481 VM_OBJECT_LOCK_ASSERT_OWNED(object); 2482 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 2483 if (m->valid != VM_PAGE_BITS_ALL) { 2484 if (vm_pager_has_page(object, pindex)) { 2485 rv = vm_pager_get_page(object, &m, 1); 2486 m = vm_page_lookup(object, pindex); 2487 if (m == NULL) 2488 return (NULL); 2489 if (rv != VM_PAGER_OK) { 2490 vm_page_free(m); 2491 return (NULL); 2492 } 2493 } else { 2494 pmap_zero_page(VM_PAGE_TO_PHYS(m)); 2495 m->valid = VM_PAGE_BITS_ALL; 2496 m->dirty = 0; 2497 } 2498 } 2499 vm_page_wire(m); 2500 vm_page_wakeup(m); 2501 atomic_add_long(&i915_gem_wired_pages_cnt, 1); 2502 return (m); 2503 } 2504 2505 int 2506 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, 2507 uint32_t flush_domains) 2508 { 2509 int ret; 2510 2511 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) 2512 return 0; 2513 2514 ret = ring->flush(ring, invalidate_domains, flush_domains); 2515 if (ret) 2516 return ret; 2517 2518 if (flush_domains & I915_GEM_GPU_DOMAINS) 2519 i915_gem_process_flushing_list(ring, flush_domains); 2520 return 0; 2521 } 2522 2523 static int 2524 i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire) 2525 { 2526 int ret; 2527 2528 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 2529 return 0; 2530 2531 if (!list_empty(&ring->gpu_write_list)) { 2532 ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS, 2533 I915_GEM_GPU_DOMAINS); 2534 if (ret != 0) 2535 return ret; 2536 } 2537 2538 return (i915_wait_request(ring, i915_gem_next_request_seqno(ring), 2539 do_retire)); 2540 } 2541 2542 int 2543 i915_gpu_idle(struct drm_device *dev, bool do_retire) 2544 { 2545 drm_i915_private_t *dev_priv = dev->dev_private; 2546 int ret, i; 2547 2548 /* Flush everything onto the inactive list. */ 2549 for (i = 0; i < I915_NUM_RINGS; i++) { 2550 ret = i915_ring_idle(&dev_priv->rings[i], do_retire); 2551 if (ret) 2552 return ret; 2553 } 2554 2555 return 0; 2556 } 2557 2558 int 2559 i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire) 2560 { 2561 drm_i915_private_t *dev_priv; 2562 struct drm_i915_gem_request *request; 2563 uint32_t ier; 2564 int flags, ret; 2565 bool recovery_complete; 2566 2567 KASSERT(seqno != 0, ("Zero seqno")); 2568 2569 dev_priv = ring->dev->dev_private; 2570 ret = 0; 2571 2572 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 2573 /* Give the error handler a chance to run. */ 2574 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 2575 recovery_complete = (&dev_priv->error_completion) > 0; 2576 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 2577 return (recovery_complete ? -EIO : -EAGAIN); 2578 } 2579 2580 if (seqno == ring->outstanding_lazy_request) { 2581 request = kmalloc(sizeof(*request), DRM_I915_GEM, 2582 M_WAITOK | M_ZERO); 2583 if (request == NULL) 2584 return (-ENOMEM); 2585 2586 ret = i915_add_request(ring, NULL, request); 2587 if (ret != 0) { 2588 drm_free(request, DRM_I915_GEM); 2589 return (ret); 2590 } 2591 2592 seqno = request->seqno; 2593 } 2594 2595 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 2596 if (HAS_PCH_SPLIT(ring->dev)) 2597 ier = I915_READ(DEIER) | I915_READ(GTIER); 2598 else 2599 ier = I915_READ(IER); 2600 if (!ier) { 2601 DRM_ERROR("something (likely vbetool) disabled " 2602 "interrupts, re-enabling\n"); 2603 ring->dev->driver->irq_preinstall(ring->dev); 2604 ring->dev->driver->irq_postinstall(ring->dev); 2605 } 2606 2607 ring->waiting_seqno = seqno; 2608 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 2609 if (ring->irq_get(ring)) { 2610 flags = dev_priv->mm.interruptible ? PCATCH : 0; 2611 while (!i915_seqno_passed(ring->get_seqno(ring), seqno) 2612 && !atomic_load_acq_int(&dev_priv->mm.wedged) && 2613 ret == 0) { 2614 ret = -lksleep(ring, &ring->irq_lock, flags, 2615 "915gwr", 0); 2616 } 2617 ring->irq_put(ring); 2618 lockmgr(&ring->irq_lock, LK_RELEASE); 2619 } else { 2620 lockmgr(&ring->irq_lock, LK_RELEASE); 2621 if (_intel_wait_for(ring->dev, 2622 i915_seqno_passed(ring->get_seqno(ring), seqno) || 2623 atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 2624 0, "i915wrq") != 0) 2625 ret = -EBUSY; 2626 } 2627 ring->waiting_seqno = 0; 2628 2629 } 2630 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 2631 ret = -EAGAIN; 2632 2633 /* Directly dispatch request retiring. While we have the work queue 2634 * to handle this, the waiter on a request often wants an associated 2635 * buffer to have made it to the inactive list, and we would need 2636 * a separate wait queue to handle that. 2637 */ 2638 if (ret == 0 && do_retire) 2639 i915_gem_retire_requests_ring(ring); 2640 2641 return (ret); 2642 } 2643 2644 static u32 2645 i915_gem_get_seqno(struct drm_device *dev) 2646 { 2647 drm_i915_private_t *dev_priv = dev->dev_private; 2648 u32 seqno = dev_priv->next_seqno; 2649 2650 /* reserve 0 for non-seqno */ 2651 if (++dev_priv->next_seqno == 0) 2652 dev_priv->next_seqno = 1; 2653 2654 return seqno; 2655 } 2656 2657 u32 2658 i915_gem_next_request_seqno(struct intel_ring_buffer *ring) 2659 { 2660 if (ring->outstanding_lazy_request == 0) 2661 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); 2662 2663 return ring->outstanding_lazy_request; 2664 } 2665 2666 int 2667 i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, 2668 struct drm_i915_gem_request *request) 2669 { 2670 drm_i915_private_t *dev_priv; 2671 struct drm_i915_file_private *file_priv; 2672 uint32_t seqno; 2673 u32 request_ring_position; 2674 int was_empty; 2675 int ret; 2676 2677 KASSERT(request != NULL, ("NULL request in add")); 2678 DRM_LOCK_ASSERT(ring->dev); 2679 dev_priv = ring->dev->dev_private; 2680 2681 seqno = i915_gem_next_request_seqno(ring); 2682 request_ring_position = intel_ring_get_tail(ring); 2683 2684 ret = ring->add_request(ring, &seqno); 2685 if (ret != 0) 2686 return ret; 2687 2688 request->seqno = seqno; 2689 request->ring = ring; 2690 request->tail = request_ring_position; 2691 request->emitted_jiffies = ticks; 2692 was_empty = list_empty(&ring->request_list); 2693 list_add_tail(&request->list, &ring->request_list); 2694 2695 if (file != NULL) { 2696 file_priv = file->driver_priv; 2697 2698 spin_lock(&file_priv->mm.lock); 2699 request->file_priv = file_priv; 2700 list_add_tail(&request->client_list, 2701 &file_priv->mm.request_list); 2702 spin_unlock(&file_priv->mm.lock); 2703 } 2704 2705 ring->outstanding_lazy_request = 0; 2706 2707 if (!dev_priv->mm.suspended) { 2708 if (i915_enable_hangcheck) { 2709 callout_reset(&dev_priv->hangcheck_timer, 2710 DRM_I915_HANGCHECK_PERIOD, i915_hangcheck_elapsed, ring->dev); 2711 } 2712 if (was_empty) 2713 taskqueue_enqueue_timeout(dev_priv->tq, 2714 &dev_priv->mm.retire_task, hz); 2715 } 2716 return (0); 2717 } 2718 2719 static inline void 2720 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 2721 { 2722 struct drm_i915_file_private *file_priv = request->file_priv; 2723 2724 if (!file_priv) 2725 return; 2726 2727 DRM_LOCK_ASSERT(request->ring->dev); 2728 2729 spin_lock(&file_priv->mm.lock); 2730 if (request->file_priv != NULL) { 2731 list_del(&request->client_list); 2732 request->file_priv = NULL; 2733 } 2734 spin_unlock(&file_priv->mm.lock); 2735 } 2736 2737 void 2738 i915_gem_release(struct drm_device *dev, struct drm_file *file) 2739 { 2740 struct drm_i915_file_private *file_priv; 2741 struct drm_i915_gem_request *request; 2742 2743 file_priv = file->driver_priv; 2744 2745 /* Clean up our request list when the client is going away, so that 2746 * later retire_requests won't dereference our soon-to-be-gone 2747 * file_priv. 2748 */ 2749 spin_lock(&file_priv->mm.lock); 2750 while (!list_empty(&file_priv->mm.request_list)) { 2751 request = list_first_entry(&file_priv->mm.request_list, 2752 struct drm_i915_gem_request, 2753 client_list); 2754 list_del(&request->client_list); 2755 request->file_priv = NULL; 2756 } 2757 spin_unlock(&file_priv->mm.lock); 2758 } 2759 2760 static void 2761 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 2762 struct intel_ring_buffer *ring) 2763 { 2764 2765 if (ring->dev != NULL) 2766 DRM_LOCK_ASSERT(ring->dev); 2767 2768 while (!list_empty(&ring->request_list)) { 2769 struct drm_i915_gem_request *request; 2770 2771 request = list_first_entry(&ring->request_list, 2772 struct drm_i915_gem_request, list); 2773 2774 list_del(&request->list); 2775 i915_gem_request_remove_from_client(request); 2776 drm_free(request, DRM_I915_GEM); 2777 } 2778 2779 while (!list_empty(&ring->active_list)) { 2780 struct drm_i915_gem_object *obj; 2781 2782 obj = list_first_entry(&ring->active_list, 2783 struct drm_i915_gem_object, ring_list); 2784 2785 obj->base.write_domain = 0; 2786 list_del_init(&obj->gpu_write_list); 2787 i915_gem_object_move_to_inactive(obj); 2788 } 2789 } 2790 2791 static void 2792 i915_gem_reset_fences(struct drm_device *dev) 2793 { 2794 struct drm_i915_private *dev_priv = dev->dev_private; 2795 int i; 2796 2797 for (i = 0; i < dev_priv->num_fence_regs; i++) { 2798 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2799 struct drm_i915_gem_object *obj = reg->obj; 2800 2801 if (!obj) 2802 continue; 2803 2804 if (obj->tiling_mode) 2805 i915_gem_release_mmap(obj); 2806 2807 reg->obj->fence_reg = I915_FENCE_REG_NONE; 2808 reg->obj->fenced_gpu_access = false; 2809 reg->obj->last_fenced_seqno = 0; 2810 reg->obj->last_fenced_ring = NULL; 2811 i915_gem_clear_fence_reg(dev, reg); 2812 } 2813 } 2814 2815 void 2816 i915_gem_reset(struct drm_device *dev) 2817 { 2818 struct drm_i915_private *dev_priv = dev->dev_private; 2819 struct drm_i915_gem_object *obj; 2820 int i; 2821 2822 for (i = 0; i < I915_NUM_RINGS; i++) 2823 i915_gem_reset_ring_lists(dev_priv, &dev_priv->rings[i]); 2824 2825 /* Remove anything from the flushing lists. The GPU cache is likely 2826 * to be lost on reset along with the data, so simply move the 2827 * lost bo to the inactive list. 2828 */ 2829 while (!list_empty(&dev_priv->mm.flushing_list)) { 2830 obj = list_first_entry(&dev_priv->mm.flushing_list, 2831 struct drm_i915_gem_object, 2832 mm_list); 2833 2834 obj->base.write_domain = 0; 2835 list_del_init(&obj->gpu_write_list); 2836 i915_gem_object_move_to_inactive(obj); 2837 } 2838 2839 /* Move everything out of the GPU domains to ensure we do any 2840 * necessary invalidation upon reuse. 2841 */ 2842 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { 2843 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2844 } 2845 2846 /* The fence registers are invalidated so clear them out */ 2847 i915_gem_reset_fences(dev); 2848 } 2849 2850 /** 2851 * This function clears the request list as sequence numbers are passed. 2852 */ 2853 void 2854 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 2855 { 2856 uint32_t seqno; 2857 int i; 2858 2859 if (list_empty(&ring->request_list)) 2860 return; 2861 2862 seqno = ring->get_seqno(ring); 2863 for (i = 0; i < DRM_ARRAY_SIZE(ring->sync_seqno); i++) 2864 if (seqno >= ring->sync_seqno[i]) 2865 ring->sync_seqno[i] = 0; 2866 2867 while (!list_empty(&ring->request_list)) { 2868 struct drm_i915_gem_request *request; 2869 2870 request = list_first_entry(&ring->request_list, 2871 struct drm_i915_gem_request, 2872 list); 2873 2874 if (!i915_seqno_passed(seqno, request->seqno)) 2875 break; 2876 2877 ring->last_retired_head = request->tail; 2878 2879 list_del(&request->list); 2880 i915_gem_request_remove_from_client(request); 2881 drm_free(request, DRM_I915_GEM); 2882 } 2883 2884 /* Move any buffers on the active list that are no longer referenced 2885 * by the ringbuffer to the flushing/inactive lists as appropriate. 2886 */ 2887 while (!list_empty(&ring->active_list)) { 2888 struct drm_i915_gem_object *obj; 2889 2890 obj = list_first_entry(&ring->active_list, 2891 struct drm_i915_gem_object, 2892 ring_list); 2893 2894 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) 2895 break; 2896 2897 if (obj->base.write_domain != 0) 2898 i915_gem_object_move_to_flushing(obj); 2899 else 2900 i915_gem_object_move_to_inactive(obj); 2901 } 2902 2903 if (ring->trace_irq_seqno && 2904 i915_seqno_passed(seqno, ring->trace_irq_seqno)) { 2905 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 2906 ring->irq_put(ring); 2907 lockmgr(&ring->irq_lock, LK_RELEASE); 2908 ring->trace_irq_seqno = 0; 2909 } 2910 } 2911 2912 void 2913 i915_gem_retire_requests(struct drm_device *dev) 2914 { 2915 drm_i915_private_t *dev_priv = dev->dev_private; 2916 struct drm_i915_gem_object *obj, *next; 2917 int i; 2918 2919 if (!list_empty(&dev_priv->mm.deferred_free_list)) { 2920 list_for_each_entry_safe(obj, next, 2921 &dev_priv->mm.deferred_free_list, mm_list) 2922 i915_gem_free_object_tail(obj); 2923 } 2924 2925 for (i = 0; i < I915_NUM_RINGS; i++) 2926 i915_gem_retire_requests_ring(&dev_priv->rings[i]); 2927 } 2928 2929 static int 2930 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj, 2931 struct intel_ring_buffer *pipelined) 2932 { 2933 struct drm_device *dev = obj->base.dev; 2934 drm_i915_private_t *dev_priv = dev->dev_private; 2935 u32 size = obj->gtt_space->size; 2936 int regnum = obj->fence_reg; 2937 uint64_t val; 2938 2939 val = (uint64_t)((obj->gtt_offset + size - 4096) & 2940 0xfffff000) << 32; 2941 val |= obj->gtt_offset & 0xfffff000; 2942 val |= (uint64_t)((obj->stride / 128) - 1) << 2943 SANDYBRIDGE_FENCE_PITCH_SHIFT; 2944 2945 if (obj->tiling_mode == I915_TILING_Y) 2946 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2947 val |= I965_FENCE_REG_VALID; 2948 2949 if (pipelined) { 2950 int ret = intel_ring_begin(pipelined, 6); 2951 if (ret) 2952 return ret; 2953 2954 intel_ring_emit(pipelined, MI_NOOP); 2955 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 2956 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8); 2957 intel_ring_emit(pipelined, (u32)val); 2958 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4); 2959 intel_ring_emit(pipelined, (u32)(val >> 32)); 2960 intel_ring_advance(pipelined); 2961 } else 2962 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val); 2963 2964 return 0; 2965 } 2966 2967 static int 2968 i965_write_fence_reg(struct drm_i915_gem_object *obj, 2969 struct intel_ring_buffer *pipelined) 2970 { 2971 struct drm_device *dev = obj->base.dev; 2972 drm_i915_private_t *dev_priv = dev->dev_private; 2973 u32 size = obj->gtt_space->size; 2974 int regnum = obj->fence_reg; 2975 uint64_t val; 2976 2977 val = (uint64_t)((obj->gtt_offset + size - 4096) & 2978 0xfffff000) << 32; 2979 val |= obj->gtt_offset & 0xfffff000; 2980 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 2981 if (obj->tiling_mode == I915_TILING_Y) 2982 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2983 val |= I965_FENCE_REG_VALID; 2984 2985 if (pipelined) { 2986 int ret = intel_ring_begin(pipelined, 6); 2987 if (ret) 2988 return ret; 2989 2990 intel_ring_emit(pipelined, MI_NOOP); 2991 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 2992 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8); 2993 intel_ring_emit(pipelined, (u32)val); 2994 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4); 2995 intel_ring_emit(pipelined, (u32)(val >> 32)); 2996 intel_ring_advance(pipelined); 2997 } else 2998 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val); 2999 3000 return 0; 3001 } 3002 3003 static int 3004 i915_write_fence_reg(struct drm_i915_gem_object *obj, 3005 struct intel_ring_buffer *pipelined) 3006 { 3007 struct drm_device *dev = obj->base.dev; 3008 drm_i915_private_t *dev_priv = dev->dev_private; 3009 u32 size = obj->gtt_space->size; 3010 u32 fence_reg, val, pitch_val; 3011 int tile_width; 3012 3013 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) || 3014 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3015 kprintf( 3016 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 3017 obj->gtt_offset, obj->map_and_fenceable, size); 3018 return -EINVAL; 3019 } 3020 3021 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 3022 tile_width = 128; 3023 else 3024 tile_width = 512; 3025 3026 /* Note: pitch better be a power of two tile widths */ 3027 pitch_val = obj->stride / tile_width; 3028 pitch_val = ffs(pitch_val) - 1; 3029 3030 val = obj->gtt_offset; 3031 if (obj->tiling_mode == I915_TILING_Y) 3032 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3033 val |= I915_FENCE_SIZE_BITS(size); 3034 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3035 val |= I830_FENCE_REG_VALID; 3036 3037 fence_reg = obj->fence_reg; 3038 if (fence_reg < 8) 3039 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3040 else 3041 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3042 3043 if (pipelined) { 3044 int ret = intel_ring_begin(pipelined, 4); 3045 if (ret) 3046 return ret; 3047 3048 intel_ring_emit(pipelined, MI_NOOP); 3049 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3050 intel_ring_emit(pipelined, fence_reg); 3051 intel_ring_emit(pipelined, val); 3052 intel_ring_advance(pipelined); 3053 } else 3054 I915_WRITE(fence_reg, val); 3055 3056 return 0; 3057 } 3058 3059 static int 3060 i830_write_fence_reg(struct drm_i915_gem_object *obj, 3061 struct intel_ring_buffer *pipelined) 3062 { 3063 struct drm_device *dev = obj->base.dev; 3064 drm_i915_private_t *dev_priv = dev->dev_private; 3065 u32 size = obj->gtt_space->size; 3066 int regnum = obj->fence_reg; 3067 uint32_t val; 3068 uint32_t pitch_val; 3069 3070 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) || 3071 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3072 kprintf( 3073 "object 0x%08x not 512K or pot-size 0x%08x aligned\n", 3074 obj->gtt_offset, size); 3075 return -EINVAL; 3076 } 3077 3078 pitch_val = obj->stride / 128; 3079 pitch_val = ffs(pitch_val) - 1; 3080 3081 val = obj->gtt_offset; 3082 if (obj->tiling_mode == I915_TILING_Y) 3083 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3084 val |= I830_FENCE_SIZE_BITS(size); 3085 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3086 val |= I830_FENCE_REG_VALID; 3087 3088 if (pipelined) { 3089 int ret = intel_ring_begin(pipelined, 4); 3090 if (ret) 3091 return ret; 3092 3093 intel_ring_emit(pipelined, MI_NOOP); 3094 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3095 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4); 3096 intel_ring_emit(pipelined, val); 3097 intel_ring_advance(pipelined); 3098 } else 3099 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val); 3100 3101 return 0; 3102 } 3103 3104 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno) 3105 { 3106 return i915_seqno_passed(ring->get_seqno(ring), seqno); 3107 } 3108 3109 static int 3110 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, 3111 struct intel_ring_buffer *pipelined) 3112 { 3113 int ret; 3114 3115 if (obj->fenced_gpu_access) { 3116 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 3117 ret = i915_gem_flush_ring(obj->last_fenced_ring, 0, 3118 obj->base.write_domain); 3119 if (ret) 3120 return ret; 3121 } 3122 3123 obj->fenced_gpu_access = false; 3124 } 3125 3126 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) { 3127 if (!ring_passed_seqno(obj->last_fenced_ring, 3128 obj->last_fenced_seqno)) { 3129 ret = i915_wait_request(obj->last_fenced_ring, 3130 obj->last_fenced_seqno, 3131 true); 3132 if (ret) 3133 return ret; 3134 } 3135 3136 obj->last_fenced_seqno = 0; 3137 obj->last_fenced_ring = NULL; 3138 } 3139 3140 /* Ensure that all CPU reads are completed before installing a fence 3141 * and all writes before removing the fence. 3142 */ 3143 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 3144 cpu_mfence(); 3145 3146 return 0; 3147 } 3148 3149 int 3150 i915_gem_object_put_fence(struct drm_i915_gem_object *obj) 3151 { 3152 int ret; 3153 3154 if (obj->tiling_mode) 3155 i915_gem_release_mmap(obj); 3156 3157 ret = i915_gem_object_flush_fence(obj, NULL); 3158 if (ret) 3159 return ret; 3160 3161 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3162 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3163 3164 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0) 3165 kprintf("%s: pin_count %d\n", __func__, 3166 dev_priv->fence_regs[obj->fence_reg].pin_count); 3167 i915_gem_clear_fence_reg(obj->base.dev, 3168 &dev_priv->fence_regs[obj->fence_reg]); 3169 3170 obj->fence_reg = I915_FENCE_REG_NONE; 3171 } 3172 3173 return 0; 3174 } 3175 3176 static struct drm_i915_fence_reg * 3177 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined) 3178 { 3179 struct drm_i915_private *dev_priv = dev->dev_private; 3180 struct drm_i915_fence_reg *reg, *first, *avail; 3181 int i; 3182 3183 /* First try to find a free reg */ 3184 avail = NULL; 3185 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 3186 reg = &dev_priv->fence_regs[i]; 3187 if (!reg->obj) 3188 return reg; 3189 3190 if (!reg->pin_count) 3191 avail = reg; 3192 } 3193 3194 if (avail == NULL) 3195 return NULL; 3196 3197 /* None available, try to steal one or wait for a user to finish */ 3198 avail = first = NULL; 3199 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 3200 if (reg->pin_count) 3201 continue; 3202 3203 if (first == NULL) 3204 first = reg; 3205 3206 if (!pipelined || 3207 !reg->obj->last_fenced_ring || 3208 reg->obj->last_fenced_ring == pipelined) { 3209 avail = reg; 3210 break; 3211 } 3212 } 3213 3214 if (avail == NULL) 3215 avail = first; 3216 3217 return avail; 3218 } 3219 3220 int 3221 i915_gem_object_get_fence(struct drm_i915_gem_object *obj, 3222 struct intel_ring_buffer *pipelined) 3223 { 3224 struct drm_device *dev = obj->base.dev; 3225 struct drm_i915_private *dev_priv = dev->dev_private; 3226 struct drm_i915_fence_reg *reg; 3227 int ret; 3228 3229 pipelined = NULL; 3230 ret = 0; 3231 3232 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3233 reg = &dev_priv->fence_regs[obj->fence_reg]; 3234 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3235 3236 if (obj->tiling_changed) { 3237 ret = i915_gem_object_flush_fence(obj, pipelined); 3238 if (ret) 3239 return ret; 3240 3241 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3242 pipelined = NULL; 3243 3244 if (pipelined) { 3245 reg->setup_seqno = 3246 i915_gem_next_request_seqno(pipelined); 3247 obj->last_fenced_seqno = reg->setup_seqno; 3248 obj->last_fenced_ring = pipelined; 3249 } 3250 3251 goto update; 3252 } 3253 3254 if (!pipelined) { 3255 if (reg->setup_seqno) { 3256 if (!ring_passed_seqno(obj->last_fenced_ring, 3257 reg->setup_seqno)) { 3258 ret = i915_wait_request( 3259 obj->last_fenced_ring, 3260 reg->setup_seqno, 3261 true); 3262 if (ret) 3263 return ret; 3264 } 3265 3266 reg->setup_seqno = 0; 3267 } 3268 } else if (obj->last_fenced_ring && 3269 obj->last_fenced_ring != pipelined) { 3270 ret = i915_gem_object_flush_fence(obj, pipelined); 3271 if (ret) 3272 return ret; 3273 } 3274 3275 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3276 pipelined = NULL; 3277 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined")); 3278 3279 if (obj->tiling_changed) { 3280 if (pipelined) { 3281 reg->setup_seqno = 3282 i915_gem_next_request_seqno(pipelined); 3283 obj->last_fenced_seqno = reg->setup_seqno; 3284 obj->last_fenced_ring = pipelined; 3285 } 3286 goto update; 3287 } 3288 3289 return 0; 3290 } 3291 3292 reg = i915_find_fence_reg(dev, pipelined); 3293 if (reg == NULL) 3294 return -EDEADLK; 3295 3296 ret = i915_gem_object_flush_fence(obj, pipelined); 3297 if (ret) 3298 return ret; 3299 3300 if (reg->obj) { 3301 struct drm_i915_gem_object *old = reg->obj; 3302 3303 drm_gem_object_reference(&old->base); 3304 3305 if (old->tiling_mode) 3306 i915_gem_release_mmap(old); 3307 3308 ret = i915_gem_object_flush_fence(old, pipelined); 3309 if (ret) { 3310 drm_gem_object_unreference(&old->base); 3311 return ret; 3312 } 3313 3314 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0) 3315 pipelined = NULL; 3316 3317 old->fence_reg = I915_FENCE_REG_NONE; 3318 old->last_fenced_ring = pipelined; 3319 old->last_fenced_seqno = 3320 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3321 3322 drm_gem_object_unreference(&old->base); 3323 } else if (obj->last_fenced_seqno == 0) 3324 pipelined = NULL; 3325 3326 reg->obj = obj; 3327 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3328 obj->fence_reg = reg - dev_priv->fence_regs; 3329 obj->last_fenced_ring = pipelined; 3330 3331 reg->setup_seqno = 3332 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3333 obj->last_fenced_seqno = reg->setup_seqno; 3334 3335 update: 3336 obj->tiling_changed = false; 3337 switch (INTEL_INFO(dev)->gen) { 3338 case 7: 3339 case 6: 3340 ret = sandybridge_write_fence_reg(obj, pipelined); 3341 break; 3342 case 5: 3343 case 4: 3344 ret = i965_write_fence_reg(obj, pipelined); 3345 break; 3346 case 3: 3347 ret = i915_write_fence_reg(obj, pipelined); 3348 break; 3349 case 2: 3350 ret = i830_write_fence_reg(obj, pipelined); 3351 break; 3352 } 3353 3354 return ret; 3355 } 3356 3357 static void 3358 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg) 3359 { 3360 drm_i915_private_t *dev_priv = dev->dev_private; 3361 uint32_t fence_reg = reg - dev_priv->fence_regs; 3362 3363 switch (INTEL_INFO(dev)->gen) { 3364 case 7: 3365 case 6: 3366 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0); 3367 break; 3368 case 5: 3369 case 4: 3370 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0); 3371 break; 3372 case 3: 3373 if (fence_reg >= 8) 3374 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3375 else 3376 case 2: 3377 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3378 3379 I915_WRITE(fence_reg, 0); 3380 break; 3381 } 3382 3383 list_del_init(®->lru_list); 3384 reg->obj = NULL; 3385 reg->setup_seqno = 0; 3386 reg->pin_count = 0; 3387 } 3388 3389 int 3390 i915_gem_init_object(struct drm_gem_object *obj) 3391 { 3392 3393 kprintf("i915_gem_init_object called\n"); 3394 return (0); 3395 } 3396 3397 static bool 3398 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 3399 { 3400 3401 return (obj->gtt_space && !obj->active && obj->pin_count == 0); 3402 } 3403 3404 static void 3405 i915_gem_retire_task_handler(void *arg, int pending) 3406 { 3407 drm_i915_private_t *dev_priv; 3408 struct drm_device *dev; 3409 bool idle; 3410 int i; 3411 3412 dev_priv = arg; 3413 dev = dev_priv->dev; 3414 3415 /* Come back later if the device is busy... */ 3416 if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) { 3417 taskqueue_enqueue_timeout(dev_priv->tq, 3418 &dev_priv->mm.retire_task, hz); 3419 return; 3420 } 3421 3422 i915_gem_retire_requests(dev); 3423 3424 /* Send a periodic flush down the ring so we don't hold onto GEM 3425 * objects indefinitely. 3426 */ 3427 idle = true; 3428 for (i = 0; i < I915_NUM_RINGS; i++) { 3429 struct intel_ring_buffer *ring = &dev_priv->rings[i]; 3430 3431 if (!list_empty(&ring->gpu_write_list)) { 3432 struct drm_i915_gem_request *request; 3433 int ret; 3434 3435 ret = i915_gem_flush_ring(ring, 3436 0, I915_GEM_GPU_DOMAINS); 3437 request = kmalloc(sizeof(*request), DRM_I915_GEM, 3438 M_WAITOK | M_ZERO); 3439 if (ret || request == NULL || 3440 i915_add_request(ring, NULL, request)) 3441 drm_free(request, DRM_I915_GEM); 3442 } 3443 3444 idle &= list_empty(&ring->request_list); 3445 } 3446 3447 if (!dev_priv->mm.suspended && !idle) 3448 taskqueue_enqueue_timeout(dev_priv->tq, 3449 &dev_priv->mm.retire_task, hz); 3450 3451 DRM_UNLOCK(dev); 3452 } 3453 3454 void 3455 i915_gem_lastclose(struct drm_device *dev) 3456 { 3457 int ret; 3458 3459 if (drm_core_check_feature(dev, DRIVER_MODESET)) 3460 return; 3461 3462 ret = i915_gem_idle(dev); 3463 if (ret != 0) 3464 DRM_ERROR("failed to idle hardware: %d\n", ret); 3465 } 3466 3467 static int 3468 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align) 3469 { 3470 drm_i915_private_t *dev_priv; 3471 struct drm_i915_gem_phys_object *phys_obj; 3472 int ret; 3473 3474 dev_priv = dev->dev_private; 3475 if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0) 3476 return (0); 3477 3478 phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM, 3479 M_WAITOK | M_ZERO); 3480 3481 phys_obj->id = id; 3482 3483 phys_obj->handle = drm_pci_alloc(dev, size, align, ~0); 3484 if (phys_obj->handle == NULL) { 3485 ret = -ENOMEM; 3486 goto free_obj; 3487 } 3488 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 3489 size / PAGE_SIZE, PAT_WRITE_COMBINING); 3490 3491 dev_priv->mm.phys_objs[id - 1] = phys_obj; 3492 3493 return (0); 3494 3495 free_obj: 3496 drm_free(phys_obj, DRM_I915_GEM); 3497 return (ret); 3498 } 3499 3500 static void 3501 i915_gem_free_phys_object(struct drm_device *dev, int id) 3502 { 3503 drm_i915_private_t *dev_priv; 3504 struct drm_i915_gem_phys_object *phys_obj; 3505 3506 dev_priv = dev->dev_private; 3507 if (dev_priv->mm.phys_objs[id - 1] == NULL) 3508 return; 3509 3510 phys_obj = dev_priv->mm.phys_objs[id - 1]; 3511 if (phys_obj->cur_obj != NULL) 3512 i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 3513 3514 drm_pci_free(dev, phys_obj->handle); 3515 drm_free(phys_obj, DRM_I915_GEM); 3516 dev_priv->mm.phys_objs[id - 1] = NULL; 3517 } 3518 3519 void 3520 i915_gem_free_all_phys_object(struct drm_device *dev) 3521 { 3522 int i; 3523 3524 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 3525 i915_gem_free_phys_object(dev, i); 3526 } 3527 3528 void 3529 i915_gem_detach_phys_object(struct drm_device *dev, 3530 struct drm_i915_gem_object *obj) 3531 { 3532 vm_page_t m; 3533 struct sf_buf *sf; 3534 char *vaddr, *dst; 3535 int i, page_count; 3536 3537 if (obj->phys_obj == NULL) 3538 return; 3539 vaddr = obj->phys_obj->handle->vaddr; 3540 3541 page_count = obj->base.size / PAGE_SIZE; 3542 VM_OBJECT_LOCK(obj->base.vm_obj); 3543 for (i = 0; i < page_count; i++) { 3544 m = i915_gem_wire_page(obj->base.vm_obj, i); 3545 if (m == NULL) 3546 continue; /* XXX */ 3547 3548 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3549 sf = sf_buf_alloc(m); 3550 if (sf != NULL) { 3551 dst = (char *)sf_buf_kva(sf); 3552 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 3553 sf_buf_free(sf); 3554 } 3555 drm_clflush_pages(&m, 1); 3556 3557 VM_OBJECT_LOCK(obj->base.vm_obj); 3558 vm_page_reference(m); 3559 vm_page_dirty(m); 3560 vm_page_busy_wait(m, FALSE, "i915gem"); 3561 vm_page_unwire(m, 0); 3562 vm_page_wakeup(m); 3563 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3564 } 3565 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3566 intel_gtt_chipset_flush(); 3567 3568 obj->phys_obj->cur_obj = NULL; 3569 obj->phys_obj = NULL; 3570 } 3571 3572 int 3573 i915_gem_attach_phys_object(struct drm_device *dev, 3574 struct drm_i915_gem_object *obj, int id, int align) 3575 { 3576 drm_i915_private_t *dev_priv; 3577 vm_page_t m; 3578 struct sf_buf *sf; 3579 char *dst, *src; 3580 int i, page_count, ret; 3581 3582 if (id > I915_MAX_PHYS_OBJECT) 3583 return (-EINVAL); 3584 3585 if (obj->phys_obj != NULL) { 3586 if (obj->phys_obj->id == id) 3587 return (0); 3588 i915_gem_detach_phys_object(dev, obj); 3589 } 3590 3591 dev_priv = dev->dev_private; 3592 if (dev_priv->mm.phys_objs[id - 1] == NULL) { 3593 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align); 3594 if (ret != 0) { 3595 DRM_ERROR("failed to init phys object %d size: %zu\n", 3596 id, obj->base.size); 3597 return (ret); 3598 } 3599 } 3600 3601 /* bind to the object */ 3602 obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 3603 obj->phys_obj->cur_obj = obj; 3604 3605 page_count = obj->base.size / PAGE_SIZE; 3606 3607 VM_OBJECT_LOCK(obj->base.vm_obj); 3608 ret = 0; 3609 for (i = 0; i < page_count; i++) { 3610 m = i915_gem_wire_page(obj->base.vm_obj, i); 3611 if (m == NULL) { 3612 ret = -EIO; 3613 break; 3614 } 3615 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3616 sf = sf_buf_alloc(m); 3617 src = (char *)sf_buf_kva(sf); 3618 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 3619 memcpy(dst, src, PAGE_SIZE); 3620 sf_buf_free(sf); 3621 3622 VM_OBJECT_LOCK(obj->base.vm_obj); 3623 3624 vm_page_reference(m); 3625 vm_page_busy_wait(m, FALSE, "i915gem"); 3626 vm_page_unwire(m, 0); 3627 vm_page_wakeup(m); 3628 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3629 } 3630 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3631 3632 return (0); 3633 } 3634 3635 static int 3636 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, 3637 uint64_t data_ptr, uint64_t offset, uint64_t size, 3638 struct drm_file *file_priv) 3639 { 3640 char *user_data, *vaddr; 3641 int ret; 3642 3643 vaddr = (char *)obj->phys_obj->handle->vaddr + offset; 3644 user_data = (char *)(uintptr_t)data_ptr; 3645 3646 if (copyin_nofault(user_data, vaddr, size) != 0) { 3647 /* The physical object once assigned is fixed for the lifetime 3648 * of the obj, so we can safely drop the lock and continue 3649 * to access vaddr. 3650 */ 3651 DRM_UNLOCK(dev); 3652 ret = -copyin(user_data, vaddr, size); 3653 DRM_LOCK(dev); 3654 if (ret != 0) 3655 return (ret); 3656 } 3657 3658 intel_gtt_chipset_flush(); 3659 return (0); 3660 } 3661 3662 static int 3663 i915_gpu_is_active(struct drm_device *dev) 3664 { 3665 drm_i915_private_t *dev_priv; 3666 3667 dev_priv = dev->dev_private; 3668 return (!list_empty(&dev_priv->mm.flushing_list) || 3669 !list_empty(&dev_priv->mm.active_list)); 3670 } 3671 3672 static void 3673 i915_gem_lowmem(void *arg) 3674 { 3675 struct drm_device *dev; 3676 struct drm_i915_private *dev_priv; 3677 struct drm_i915_gem_object *obj, *next; 3678 int cnt, cnt_fail, cnt_total; 3679 3680 dev = arg; 3681 dev_priv = dev->dev_private; 3682 3683 if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) 3684 return; 3685 3686 rescan: 3687 /* first scan for clean buffers */ 3688 i915_gem_retire_requests(dev); 3689 3690 cnt_total = cnt_fail = cnt = 0; 3691 3692 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3693 mm_list) { 3694 if (i915_gem_object_is_purgeable(obj)) { 3695 if (i915_gem_object_unbind(obj) != 0) 3696 cnt_total++; 3697 } else 3698 cnt_total++; 3699 } 3700 3701 /* second pass, evict/count anything still on the inactive list */ 3702 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3703 mm_list) { 3704 if (i915_gem_object_unbind(obj) == 0) 3705 cnt++; 3706 else 3707 cnt_fail++; 3708 } 3709 3710 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) { 3711 /* 3712 * We are desperate for pages, so as a last resort, wait 3713 * for the GPU to finish and discard whatever we can. 3714 * This has a dramatic impact to reduce the number of 3715 * OOM-killer events whilst running the GPU aggressively. 3716 */ 3717 if (i915_gpu_idle(dev, true) == 0) 3718 goto rescan; 3719 } 3720 DRM_UNLOCK(dev); 3721 } 3722 3723 void 3724 i915_gem_unload(struct drm_device *dev) 3725 { 3726 struct drm_i915_private *dev_priv; 3727 3728 dev_priv = dev->dev_private; 3729 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem); 3730 } 3731